[cig-commits] [commit] devel: adios for undo attenuation (88137d7)
cig_noreply at geodynamics.org
cig_noreply at geodynamics.org
Wed Jun 18 05:17:00 PDT 2014
Repository : https://github.com/geodynamics/specfem3d_globe
On branch : devel
Link : https://github.com/geodynamics/specfem3d_globe/compare/23b1b2264384aa7327ddfe20497ffac3565375d3...4c1921a1affc49b468e18ac9949368c811b3513c
>---------------------------------------------------------------
commit 88137d7951763bd77bf5ec32088e4c8d271caa03
Author: Matthieu Lefebvre <ml15 at princeton.edu>
Date: Wed Jun 18 07:44:01 2014 -0400
adios for undo attenuation
>---------------------------------------------------------------
88137d7951763bd77bf5ec32088e4c8d271caa03
DATA | 2 +-
EXAMPLES | 2 +-
m4 | 2 +-
src/shared/adios_method_stubs.c | 4 +
src/shared/broadcast_computed_parameters.f90 | 5 +-
src/shared/read_parameter_file.f90 | 2 +
src/shared/shared_par.f90 | 3 +-
src/specfem3D/read_forward_arrays.f90 | 74 ++++++------
src/specfem3D/read_forward_arrays_adios.F90 | 163 +++++++++++++++++++++++++++
src/specfem3D/read_mesh_databases_adios.f90 | 1 -
src/specfem3D/save_forward_arrays.f90 | 76 +++++++------
src/specfem3D/save_forward_arrays_adios.F90 | 161 +++++++++++++++++++++++++-
12 files changed, 416 insertions(+), 79 deletions(-)
diff --git a/DATA b/DATA
index c01212c..414b996 160000
--- a/DATA
+++ b/DATA
@@ -1 +1 @@
-Subproject commit c01212ccd2afcc315335f8adf27168fe512bc7e2
+Subproject commit 414b9967a8db10aaddbcda86322e651a49a4f9c3
diff --git a/EXAMPLES b/EXAMPLES
index f686425..76f64d8 160000
--- a/EXAMPLES
+++ b/EXAMPLES
@@ -1 +1 @@
-Subproject commit f6864259d91072dda7f040916359143beb52916a
+Subproject commit 76f64d82951c78d0d06f7fd39d65768cef8cebc5
diff --git a/m4 b/m4
index b58c6fb..09ca47b 160000
--- a/m4
+++ b/m4
@@ -1 +1 @@
-Subproject commit b58c6fb1cdfc650eeca28274913dc239888a09d4
+Subproject commit 09ca47be4e87a6133563b7e8e32974a6da7c5d58
diff --git a/src/shared/adios_method_stubs.c b/src/shared/adios_method_stubs.c
index bb2ae78..31f9110 100644
--- a/src/shared/adios_method_stubs.c
+++ b/src/shared/adios_method_stubs.c
@@ -82,6 +82,8 @@ void FC_FUNC_(read_attenuation_adios,READ_ATTENUATION_ADIOS)(void){}
void FC_FUNC_(read_forward_arrays_adios,READ_FORWARD_ARRAYS_ADIOS)(void){}
+void FC_FUNC_(read_forward_arrays_undoatt_adios,READ_FORWARD_ARRAYS_UNDOATT_ADIOS)(void){}
+
void FC_FUNC_(read_intermediate_forward_arrays_adios,READ_INTERMEDIATE_FORWARD_ARRAYS_ADIOS)(void){}
void FC_FUNC_(read_mesh_databases_coupling_adios,READ_MESH_DATABASES_COUPLING_ADIOS)(void){}
@@ -96,6 +98,8 @@ void FC_FUNC_(read_mesh_databases_stacey_adios,READ_MESH_DATABASES_STACEY_ADIOS)
void FC_FUNC_(save_forward_arrays_adios,SAVE_FORWARD_ARRAYS_ADIOS)(void){}
+void FC_FUNC_(save_forward_arrays_undoatt_adios,SAVE_FORWARD_ARRAYS_UNDOATT_ADIOS)(void){}
+
void FC_FUNC_(save_intermediate_forward_arrays_adios,SAVE_INTERMEDIATE_FORWARD_ARRAYS_ADIOS)(void){}
void FC_FUNC_(perform_write_adios_kernels,PERFORM_WRITE_ADIOS_KERNELS)(void){}
diff --git a/src/shared/broadcast_computed_parameters.f90 b/src/shared/broadcast_computed_parameters.f90
index da54635..5c84748 100644
--- a/src/shared/broadcast_computed_parameters.f90
+++ b/src/shared/broadcast_computed_parameters.f90
@@ -39,7 +39,7 @@
integer, parameter :: nparam_i = 44
integer, dimension(nparam_i) :: bcast_integer
- integer, parameter :: nparam_l = 57
+ integer, parameter :: nparam_l = 58
logical, dimension(nparam_l) :: bcast_logical
integer, parameter :: nparam_dp = 34
@@ -99,7 +99,7 @@
ADIOS_ENABLED,ADIOS_FOR_FORWARD_ARRAYS, &
ADIOS_FOR_MPI_ARRAYS,ADIOS_FOR_ARRAYS_SOLVER, &
ADIOS_FOR_SOLVER_MESHFILES,ADIOS_FOR_AVS_DX,&
- ADIOS_FOR_KERNELS,ADIOS_FOR_MODELS &
+ ADIOS_FOR_KERNELS,ADIOS_FOR_MODELS, ADIOS_FOR_UNDO_ATTENUATION &
/)
bcast_double_precision = (/ &
@@ -265,6 +265,7 @@
ADIOS_FOR_AVS_DX = bcast_logical(55)
ADIOS_FOR_KERNELS = bcast_logical(56)
ADIOS_FOR_MODELS = bcast_logical(57)
+ ADIOS_FOR_UNDO_ATTENUATION = bcast_logical(58)
! double precisions
DT = bcast_double_precision(1)
diff --git a/src/shared/read_parameter_file.f90 b/src/shared/read_parameter_file.f90
index ee27e54..2fad2e9 100644
--- a/src/shared/read_parameter_file.f90
+++ b/src/shared/read_parameter_file.f90
@@ -255,6 +255,8 @@
if (ierr /= 0) stop 'an error occurred while reading the parameter file: ADIOS_FOR_KERNELS'
call read_value_logical(ADIOS_FOR_MODELS, 'ADIOS_FOR_MODELS', ierr)
if (ierr /= 0) stop 'an error occurred while reading the parameter file: ADIOS_FOR_MODELS'
+ call read_value_logical(ADIOS_FOR_UNDO_ATTENUATION, 'ADIOS_FOR_UNDO_ATTENUATION', ierr)
+ if (ierr /= 0) stop 'an error occurred while reading the parameter file: ADIOS_FOR_UNDO_ATT'
! closes parameter file
call close_parameter_file()
diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90
index 3ca7e36..2fd38b4 100644
--- a/src/shared/shared_par.f90
+++ b/src/shared/shared_par.f90
@@ -112,7 +112,8 @@
! adios file output
logical :: ADIOS_ENABLED,ADIOS_FOR_FORWARD_ARRAYS, &
ADIOS_FOR_MPI_ARRAYS,ADIOS_FOR_ARRAYS_SOLVER,ADIOS_FOR_SOLVER_MESHFILES, &
- ADIOS_FOR_AVS_DX,ADIOS_FOR_KERNELS,ADIOS_FOR_MODELS
+ ADIOS_FOR_AVS_DX,ADIOS_FOR_KERNELS,ADIOS_FOR_MODELS, &
+ ADIOS_FOR_UNDO_ATTENUATION
end module shared_input_parameters
diff --git a/src/specfem3D/read_forward_arrays.f90 b/src/specfem3D/read_forward_arrays.f90
index 991c01a..4f282de 100644
--- a/src/specfem3D/read_forward_arrays.f90
+++ b/src/specfem3D/read_forward_arrays.f90
@@ -264,50 +264,54 @@
! current subset iteration
iteration_on_subset_tmp = NSTEP/NT_DUMP_ATTENUATION - iteration_on_subset + 1
- ! reads in saved wavefield
- write(outputname,'(a,i6.6,a,i6.6,a)') 'proc',myrank,'_save_frame_at',iteration_on_subset_tmp,'.bin'
+ if (ADIOS_ENABLED .and. ADIOS_FOR_UNDO_ATTENUATION) then
+ call read_forward_arrays_undoatt_adios(iteration_on_subset_tmp)
+ else
+ ! reads in saved wavefield
+ write(outputname,'(a,i6.6,a,i6.6,a)') 'proc',myrank,'_save_frame_at',iteration_on_subset_tmp,'.bin'
+
+ ! debug
+ !if(myrank == 0 ) print*,'reading in: ',trim(LOCAL_PATH)//'/'//outputname, NSTEP/NT_DUMP_ATTENUATION,iteration_on_subset
- ! debug
- !if(myrank == 0 ) print*,'reading in: ',trim(LOCAL_PATH)//'/'//outputname, NSTEP/NT_DUMP_ATTENUATION,iteration_on_subset
+ ! opens corresponding snapshot file for reading
+ open(unit=IIN,file=trim(LOCAL_PATH)//'/'//outputname, &
+ status='old',action='read',form='unformatted',iostat=ier)
+ if( ier /= 0 ) call exit_MPI(myrank,'error opening file proc***_save_frame_at** for reading')
- ! opens corresponding snapshot file for reading
- open(unit=IIN,file=trim(LOCAL_PATH)//'/'//outputname, &
- status='old',action='read',form='unformatted',iostat=ier)
- if( ier /= 0 ) call exit_MPI(myrank,'error opening file proc***_save_frame_at** for reading')
+ read(IIN) b_displ_crust_mantle
+ read(IIN) b_veloc_crust_mantle
+ read(IIN) b_accel_crust_mantle
- read(IIN) b_displ_crust_mantle
- read(IIN) b_veloc_crust_mantle
- read(IIN) b_accel_crust_mantle
+ read(IIN) b_displ_inner_core
+ read(IIN) b_veloc_inner_core
+ read(IIN) b_accel_inner_core
- read(IIN) b_displ_inner_core
- read(IIN) b_veloc_inner_core
- read(IIN) b_accel_inner_core
+ read(IIN) b_displ_outer_core
+ read(IIN) b_veloc_outer_core
+ read(IIN) b_accel_outer_core
- read(IIN) b_displ_outer_core
- read(IIN) b_veloc_outer_core
- read(IIN) b_accel_outer_core
+ if (ROTATION_VAL) then
+ read(IIN) b_A_array_rotation
+ read(IIN) b_B_array_rotation
+ endif
- if (ROTATION_VAL) then
- read(IIN) b_A_array_rotation
- read(IIN) b_B_array_rotation
- endif
+ if (ATTENUATION_VAL) then
+ read(IIN) b_R_xx_crust_mantle
+ read(IIN) b_R_yy_crust_mantle
+ read(IIN) b_R_xy_crust_mantle
+ read(IIN) b_R_xz_crust_mantle
+ read(IIN) b_R_yz_crust_mantle
+
+ read(IIN) b_R_xx_inner_core
+ read(IIN) b_R_yy_inner_core
+ read(IIN) b_R_xy_inner_core
+ read(IIN) b_R_xz_inner_core
+ read(IIN) b_R_yz_inner_core
+ endif
- if (ATTENUATION_VAL) then
- read(IIN) b_R_xx_crust_mantle
- read(IIN) b_R_yy_crust_mantle
- read(IIN) b_R_xy_crust_mantle
- read(IIN) b_R_xz_crust_mantle
- read(IIN) b_R_yz_crust_mantle
-
- read(IIN) b_R_xx_inner_core
- read(IIN) b_R_yy_inner_core
- read(IIN) b_R_xy_inner_core
- read(IIN) b_R_xz_inner_core
- read(IIN) b_R_yz_inner_core
+ close(IIN)
endif
- close(IIN)
-
! transfers fields onto GPU
if( GPU_MODE ) then
! transfers initialized wavefields to GPU device
diff --git a/src/specfem3D/read_forward_arrays_adios.F90 b/src/specfem3D/read_forward_arrays_adios.F90
index 7c12ec9..4af380c 100644
--- a/src/specfem3D/read_forward_arrays_adios.F90
+++ b/src/specfem3D/read_forward_arrays_adios.F90
@@ -433,3 +433,166 @@ subroutine read_forward_arrays_adios()
call synchronize_all_comm(comm)
end subroutine read_forward_arrays_adios
+
+
+!-------------------------------------------------------------------------------
+!> \brief Read forward arrays for undo attenuation from an ADIOS file.
+subroutine read_forward_arrays_undoatt_adios(iteration_on_subset_tmp)
+
+ ! External imports
+ use adios_read_mod
+ ! Internal imports
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ use adios_helpers_mod, only: check_adios_err
+
+ implicit none
+ ! Arguments
+ integer, intent(in) :: iteration_on_subset_tmp
+ ! Local parameters
+ integer :: comm, ierr
+ character(len=256) :: file_name
+ integer :: local_dim
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_handle, sel
+ integer(kind=8), dimension(1) :: start, count
+
+
+ write(file_name,'(a,a,i6.6,a)') trim(LOCAL_TMP_PATH), '/save_frame_at', &
+ iteration_on_subset_tmp,'.bp'
+
+ call world_duplicate(comm)
+
+ ! opens adios file
+ call adios_read_init_method (ADIOS_READ_METHOD_BP, comm, &
+ "verbose=1", adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_open_file (adios_handle, file_name, 0, comm, ierr)
+ call check_adios_err(myrank,adios_err)
+
+ ! reads in arrays
+ local_dim = NDIM * NGLOB_CRUST_MANTLE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_crust_mantle/array", 0, 1, &
+ b_displ_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_crust_mantle/array", 0, 1, &
+ b_veloc_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_crust_mantle/array", 0, 1, &
+ b_accel_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ ! NOTE: perform reads before changing selection, otherwise it will segfault
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = NDIM * NGLOB_INNER_CORE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_inner_core/array", 0, 1, &
+ b_displ_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_inner_core/array", 0, 1, &
+ b_veloc_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_inner_core/array", 0, 1, &
+ b_accel_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = NGLOB_OUTER_CORE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_outer_core/array", 0, 1, &
+ b_displ_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_outer_core/array", 0, 1, &
+ b_veloc_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_outer_core/array", 0, 1, &
+ b_accel_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ if (ROTATION_VAL) then
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_OUTER_CORE_ROTATION
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "A_array_rotation/array", 0, 1, &
+ b_A_array_rotation, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "B_array_rotation/array", 0, 1, &
+ b_B_array_rotation, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ if (ATTENUATION_VAL) then
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_CRUST_MANTLE_ATTENUATION
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "R_xx_crust_mantle/array", 0, 1, &
+ b_R_xx_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yy_crust_mantle/array", 0, 1, &
+ b_R_yy_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xy_crust_mantle/array", 0, 1, &
+ b_R_xy_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xz_crust_mantle/array", 0, 1, &
+ b_R_xz_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yz_crust_mantle/array", 0, 1, &
+ b_R_yz_crust_mantle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_INNER_CORE_ATTENUATION
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "R_xx_inner_core/array", 0, 1, &
+ b_R_xx_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yy_inner_core/array", 0, 1, &
+ b_R_yy_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xy_inner_core/array", 0, 1, &
+ b_R_xy_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xz_inner_core/array", 0, 1, &
+ b_R_xz_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yz_inner_core/array", 0, 1, &
+ b_R_yz_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ ! Close ADIOS handler to the restart file.
+ call adios_selection_delete(sel)
+ call adios_read_close(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_finalize_method(ADIOS_READ_METHOD_BP, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call synchronize_all_comm(comm)
+
+
+end subroutine read_forward_arrays_undoatt_adios
diff --git a/src/specfem3D/read_mesh_databases_adios.f90 b/src/specfem3D/read_mesh_databases_adios.f90
index 6c4cbd7..0d9ea67 100644
--- a/src/specfem3D/read_mesh_databases_adios.f90
+++ b/src/specfem3D/read_mesh_databases_adios.f90
@@ -627,7 +627,6 @@ subroutine read_mesh_databases_addressing_adios()
implicit none
! local parameters
- integer, dimension(NCHUNKS_VAL,0:NPROC_XI_VAL-1,0:NPROC_ETA_VAL-1) :: addressing
integer, dimension(0:NPROCTOT_VAL-1) :: ichunk_slice,iproc_xi_slice,iproc_eta_slice
integer :: ierr,iproc,iproc_read,iproc_xi,iproc_eta
diff --git a/src/specfem3D/save_forward_arrays.f90 b/src/specfem3D/save_forward_arrays.f90
index ec6be31..139c95a 100644
--- a/src/specfem3D/save_forward_arrays.f90
+++ b/src/specfem3D/save_forward_arrays.f90
@@ -203,52 +203,56 @@
endif
endif
- ! current subset iteration
- iteration_on_subset_tmp = iteration_on_subset
- ! saves frame of the forward simulation
+ if( ADIOS_ENABLED .and. ADIOS_FOR_UNDO_ATTENUATION) then
+ call save_forward_arrays_undoatt_adios()
+ else
+ ! current subset iteration
+ iteration_on_subset_tmp = iteration_on_subset
- write(outputname,'(a,i6.6,a,i6.6,a)') 'proc',myrank,'_save_frame_at',iteration_on_subset_tmp,'.bin'
+ ! saves frame of the forward simulation
- ! debug
- !if(myrank == 0 ) print*,'saving in: ',trim(LOCAL_PATH)//'/'//outputname, NSTEP/NT_DUMP_ATTENUATION
+ write(outputname,'(a,i6.6,a,i6.6,a)') 'proc',myrank,'_save_frame_at',iteration_on_subset_tmp,'.bin'
- open(unit=IOUT,file=trim(LOCAL_PATH)//'/'//outputname, &
- status='unknown',form='unformatted',action='write',iostat=ier)
- if( ier /= 0 ) call exit_MPI(myrank,'error opening file proc***_save_frame_at** for writing')
+ ! debug
+ !if(myrank == 0 ) print*,'saving in: ',trim(LOCAL_PATH)//'/'//outputname, NSTEP/NT_DUMP_ATTENUATION
+ open(unit=IOUT,file=trim(LOCAL_PATH)//'/'//outputname, &
+ status='unknown',form='unformatted',action='write',iostat=ier)
+ if( ier /= 0 ) call exit_MPI(myrank,'error opening file proc***_save_frame_at** for writing')
- write(IOUT) displ_crust_mantle
- write(IOUT) veloc_crust_mantle
- write(IOUT) accel_crust_mantle
+ write(IOUT) displ_crust_mantle
+ write(IOUT) veloc_crust_mantle
+ write(IOUT) accel_crust_mantle
- write(IOUT) displ_inner_core
- write(IOUT) veloc_inner_core
- write(IOUT) accel_inner_core
+ write(IOUT) displ_inner_core
+ write(IOUT) veloc_inner_core
+ write(IOUT) accel_inner_core
- write(IOUT) displ_outer_core
- write(IOUT) veloc_outer_core
- write(IOUT) accel_outer_core
+ write(IOUT) displ_outer_core
+ write(IOUT) veloc_outer_core
+ write(IOUT) accel_outer_core
- if (ROTATION_VAL) then
- write(IOUT) A_array_rotation
- write(IOUT) B_array_rotation
- endif
+ if (ROTATION_VAL) then
+ write(IOUT) A_array_rotation
+ write(IOUT) B_array_rotation
+ endif
- if (ATTENUATION_VAL) then
- write(IOUT) R_xx_crust_mantle
- write(IOUT) R_yy_crust_mantle
- write(IOUT) R_xy_crust_mantle
- write(IOUT) R_xz_crust_mantle
- write(IOUT) R_yz_crust_mantle
-
- write(IOUT) R_xx_inner_core
- write(IOUT) R_yy_inner_core
- write(IOUT) R_xy_inner_core
- write(IOUT) R_xz_inner_core
- write(IOUT) R_yz_inner_core
- endif
+ if (ATTENUATION_VAL) then
+ write(IOUT) R_xx_crust_mantle
+ write(IOUT) R_yy_crust_mantle
+ write(IOUT) R_xy_crust_mantle
+ write(IOUT) R_xz_crust_mantle
+ write(IOUT) R_yz_crust_mantle
+
+ write(IOUT) R_xx_inner_core
+ write(IOUT) R_yy_inner_core
+ write(IOUT) R_xy_inner_core
+ write(IOUT) R_xz_inner_core
+ write(IOUT) R_yz_inner_core
+ endif
- close(IOUT)
+ close(IOUT)
+ endif
end subroutine save_forward_arrays_undoatt
diff --git a/src/specfem3D/save_forward_arrays_adios.F90 b/src/specfem3D/save_forward_arrays_adios.F90
index f133dfe..6300fe4 100644
--- a/src/specfem3D/save_forward_arrays_adios.F90
+++ b/src/specfem3D/save_forward_arrays_adios.F90
@@ -72,6 +72,7 @@ subroutine save_intermediate_forward_arrays_adios()
! Define ADIOS variables
call define_common_forward_arrays_adios(adios_group, group_size_inc)
+ call define_epsilon_forward_arrays_adios(adios_group, group_size_inc)
call define_rotation_forward_arrays_adios(adios_group, group_size_inc)
call define_attenuation_forward_arrays_adios(adios_group, group_size_inc)
@@ -83,6 +84,7 @@ subroutine save_intermediate_forward_arrays_adios()
! Issue the order to write the previously defined variable to the ADIOS file
call write_common_forward_arrays_adios(adios_handle)
+ call write_epsilon_forward_arrays_adios(adios_handle)
call write_rotation_forward_arrays_adios(adios_handle)
call write_attenuation_forward_arrays_adios(adios_handle)
! Reset the path to its original value to avoid bugs.
@@ -135,6 +137,7 @@ subroutine save_forward_arrays_adios()
! Define ADIOS variables
call define_common_forward_arrays_adios(adios_group, group_size_inc)
+ call define_epsilon_forward_arrays_adios(adios_group, group_size_inc)
! TODO check following:
! conditional definition of vars seem to mess with the group size,
! even if the variables are conditionally written.
@@ -153,6 +156,7 @@ subroutine save_forward_arrays_adios()
! Issue the order to write the previously defined variable to the ADIOS file
call write_common_forward_arrays_adios(adios_handle)
+ call write_epsilon_forward_arrays_adios(adios_handle)
if (ROTATION_VAL) then
call write_rotation_forward_arrays_adios(adios_handle)
endif
@@ -238,6 +242,55 @@ subroutine define_common_forward_arrays_adios(adios_group, group_size_inc)
"", STRINGIFY_VAR(epsilondev_yz_inner_core))
end subroutine define_common_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Define ADIOS forward arrays that are always dumped
+!! except for undo attenuation.
+!! \param adios_group The adios group where the variables belongs
+!! \param group_size_inc The inout adios group size to increment
+!! with the size of the variable
+subroutine define_epsilon_forward_arrays_adios(adios_group, group_size_inc)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ use adios_helpers_mod
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_group
+ integer(kind=8), intent(inout) :: group_size_inc
+
+ integer :: local_dim
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_CRUST_MANTLE_STR_OR_ATT
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_xx_crust_mantle))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_yy_crust_mantle))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_xy_crust_mantle))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_xz_crust_mantle))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_yz_crust_mantle))
+
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_INNER_CORE_STR_OR_ATT
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_xx_inner_core))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_yy_inner_core))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_xy_inner_core))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_xz_inner_core))
+ call define_adios_global_array1D(adios_group, group_size_inc, local_dim, &
+ "", STRINGIFY_VAR(epsilondev_yz_inner_core))
+end subroutine define_epsilon_forward_arrays_adios
+
+
!-------------------------------------------------------------------------------
!> Define ADIOS forward arrays that are dumped if ROTATION is true.
!! \param adios_group The adios group where the variables belongs
@@ -360,6 +413,33 @@ subroutine write_common_forward_arrays_adios(adios_handle)
local_dim, STRINGIFY_VAR(veloc_outer_core))
call write_adios_global_1d_array(adios_handle, myrank, sizeprocs, &
local_dim, STRINGIFY_VAR(accel_outer_core))
+end subroutine write_common_forward_arrays_adios
+
+
+!-------------------------------------------------------------------------------
+!> Schedule writes of ADIOS forward arrays that are always dumped.
+!! \param adios_handle The handle to the adios bp file
+!! \param group_size_inc The number of MPI processes involved in the writing
+subroutine write_epsilon_forward_arrays_adios(adios_handle)
+
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ use adios_helpers_mod
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_handle
+
+ integer :: local_dim !, adios_err
+
+ integer :: sizeprocs
+
+ ! number of MPI processes
+ call world_size(sizeprocs)
local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_CRUST_MANTLE_STR_OR_ATT
call write_adios_global_1d_array(adios_handle, myrank, sizeprocs, &
@@ -384,7 +464,8 @@ subroutine write_common_forward_arrays_adios(adios_handle)
local_dim, STRINGIFY_VAR(epsilondev_xz_inner_core))
call write_adios_global_1d_array(adios_handle, myrank, sizeprocs, &
local_dim, STRINGIFY_VAR(epsilondev_yz_inner_core))
-end subroutine write_common_forward_arrays_adios
+end subroutine write_epsilon_forward_arrays_adios
+
!-------------------------------------------------------------------------------
!> Schedule writes of ADIOS forward arrays that are dumped if ROTATION is true.
@@ -492,3 +573,81 @@ end subroutine write_attenuation_forward_arrays_adios
!call check_adios_err(myrank,adios_err)
!end subroutine write_1D_global_array_adios_dims
+!-------------------------------------------------------------------------------
+!> \brief Write selected forward arrays in an ADIOS file.
+!!
+!! This subroutine is only used for forward simulations when
+!! SAVE_FORWARD is set to .true. It dumps the same arrays than
+!! save_intermediate_forward_arrays_adios() except than some arrays
+!! are only dumped if ROTATION and ATTENUATION are set to .true.
+subroutine save_forward_arrays_undoatt_adios()
+
+ ! External imports
+ use adios_write_mod
+ ! Internal imports
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ use adios_helpers_mod, only: check_adios_err
+
+
+ implicit none
+ ! Local parameters
+ integer :: comm, iteration_on_subset_tmp
+ character(len=150) :: outputname
+ integer(kind=8) :: group_size_inc
+ ! ADIOS variables
+ character(len=150) :: group_name
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle
+ integer(kind=8) :: adios_totalsize
+
+ ! current subset iteration
+ iteration_on_subset_tmp = iteration_on_subset
+
+ write(outputname,'(a, a, i6.6, a)') trim(LOCAL_PATH), '/save_frame_at', &
+ iteration_on_subset_tmp,'.bp'
+ write(group_name, '(a, i6)') "SPECFEM3D_GLOBE_FORWARD_ARRAYS", &
+ iteration_on_subset_tmp
+
+ call world_duplicate(comm)
+
+ group_size_inc = 0
+
+ call adios_declare_group(adios_group, group_name, "iter", 1, adios_err)
+ call adios_select_method(adios_group, "MPI", "", "", adios_err)
+
+ ! Define ADIOS variables
+ call define_common_forward_arrays_adios(adios_group, group_size_inc)
+ ! TODO check following:
+ ! conditional definition of vars seem to mess with the group size,
+ ! even if the variables are conditionally written.
+ !if (ROTATION_VAL) then
+ call define_rotation_forward_arrays_adios(adios_group, group_size_inc)
+ !endif
+ !if (ATTENUATION_VAL) then
+ call define_attenuation_forward_arrays_adios(adios_group, group_size_inc)
+ !endif
+
+ ! Open an ADIOS handler to the restart file.
+ call adios_open (adios_handle, group_name, & outputname, "w", comm, adios_err)
+ call adios_group_size (adios_handle, group_size_inc, &
+ adios_totalsize, adios_err)
+
+ ! Issue the order to write the previously defined variable to the ADIOS file
+ call write_common_forward_arrays_adios(adios_handle)
+ if (ROTATION_VAL) then
+ call write_rotation_forward_arrays_adios(adios_handle)
+ endif
+ if (ATTENUATION_VAL) then
+ call write_attenuation_forward_arrays_adios(adios_handle)
+ endif
+ ! Reset the path to its original value to avoid bugs.
+ call adios_set_path (adios_handle, "", adios_err)
+
+ ! Close ADIOS handler to the restart file.
+ call adios_close(adios_handle, adios_err)
+
+end subroutine save_forward_arrays_undoatt_adios
More information about the CIG-COMMITS
mailing list