[cig-commits] r21990 - in seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS: DATA EXAMPLES/regional_Greece_small/DATA src/shared src/specfem3D
lefebvre at geodynamics.org
lefebvre at geodynamics.org
Mon May 6 11:42:06 PDT 2013
Author: lefebvre
Date: 2013-05-06 11:42:06 -0700 (Mon, 06 May 2013)
New Revision: 21990
Modified:
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/adios_helpers.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/write_specfem_adios_header.F90
Log:
Read the ADIOS_FOR_FORWARD_ARRAYS flag and branch in forward function to call adios routines.
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file 2013-05-06 18:42:06 UTC (rev 21990)
@@ -2,7 +2,7 @@
# forward or adjoint simulation
SIMULATION_TYPE = 1
NOISE_TOMOGRAPHY = 0 # flag of noise tomography, three steps (1,2,3). If earthquake simulation, set it to 0.
-SAVE_FORWARD = .false. # save last frame of forward simulation or not
+SAVE_FORWARD = .true. # save last frame of forward simulation or not
# number of chunks (1,2,3 or 6)
NCHUNKS = 1
@@ -48,7 +48,7 @@
ABSORBING_CONDITIONS = .true.
# record length in minutes
-RECORD_LENGTH_IN_MINUTES = 15.0d0
+RECORD_LENGTH_IN_MINUTES = 0.1d0
# save AVS or OpenDX movies
MOVIE_SURFACE = .false.
@@ -121,4 +121,5 @@
# set to true to use GPUs
ADIOS_ENABLED = .true.
+ADIOS_FOR_FORWARD_ARRAYS = .true.
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file 2013-05-06 18:42:06 UTC (rev 21990)
@@ -2,7 +2,7 @@
# forward or adjoint simulation
SIMULATION_TYPE = 1
NOISE_TOMOGRAPHY = 0 # flag of noise tomography, three steps (1,2,3). If earthquake simulation, set it to 0.
-SAVE_FORWARD = .false. # save last frame of forward simulation or not
+SAVE_FORWARD = .true. # save last frame of forward simulation or not
# number of chunks (1,2,3 or 6)
NCHUNKS = 1
@@ -48,7 +48,7 @@
ABSORBING_CONDITIONS = .true.
# record length in minutes
-RECORD_LENGTH_IN_MINUTES = 15.0d0
+RECORD_LENGTH_IN_MINUTES = 0.1d0
# save AVS or OpenDX movies
MOVIE_SURFACE = .false.
@@ -121,4 +121,5 @@
# set to true to use GPUs
ADIOS_ENABLED = .true.
+ADIOS_FOR_FORWARD_ARRAYS = .true.
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -320,10 +320,11 @@
end subroutine broadcast_compute_parameters
-!
+
!-------------------------------------------------------------------------------------------------
-!
-
+!> Broadcast the parameters relative to GPU computing settings.
+!! \param myrank The rank of the MPI process in COMM_WORLD
+!! \param GPU_MODE Flag to indicate that GPU computing is ON or OFF.
subroutine broadcast_gpu_parameters(myrank,GPU_MODE)
implicit none
@@ -342,25 +343,32 @@
if( ier /= 0 ) call exit_MPI(myrank,'error broadcasting GPU_MODE')
end subroutine broadcast_gpu_parameters
-!
+
!-------------------------------------------------------------------------------------------------
-!
+!> Broadcast the parameters relative to ADIOS output settings.
+!! \param myrank The rank of the MPI process in COMM_WORLD
+!! \param ADIOS_ENABLED Flag to indicate ADIOS output for seismograms.
+!! \param ADIOS_FOR_FORWARD_ARRAYS Flag to indicate that intermediate and
+!1 forward arrays are stored with the help of ADIOS.
+subroutine broadcast_adios_parameters(myrank,ADIOS_ENABLED, &
+ ADIOS_FOR_FORWARD_ARRAYS)
- subroutine broadcast_adios_parameters(myrank,ADIOS_ENABLED)
-
implicit none
-! standard include of the MPI library
+ ! standard include of the MPI library
include 'mpif.h'
include "constants.h"
include "precision.h"
-
+
integer:: myrank
- logical:: ADIOS_ENABLED
+ logical:: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS
! local parameters
integer :: ier
call MPI_BCAST(ADIOS_ENABLED,1,MPI_LOGICAL,0,MPI_COMM_WORLD,ier)
if( ier /= 0 ) call exit_MPI(myrank,'error broadcasting ADIOS_ENABLED')
+ call MPI_BCAST(ADIOS_FOR_FORWARD_ARRAYS,1,MPI_LOGICAL,0,MPI_COMM_WORLD,ier)
+ if( ier /= 0 ) call exit_MPI(myrank, &
+ 'error broadcasting ADIOS_FOR_FORWARD_ARRAYS')
- end subroutine broadcast_adios_parameters
+end subroutine broadcast_adios_parameters
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -236,7 +236,6 @@
!-------------------------------------------------------------------------------------------------
!
-
subroutine read_adios_enabled(ADIOS_ENABLED)
implicit none
@@ -246,13 +245,28 @@
! initializes flags
ADIOS_ENABLED = .false.
-
! opens file Par_file
call open_parameter_file()
-
call read_value_logical(ADIOS_ENABLED, 'solver.ADIOS_ENABLED')
+ call close_parameter_file()
- ! close parameter file
+ end subroutine read_adios_enabled
+
+!-------------------------------------------------------------------------------------------------
+!
+ subroutine read_adios_for_forward_arrays(ADIOS_FOR_FORWARD_ARRAYS)
+
+ implicit none
+ include "constants.h"
+
+ logical :: ADIOS_FOR_FORWARD_ARRAYS
+
+ ! initializes flags
+ ADIOS_FOR_FORWARD_ARRAYS = .false.
+ ! opens file Par_file
+ call open_parameter_file()
+ call read_value_logical(ADIOS_FOR_FORWARD_ARRAYS, &
+ 'solver.ADIOS_FOR_FORWARD_ARRAYS')
call close_parameter_file()
- end subroutine read_adios_enabled
+ end subroutine read_adios_for_forward_arrays
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in 2013-05-06 18:42:06 UTC (rev 21990)
@@ -249,11 +249,13 @@
# using ADIOS files
ADIOS_OBJECTS= \
+ $O/save_forward_arrays_adios.adios.o \
+ $O/read_forward_arrays_adios.adios.o \
$O/write_specfem_adios_header.adios.o \
$O/adios_manager.adios.o \
$O/adios_helpers.adios.o
ADIOS_STUBS = \
- $O/write_par_header_ADIOS.noadios.o
+ $O/adios_empty_stubs.noadios.o
@COND_ADIOS_TRUE at XSPECFEM_OBJECTS_ADIOS = $(XSPECFEM_OBJECTS_PRE) $(ADIOS_OBJECTS)
@COND_ADIOS_FALSE at XSPECFEM_OBJECTS_ADIOS = $(XSPECFEM_OBJECTS_PRE) $(ADIOS_STUBS)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/adios_helpers.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/adios_helpers.f90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/adios_helpers.f90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -26,11 +26,31 @@
!=====================================================================
+!-------------------------------------------------------------------------------
!> \file adios_helpers.f90
!! \brief Helpers to set up adios features.
!! \author MPBL
+!-------------------------------------------------------------------------------
+!-------------------------------------------------------------------------------
+!> Get the ADIOS error message from an adios error number if there is an error.
+!! \param adios_err The error code considered.
+subroutine check_adios_err(adios_err)
+ use adios_read_mod
+ use specfem_par
+ implicit none
+ integer, intent(in) :: adios_err
+ character(len=1024) :: msg
+ if (adios_err /= 0) then
+ call adios_errmsg(msg)
+ print *, "process: ", myrank, ", error: ", msg
+ stop
+ endif
+end subroutine check_adios_err
+
+
+!-------------------------------------------------------------------------------
!> Define an ADIOS scalar double precision variable and autoincrement
!! the adios group size by (8).
!! \param adios_group The adios group where the variables belongs
@@ -54,6 +74,7 @@
group_size_inc = group_size_inc + 8
end subroutine define_adios_double_scalar
+!-------------------------------------------------------------------------------
!> Define an ADIOS scalar integer variable and autoincrement the adios
!! group size by (4).
!! \param adios_group The adios group where the variables belongs
@@ -73,10 +94,11 @@
integer(kind=8) :: varid ! dummy variable, adios use var name
! adios: 2 == integer(kind=4)
- call adios_define_var (adios_group, name, path, 2, "", "", "", varid)
+ call adios_define_var (adios_group, name, path, adios_integer, "", "", "", varid)
group_size_inc = group_size_inc + 4
end subroutine define_adios_integer_scalar
+!-------------------------------------------------------------------------------
!> Define an ADIOS scalar byte variable and autoincrement the adios
!! group size by (1).
!! \param adios_group The adios group where the variables belongs
@@ -100,6 +122,7 @@
group_size_inc = group_size_inc + 1
end subroutine define_adios_byte_scalar
+!-------------------------------------------------------------------------------
!> Define a local ADIOS array of integers and autoincrement the adios
!! group size by (4 * number of elements).
!! \param adios_group The adios group where the variables belongs
@@ -128,6 +151,7 @@
group_size_inc = group_size_inc + 4*dim
end subroutine define_adios_integer_local_array1D
+!-------------------------------------------------------------------------------
!> Define a local ADIOS array of doubles and autoincrement the adios
!! group size by (8 * number of elements).
!! \param adios_group The adios group where the variables belongs
@@ -156,6 +180,7 @@
group_size_inc = group_size_inc + 8*dim
end subroutine define_adios_double_local_array1D
+!-------------------------------------------------------------------------------
!> Define a local ADIOS string and autoincrement the adios
!! group size by (1 * string's length).
!! \param adios_group The adios group where the variables belongs
@@ -184,3 +209,41 @@
call adios_define_var (adios_group, name, path, 9, "", "", "", varid)
group_size_inc = group_size_inc + 1*length
end subroutine define_adios_string
+
+!-------------------------------------------------------------------------------
+!> Define a global ADIOS 1D real array and autoincrement the adios
+!! group size.
+!! \param adios_group The adios group where the variables belongs
+!! \param array_name The variable to be defined. This is actually the path for
+!! ADIOS. The values are stored in array_name/array
+!! \param len The local dimension of the array.
+!! \param group_size_inc The inout adios group size to increment
+!! with the size of the variable
+!! \note This function define local, global and offset sizes as well as the
+!! array to store the values in.
+subroutine define_adios_global_real_1d_array(adios_group, array_name, &
+ local_dim, group_size_inc)
+ use adios_write_mod
+ implicit none
+ ! Parameters
+ integer(kind=8), intent(in) :: adios_group
+ character(len=*), intent(in) :: array_name
+ integer, intent(in) :: local_dim
+ integer(kind=8), intent(inout) :: group_size_inc
+ ! Variables
+ integer(kind=8) :: var_id
+
+ call define_adios_integer_scalar (adios_group, "local_dim", array_name, &
+ group_size_inc)
+ call define_adios_integer_scalar (adios_group, "global_dim", array_name, &
+ group_size_inc)
+ call define_adios_integer_scalar (adios_group, "offset", array_name, &
+ group_size_inc)
+! call adios_define_var(adios_group, "array", array_name, 6, &
+! "local_dim", "global_dim", "offset", var_id)
+ call adios_define_var(adios_group, "array", array_name, 5, &
+ array_name // "/local_dim", array_name // "/global_dim", &
+ array_name // "/offset", var_id)
+ group_size_inc = group_size_inc + local_dim*4
+end subroutine define_adios_global_real_1d_array
+
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -109,6 +109,7 @@
call read_gpu_mode(GPU_MODE)
! ADIOS_ENABLED: parameter is optional, may not be in the Par_file
call read_adios_enabled(ADIOS_ENABLED)
+ call read_adios_for_forward_arrays(ADIOS_FOR_FORWARD_ARRAYS)
endif
! distributes parameters from master to all processes
@@ -149,7 +150,7 @@
! broadcasts optional GPU_MODE
call broadcast_gpu_parameters(myrank,GPU_MODE)
! broadcasts optional ADIOS_ENABLED
- call broadcast_adios_parameters(myrank,ADIOS_ENABLED)
+ call broadcast_adios_parameters(myrank,ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS)
! get the base pathname for output files
call get_value_string(OUTPUT_FILES, 'OUTPUT_FILES', 'OUTPUT_FILES')
@@ -322,13 +323,13 @@
if (ADIOS_ENABLED) then
call adios_setup()
endif
- if (ADIOS_ENABLED) then
+ !if (ADIOS_ENABLED) then
! TODO use only one ADIOS group to write simulation parameters
! i.e. merge write_solver... write_par_... into
! write_specfem3D_globe_adios_header()
!call write_solver_info_header_ADIOS()
- call write_specfem_header_adios()
- endif
+ !call write_specfem_header_adios()
+ !endif
end subroutine initialize_simulation
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -71,47 +71,51 @@
! read files back from local disk or MT tape system if restart file
if(NUMBER_OF_THIS_RUN > 1) then
- write(outputname,"('dump_all_arrays',i6.6)") myrank
- open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='old',action='read',form='unformatted')
+ if (ADIOS_FOR_FORWARD_ARRAYS) then
+ call read_intermediate_forward_arrays_adios()
+ else
+ write(outputname,"('dump_all_arrays',i6.6)") myrank
+ open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='old',action='read',form='unformatted')
- read(55) displ_crust_mantle
- read(55) veloc_crust_mantle
- read(55) accel_crust_mantle
- read(55) displ_inner_core
- read(55) veloc_inner_core
- read(55) accel_inner_core
- read(55) displ_outer_core
- read(55) veloc_outer_core
- read(55) accel_outer_core
+ read(55) displ_crust_mantle
+ read(55) veloc_crust_mantle
+ read(55) accel_crust_mantle
+ read(55) displ_inner_core
+ read(55) veloc_inner_core
+ read(55) accel_inner_core
+ read(55) displ_outer_core
+ read(55) veloc_outer_core
+ read(55) accel_outer_core
- read(55) epsilondev_xx_crust_mantle
- read(55) epsilondev_yy_crust_mantle
- read(55) epsilondev_xy_crust_mantle
- read(55) epsilondev_xz_crust_mantle
- read(55) epsilondev_yz_crust_mantle
+ read(55) epsilondev_xx_crust_mantle
+ read(55) epsilondev_yy_crust_mantle
+ read(55) epsilondev_xy_crust_mantle
+ read(55) epsilondev_xz_crust_mantle
+ read(55) epsilondev_yz_crust_mantle
- read(55) epsilondev_xx_inner_core
- read(55) epsilondev_yy_inner_core
- read(55) epsilondev_xy_inner_core
- read(55) epsilondev_xz_inner_core
- read(55) epsilondev_yz_inner_core
+ read(55) epsilondev_xx_inner_core
+ read(55) epsilondev_yy_inner_core
+ read(55) epsilondev_xy_inner_core
+ read(55) epsilondev_xz_inner_core
+ read(55) epsilondev_yz_inner_core
- read(55) A_array_rotation
- read(55) B_array_rotation
+ read(55) A_array_rotation
+ read(55) B_array_rotation
- read(55) R_xx_crust_mantle
- read(55) R_yy_crust_mantle
- read(55) R_xy_crust_mantle
- read(55) R_xz_crust_mantle
- read(55) R_yz_crust_mantle
+ read(55) R_xx_crust_mantle
+ read(55) R_yy_crust_mantle
+ read(55) R_xy_crust_mantle
+ read(55) R_xz_crust_mantle
+ read(55) R_yz_crust_mantle
- read(55) R_xx_inner_core
- read(55) R_yy_inner_core
- read(55) R_xy_inner_core
- read(55) R_xz_inner_core
- read(55) R_yz_inner_core
+ read(55) R_xx_inner_core
+ read(55) R_yy_inner_core
+ read(55) R_xy_inner_core
+ read(55) R_xz_inner_core
+ read(55) R_yz_inner_core
- close(55)
+ close(55)
+ endif
endif
! initializes backward/reconstructed arrays
@@ -181,39 +185,63 @@
integer :: ier
character(len=150) outputname
- write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin'
- open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname, &
- status='old',action='read',form='unformatted',iostat=ier)
- if( ier /= 0 ) then
- print*,'error: opening proc_****_save_forward_arrays.bin'
- print*,'path: ',trim(LOCAL_TMP_PATH)//'/'//outputname
- call exit_mpi(myrank,'error open file save_forward_arrays.bin')
- endif
+! if (ADIOS_FOR_FORWARD_ARRAYS) then
+! call read_forward_arrays_adios()
+! else
+ write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin'
+ open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname, &
+ status='old',action='read',form='unformatted',iostat=ier)
+ if( ier /= 0 ) then
+ print*,'error: opening proc_****_save_forward_arrays.bin'
+ print*,'path: ',trim(LOCAL_TMP_PATH)//'/'//outputname
+ call exit_mpi(myrank,'error open file save_forward_arrays.bin')
+ endif
- read(55) b_displ_crust_mantle
- read(55) b_veloc_crust_mantle
- read(55) b_accel_crust_mantle
+ read(55) b_displ_crust_mantle
+ read(55) b_veloc_crust_mantle
+ read(55) b_accel_crust_mantle
- read(55) b_displ_inner_core
- read(55) b_veloc_inner_core
- read(55) b_accel_inner_core
+ read(55) b_displ_inner_core
+ read(55) b_veloc_inner_core
+ read(55) b_accel_inner_core
- read(55) b_displ_outer_core
- read(55) b_veloc_outer_core
- read(55) b_accel_outer_core
+ read(55) b_displ_outer_core
+ read(55) b_veloc_outer_core
+ read(55) b_accel_outer_core
- read(55) b_epsilondev_xx_crust_mantle
- read(55) b_epsilondev_yy_crust_mantle
- read(55) b_epsilondev_xy_crust_mantle
- read(55) b_epsilondev_xz_crust_mantle
- read(55) b_epsilondev_yz_crust_mantle
+ read(55) b_epsilondev_xx_crust_mantle
+ read(55) b_epsilondev_yy_crust_mantle
+ read(55) b_epsilondev_xy_crust_mantle
+ read(55) b_epsilondev_xz_crust_mantle
+ read(55) b_epsilondev_yz_crust_mantle
- read(55) b_epsilondev_xx_inner_core
- read(55) b_epsilondev_yy_inner_core
- read(55) b_epsilondev_xy_inner_core
- read(55) b_epsilondev_xz_inner_core
- read(55) b_epsilondev_yz_inner_core
+ read(55) b_epsilondev_xx_inner_core
+ read(55) b_epsilondev_yy_inner_core
+ read(55) b_epsilondev_xy_inner_core
+ read(55) b_epsilondev_xz_inner_core
+ read(55) b_epsilondev_yz_inner_core
+ if (ROTATION_VAL) then
+ read(55) b_A_array_rotation
+ read(55) b_B_array_rotation
+ endif
+
+ if (ATTENUATION_VAL) then
+ read(55) b_R_xx_crust_mantle
+ read(55) b_R_yy_crust_mantle
+ read(55) b_R_xy_crust_mantle
+ read(55) b_R_xz_crust_mantle
+ read(55) b_R_yz_crust_mantle
+
+ read(55) b_R_xx_inner_core
+ read(55) b_R_yy_inner_core
+ read(55) b_R_xy_inner_core
+ read(55) b_R_xz_inner_core
+ read(55) b_R_yz_inner_core
+ endif
+ close(55)
+! endif ! ADIOS_FOR_FORWARD_ARRAYS
+
! transfers fields onto GPU
if(GPU_MODE) then
call transfer_b_fields_cm_to_device(NDIM*NGLOB_CRUST_MANTLE, &
@@ -237,37 +265,11 @@
b_epsilondev_xx_inner_core,b_epsilondev_yy_inner_core, &
b_epsilondev_xy_inner_core,b_epsilondev_xz_inner_core, &
b_epsilondev_yz_inner_core)
- endif
-
-
- if (ROTATION_VAL) then
- read(55) b_A_array_rotation
- read(55) b_B_array_rotation
- ! transfers to GPU
- if(GPU_MODE) then
+ if (ROTATION_VAL) then
call transfer_b_rotation_to_device(Mesh_pointer,b_A_array_rotation,b_B_array_rotation)
endif
- endif
- if (ATTENUATION_VAL) then
- read(55) b_R_xx_crust_mantle
- read(55) b_R_yy_crust_mantle
- read(55) b_R_xy_crust_mantle
- read(55) b_R_xz_crust_mantle
- read(55) b_R_yz_crust_mantle
-
- read(55) b_R_xx_inner_core
- read(55) b_R_yy_inner_core
- read(55) b_R_xy_inner_core
- read(55) b_R_xz_inner_core
- read(55) b_R_yz_inner_core
-
- ! note: for kernel simulations (SIMULATION_TYPE == 3), attenuation is by default
- ! only mimicking effects on phase shifts, but not on amplitudes.
- ! flag USE_ATTENUATION_MIMIC will have to be set to true in this case.
- !
- ! arrays b_R_xx, ... are not used when USE_ATTENUATION_MIMIC is set
- if(GPU_MODE) then
+ if (ATTENUATION_VAL) then
call transfer_b_rmemory_cm_to_device(Mesh_pointer, &
b_R_xx_crust_mantle,b_R_yy_crust_mantle, &
b_R_xy_crust_mantle,b_R_xz_crust_mantle, &
@@ -276,9 +278,7 @@
b_R_xx_inner_core,b_R_yy_inner_core, &
b_R_xy_inner_core,b_R_xz_inner_core, &
b_R_yz_inner_core)
- endif
-
+ endif
endif
- close(55)
- end subroutine read_forward_arrays
+end subroutine read_forward_arrays
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays.f90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays.f90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -25,7 +25,16 @@
!
!=====================================================================
- subroutine save_forward_arrays()
+!> Save the values of the arrays used for forward simulations.
+!!
+!! Two different options allow to save arrays used for forward simulations.
+!! - setting SAVE_FORWARD to true. That save the last frame.
+!! - setting NUMBERS_OF_RUNS to 2 or 3. This save intermadiate frame to
+!! restart the remaining of the simulation later.
+!! Moreover, one can choose to save these arrays with the help of the ADIOS
+!! library. Set ADIOS_FOR_FORWARD_ARRAYS to true in the Par_file for this
+!! feature.
+subroutine save_forward_arrays()
use specfem_par
use specfem_par_crustmantle
@@ -39,98 +48,108 @@
! save files to local disk or tape system if restart file
if(NUMBER_OF_RUNS > 1 .and. NUMBER_OF_THIS_RUN < NUMBER_OF_RUNS) then
- write(outputname,"('dump_all_arrays',i6.6)") myrank
- open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='unknown',form='unformatted',action='write')
+ if (ADIOS_FOR_FORWARD_ARRAYS) then
+ call save_intermediate_forward_arrays_adios()
+ else
+ write(outputname,"('dump_all_arrays',i6.6)") myrank
+ open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname, &
+ status='unknown',form='unformatted',action='write')
- write(55) displ_crust_mantle
- write(55) veloc_crust_mantle
- write(55) accel_crust_mantle
- write(55) displ_inner_core
- write(55) veloc_inner_core
- write(55) accel_inner_core
- write(55) displ_outer_core
- write(55) veloc_outer_core
- write(55) accel_outer_core
+ write(55) displ_crust_mantle
+ write(55) veloc_crust_mantle
+ write(55) accel_crust_mantle
+ write(55) displ_inner_core
+ write(55) veloc_inner_core
+ write(55) accel_inner_core
+ write(55) displ_outer_core
+ write(55) veloc_outer_core
+ write(55) accel_outer_core
- write(55) epsilondev_xx_crust_mantle
- write(55) epsilondev_yy_crust_mantle
- write(55) epsilondev_xy_crust_mantle
- write(55) epsilondev_xz_crust_mantle
- write(55) epsilondev_yz_crust_mantle
+ write(55) epsilondev_xx_crust_mantle
+ write(55) epsilondev_yy_crust_mantle
+ write(55) epsilondev_xy_crust_mantle
+ write(55) epsilondev_xz_crust_mantle
+ write(55) epsilondev_yz_crust_mantle
- write(55) epsilondev_xx_inner_core
- write(55) epsilondev_yy_inner_core
- write(55) epsilondev_xy_inner_core
- write(55) epsilondev_xz_inner_core
- write(55) epsilondev_yz_inner_core
+ write(55) epsilondev_xx_inner_core
+ write(55) epsilondev_yy_inner_core
+ write(55) epsilondev_xy_inner_core
+ write(55) epsilondev_xz_inner_core
+ write(55) epsilondev_yz_inner_core
- write(55) A_array_rotation
- write(55) B_array_rotation
+ write(55) A_array_rotation
+ write(55) B_array_rotation
- write(55) R_xx_crust_mantle
- write(55) R_yy_crust_mantle
- write(55) R_xy_crust_mantle
- write(55) R_xz_crust_mantle
- write(55) R_yz_crust_mantle
+ write(55) R_xx_crust_mantle
+ write(55) R_yy_crust_mantle
+ write(55) R_xy_crust_mantle
+ write(55) R_xz_crust_mantle
+ write(55) R_yz_crust_mantle
- write(55) R_xx_inner_core
- write(55) R_yy_inner_core
- write(55) R_xy_inner_core
- write(55) R_xz_inner_core
- write(55) R_yz_inner_core
+ write(55) R_xx_inner_core
+ write(55) R_yy_inner_core
+ write(55) R_xy_inner_core
+ write(55) R_xz_inner_core
+ write(55) R_yz_inner_core
- close(55)
+ close(55)
+ endif
endif
! save last frame of the forward simulation
if (SIMULATION_TYPE == 1 .and. SAVE_FORWARD) then
- write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin'
- open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='unknown',form='unformatted',action='write')
+ if (ADIOS_FOR_FORWARD_ARRAYS) then
+ call save_forward_arrays_adios()
+ else
+ write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin'
+ open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='unknown', &
+ form='unformatted',action='write')
- write(55) displ_crust_mantle
- write(55) veloc_crust_mantle
- write(55) accel_crust_mantle
+ write(55) displ_crust_mantle
+ write(55) veloc_crust_mantle
+ write(55) accel_crust_mantle
- write(55) displ_inner_core
- write(55) veloc_inner_core
- write(55) accel_inner_core
+ write(55) displ_inner_core
+ write(55) veloc_inner_core
+ write(55) accel_inner_core
- write(55) displ_outer_core
- write(55) veloc_outer_core
- write(55) accel_outer_core
+ write(55) displ_outer_core
+ write(55) veloc_outer_core
+ write(55) accel_outer_core
- write(55) epsilondev_xx_crust_mantle
- write(55) epsilondev_yy_crust_mantle
- write(55) epsilondev_xy_crust_mantle
- write(55) epsilondev_xz_crust_mantle
- write(55) epsilondev_yz_crust_mantle
+ write(55) epsilondev_xx_crust_mantle
+ write(55) epsilondev_yy_crust_mantle
+ write(55) epsilondev_xy_crust_mantle
+ write(55) epsilondev_xz_crust_mantle
+ write(55) epsilondev_yz_crust_mantle
- write(55) epsilondev_xx_inner_core
- write(55) epsilondev_yy_inner_core
- write(55) epsilondev_xy_inner_core
- write(55) epsilondev_xz_inner_core
- write(55) epsilondev_yz_inner_core
+ write(55) epsilondev_xx_inner_core
+ write(55) epsilondev_yy_inner_core
+ write(55) epsilondev_xy_inner_core
+ write(55) epsilondev_xz_inner_core
+ write(55) epsilondev_yz_inner_core
- if (ROTATION_VAL) then
- write(55) A_array_rotation
- write(55) B_array_rotation
- endif
+ if (ROTATION_VAL) then
+ write(55) A_array_rotation
+ write(55) B_array_rotation
+ endif
- if (ATTENUATION_VAL) then
- write(55) R_xx_crust_mantle
- write(55) R_yy_crust_mantle
- write(55) R_xy_crust_mantle
- write(55) R_xz_crust_mantle
- write(55) R_yz_crust_mantle
+ if (ATTENUATION_VAL) then
+ write(55) R_xx_crust_mantle
+ write(55) R_yy_crust_mantle
+ write(55) R_xy_crust_mantle
+ write(55) R_xz_crust_mantle
+ write(55) R_yz_crust_mantle
- write(55) R_xx_inner_core
- write(55) R_yy_inner_core
- write(55) R_xy_inner_core
- write(55) R_xz_inner_core
- write(55) R_yz_inner_core
+ write(55) R_xx_inner_core
+ write(55) R_yy_inner_core
+ write(55) R_xy_inner_core
+ write(55) R_xz_inner_core
+ write(55) R_yz_inner_core
+ endif
+
+ close(55)
endif
-
- close(55)
endif
- end subroutine save_forward_arrays
+end subroutine save_forward_arrays
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -307,7 +307,7 @@
! ADIOS
!-----------------------------------------------------------------
- logical :: ADIOS_ENABLED
+ logical :: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS
!-----------------------------------------------------------------
! time scheme
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/write_specfem_adios_header.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/write_specfem_adios_header.F90 2013-05-06 18:41:59 UTC (rev 21989)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/write_specfem_adios_header.F90 2013-05-06 18:42:06 UTC (rev 21990)
@@ -224,10 +224,14 @@
pkg_str_len = len_trim(PACKAGE_STRING)
conf_flags_len = len_trim(CONFIGURE_FLAGS)
- call define_adios_integer_scalar (adios_group, "package_string_length", "/solver_info", group_size_inc)
- call define_adios_string (adios_group, "package_name", "/solver_info", pkg_str_len, group_size_inc)
- call define_adios_integer_scalar (adios_group, "conf_flags_len", "/solver_info", group_size_inc)
- call define_adios_string (adios_group, "conf_flags", "/solver_info", conf_flags_len, group_size_inc)
+ call define_adios_integer_scalar (adios_group, "package_string_length", &
+ "/solver_info", group_size_inc)
+ call define_adios_string (adios_group, "package_name", "/solver_info", &
+ pkg_str_len, group_size_inc)
+ call define_adios_integer_scalar (adios_group, "conf_flags_len", &
+ "/solver_info", group_size_inc)
+ call define_adios_string (adios_group, "conf_flags", "/solver_info", &
+ conf_flags_len, group_size_inc)
end subroutine define_solver_info_variables
!> \brief Define ADIOS variable to store values from the Par_file
@@ -243,21 +247,35 @@
integer, intent(in) :: model_length ! for later reading of MODEL
!-- double precision variables
- call define_adios_double_scalar (adios_group, "ANGULAR_WIDTH_XI_IN_DEGREES", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "ANGULAR_WIDTH_ETA_IN_DEGREES", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "CENTER_LONGITUDE_IN_DEGREES", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "CENTER_LATITUDE_IN_DEGREES", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "GAMMA_ROTATION_AZIMUTH", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "HDUR_MOVIE", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "MOVIE_TOP_KM", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "MOVIE_BOTTOM_KM", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "MOVIE_EAST_DEG", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "MOVIE_WEST_DEG", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "MOVIE_NORTH_DEG", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "MOVIE_SOUTH_DEG", "/specfem3D_globe_parameter_file", group_size_inc)
- call define_adios_double_scalar (adios_group, "RECORD_LENGTH_IN_MINUTES", "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "ANGULAR_WIDTH_XI_IN_DEGREES", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "ANGULAR_WIDTH_ETA_IN_DEGREES", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "CENTER_LONGITUDE_IN_DEGREES", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "CENTER_LATITUDE_IN_DEGREES", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "GAMMA_ROTATION_AZIMUTH", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "HDUR_MOVIE", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "MOVIE_TOP_KM", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "MOVIE_BOTTOM_KM", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "MOVIE_EAST_DEG", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "MOVIE_WEST_DEG", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "MOVIE_NORTH_DEG", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "MOVIE_SOUTH_DEG", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_double_scalar (adios_group, "RECORD_LENGTH_IN_MINUTES", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
!-- integer variables
- call define_adios_integer_scalar (adios_group, "NTSTEP_BETWEEN_OUTPUT_SEISMOS", "/specfem3D_globe_parameter_file", group_size_inc)
+ call define_adios_integer_scalar (adios_group, "NTSTEP_BETWEEN_OUTPUT_SEISMOS", &
+ "/specfem3D_globe_parameter_file", group_size_inc)
call define_adios_integer_scalar (adios_group, "NTSTEP_BETWEEN_READ_ADJSRC", "/specfem3D_globe_parameter_file", group_size_inc)
call define_adios_integer_scalar (adios_group, "NTSTEP_BETWEEN_FRAMES", "/specfem3D_globe_parameter_file", group_size_inc)
call define_adios_integer_scalar (adios_group, "NTSTEP_BETWEEN_OUTPUT_INFO", "/specfem3D_globe_parameter_file", group_size_inc)
More information about the CIG-COMMITS
mailing list