[cig-commits] r21991 - seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D
lefebvre at geodynamics.org
lefebvre at geodynamics.org
Mon May 6 11:42:12 PDT 2013
Author: lefebvre
Date: 2013-05-06 11:42:12 -0700 (Mon, 06 May 2013)
New Revision: 21991
Modified:
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90
Log:
Ability to read forward arrays for adjoint simulations. No conditional groupsize definition for saving forward arrays otherwise it messes with the buffersize.
modified: ../../src/specfem3D/read_forward_arrays.f90
modified: ../../src/specfem3D/read_forward_arrays_adios.F90
modified: ../../src/specfem3D/save_forward_arrays_adios.F90
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90 2013-05-06 18:42:06 UTC (rev 21990)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays.f90 2013-05-06 18:42:12 UTC (rev 21991)
@@ -150,7 +150,7 @@
b_B_array_rotation = 0._CUSTOM_REAL
endif
- if (ATTENUATION_VAL) then
+ if(ATTENUATION_VAL) then
b_R_xx_crust_mantle = 0._CUSTOM_REAL
b_R_yy_crust_mantle = 0._CUSTOM_REAL
b_R_xy_crust_mantle = 0._CUSTOM_REAL
@@ -185,9 +185,9 @@
integer :: ier
character(len=150) outputname
-! if (ADIOS_FOR_FORWARD_ARRAYS) then
-! call read_forward_arrays_adios()
-! else
+ if (ADIOS_FOR_FORWARD_ARRAYS) then
+ call read_forward_arrays_adios()
+ else
write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin'
open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname, &
status='old',action='read',form='unformatted',iostat=ier)
@@ -240,7 +240,7 @@
read(55) b_R_yz_inner_core
endif
close(55)
-! endif ! ADIOS_FOR_FORWARD_ARRAYS
+ endif ! ADIOS_FOR_FORWARD_ARRAYS
! transfers fields onto GPU
if(GPU_MODE) then
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90 2013-05-06 18:42:06 UTC (rev 21990)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90 2013-05-06 18:42:12 UTC (rev 21991)
@@ -32,6 +32,9 @@
!-------------------------------------------------------------------------------
!> \brief Read forward arrays from an ADIOS file.
+!> \note read_intermediate_forward_arrays_adios()
+!! and read_forward_arrays_adios() are not factorized, because
+!> the latest read the bp file in "b_" prefixed arrays
subroutine read_intermediate_forward_arrays_adios()
! External imports
use mpi
@@ -232,3 +235,211 @@
end subroutine read_intermediate_forward_arrays_adios
+!-------------------------------------------------------------------------------
+!> \brief Read forward arrays from an ADIOS file.
+!> \note read_intermediate_forward_arrays_adios()
+!! and read_forward_arrays_adios() are not factorized, because
+!> the latest read the bp file in "b_" prefixed arrays
+subroutine read_forward_arrays_adios()
+ ! External imports
+ use mpi
+ use adios_read_mod
+ ! Internal imports
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+ ! Local parameters
+ integer :: sizeprocs, comm, ierr
+ character(len=150) :: file_name
+ integer(kind=8) :: group_size_inc
+ integer :: local_dim, global_dim, offset
+! integer, parameter :: num_arrays = 9 ! TODO correct number
+! character(len=256), dimension(num_arrays) :: local_dims1, local_dims2, &
+! global_dims1, global_dims2, offsets1, offsets2, array_name
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid, sel
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+ integer :: vars_count, attrs_count, current_step, last_step, vsteps
+ character(len=128), dimension(:), allocatable :: adios_names
+ integer(kind=8), dimension(1) :: start, count
+
+
+ file_name = "OUTPUT_FILES/save_forward_arrays.bp"
+ call world_size(sizeprocs)
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+
+ call adios_read_init_method (ADIOS_READ_METHOD_BP, comm, &
+ "verbose=1", adios_err)
+ call check_adios_err(adios_err)
+ call adios_read_open_file (adios_handle, file_name, 0, comm, ierr)
+ call check_adios_err(adios_err)
+
+
+ local_dim = NDIM * NGLOB_CRUST_MANTLE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_crust_mantle/array", 0, 1, &
+ b_displ_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_crust_mantle/array", 0, 1, &
+ b_veloc_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_crust_mantle/array", 0, 1, &
+ b_accel_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ ! NOTE: perform reads before changing selection, otherwise it will segfault
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NDIM * NGLOB_INNER_CORE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_inner_core/array", 0, 1, &
+ b_displ_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_inner_core/array", 0, 1, &
+ b_veloc_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_inner_core/array", 0, 1, &
+ b_accel_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLOB_OUTER_CORE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_outer_core/array", 0, 1, &
+ b_displ_outer_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_outer_core/array", 0, 1, &
+ b_veloc_outer_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_outer_core/array", 0, 1, &
+ b_accel_outer_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_CRUST_MANTLE_STR_OR_ATT
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xx_crust_mantle/array",&
+ 0, 1, b_epsilondev_xx_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yy_crust_mantle/array",&
+ 0, 1, b_epsilondev_yy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xy_crust_mantle/array",&
+ 0, 1, b_epsilondev_xy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xz_crust_mantle/array",&
+ 0, 1, b_epsilondev_xz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yz_crust_mantle/array",&
+ 0, 1, b_epsilondev_yz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_INNER_CORE_STR_OR_ATT
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xx_inner_core/array",&
+ 0, 1, b_epsilondev_xx_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yy_inner_core/array",&
+ 0, 1, b_epsilondev_yy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xy_inner_core/array",&
+ 0, 1, b_epsilondev_xy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xz_inner_core/array",&
+ 0, 1, b_epsilondev_xz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yz_inner_core/array",&
+ 0, 1, b_epsilondev_yz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ if (ROTATION_VAL) then
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_OUTER_CORE_ROTATION
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "A_array_rotation/array", 0, 1, &
+ b_A_array_rotation, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "B_array_rotation/array", 0, 1, &
+ b_B_array_rotation, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+ endif
+
+ if (ATTENUATION_VAL) then
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_CRUST_MANTLE_ATTENUAT
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "R_xx_crust_mantle/array", 0, 1, &
+ b_R_xx_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yy_crust_mantle/array", 0, 1, &
+ b_R_yy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xy_crust_mantle/array", 0, 1, &
+ b_R_xy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xz_crust_mantle/array", 0, 1, &
+ b_R_xz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yz_crust_mantle/array", 0, 1, &
+ b_R_yz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_INNER_CORE_ATTENUATION
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "R_xx_inner_core/array", 0, 1, &
+ b_R_xx_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yy_inner_core/array", 0, 1, &
+ b_R_yy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xy_inner_core/array", 0, 1, &
+ b_R_xy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xz_inner_core/array", 0, 1, &
+ b_R_xz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yz_inner_core/array", 0, 1, &
+ b_R_yz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+ endif
+
+ ! Close ADIOS handler to the restart file.
+ call adios_selection_delete(sel)
+ call adios_read_close(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_read_finalize_method(ADIOS_READ_METHOD_BP, adios_err)
+ call check_adios_err(adios_err)
+
+ call MPI_Barrier(comm, ierr)
+
+end subroutine read_forward_arrays_adios
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90 2013-05-06 18:42:06 UTC (rev 21990)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90 2013-05-06 18:42:12 UTC (rev 21991)
@@ -140,12 +140,14 @@
! Define ADIOS variables
call define_common_forward_arrays_adios(adios_group, group_size_inc)
- if (ROTATION_VAL) then
+ ! conditional definition of vars seem to mess with the group size,
+ ! even if the variables are conditionnaly written.
+! if (ROTATION_VAL) then
call define_rotation_forward_arrays_adios(adios_group, group_size_inc)
- endif
- if (ATTENUATION_VAL) then
+! endif
+! if (ATTENUATION_VAL) then
call define_attenuation_forward_arrays_adios(adios_group, group_size_inc)
- endif
+! endif
! Open an ADIOS handler to the restart file.
call adios_open (adios_handle, "SPECFEM3D_GLOBE_FORWARD_ARRAYS", &
More information about the CIG-COMMITS
mailing list