[cig-commits] r21989 - seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D
lefebvre at geodynamics.org
lefebvre at geodynamics.org
Mon May 6 11:41:59 PDT 2013
Author: lefebvre
Date: 2013-05-06 11:41:59 -0700 (Mon, 06 May 2013)
New Revision: 21989
Added:
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90
Log:
Forward arrays (intermediate and finals) are written with ADIOS. intermediate forward arrays are read with adios. ADIOS_FOR_FORWARD_ARRAYS flags addded to Par_file.
new file: ../../src/specfem3D/read_forward_arrays_adios.F90
new file: ../../src/specfem3D/save_forward_arrays_adios.F90
modified: ../../DATA/Par_file
modified: DATA/Par_file
modified: ../../src/shared/broadcast_compute_parameters.f90
modified: ../../src/shared/read_parameter_file.f90
modified: ../../src/specfem3D/Makefile.in
modified: ../../src/specfem3D/adios_helpers.f90
modified: ../../src/specfem3D/initialize_simulation.f90
modified: ../../src/specfem3D/read_forward_arrays.f90
modified: ../../src/specfem3D/save_forward_arrays.f90
modified: ../../src/specfem3D/specfem3D_par.F90
modified: ../../src/specfem3D/write_specfem_adios_header.F90
Added: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90 (rev 0)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_forward_arrays_adios.F90 2013-05-06 18:41:59 UTC (rev 21989)
@@ -0,0 +1,234 @@
+!=====================================================================
+!
+! S p e c f e m 3 D G l o b e V e r s i o n 5 . 1
+! --------------------------------------------------
+!
+! Main authors: Dimitri Komatitsch and Jeroen Tromp
+! Princeton University, USA
+! and University of Pau / CNRS / INRIA, France
+! (c) Princeton University / California Institute of Technology and University of Pau / CNRS / INRIA
+! April 2011
+!
+! This program is free software; you can redistribute it and/or modify
+! it under the terms of the GNU General Public License as published by
+! the Free Software Foundation; either version 2 of the License, or
+! (at your option) any later version.
+!
+! This program is distributed in the hope that it will be useful,
+! but WITHOUT ANY WARRANTY; without even the implied warranty of
+! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+! GNU General Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along
+! with this program; if not, write to the Free Software Foundation, Inc.,
+! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+!
+!=====================================================================
+
+!-------------------------------------------------------------------------------
+!> \file read_forward_arrays_adios.F90
+!! \brief Read saved forward arrays with the help of the ADIOS library.
+!-------------------------------------------------------------------------------
+
+!-------------------------------------------------------------------------------
+!> \brief Read forward arrays from an ADIOS file.
+subroutine read_intermediate_forward_arrays_adios()
+ ! External imports
+ use mpi
+ use adios_read_mod
+ ! Internal imports
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+ ! Local parameters
+ integer :: sizeprocs, comm, ierr
+ character(len=150) :: file_name
+ integer(kind=8) :: group_size_inc
+ integer :: local_dim, global_dim, offset
+! integer, parameter :: num_arrays = 9 ! TODO correct number
+! character(len=256), dimension(num_arrays) :: local_dims1, local_dims2, &
+! global_dims1, global_dims2, offsets1, offsets2, array_name
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid, sel
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+ integer :: vars_count, attrs_count, current_step, last_step, vsteps
+ character(len=128), dimension(:), allocatable :: adios_names
+ integer(kind=8), dimension(1) :: start, count
+
+
+ file_name = "OUTPUT_FILES/dump_all_arrays_adios.bp"
+ call world_size(sizeprocs)
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+
+ call adios_read_init_method (ADIOS_READ_METHOD_BP, comm, &
+ "verbose=1", adios_err)
+ call check_adios_err(adios_err)
+ call adios_read_open_file (adios_handle, file_name, 0, comm, ierr)
+ call check_adios_err(adios_err)
+
+
+ local_dim = NDIM * NGLOB_CRUST_MANTLE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_crust_mantle/array", 0, 1, &
+ displ_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_crust_mantle/array", 0, 1, &
+ veloc_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_crust_mantle/array", 0, 1, &
+ accel_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ ! NOTE: perform reads before changing selection, otherwise it will segfault
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NDIM * NGLOB_INNER_CORE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_inner_core/array", 0, 1, &
+ displ_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_inner_core/array", 0, 1, &
+ veloc_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_inner_core/array", 0, 1, &
+ accel_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLOB_OUTER_CORE
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "displ_outer_core/array", 0, 1, &
+ displ_outer_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "veloc_outer_core/array", 0, 1, &
+ veloc_outer_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "accel_outer_core/array", 0, 1, &
+ accel_outer_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_CRUST_MANTLE_STR_OR_ATT
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xx_crust_mantle/array",&
+ 0, 1, epsilondev_xx_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yy_crust_mantle/array",&
+ 0, 1, epsilondev_yy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xy_crust_mantle/array",&
+ 0, 1, epsilondev_xy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xz_crust_mantle/array",&
+ 0, 1, epsilondev_xz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yz_crust_mantle/array",&
+ 0, 1, epsilondev_yz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_INNER_CORE_STR_OR_ATT
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xx_inner_core/array",&
+ 0, 1, epsilondev_xx_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yy_inner_core/array",&
+ 0, 1, epsilondev_yy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xy_inner_core/array",&
+ 0, 1, epsilondev_xy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_xz_inner_core/array",&
+ 0, 1, epsilondev_xz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "epsilondev_yz_inner_core/array",&
+ 0, 1, epsilondev_yz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_OUTER_CORE_ROTATION
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "A_array_rotation/array", 0, 1, &
+ A_array_rotation, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "B_array_rotation/array", 0, 1, &
+ B_array_rotation, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_CRUST_MANTLE_ATTENUAT
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "R_xx_crust_mantle/array", 0, 1, &
+ R_xx_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yy_crust_mantle/array", 0, 1, &
+ R_yy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xy_crust_mantle/array", 0, 1, &
+ R_xy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xz_crust_mantle/array", 0, 1, &
+ R_xz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yz_crust_mantle/array", 0, 1, &
+ R_yz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_INNER_CORE_ATTENUATION
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "R_xx_inner_core/array", 0, 1, &
+ R_xx_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yy_inner_core/array", 0, 1, &
+ R_yy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xy_inner_core/array", 0, 1, &
+ R_xy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_xz_inner_core/array", 0, 1, &
+ R_xz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_schedule_read(adios_handle, sel, "R_yz_inner_core/array", 0, 1, &
+ R_yz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+
+ ! Close ADIOS handler to the restart file.
+ call adios_selection_delete(sel)
+ call adios_read_close(adios_handle, adios_err)
+ call check_adios_err(adios_err)
+ call adios_read_finalize_method(ADIOS_READ_METHOD_BP, adios_err)
+ call check_adios_err(adios_err)
+
+ call MPI_Barrier(comm, ierr)
+
+end subroutine read_intermediate_forward_arrays_adios
+
Added: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90 (rev 0)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/save_forward_arrays_adios.F90 2013-05-06 18:41:59 UTC (rev 21989)
@@ -0,0 +1,593 @@
+!=====================================================================
+!
+! S p e c f e m 3 D G l o b e V e r s i o n 5 . 1
+! --------------------------------------------------
+!
+! Main authors: Dimitri Komatitsch and Jeroen Tromp
+! Princeton University, USA
+! and University of Pau / CNRS / INRIA, France
+! (c) Princeton University / California Institute of Technology and University of Pau / CNRS / INRIA
+! April 2011
+!
+! This program is free software; you can redistribute it and/or modify
+! it under the terms of the GNU General Public License as published by
+! the Free Software Foundation; either version 2 of the License, or
+! (at your option) any later version.
+!
+! This program is distributed in the hope that it will be useful,
+! but WITHOUT ANY WARRANTY; without even the implied warranty of
+! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+! GNU General Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along
+! with this program; if not, write to the Free Software Foundation, Inc.,
+! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+!
+!=====================================================================
+
+!-------------------------------------------------------------------------------
+!> \file save_forward_arrays_adios.F90
+!! \brief Save forward arrays with the help of the ADIOS library.
+!! \author MPBL
+!-------------------------------------------------------------------------------
+
+!-------------------------------------------------------------------------------
+!> \brief Write intermediate forward arrays in an ADIOS file.
+!!
+!! This subroutine is only used when NUMBER_OF_RUNS > 1 and
+!! NUMBER_OF_THIS_RUN < NUMBER_OF_RUNS.
+subroutine save_intermediate_forward_arrays_adios()
+ ! External imports
+ use mpi
+ use adios_write_mod
+ ! Internal imports
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+ ! Local parameters
+ integer :: sizeprocs, comm, ierr
+ character(len=150) :: outputname
+ integer(kind=8) :: group_size_inc
+ integer :: local_dim, global_dim, offset
+! integer, parameter :: num_arrays = 9 ! TODO correct number
+! character(len=256), dimension(num_arrays) :: local_dims1, local_dims2, &
+! global_dims1, global_dims2, offsets1, offsets2, array_name
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+
+ outputname = "OUTPUT_FILES/dump_all_arrays_adios.bp"
+ call world_size(sizeprocs)
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+ group_size_inc = 0
+ call adios_declare_group(adios_group, "SPECFEM3D_GLOBE_FORWARD_ARRAYS", &
+ "", 1, adios_err)
+! call check_adios_err(adios_err)
+ call adios_select_method(adios_group, "MPI", "", "", adios_err)
+! call check_adios_err(adios_err)
+
+ ! Define ADIOS variables
+ call define_common_forward_arrays_adios(adios_group, group_size_inc)
+ call define_rotation_forward_arrays_adios(adios_group, group_size_inc)
+ call define_attenuation_forward_arrays_adios(adios_group, group_size_inc)
+
+ ! Open an ADIOS handler to the restart file.
+ call adios_open (adios_handle, "SPECFEM3D_GLOBE_FORWARD_ARRAYS", &
+ outputname, "w", comm, adios_err);
+! call check_adios_err(adios_err)
+ call adios_group_size (adios_handle, group_size_inc, &
+ adios_totalsize, adios_err)
+! call check_adios_err(adios_err)
+
+ ! Issue the order to write the previously defined variable to the ADIOS file
+ call write_common_forward_arrays_adios(adios_handle, sizeprocs)
+ call write_rotation_forward_arrays_adios(adios_handle, sizeprocs)
+ call write_attenuation_forward_arrays_adios(adios_handle, sizeprocs)
+ ! Reset the path to its original value to avoid bugs.
+ call adios_set_path (adios_handle, "", adios_err)
+! call check_adios_err(adios_err)
+
+ ! Close ADIOS handler to the restart file.
+ call adios_close(adios_handle, adios_err)
+! call check_adios_err(adios_err)
+end subroutine save_intermediate_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> \brief Write selected forward arrays in an ADIOS file.
+!!
+!! This subroutine is only used for forward simualtions when
+!! SAVE_FORWARD is set to .true. It dumps the same arrays than
+!! save_intermediate_forward_arrays_adios() execpt than some arrays
+!! are only dumped if ROTATION and ATTENUATION are set to .true.
+subroutine save_forward_arrays_adios()
+ ! External imports
+ use mpi
+ use adios_write_mod
+ ! Internal imports
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+ ! Local parameters
+ integer :: sizeprocs, comm, ierr
+ character(len=150) :: outputname
+ integer(kind=8) :: group_size_inc
+ integer :: local_dim, global_dim, offset
+! integer, parameter :: num_arrays = 9 ! TODO correct number
+! character(len=256), dimension(num_arrays) :: local_dims1, local_dims2, &
+! global_dims1, global_dims2, offsets1, offsets2, array_name
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+
+ outputname = "OUTPUT_FILES/save_forward_arrays.bp"
+ call world_size(sizeprocs)
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+ group_size_inc = 0
+
+ call adios_declare_group(adios_group, "SPECFEM3D_GLOBE_FORWARD_ARRAYS", &
+ "", 1, adios_err)
+! call check_adios_err(adios_err)
+ call adios_select_method(adios_group, "MPI", "", "", adios_err)
+! call check_adios_err(adios_err)
+
+ ! Define ADIOS variables
+ call define_common_forward_arrays_adios(adios_group, group_size_inc)
+ if (ROTATION_VAL) then
+ call define_rotation_forward_arrays_adios(adios_group, group_size_inc)
+ endif
+ if (ATTENUATION_VAL) then
+ call define_attenuation_forward_arrays_adios(adios_group, group_size_inc)
+ endif
+
+ ! Open an ADIOS handler to the restart file.
+ call adios_open (adios_handle, "SPECFEM3D_GLOBE_FORWARD_ARRAYS", &
+ outputname, "w", comm, adios_err);
+! call check_adios_err(adios_err)
+ call adios_group_size (adios_handle, group_size_inc, &
+ adios_totalsize, adios_err)
+! call check_adios_err(adios_err)
+
+ ! Issue the order to write the previously defined variable to the ADIOS file
+ call write_common_forward_arrays_adios(adios_handle, sizeprocs)
+ if (ROTATION_VAL) then
+ call write_rotation_forward_arrays_adios(adios_handle, sizeprocs)
+ endif
+ if (ATTENUATION_VAL) then
+ call write_attenuation_forward_arrays_adios(adios_handle, sizeprocs)
+ endif
+ ! Reset the path to its original value to avoid bugs.
+ call adios_set_path (adios_handle, "", adios_err)
+! call check_adios_err(adios_err)
+
+ ! Close ADIOS handler to the restart file.
+ call adios_close(adios_handle, adios_err)
+! call check_adios_err(adios_err)
+end subroutine save_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Define ADIOS forward arrays that are always dumped.
+!! \param adios_group The adios group where the variables belongs
+!! \param group_size_inc The inout adios group size to increment
+!! with the size of the variable
+subroutine define_common_forward_arrays_adios(adios_group, group_size_inc)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_group
+ integer(kind=8), intent(inout) :: group_size_inc
+
+ integer :: local_dim
+
+ local_dim = NDIM * NGLOB_CRUST_MANTLE
+ call define_adios_global_real_1d_array(adios_group, "displ_crust_mantle", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "veloc_crust_mantle", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "accel_crust_mantle", &
+ local_dim, group_size_inc)
+
+ local_dim = NDIM * NGLOB_INNER_CORE
+ call define_adios_global_real_1d_array(adios_group, "displ_inner_core", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "veloc_inner_core", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "accel_inner_core", &
+ local_dim, group_size_inc)
+
+ local_dim = NGLOB_OUTER_CORE
+ call define_adios_global_real_1d_array(adios_group, "displ_outer_core", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "veloc_outer_core", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "accel_outer_core", &
+ local_dim, group_size_inc)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_CRUST_MANTLE_STR_OR_ATT
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_xx_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_yy_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_xy_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_xz_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_yz_crust_mantle", local_dim, group_size_inc)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_INNER_CORE_STR_OR_ATT
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_xx_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_yy_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_xy_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_xz_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "epsilondev_yz_inner_core", local_dim, group_size_inc)
+end subroutine define_common_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Define ADIOS forward arrays that are dumped if ROTATION is true.
+!! \param adios_group The adios group where the variables belongs
+!! \param group_size_inc The inout adios group size to increment
+!! with the size of the variable
+subroutine define_rotation_forward_arrays_adios(adios_group, group_size_inc)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_group
+ integer(kind=8), intent(inout) :: group_size_inc
+
+ integer :: local_dim
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_OUTER_CORE_ROTATION
+ call define_adios_global_real_1d_array(adios_group, &
+ "A_array_rotation", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "B_array_rotation", local_dim, group_size_inc)
+end subroutine define_rotation_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Define ADIOS forward arrays that are dumped if ATTENUATION is true.
+!! \param adios_group The adios group where the variables belongs
+!! \param group_size_inc The inout adios group size to increment
+!! with the size of the variable
+subroutine define_attenuation_forward_arrays_adios(adios_group, group_size_inc)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_group
+ integer(kind=8), intent(inout) :: group_size_inc
+
+ integer :: local_dim
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_CRUST_MANTLE_ATTENUAT
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_xx_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_yy_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_xy_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_xz_crust_mantle", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_yz_crust_mantle", local_dim, group_size_inc)
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_INNER_CORE_ATTENUATION
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_xx_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_yy_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_xy_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_xz_inner_core", local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, &
+ "R_yz_inner_core", local_dim, group_size_inc)
+end subroutine define_attenuation_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Schedule writes of ADIOS forward arrays that are always dumped.
+!! \param adios_handle The handle to the adios bp file
+!! \param group_size_inc The number of MPI processes involved in the writting
+subroutine write_common_forward_arrays_adios(adios_handle, sizeprocs)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_handle
+ integer, intent(in) :: sizeprocs
+
+ integer :: local_dim, adios_err
+
+ local_dim = NDIM * NGLOB_CRUST_MANTLE
+ call adios_set_path (adios_handle, "displ_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", displ_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "veloc_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", veloc_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "accel_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", accel_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NDIM * NGLOB_INNER_CORE
+ call adios_set_path (adios_handle, "displ_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", displ_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "veloc_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", veloc_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "accel_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", accel_inner_core, adios_err)
+ call check_adios_err(adios_err)
+ call adios_set_path (adios_handle, "", adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLOB_OUTER_CORE
+ call adios_set_path (adios_handle, "displ_outer_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", displ_outer_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "veloc_outer_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", veloc_outer_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "accel_outer_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", accel_outer_core, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_CRUST_MANTLE_STR_OR_ATT
+ call adios_set_path (adios_handle, "epsilondev_xx_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_xx_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_yy_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_yy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_xy_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_xy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_xz_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_xz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_yz_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_yz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_INNER_CORE_STR_OR_ATT
+ call adios_set_path (adios_handle, "epsilondev_xx_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_xx_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_yy_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_yy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_xy_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_xy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_xz_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_xz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "epsilondev_yz_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", epsilondev_yz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+end subroutine write_common_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Schedule writes of ADIOS forward arrays that are dumped if ROTATION is true.
+!! \param adios_handle The handle to the adios bp file
+!! \param group_size_inc The number of MPI processes involved in the writting
+subroutine write_rotation_forward_arrays_adios(adios_handle, sizeprocs)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_handle
+ integer, intent(in) :: sizeprocs
+
+ integer :: local_dim, adios_err
+
+ local_dim = NGLLX * NGLLY * NGLLZ * NSPEC_OUTER_CORE_ROTATION
+ call adios_set_path (adios_handle, "A_array_rotation", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", A_array_rotation, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "B_array_rotation", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", B_array_rotation, adios_err)
+ call check_adios_err(adios_err)
+end subroutine write_rotation_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Schedule writes of ADIOS forward arrays that are dumped if ATTENUATION
+!! is true.
+!! \param adios_handle The handle to the adios bp file
+!! \param group_size_inc The number of MPI processes involved in the writting
+subroutine write_attenuation_forward_arrays_adios(adios_handle, sizeprocs)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_handle
+ integer, intent(in) :: sizeprocs
+
+ integer :: local_dim, adios_err
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_CRUST_MANTLE_ATTENUAT
+ call adios_set_path (adios_handle, "R_xx_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_xx_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_yy_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_yy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_xy_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_xy_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_xz_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_xz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_yz_crust_mantle", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_yz_crust_mantle, adios_err)
+ call check_adios_err(adios_err)
+
+ local_dim = N_SLS*NGLLX*NGLLY*NGLLZ*NSPEC_INNER_CORE_ATTENUATION
+ call adios_set_path (adios_handle, "R_xx_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_xx_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_yy_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_yy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_xy_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_xy_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_xz_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_xz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+
+ call adios_set_path (adios_handle, "R_yz_inner_core", adios_err)
+ call check_adios_err(adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", R_yz_inner_core, adios_err)
+ call check_adios_err(adios_err)
+end subroutine write_attenuation_forward_arrays_adios
+
+!-------------------------------------------------------------------------------
+!> Write local, global and offset dimensions to ADIOS
+!! \param adios_handle Handle to the adios file
+!! \param local_dim Number of elements to be written by one process
+!! \param sizeprocs Number of MPI processes
+subroutine write_1D_global_array_adios_dims(adios_handle, local_dim, sizeprocs)
+ use adios_write_mod
+ use specfem_par
+ use specfem_par_crustmantle
+ use specfem_par_innercore
+ use specfem_par_outercore
+
+ implicit none
+
+ integer(kind=8), intent(in) :: adios_handle
+ integer, intent(in) :: sizeprocs, local_dim
+
+ integer :: adios_err
+
+ call adios_write(adios_handle, "local_dim", local_dim, adios_err)
+ call check_adios_err(adios_err)
+ call adios_write(adios_handle, "global_dim", local_dim*sizeprocs, adios_err)
+ call check_adios_err(adios_err)
+ call adios_write(adios_handle, "offset", local_dim*myrank, adios_err)
+ call check_adios_err(adios_err)
+end subroutine write_1D_global_array_adios_dims
+
More information about the CIG-COMMITS
mailing list