[cig-commits] r21993 - in seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS: . DATA EXAMPLES/regional_Greece_small/DATA src/meshfem3D src/shared src/specfem3D
lefebvre at geodynamics.org
lefebvre at geodynamics.org
Mon May 6 11:42:27 PDT 2013
Author: lefebvre
Date: 2013-05-06 11:42:27 -0700 (Mon, 06 May 2013)
New Revision: 21993
Added:
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_arrays_solver_adios.f90
Modified:
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/.gitignore
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/create_regions_mesh.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/initialize_mesher.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/meshfem3D_par.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver_adios.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/adios_helpers.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases_adios.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90
Log:
procxxx_regy_solver_data.bin files saved and read with ADIOS as regy_solver_data.bp. New files added to Makefile.in.
Modifications for meshfem3d/ and specfem3d/.
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/.gitignore
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/.gitignore 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/.gitignore 2013-05-06 18:42:27 UTC (rev 21993)
@@ -1,5 +1,7 @@
# List files that should not be committed
.*swp
.*swo
+.*swn
+*~
*.mod
obj/*
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/DATA/Par_file 2013-05-06 18:42:27 UTC (rev 21993)
@@ -123,4 +123,5 @@
ADIOS_ENABLED = .true.
ADIOS_FOR_FORWARD_ARRAYS = .true.
ADIOS_FOR_MPI_ARRAYS = .true.
+ADIOS_FOR_ARRAYS_SOLVER = .true.
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/EXAMPLES/regional_Greece_small/DATA/Par_file 2013-05-06 18:42:27 UTC (rev 21993)
@@ -48,7 +48,7 @@
ABSORBING_CONDITIONS = .true.
# record length in minutes
-RECORD_LENGTH_IN_MINUTES = 0.1d0
+RECORD_LENGTH_IN_MINUTES = 5.1d0
# save AVS or OpenDX movies
MOVIE_SURFACE = .false.
@@ -123,4 +123,4 @@
ADIOS_ENABLED = .true.
ADIOS_FOR_FORWARD_ARRAYS = .true.
ADIOS_FOR_MPI_ARRAYS = .true.
-
+ADIOS_FOR_ARRAYS_SOLVER = .true.
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/create_regions_mesh.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/create_regions_mesh.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/create_regions_mesh.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -39,12 +39,14 @@
use meshfem3D_par,only: &
ibool,idoubling,xstore,ystore,zstore, &
IMAIN,volume_total,myrank,LOCAL_PATH, &
- IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE,IFLAG_IN_FICTITIOUS_CUBE, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE, &
+ IFLAG_IN_FICTITIOUS_CUBE, &
NCHUNKS,SAVE_MESH_FILES,ABSORBING_CONDITIONS, &
R_CENTRAL_CUBE,RICB,RCMB, &
MAX_NUMBER_OF_MESH_LAYERS,MAX_NUM_REGIONS,NB_SQUARE_CORNERS, &
NGLOB1D_RADIAL_CORNER, &
- NGLOB2DMAX_XMIN_XMAX,NGLOB2DMAX_YMIN_YMAX
+ NGLOB2DMAX_XMIN_XMAX,NGLOB2DMAX_YMIN_YMAX, &
+ ADIOS_FOR_ARRAYS_SOLVER
use meshfem3D_models_par,only: &
SAVE_BOUNDARY_MESH,SUPPRESS_CRUSTAL_MESH,REGIONAL_MOHO_MESH, &
@@ -334,9 +336,13 @@
call flush_IMAIN()
endif
! saves mesh and model parameters
- call save_arrays_solver(myrank,nspec,nglob,idoubling,ibool, &
- iregion_code,xstore,ystore,zstore, &
- NSPEC2D_TOP,NSPEC2D_BOTTOM)
+ if (ADIOS_FOR_ARRAYS_SOLVER) then
+ call save_arrays_solver_adios(myrank,nspec,nglob,idoubling,ibool, &
+ iregion_code,xstore,ystore,zstore, NSPEC2D_TOP,NSPEC2D_BOTTOM)
+ else
+ call save_arrays_solver(myrank,nspec,nglob,idoubling,ibool, &
+ iregion_code,xstore,ystore,zstore, NSPEC2D_TOP,NSPEC2D_BOTTOM)
+ endif
! frees memory
deallocate(rmassx,rmassy,rmassz)
@@ -366,15 +372,13 @@
! compute volume, bottom and top area of that part of the slice
call compute_volumes(volume_local,area_local_bottom,area_local_top, &
- nspec,wxgll,wygll,wzgll,xixstore,xiystore,xizstore, &
- etaxstore,etaystore,etazstore,gammaxstore,gammaystore,gammazstore, &
- NSPEC2D_BOTTOM,jacobian2D_bottom,NSPEC2D_TOP,jacobian2D_top)
+ nspec,wxgll,wygll,wzgll,xixstore,xiystore,xizstore, &
+ etaxstore,etaystore,etazstore,gammaxstore,gammaystore,gammazstore, &
+ NSPEC2D_BOTTOM,jacobian2D_bottom,NSPEC2D_TOP,jacobian2D_top)
! computes total area and volume
- call compute_area(myrank,NCHUNKS,iregion_code, &
- area_local_bottom,area_local_top,&
- volume_local,volume_total, &
- RCMB,RICB,R_CENTRAL_CUBE)
+ call compute_area(myrank,NCHUNKS,iregion_code, area_local_bottom, &
+ area_local_top, volume_local,volume_total, RCMB,RICB,R_CENTRAL_CUBE)
! create AVS or DX mesh data for the slices
if(SAVE_MESH_FILES) then
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/initialize_mesher.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/initialize_mesher.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/initialize_mesher.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -104,7 +104,7 @@
! ADIOS_ENABLED: parameter is optional, may not be in the Par_file
call read_adios_parameters(ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, &
- ADIOS_FOR_MPI_ARRAYS)
+ ADIOS_FOR_MPI_ARRAYS, ADIOS_FOR_ARRAYS_SOLVER)
endif
@@ -143,7 +143,7 @@
ATTENUATION,ATTENUATION_NEW,ATTENUATION_3D,ANISOTROPIC_INNER_CORE,NOISE_TOMOGRAPHY)
! broadcasts optional ADIOS_ENABLED
call broadcast_adios_parameters(myrank,ADIOS_ENABLED, &
- ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS)
+ ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS, ADIOS_FOR_ARRAYS_SOLVER)
! check that the code is running with the requested number of processes
if(sizeprocs /= NPROCTOT) call exit_MPI(myrank,'wrong number of MPI processes')
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/meshfem3D_par.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/meshfem3D_par.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/meshfem3D_par.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -268,7 +268,8 @@
! ADIOS
!-----------------------------------------------------------------
- logical :: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS
+ logical :: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS, &
+ ADIOS_FOR_ARRAYS_SOLVER
end module meshfem3D_par
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -489,7 +489,6 @@
select case( iregion_code )
case( IREGION_CRUST_MANTLE )
- print *, myrank, " region crust mantle"
! crust mantle
if (ADIOS_FOR_MPI_ARRAYS) then
call save_MPI_arrays_adios(myrank,IREGION_CRUST_MANTLE,LOCAL_PATH, &
@@ -500,7 +499,6 @@
num_phase_ispec_crust_mantle,phase_ispec_inner_crust_mantle, &
num_colors_outer_crust_mantle,num_colors_inner_crust_mantle, &
num_elem_colors_crust_mantle)
- print *, myrank, "adios crust passed"
else
call save_MPI_arrays(myrank,IREGION_CRUST_MANTLE,LOCAL_PATH, &
num_interfaces_crust_mantle,max_nibool_interfaces_cm, &
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver_adios.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver_adios.F90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/meshfem3D/save_arrays_solver_adios.F90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -25,10 +25,13 @@
!
!=====================================================================
- subroutine save_arrays_solver_adios(myrank,nspec,nglob,idoubling,ibool, &
+subroutine save_arrays_solver_adios(myrank,nspec,nglob,idoubling,ibool, &
iregion_code,xstore,ystore,zstore, &
NSPEC2D_TOP,NSPEC2D_BOTTOM)
+ use mpi
+ use adios_write_mod
+
use constants
use meshfem3D_models_par,only: &
@@ -36,7 +39,7 @@
ANISOTROPIC_INNER_CORE,ATTENUATION
use meshfem3D_par,only: &
- NCHUNKS,ABSORBING_CONDITIONS,SAVE_MESH_FILES
+ NCHUNKS,ABSORBING_CONDITIONS,SAVE_MESH_FILES, LOCAL_PATH
use create_regions_mesh_par2,only: &
xixstore,xiystore,xizstore,etaxstore,etaystore,etazstore, &
@@ -53,7 +56,7 @@
rho_vp,rho_vs, &
nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, &
ispec_is_tiso,tau_s,T_c_source,tau_e_store,Qmu_store, &
- prname
+ prname, nspec_actually, nspec_ani, nspec_stacey, nglob_xy, nglob_oceans
implicit none
@@ -74,27 +77,211 @@
! local parameters
integer :: i,j,k,ispec,iglob,ier
- real(kind=CUSTOM_REAL),dimension(:),allocatable :: tmp_array
+ real(kind=CUSTOM_REAL),dimension(:),allocatable :: tmp_array_x, &
+ tmp_array_y, tmp_array_z
- ! mesh databases
- open(unit=27,file=prname(1:len_trim(prname))//'solver_data.bin', &
- status='unknown',form='unformatted',action='write',iostat=ier)
- if( ier /= 0 ) call exit_mpi(myrank,'error opening solver_data.bin file')
+ ! local parameters
+ character(len=150) :: reg_name, outputname, group_name
+ integer :: ierr, sizeprocs, comm, local_dim
+ integer(kind=8) :: group_size_inc
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+ ! create the name for the database of the current slide and region
+ call create_name_database_adios(reg_name,iregion_code,LOCAL_PATH)
+
+ outputname = trim(reg_name) // "solver_data.bp"
+
+ write(group_name,"('SPECFEM3D_GLOBE_ARRAYS_SOLVER_reg',i1)") iregion_code
+ call world_size(sizeprocs) ! TODO keep it in parameters
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+ group_size_inc = 0
+ call adios_declare_group(adios_group, group_name, &
+ "", 0, adios_err)
+ call adios_select_method(adios_group, "MPI", "", "", adios_err)
+
! save nspec and nglob, to be used in combine_paraview_data
- write(27) nspec
- write(27) nglob
+ call define_adios_integer_scalar (adios_group, "nspec", "", &
+ group_size_inc)
+ call define_adios_integer_scalar (adios_group, "nglob", "", &
+ group_size_inc)
+ local_dim = nglob
+ call define_adios_global_real_1d_array(adios_group, "xstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "ystore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "zstore", &
+ local_dim, group_size_inc)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec
+ call define_adios_global_real_1d_array(adios_group, "rhostore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "kappavstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_integer_1d_array(adios_group, "ibool", &
+ local_dim, group_size_inc)
+ if(iregion_code /= IREGION_OUTER_CORE) then
+ if(.not. (ANISOTROPIC_3D_MANTLE .and. &
+ iregion_code == IREGION_CRUST_MANTLE)) then
+ call define_adios_global_real_1d_array(adios_group, "muvstore", &
+ local_dim, group_size_inc)
+ endif
+ if(TRANSVERSE_ISOTROPY) then
+ if(iregion_code == IREGION_CRUST_MANTLE .and. &
+ .not. ANISOTROPIC_3D_MANTLE) then
+ call define_adios_global_real_1d_array(adios_group, "kappahstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "muhstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "eta_anisostore", &
+ local_dim, group_size_inc)
+ endif
+ endif
+ endif
+
+ local_dim = nspec
+ call define_adios_global_integer_1d_array(adios_group, "idoubling", &
+ local_dim, group_size_inc)
+ call define_adios_global_integer_1d_array(adios_group, "ispec_is_tiso", &
+ local_dim, group_size_inc)
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_actually
+ call define_adios_global_real_1d_array(adios_group, "xixstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "xiystore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "xizstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "etaxstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "etaystore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "etazstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "gammaxstore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "gammaystore", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "gammazstore", &
+ local_dim, group_size_inc)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_ani
+ if(iregion_code /= IREGION_OUTER_CORE) then
+ ! save anisotropy in the inner core only
+ if(ANISOTROPIC_INNER_CORE .and. iregion_code == IREGION_INNER_CORE) then
+ call define_adios_global_real_1d_array(adios_group, "c11store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c33store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c12store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c13store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c44store", &
+ local_dim, group_size_inc)
+ endif
+ if(ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then
+ call define_adios_global_real_1d_array(adios_group, "c11store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c12store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c13store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c14store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c15store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c16store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c22store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c23store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c24store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c25store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c26store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c33store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c34store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c35store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c36store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c44store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c45store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c46store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c55store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c56store", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "c66store", &
+ local_dim, group_size_inc)
+ endif
+ endif
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_stacey
+ if(ABSORBING_CONDITIONS) then
+ if(iregion_code == IREGION_CRUST_MANTLE) then
+ call define_adios_global_real_1d_array(adios_group, "rho_vp", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "rho_vs", &
+ local_dim, group_size_inc)
+ else if(iregion_code == IREGION_OUTER_CORE) then
+ call define_adios_global_real_1d_array(adios_group, "rho_vp", &
+ local_dim, group_size_inc)
+ endif
+ endif
+
+ local_dim = nglob_xy
+ if(NCHUNKS /= 6 .and. ABSORBING_CONDITIONS .and. &
+ iregion_code == IREGION_CRUST_MANTLE) then
+ call define_adios_global_real_1d_array(adios_group, "rmassx", &
+ local_dim, group_size_inc)
+ call define_adios_global_real_1d_array(adios_group, "rmassy", &
+ local_dim, group_size_inc)
+ endif
+ local_dim = nglob
+ call define_adios_global_real_1d_array(adios_group, "rmassz", &
+ local_dim, group_size_inc)
+
+ local_dim = nglob_oceans
+ if(OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then
+ call define_adios_global_real_1d_array(adios_group, "rmass_ocean_load", &
+ local_dim, group_size_inc)
+ endif
+
+ ! Open an ADIOS handler to the restart file.
+ call adios_open (adios_handle, group_name, &
+ outputname, "w", comm, adios_err);
+ call adios_group_size (adios_handle, group_size_inc, &
+ adios_totalsize, adios_err)
+
! mesh topology
! mesh arrays used in the solver to locate source and receivers
! and for anisotropy and gravity, save in single precision
! use tmp_array for temporary storage to perform conversion
- allocate(tmp_array(nglob),stat=ier)
- if( ier /=0 ) call exit_MPI(myrank,'error allocating temporary array for mesh topology')
+ allocate(tmp_array_x(nglob),stat=ier)
+ if( ier /=0 ) call exit_MPI(myrank,&
+ 'error allocating temporary array for mesh topology')
+ allocate(tmp_array_y(nglob),stat=ier)
+ if( ier /=0 ) call exit_MPI(myrank,&
+ 'error allocating temporary array for mesh topology')
+ allocate(tmp_array_z(nglob),stat=ier)
+ if( ier /=0 ) call exit_MPI(myrank,&
+ 'error allocating temporary array for mesh topology')
!--- x coordinate
- tmp_array(:) = 0._CUSTOM_REAL
+ tmp_array_x(:) = 0._CUSTOM_REAL
do ispec = 1,nspec
do k = 1,NGLLZ
do j = 1,NGLLY
@@ -102,18 +289,16 @@
iglob = ibool(i,j,k,ispec)
! distinguish between single and double precision for reals
if(CUSTOM_REAL == SIZE_REAL) then
- tmp_array(iglob) = sngl(xstore(i,j,k,ispec))
+ tmp_array_x(iglob) = sngl(xstore(i,j,k,ispec))
else
- tmp_array(iglob) = xstore(i,j,k,ispec)
+ tmp_array_x(iglob) = xstore(i,j,k,ispec)
endif
enddo
enddo
enddo
enddo
- write(27) tmp_array ! xstore
-
!--- y coordinate
- tmp_array(:) = 0._CUSTOM_REAL
+ tmp_array_y(:) = 0._CUSTOM_REAL
do ispec = 1,nspec
do k = 1,NGLLZ
do j = 1,NGLLY
@@ -121,18 +306,16 @@
iglob = ibool(i,j,k,ispec)
! distinguish between single and double precision for reals
if(CUSTOM_REAL == SIZE_REAL) then
- tmp_array(iglob) = sngl(ystore(i,j,k,ispec))
+ tmp_array_y(iglob) = sngl(ystore(i,j,k,ispec))
else
- tmp_array(iglob) = ystore(i,j,k,ispec)
+ tmp_array_y(iglob) = ystore(i,j,k,ispec)
endif
enddo
enddo
enddo
enddo
- write(27) tmp_array ! ystore
-
!--- z coordinate
- tmp_array(:) = 0._CUSTOM_REAL
+ tmp_array_z(:) = 0._CUSTOM_REAL
do ispec = 1,nspec
do k = 1,NGLLZ
do j = 1,NGLLY
@@ -140,122 +323,337 @@
iglob = ibool(i,j,k,ispec)
! distinguish between single and double precision for reals
if(CUSTOM_REAL == SIZE_REAL) then
- tmp_array(iglob) = sngl(zstore(i,j,k,ispec))
+ tmp_array_z(iglob) = sngl(zstore(i,j,k,ispec))
else
- tmp_array(iglob) = zstore(i,j,k,ispec)
+ tmp_array_z(iglob) = zstore(i,j,k,ispec)
endif
enddo
enddo
enddo
enddo
- write(27) tmp_array ! zstore
- deallocate(tmp_array)
+ ! save nspec and nglob, to be used in combine_paraview_data
+ call adios_write(adios_handle, "nspec", nspec, adios_err)
+ call adios_write(adios_handle, "nglob", nglob, adios_err)
- ! local to global indexing
- write(27) ibool
- write(27) idoubling
- write(27) ispec_is_tiso
+ local_dim = nglob
+ call adios_set_path (adios_handle, "xstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", tmp_array_x, adios_err)
- ! local GLL points
- write(27) xixstore
- write(27) xiystore
- write(27) xizstore
- write(27) etaxstore
- write(27) etaystore
- write(27) etazstore
- write(27) gammaxstore
- write(27) gammaystore
- write(27) gammazstore
+ call adios_set_path (adios_handle, "ystore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", tmp_array_y, adios_err)
+
+ call adios_set_path (adios_handle, "zstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", tmp_array_z, adios_err)
- write(27) rhostore
- write(27) kappavstore
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec
+ call adios_set_path (adios_handle, "rhostore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rhostore, adios_err)
- ! other terms needed in the solid regions only
+ call adios_set_path (adios_handle, "kappavstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", kappavstore, adios_err)
+
+ call adios_set_path (adios_handle, "ibool", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", ibool, adios_err)
+
if(iregion_code /= IREGION_OUTER_CORE) then
-
- ! note: muvstore needed for Q_mu shear attenuation in inner core
- if(.not. (ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE)) then
- write(27) muvstore
+ if(.not. (ANISOTROPIC_3D_MANTLE .and. &
+ iregion_code == IREGION_CRUST_MANTLE)) then
+ call adios_set_path (adios_handle, "muvstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", muvstore, adios_err)
endif
+ if(TRANSVERSE_ISOTROPY) then
+ if(iregion_code == IREGION_CRUST_MANTLE .and. &
+ .not. ANISOTROPIC_3D_MANTLE) then
+ call adios_set_path (adios_handle, "kappahstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", kappahstore, adios_err)
- ! save anisotropy in the mantle only
- if(TRANSVERSE_ISOTROPY) then
- if(iregion_code == IREGION_CRUST_MANTLE .and. .not. ANISOTROPIC_3D_MANTLE) then
- write(27) kappahstore
- write(27) muhstore
- write(27) eta_anisostore
+ call adios_set_path (adios_handle, "muhstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", muhstore, adios_err)
+
+ call adios_set_path (adios_handle, "eta_anisostore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", eta_anisostore, adios_err)
endif
endif
+ endif
+ local_dim = nspec
+ call adios_set_path (adios_handle, "idoubling", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", idoubling, adios_err)
+
+ call adios_set_path (adios_handle, "ispec_is_tiso", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", ispec_is_tiso, adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_actually
+ call adios_set_path (adios_handle, "xixstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", xixstore, adios_err)
+
+ call adios_set_path (adios_handle, "xiystore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", xiystore, adios_err)
+
+ call adios_set_path (adios_handle, "xizstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", xizstore, adios_err)
+
+ call adios_set_path (adios_handle, "etaxstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", etaxstore, adios_err)
+
+ call adios_set_path (adios_handle, "etaystore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", etaystore, adios_err)
+
+ call adios_set_path (adios_handle, "etazstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", etazstore, adios_err)
+
+ call adios_set_path (adios_handle, "gammaxstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", gammaxstore, adios_err)
+
+ call adios_set_path (adios_handle, "gammaystore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", gammaystore, adios_err)
+
+ call adios_set_path (adios_handle, "gammazstore", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", gammazstore, adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_ani
+ if(iregion_code /= IREGION_OUTER_CORE) then
! save anisotropy in the inner core only
if(ANISOTROPIC_INNER_CORE .and. iregion_code == IREGION_INNER_CORE) then
- write(27) c11store
- write(27) c33store
- write(27) c12store
- write(27) c13store
- write(27) c44store
+ call adios_set_path (adios_handle, "c11store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c11store, adios_err)
+
+ call adios_set_path (adios_handle, "c33store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c33store, adios_err)
+
+ call adios_set_path (adios_handle, "c12store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c12store, adios_err)
+
+ call adios_set_path (adios_handle, "c13store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c13store, adios_err)
+
+ call adios_set_path (adios_handle, "c44store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c44store, adios_err)
endif
+ if(ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then
+ call adios_set_path (adios_handle, "c11store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c11store, adios_err)
- if(ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then
- write(27) c11store
- write(27) c12store
- write(27) c13store
- write(27) c14store
- write(27) c15store
- write(27) c16store
- write(27) c22store
- write(27) c23store
- write(27) c24store
- write(27) c25store
- write(27) c26store
- write(27) c33store
- write(27) c34store
- write(27) c35store
- write(27) c36store
- write(27) c44store
- write(27) c45store
- write(27) c46store
- write(27) c55store
- write(27) c56store
- write(27) c66store
+ call adios_set_path (adios_handle, "c12store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c12store, adios_err)
+
+ call adios_set_path (adios_handle, "c13store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c13store, adios_err)
+
+ call adios_set_path (adios_handle, "c14store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c14store, adios_err)
+
+ call adios_set_path (adios_handle, "c15store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c15store, adios_err)
+
+ call adios_set_path (adios_handle, "c16store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c16store, adios_err)
+
+ call adios_set_path (adios_handle, "c22store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c22store, adios_err)
+
+ call adios_set_path (adios_handle, "c23store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c23store, adios_err)
+
+ call adios_set_path (adios_handle, "c24store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c24store, adios_err)
+
+ call adios_set_path (adios_handle, "c25store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c25store, adios_err)
+
+ call adios_set_path (adios_handle, "c26store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c26store, adios_err)
+
+ call adios_set_path (adios_handle, "c33store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c33store, adios_err)
+
+ call adios_set_path (adios_handle, "c34store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c34store, adios_err)
+
+ call adios_set_path (adios_handle, "c35store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c35store, adios_err)
+
+ call adios_set_path (adios_handle, "c36store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c36store, adios_err)
+
+ call adios_set_path (adios_handle, "c44store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c44store, adios_err)
+
+ call adios_set_path (adios_handle, "c45store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c45store, adios_err)
+
+ call adios_set_path (adios_handle, "c46store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c46store, adios_err)
+
+ call adios_set_path (adios_handle, "c55store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c55store, adios_err)
+
+ call adios_set_path (adios_handle, "c56store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c56store, adios_err)
+
+ call adios_set_path (adios_handle, "c66store", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", c66store, adios_err)
endif
-
endif
- ! Stacey
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_stacey
if(ABSORBING_CONDITIONS) then
+ if(iregion_code == IREGION_CRUST_MANTLE) then
+ call adios_set_path (adios_handle, "rho_vp", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rho_vp, adios_err)
- if(iregion_code == IREGION_CRUST_MANTLE) then
- write(27) rho_vp
- write(27) rho_vs
+ call adios_set_path (adios_handle, "rho_vs", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rho_vs, adios_err)
+
else if(iregion_code == IREGION_OUTER_CORE) then
- write(27) rho_vp
+ call adios_set_path (adios_handle, "rho_vp", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rho_vp, adios_err)
endif
+ endif
+ local_dim = nglob_xy
+ if(NCHUNKS /= 6 .and. ABSORBING_CONDITIONS .and. &
+ iregion_code == IREGION_CRUST_MANTLE) then
+ call adios_set_path (adios_handle, "rmassx", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rmassx, adios_err)
+
+ call adios_set_path (adios_handle, "rmassy", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rmassy, adios_err)
endif
- ! mass matrices
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
- ! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
- ! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
- ! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
- ! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
- if(NCHUNKS /= 6 .and. ABSORBING_CONDITIONS .and. iregion_code == IREGION_CRUST_MANTLE) then
- write(27) rmassx
- write(27) rmassy
+ local_dim = nglob
+ call adios_set_path (adios_handle, "rmassz", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rmassz, adios_err)
+
+ local_dim = nglob_oceans
+ if(OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then
+ call adios_set_path (adios_handle, "rmass_ocean_load", adios_err)
+ call write_1D_global_array_adios_dims(adios_handle, myrank, &
+ local_dim, sizeprocs)
+ call adios_write(adios_handle, "array", rmass_ocean_load, adios_err)
+ if(minval(rmass_ocean_load) <= 0._CUSTOM_REAL) &
+ call exit_MPI(myrank,'negative mass matrix term for the oceans')
endif
- write(27) rmassz
+ call adios_set_path (adios_handle, "", adios_err)
+ call adios_close(adios_handle, adios_err)
- ! additional ocean load mass matrix if oceans and if we are in the crust
- if(OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) write(27) rmass_ocean_load
+ deallocate(tmp_array_x)
+ deallocate(tmp_array_y)
+ deallocate(tmp_array_z)
- close(27) ! solver_data.bin
+!@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+!**** STOP HERE FOR WRITING ADIOS ARRAYS SOLVER ********************
+!#### TODO REMOVE WHEN ADIOS WRITE IS CODED ########################
+!@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
-
! absorbing boundary parameters
open(unit=27,file=prname(1:len_trim(prname))//'boundary.bin', &
status='unknown',form='unformatted',action='write',iostat=ier)
@@ -319,15 +717,15 @@
call save_arrays_solver_meshfiles(myrank,nspec)
endif
- end subroutine save_arrays_solver_adios
+end subroutine save_arrays_solver_adios
!
!-------------------------------------------------------------------------------------------------
!
- subroutine save_arrays_solver_meshfiles_adios(myrank,nspec)
+subroutine save_arrays_solver_meshfiles_adios(myrank,nspec)
-! outputs model files in binary format
+ ! outputs model files in binary format
use constants
@@ -464,11 +862,11 @@
deallocate(temp_store)
endif ! ATTENUATION
- end subroutine save_arrays_solver_meshfiles_adios
+end subroutine save_arrays_solver_meshfiles_adios
-!
+
!------------------------------------------------------------------------------
-!
+!> \brief TODO
subroutine save_MPI_arrays_adios(myrank,iregion_code,LOCAL_PATH, &
num_interfaces,max_nibool_interfaces, my_neighbours,nibool_interfaces, &
ibool_interfaces, nspec_inner,nspec_outer, num_phase_ispec, &
@@ -620,14 +1018,12 @@
call adios_close(adios_handle, adios_err)
- end subroutine save_MPI_arrays_adios
+end subroutine save_MPI_arrays_adios
-!
-!-------------------------------------------------------------------------------------------------
-!
+!-------------------------------------------------------------------------------
- subroutine save_arrays_solver_boundary_adios()
+subroutine save_arrays_solver_boundary_adios()
! saves arrays for boundaries such as MOHO, 400 and 670 discontinuities
@@ -682,7 +1078,7 @@
close(27)
- end subroutine save_arrays_solver_boundary_adios
+end subroutine save_arrays_solver_boundary_adios
!-------------------------------------------------------------------------------
!> Write local, global and offset dimensions to ADIOS
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/adios_helpers.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/adios_helpers.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/adios_helpers.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -282,3 +282,40 @@
array_name // "/offset", var_id)
group_size_inc = group_size_inc + local_dim*4
end subroutine define_adios_global_integer_1d_array
+
+!-------------------------------------------------------------------------------
+!> Define a global ADIOS 1D logical array and autoincrement the adios
+!! group size.
+!! \param adios_group The adios group where the variables belongs
+!! \param array_name The variable to be defined. This is actually the path for
+!! ADIOS. The values are stored in array_name/array
+!! \param len The local dimension of the array.
+!! \param group_size_inc The inout adios group size to increment
+!! with the size of the variable
+!! \note This function define local, global and offset sizes as well as the
+!! array to store the values in.
+subroutine define_adios_global_logical_1d_array(adios_group, array_name, &
+ local_dim, group_size_inc)
+ use adios_write_mod
+ implicit none
+ ! Parameters
+ integer(kind=8), intent(in) :: adios_group
+ character(len=*), intent(in) :: array_name
+ integer, intent(in) :: local_dim
+ integer(kind=8), intent(inout) :: group_size_inc
+ ! Variables
+ integer(kind=8) :: var_id
+
+ call define_adios_integer_scalar (adios_group, "local_dim", array_name, &
+ group_size_inc)
+ call define_adios_integer_scalar (adios_group, "global_dim", array_name, &
+ group_size_inc)
+ call define_adios_integer_scalar (adios_group, "offset", array_name, &
+ group_size_inc)
+! call adios_define_var(adios_group, "array", array_name, 6, &
+! "local_dim", "global_dim", "offset", var_id)
+ call adios_define_var(adios_group, "array", array_name, 1, &
+ array_name // "/local_dim", array_name // "/global_dim", &
+ array_name // "/offset", var_id)
+ group_size_inc = group_size_inc + local_dim*1
+end subroutine define_adios_global_logical_1d_array
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/broadcast_compute_parameters.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -351,7 +351,8 @@
!! \param ADIOS_FOR_FORWARD_ARRAYS Flag to indicate that intermediate and
!1 forward arrays are stored with the help of ADIOS.
subroutine broadcast_adios_parameters(myrank,ADIOS_ENABLED, &
- ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS)
+ ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS, &
+ ADIOS_FOR_ARRAYS_SOLVER)
implicit none
@@ -361,10 +362,10 @@
include "precision.h"
integer:: myrank
- logical:: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS
+ logical:: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS, &
+ ADIOS_FOR_ARRAYS_SOLVER
! local parameters
integer :: ier
-
call MPI_BCAST(ADIOS_ENABLED,1,MPI_LOGICAL,0,MPI_COMM_WORLD,ier)
if( ier /= 0 ) call exit_MPI(myrank,'error broadcasting ADIOS_ENABLED')
call MPI_BCAST(ADIOS_FOR_FORWARD_ARRAYS,1,MPI_LOGICAL,0,MPI_COMM_WORLD,ier)
@@ -373,5 +374,8 @@
call MPI_BCAST(ADIOS_FOR_MPI_ARRAYS,1,MPI_LOGICAL,0,MPI_COMM_WORLD,ier)
if( ier /= 0 ) call exit_MPI(myrank, &
'error broadcasting ADIOS_FOR_MPI_ARRAYS')
+ call MPI_BCAST(ADIOS_FOR_ARRAYS_SOLVER,1,MPI_LOGICAL,0,MPI_COMM_WORLD,ier)
+ if( ier /= 0 ) call exit_MPI(myrank, &
+ 'error broadcasting ADIOS_FOR_ARRAYS_SOLVER')
end subroutine broadcast_adios_parameters
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/shared/read_parameter_file.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -237,24 +237,30 @@
!-------------------------------------------------------------------------------------------------
!
subroutine read_adios_parameters(ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, &
- ADIOS_FOR_MPI_ARRAYS)
+ ADIOS_FOR_MPI_ARRAYS, ADIOS_FOR_ARRAYS_SOLVER)
implicit none
include "constants.h"
- logical :: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS
+ logical :: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS, &
+ ADIOS_FOR_ARRAYS_SOLVER
! initializes flags
ADIOS_ENABLED = .false.
ADIOS_FOR_FORWARD_ARRAYS = .false.
ADIOS_FOR_MPI_ARRAYS = .false.
+ ADIOS_FOR_ARRAYS_SOLVER = .false.
! opens file Par_file
call open_parameter_file()
call read_value_logical(ADIOS_ENABLED, 'solver.ADIOS_ENABLED')
- call read_value_logical(ADIOS_FOR_FORWARD_ARRAYS, &
- 'solver.ADIOS_FOR_FORWARD_ARRAYS')
- call read_value_logical(ADIOS_FOR_MPI_ARRAYS, &
- 'solver.ADIOS_FOR_MPI_ARRAYS')
+ if (ADIOS_ENABLED) then
+ call read_value_logical(ADIOS_FOR_FORWARD_ARRAYS, &
+ 'solver.ADIOS_FOR_FORWARD_ARRAYS')
+ call read_value_logical(ADIOS_FOR_MPI_ARRAYS, &
+ 'solver.ADIOS_FOR_MPI_ARRAYS')
+ call read_value_logical(ADIOS_FOR_ARRAYS_SOLVER, &
+ 'solver.ADIOS_FOR_ARRAYS_SOLVER')
+ endif
call close_parameter_file()
end subroutine read_adios_parameters
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/Makefile.in 2013-05-06 18:42:27 UTC (rev 21993)
@@ -250,6 +250,7 @@
# using ADIOS files
ADIOS_OBJECTS= \
$O/read_mesh_databases_adios.adios.o \
+ $O/read_arrays_solver_adios.adios.o \
$O/save_forward_arrays_adios.adios.o \
$O/read_forward_arrays_adios.adios.o \
$O/write_specfem_adios_header.adios.o \
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/initialize_simulation.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -109,7 +109,7 @@
call read_gpu_mode(GPU_MODE)
! ADIOS_ENABLED: parameter is optional, may not be in the Par_file
call read_adios_parameters(ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, &
- ADIOS_FOR_MPI_ARRAYS)
+ ADIOS_FOR_MPI_ARRAYS, ADIOS_FOR_ARRAYS_SOLVER)
endif
! distributes parameters from master to all processes
@@ -151,8 +151,7 @@
call broadcast_gpu_parameters(myrank,GPU_MODE)
! broadcasts optional ADIOS_ENABLED
call broadcast_adios_parameters(myrank,ADIOS_ENABLED, &
- ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS)
-
+ ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS, ADIOS_FOR_ARRAYS_SOLVER)
! get the base pathname for output files
call get_value_string(OUTPUT_FILES, 'OUTPUT_FILES', 'OUTPUT_FILES')
Added: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_arrays_solver_adios.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_arrays_solver_adios.f90 (rev 0)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_arrays_solver_adios.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -0,0 +1,400 @@
+!=====================================================================
+!
+! S p e c f e m 3 D G l o b e V e r s i o n 5 . 1
+! --------------------------------------------------
+!
+! Main authors: Dimitri Komatitsch and Jeroen Tromp
+! Princeton University, USA
+! and University of Pau / CNRS / INRIA, France
+! (c) Princeton University / California Institute of Technology and University of Pau / CNRS / INRIA
+! April 2011
+!
+! This program is free software; you can redistribute it and/or modify
+! it under the terms of the GNU General Public License as published by
+! the Free Software Foundation; either version 2 of the License, or
+! (at your option) any later version.
+!
+! This program is distributed in the hope that it will be useful,
+! but WITHOUT ANY WARRANTY; without even the implied warranty of
+! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+! GNU General Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along
+! with this program; if not, write to the Free Software Foundation, Inc.,
+! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+!
+!=====================================================================
+
+! read arrays created by the mesher
+
+subroutine read_arrays_solver_adios(iregion_code,myrank, &
+ nspec,nglob,nglob_xy, &
+ nspec_iso,nspec_tiso,nspec_ani, &
+ rho_vp,rho_vs,xstore,ystore,zstore, &
+ xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz, &
+ rhostore, kappavstore,muvstore,kappahstore,muhstore,eta_anisostore, &
+ c11store,c12store,c13store,c14store,c15store,c16store,c22store, &
+ c23store,c24store,c25store,c26store,c33store,c34store,c35store, &
+ c36store,c44store,c45store,c46store,c55store,c56store,c66store, &
+ ibool,idoubling,ispec_is_tiso, &
+ rmassx,rmassy,rmassz,rmass_ocean_load, &
+ READ_KAPPA_MU,READ_TISO, &
+ ABSORBING_CONDITIONS,LOCAL_PATH)
+
+ use mpi
+ use adios_read_mod
+ implicit none
+
+ include "constants.h"
+ include "OUTPUT_FILES/values_from_mesher.h"
+
+ integer :: iregion_code,myrank
+ integer :: nspec,nglob,nglob_xy
+ integer :: nspec_iso,nspec_tiso,nspec_ani
+
+ ! Stacey
+ real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec):: rho_vp,rho_vs
+
+ real(kind=CUSTOM_REAL), dimension(nglob) :: xstore,ystore,zstore
+
+ real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: &
+ xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz
+
+ ! material properties
+ real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec_iso) :: &
+ rhostore,kappavstore,muvstore
+
+ ! additional arrays for anisotropy stored only where needed to save memory
+ real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec_tiso) :: &
+ kappahstore,muhstore,eta_anisostore
+
+ ! additional arrays for full anisotropy
+ real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec_ani) :: &
+ c11store,c12store,c13store,c14store,c15store,c16store, &
+ c22store,c23store,c24store,c25store,c26store,c33store,c34store, &
+ c35store,c36store,c44store,c45store,c46store,c55store,c56store,c66store
+
+ ! global addressing
+ integer,dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+ integer, dimension(nspec) :: idoubling
+ logical, dimension(nspec) :: ispec_is_tiso
+
+ ! mass matrices and additional ocean load mass matrix
+ real(kind=CUSTOM_REAL), dimension(nglob_xy) :: rmassx,rmassy
+ real(kind=CUSTOM_REAL), dimension(nglob) :: rmassz
+ real(kind=CUSTOM_REAL), dimension(NGLOB_CRUST_MANTLE_OCEANS) :: rmass_ocean_load
+
+ ! flags to know if we should read Vs and anisotropy arrays
+ logical :: READ_KAPPA_MU,READ_TISO,ABSORBING_CONDITIONS
+
+ character(len=150) :: LOCAL_PATH, file_name
+
+ ! local parameters
+ integer :: ierr, comm, lnspec, lnglob, local_dim
+ ! processor identification
+ character(len=150) :: prname
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid, sel
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+ integer :: vars_count, attrs_count, current_step, last_step, vsteps
+ character(len=128), dimension(:), allocatable :: adios_names
+ integer(kind=8), dimension(1) :: start, count
+
+ call create_name_database_adios(prname, iregion_code, LOCAL_PATH)
+
+ file_name= trim(prname) // "solver_data.bp"
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+
+ call adios_read_init_method (ADIOS_READ_METHOD_BP, comm, &
+ "verbose=1", adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_open_file (adios_handle, file_name, 0, comm, ierr)
+ call check_adios_err(myrank,adios_err)
+
+ ! read coordinates of the mesh
+ call adios_get_scalar(adios_handle, "nspec", lnspec, adios_err)
+ call adios_get_scalar(adios_handle, "nglob", lnglob, adios_err)
+
+ ! checks dimensions
+ if( lnspec /= nspec ) then
+ print*,'error file dimension: nspec in file = ',lnspec, &
+ ' but nspec desired:',nspec
+ print*,'please check file ', file_name
+ call exit_mpi(myrank,'error dimensions in solver_data.bp')
+ endif
+ if( lnglob /= nglob ) then
+ print*,'error file dimension: nglob in file = ',lnglob, &
+ ' but nglob desired:',nglob
+ print*,'please check file ', file_name
+ call exit_mpi(myrank,'error dimensions in solver_data.bp')
+ endif
+
+ ! mesh coordinates
+ local_dim = nglob
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "xstore/array", 0, 1, &
+ xstore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "ystore/array", 0, 1, &
+ ystore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "zstore/array", 0, 1, &
+ zstore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "rmassz/array", 0, 1, &
+ rmassz, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_iso
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_schedule_read(adios_handle, sel, "rhostore/array", 0, 1, &
+ rhostore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "kappavstore/array", 0, 1, &
+ kappavstore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ if(READ_KAPPA_MU) then
+ call adios_schedule_read(adios_handle, sel, "muvstore/array", 0, 1, &
+ muvstore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_tiso
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ if(TRANSVERSE_ISOTROPY_VAL .and. READ_TISO) then
+ call adios_schedule_read(adios_handle, sel, "kappahstore/array", 0, 1, &
+ kappahstore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "muhstore/array", 0, 1, &
+ muhstore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "eta_anisostore/array", 0, 1, &
+ eta_anisostore, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = nspec
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "idoubling/array", 0, 1, &
+ idoubling, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "ispec_is_tiso/array", 0, 1, &
+ ispec_is_tiso, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "ibool/array", 0, 1, &
+ ibool, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "xixstore/array", 0, 1, &
+ xix, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "xiystore/array", 0, 1, &
+ xiy, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "xizstore/array", 0, 1, &
+ xiz, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "etaxstore/array", 0, 1, &
+ etax, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "etaystore/array", 0, 1, &
+ etay, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "etazstore/array", 0, 1, &
+ etaz, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "gammaxstore/array", 0, 1, &
+ gammax, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "gammaystore/array", 0, 1, &
+ gammay, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "gammazstore/array", 0, 1, &
+ gammaz, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec_ani
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+
+ if(ANISOTROPIC_INNER_CORE_VAL .and. iregion_code == IREGION_INNER_CORE) then
+ call adios_schedule_read(adios_handle, sel, "c11store/array", 0, 1, &
+ c11store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c12store/array", 0, 1, &
+ c12store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c13store/array", 0, 1, &
+ c13store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c33store/array", 0, 1, &
+ c33store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c44store/array", 0, 1, &
+ c44store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ if(ANISOTROPIC_3D_MANTLE_VAL .and. iregion_code == IREGION_CRUST_MANTLE) then
+ call adios_schedule_read(adios_handle, sel, "c11store/array", 0, 1, &
+ c11store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c12store/array", 0, 1, &
+ c12store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c13store/array", 0, 1, &
+ c13store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c14store/array", 0, 1, &
+ c14store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c15store/array", 0, 1, &
+ c15store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c16store/array", 0, 1, &
+ c16store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c22store/array", 0, 1, &
+ c22store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c23store/array", 0, 1, &
+ c23store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c24store/array", 0, 1, &
+ c24store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c25store/array", 0, 1, &
+ c25store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c26store/array", 0, 1, &
+ c26store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c33store/array", 0, 1, &
+ c33store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c34store/array", 0, 1, &
+ c34store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c35store/array", 0, 1, &
+ c35store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c36store/array", 0, 1, &
+ c36store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c44store/array", 0, 1, &
+ c44store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c45store/array", 0, 1, &
+ c45store, adios_err)
+ call adios_schedule_read(adios_handle, sel, "c46store/array", 0, 1, &
+ c46store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c55store/array", 0, 1, &
+ c55store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c56store/array", 0, 1, &
+ c56store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "c66store/array", 0, 1, &
+ c66store, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = NGLLX * NGLLY * NGLLZ * nspec ! nspec_stacey in meshfem3D
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+
+ ! Stacey
+ if(ABSORBING_CONDITIONS) then
+
+ if(iregion_code == IREGION_CRUST_MANTLE) then
+ call adios_schedule_read(adios_handle, sel, "rho_vp/array", 0, 1, &
+ rho_vp, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "rho_vs/array", 0, 1, &
+ rho_vs, adios_err)
+ call check_adios_err(myrank,adios_err)
+ else if(iregion_code == IREGION_OUTER_CORE) then
+ call adios_schedule_read(adios_handle, sel, "rho_vp/array", 0, 1, &
+ rho_vp, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ endif
+
+ ! mass matrices
+ !
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ ! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
+ ! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
+ !
+ ! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
+ ! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = nglob_xy
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+
+ if(NCHUNKS_VAL /= 6 .and. ABSORBING_CONDITIONS .and. &
+ iregion_code == IREGION_CRUST_MANTLE) then
+ call adios_schedule_read(adios_handle, sel, "rmassx/array", 0, 1, &
+ rmassx, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "rmassy/array", 0, 1, &
+ rmassy, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ ! read additional ocean load mass matrix
+ if(OCEANS_VAL .and. iregion_code == IREGION_CRUST_MANTLE) then
+ local_dim = NGLOB_CRUST_MANTLE_OCEANS ! nglob_oceans
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+
+ call adios_schedule_read(adios_handle, sel, "rmass_ocean_load/array", &
+ 0, 1, rmass_ocean_load, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
+ call adios_selection_delete(sel)
+ call adios_read_close(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_finalize_method(ADIOS_READ_METHOD_BP, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call MPI_Barrier(comm, ierr)
+
+end subroutine read_arrays_solver_adios
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -148,31 +148,56 @@
if(ier /= 0) stop 'error allocating rmassz in crust_mantle'
! reads databases file
- call read_arrays_solver(IREGION_CRUST_MANTLE,myrank, &
- NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, &
- nspec_iso,nspec_tiso,nspec_ani, &
- rho_vp_crust_mantle,rho_vs_crust_mantle, &
- xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
- xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, &
- etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
- gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, &
- rhostore_crust_mantle,kappavstore_crust_mantle,muvstore_crust_mantle, &
- kappahstore_crust_mantle,muhstore_crust_mantle,eta_anisostore_crust_mantle, &
- c11store_crust_mantle,c12store_crust_mantle,c13store_crust_mantle, &
- c14store_crust_mantle,c15store_crust_mantle,c16store_crust_mantle, &
- c22store_crust_mantle,c23store_crust_mantle,c24store_crust_mantle, &
- c25store_crust_mantle,c26store_crust_mantle,c33store_crust_mantle, &
- c34store_crust_mantle,c35store_crust_mantle,c36store_crust_mantle, &
- c44store_crust_mantle,c45store_crust_mantle,c46store_crust_mantle, &
- c55store_crust_mantle,c56store_crust_mantle,c66store_crust_mantle, &
- ibool_crust_mantle,dummy_idoubling,ispec_is_tiso_crust_mantle, &
- rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle,rmass_ocean_load, &
- READ_KAPPA_MU,READ_TISO, &
- ABSORBING_CONDITIONS,LOCAL_PATH)
+ if(ADIOS_FOR_ARRAYS_SOLVER) then
+ call read_arrays_solver_adios(IREGION_CRUST_MANTLE,myrank, &
+ NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, &
+ nspec_iso,nspec_tiso,nspec_ani, &
+ rho_vp_crust_mantle,rho_vs_crust_mantle, &
+ xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
+ xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, &
+ etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
+ gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, &
+ rhostore_crust_mantle,kappavstore_crust_mantle,muvstore_crust_mantle, &
+ kappahstore_crust_mantle,muhstore_crust_mantle,eta_anisostore_crust_mantle, &
+ c11store_crust_mantle,c12store_crust_mantle,c13store_crust_mantle, &
+ c14store_crust_mantle,c15store_crust_mantle,c16store_crust_mantle, &
+ c22store_crust_mantle,c23store_crust_mantle,c24store_crust_mantle, &
+ c25store_crust_mantle,c26store_crust_mantle,c33store_crust_mantle, &
+ c34store_crust_mantle,c35store_crust_mantle,c36store_crust_mantle, &
+ c44store_crust_mantle,c45store_crust_mantle,c46store_crust_mantle, &
+ c55store_crust_mantle,c56store_crust_mantle,c66store_crust_mantle, &
+ ibool_crust_mantle,dummy_idoubling,ispec_is_tiso_crust_mantle, &
+ rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle,rmass_ocean_load, &
+ READ_KAPPA_MU,READ_TISO, &
+ ABSORBING_CONDITIONS,LOCAL_PATH)
+ else
+ call read_arrays_solver(IREGION_CRUST_MANTLE,myrank, &
+ NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, &
+ nspec_iso,nspec_tiso,nspec_ani, &
+ rho_vp_crust_mantle,rho_vs_crust_mantle, &
+ xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
+ xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, &
+ etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
+ gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, &
+ rhostore_crust_mantle,kappavstore_crust_mantle,muvstore_crust_mantle, &
+ kappahstore_crust_mantle,muhstore_crust_mantle,eta_anisostore_crust_mantle, &
+ c11store_crust_mantle,c12store_crust_mantle,c13store_crust_mantle, &
+ c14store_crust_mantle,c15store_crust_mantle,c16store_crust_mantle, &
+ c22store_crust_mantle,c23store_crust_mantle,c24store_crust_mantle, &
+ c25store_crust_mantle,c26store_crust_mantle,c33store_crust_mantle, &
+ c34store_crust_mantle,c35store_crust_mantle,c36store_crust_mantle, &
+ c44store_crust_mantle,c45store_crust_mantle,c46store_crust_mantle, &
+ c55store_crust_mantle,c56store_crust_mantle,c66store_crust_mantle, &
+ ibool_crust_mantle,dummy_idoubling,ispec_is_tiso_crust_mantle, &
+ rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle,rmass_ocean_load, &
+ READ_KAPPA_MU,READ_TISO, &
+ ABSORBING_CONDITIONS,LOCAL_PATH)
+ endif
! check that the number of points in this slice is correct
- if(minval(ibool_crust_mantle(:,:,:,:)) /= 1 .or. &
- maxval(ibool_crust_mantle(:,:,:,:)) /= NGLOB_CRUST_MANTLE) &
+ if(minval(ibool_crust_mantle(:,:,:,:)) /= 1) &
+ call exit_MPI(myrank,'incorrect global numbering: iboolmin is not equal to 1 in crust and mantle')
+ if(maxval(ibool_crust_mantle(:,:,:,:)) /= NGLOB_CRUST_MANTLE) &
call exit_MPI(myrank,'incorrect global numbering: iboolmax does not equal nglob in crust and mantle')
deallocate(dummy_idoubling)
@@ -231,34 +256,62 @@
allocate(rmass_outer_core(NGLOB_OUTER_CORE),stat=ier)
if(ier /= 0) stop 'error allocating rmass in outer core'
- call read_arrays_solver(IREGION_OUTER_CORE,myrank, &
- NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, &
- nspec_iso,nspec_tiso,nspec_ani, &
- vp_outer_core,dummy_array, &
- xstore_outer_core,ystore_outer_core,zstore_outer_core, &
- xix_outer_core,xiy_outer_core,xiz_outer_core, &
- etax_outer_core,etay_outer_core,etaz_outer_core, &
- gammax_outer_core,gammay_outer_core,gammaz_outer_core, &
- rhostore_outer_core,kappavstore_outer_core,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- ibool_outer_core,dummy_idoubling_outer_core,dummy_ispec_is_tiso, &
- dummy_rmass,dummy_rmass,rmass_outer_core,rmass_ocean_load, &
- READ_KAPPA_MU,READ_TISO, &
- ABSORBING_CONDITIONS,LOCAL_PATH)
+ if (ADIOS_FOR_ARRAYS_SOLVER) then
+ call read_arrays_solver_adios(IREGION_OUTER_CORE,myrank, &
+ NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, &
+ nspec_iso,nspec_tiso,nspec_ani, &
+ vp_outer_core,dummy_array, &
+ xstore_outer_core,ystore_outer_core,zstore_outer_core, &
+ xix_outer_core,xiy_outer_core,xiz_outer_core, &
+ etax_outer_core,etay_outer_core,etaz_outer_core, &
+ gammax_outer_core,gammay_outer_core,gammaz_outer_core, &
+ rhostore_outer_core,kappavstore_outer_core,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ ibool_outer_core,dummy_idoubling_outer_core,dummy_ispec_is_tiso, &
+ dummy_rmass,dummy_rmass,rmass_outer_core,rmass_ocean_load, &
+ READ_KAPPA_MU,READ_TISO, &
+ ABSORBING_CONDITIONS,LOCAL_PATH)
+ else
+ call read_arrays_solver(IREGION_OUTER_CORE,myrank, &
+ NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, &
+ nspec_iso,nspec_tiso,nspec_ani, &
+ vp_outer_core,dummy_array, &
+ xstore_outer_core,ystore_outer_core,zstore_outer_core, &
+ xix_outer_core,xiy_outer_core,xiz_outer_core, &
+ etax_outer_core,etay_outer_core,etaz_outer_core, &
+ gammax_outer_core,gammay_outer_core,gammaz_outer_core, &
+ rhostore_outer_core,kappavstore_outer_core,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ ibool_outer_core,dummy_idoubling_outer_core,dummy_ispec_is_tiso, &
+ dummy_rmass,dummy_rmass,rmass_outer_core,rmass_ocean_load, &
+ READ_KAPPA_MU,READ_TISO, &
+ ABSORBING_CONDITIONS,LOCAL_PATH)
+ endif
deallocate(dummy_idoubling_outer_core,dummy_ispec_is_tiso,dummy_rmass)
! check that the number of points in this slice is correct
- if(minval(ibool_outer_core(:,:,:,:)) /= 1 .or. &
- maxval(ibool_outer_core(:,:,:,:)) /= NGLOB_OUTER_CORE) &
- call exit_MPI(myrank,'incorrect global numbering: iboolmax does not equal nglob in outer core')
+ ! check that the number of points in this slice is correct
+ if(minval(ibool_outer_core(:,:,:,:)) /= 1) &
+ call exit_MPI(myrank,'incorrect global numbering: iboolmin is not equal to 1 in outer core')
+ if(maxval(ibool_outer_core(:,:,:,:)) /= NGLOB_OUTER_CORE) then
+ call exit_MPI(myrank, 'incorrect global numbering: &
+ & iboolmax does not equal nglob in outer core')
+ endif
end subroutine read_mesh_databases_OC
@@ -315,27 +368,51 @@
allocate(rmass_inner_core(NGLOB_INNER_CORE),stat=ier)
if(ier /= 0) stop 'error allocating rmass in inner core'
- call read_arrays_solver(IREGION_INNER_CORE,myrank, &
- NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_dummy, &
- nspec_iso,nspec_tiso,nspec_ani, &
- dummy_array,dummy_array, &
- xstore_inner_core,ystore_inner_core,zstore_inner_core, &
- xix_inner_core,xiy_inner_core,xiz_inner_core, &
- etax_inner_core,etay_inner_core,etaz_inner_core, &
- gammax_inner_core,gammay_inner_core,gammaz_inner_core, &
- rhostore_inner_core,kappavstore_inner_core,muvstore_inner_core, &
- dummy_array,dummy_array,dummy_array, &
- c11store_inner_core,c12store_inner_core,c13store_inner_core, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,c33store_inner_core, &
- dummy_array,dummy_array,dummy_array, &
- c44store_inner_core,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- ibool_inner_core,idoubling_inner_core,dummy_ispec_is_tiso, &
- dummy_rmass,dummy_rmass,rmass_inner_core,rmass_ocean_load, &
- READ_KAPPA_MU,READ_TISO, &
- ABSORBING_CONDITIONS,LOCAL_PATH)
+ if (ADIOS_FOR_ARRAYS_SOLVER) then
+ call read_arrays_solver_adios(IREGION_INNER_CORE,myrank, &
+ NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_dummy, &
+ nspec_iso,nspec_tiso,nspec_ani, &
+ dummy_array,dummy_array, &
+ xstore_inner_core,ystore_inner_core,zstore_inner_core, &
+ xix_inner_core,xiy_inner_core,xiz_inner_core, &
+ etax_inner_core,etay_inner_core,etaz_inner_core, &
+ gammax_inner_core,gammay_inner_core,gammaz_inner_core, &
+ rhostore_inner_core,kappavstore_inner_core,muvstore_inner_core, &
+ dummy_array,dummy_array,dummy_array, &
+ c11store_inner_core,c12store_inner_core,c13store_inner_core, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,c33store_inner_core, &
+ dummy_array,dummy_array,dummy_array, &
+ c44store_inner_core,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ ibool_inner_core,idoubling_inner_core,dummy_ispec_is_tiso, &
+ dummy_rmass,dummy_rmass,rmass_inner_core,rmass_ocean_load, &
+ READ_KAPPA_MU,READ_TISO, &
+ ABSORBING_CONDITIONS,LOCAL_PATH)
+ else
+ call read_arrays_solver(IREGION_INNER_CORE,myrank, &
+ NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_dummy, &
+ nspec_iso,nspec_tiso,nspec_ani, &
+ dummy_array,dummy_array, &
+ xstore_inner_core,ystore_inner_core,zstore_inner_core, &
+ xix_inner_core,xiy_inner_core,xiz_inner_core, &
+ etax_inner_core,etay_inner_core,etaz_inner_core, &
+ gammax_inner_core,gammay_inner_core,gammaz_inner_core, &
+ rhostore_inner_core,kappavstore_inner_core,muvstore_inner_core, &
+ dummy_array,dummy_array,dummy_array, &
+ c11store_inner_core,c12store_inner_core,c13store_inner_core, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ dummy_array,dummy_array,c33store_inner_core, &
+ dummy_array,dummy_array,dummy_array, &
+ c44store_inner_core,dummy_array,dummy_array, &
+ dummy_array,dummy_array,dummy_array, &
+ ibool_inner_core,idoubling_inner_core,dummy_ispec_is_tiso, &
+ dummy_rmass,dummy_rmass,rmass_inner_core,rmass_ocean_load, &
+ READ_KAPPA_MU,READ_TISO, &
+ ABSORBING_CONDITIONS,LOCAL_PATH)
+ endif
deallocate(dummy_ispec_is_tiso,dummy_rmass)
@@ -659,7 +736,11 @@
endif
! outer core
- call read_mesh_databases_MPI_OC()
+ if (ADIOS_FOR_MPI_ARRAYS) then
+ call read_mesh_databases_MPI_OC_adios()
+ else
+ call read_mesh_databases_MPI_OC()
+ endif
allocate(buffer_send_scalar_outer_core(max_nibool_interfaces_oc,num_interfaces_outer_core), &
buffer_recv_scalar_outer_core(max_nibool_interfaces_oc,num_interfaces_outer_core), &
@@ -678,7 +759,11 @@
endif
! inner core
- call read_mesh_databases_MPI_IC()
+ if (ADIOS_FOR_MPI_ARRAYS) then
+ call read_mesh_databases_MPI_IC_adios()
+ else
+ call read_mesh_databases_MPI_IC()
+ endif
allocate(buffer_send_vector_inner_core(NDIM,max_nibool_interfaces_ic,num_interfaces_inner_core), &
buffer_recv_vector_inner_core(NDIM,max_nibool_interfaces_ic,num_interfaces_inner_core), &
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases_adios.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases_adios.f90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/read_mesh_databases_adios.f90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -25,273 +25,10 @@
!
!=====================================================================
-!-------------------------------------------------------------------------------------------------
- subroutine read_mesh_databases_CM_adios()
-
-! mesh for CRUST MANTLE region
-
- use specfem_par
- use specfem_par_crustmantle
- implicit none
-
- ! local parameters
- integer :: nspec_iso,nspec_tiso,nspec_ani
- logical :: READ_KAPPA_MU,READ_TISO
- ! dummy array that does not need to be actually read
- integer, dimension(:),allocatable :: dummy_idoubling
- integer :: ierr
-
- ! crust and mantle
-
- if(ANISOTROPIC_3D_MANTLE_VAL) then
- READ_KAPPA_MU = .false.
- READ_TISO = .false.
- nspec_iso = NSPECMAX_ISO_MANTLE ! 1
- nspec_tiso = NSPECMAX_TISO_MANTLE ! 1
- nspec_ani = NSPEC_CRUST_MANTLE
- else
- READ_KAPPA_MU = .true.
- nspec_iso = NSPEC_CRUST_MANTLE
- if(TRANSVERSE_ISOTROPY_VAL) then
- nspec_tiso = NSPECMAX_TISO_MANTLE
- else
- nspec_tiso = 1
- endif
- nspec_ani = NSPECMAX_ANISO_MANTLE ! 1
- READ_TISO = .true.
- endif
-
- ! sets number of top elements for surface movies & noise tomography
- NSPEC_TOP = NSPEC2D_TOP(IREGION_CRUST_MANTLE)
-
- ! allocates mass matrices in this slice (will be fully assembled in the solver)
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
- ! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
- ! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
- ! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
- ! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
- if(NCHUNKS_VAL /= 6 .and. ABSORBING_CONDITIONS) then
- NGLOB_XY_CM = NGLOB_CRUST_MANTLE
- else
- NGLOB_XY_CM = 1
- endif
-
- ! allocates dummy array
- allocate(dummy_idoubling(NSPEC_CRUST_MANTLE),stat=ierr)
- if( ierr /= 0 ) call exit_mpi(myrank,'error allocating dummy idoubling in crust_mantle')
-
- ! allocates mass matrices
- allocate(rmassx_crust_mantle(NGLOB_XY_CM), &
- rmassy_crust_mantle(NGLOB_XY_CM),stat=ierr)
- if(ierr /= 0) stop 'error allocating dummy rmassx, rmassy in crust_mantle'
- allocate(rmassz_crust_mantle(NGLOB_CRUST_MANTLE),stat=ierr)
- if(ierr /= 0) stop 'error allocating rmassz in crust_mantle'
-
- ! reads databases file
- call read_arrays_solver(IREGION_CRUST_MANTLE,myrank, &
- NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, &
- nspec_iso,nspec_tiso,nspec_ani, &
- rho_vp_crust_mantle,rho_vs_crust_mantle, &
- xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
- xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, &
- etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
- gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, &
- rhostore_crust_mantle,kappavstore_crust_mantle,muvstore_crust_mantle, &
- kappahstore_crust_mantle,muhstore_crust_mantle,eta_anisostore_crust_mantle, &
- c11store_crust_mantle,c12store_crust_mantle,c13store_crust_mantle, &
- c14store_crust_mantle,c15store_crust_mantle,c16store_crust_mantle, &
- c22store_crust_mantle,c23store_crust_mantle,c24store_crust_mantle, &
- c25store_crust_mantle,c26store_crust_mantle,c33store_crust_mantle, &
- c34store_crust_mantle,c35store_crust_mantle,c36store_crust_mantle, &
- c44store_crust_mantle,c45store_crust_mantle,c46store_crust_mantle, &
- c55store_crust_mantle,c56store_crust_mantle,c66store_crust_mantle, &
- ibool_crust_mantle,dummy_idoubling,ispec_is_tiso_crust_mantle, &
- rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle,rmass_ocean_load, &
- READ_KAPPA_MU,READ_TISO, &
- ABSORBING_CONDITIONS,LOCAL_PATH)
-
- ! check that the number of points in this slice is correct
- if(minval(ibool_crust_mantle(:,:,:,:)) /= 1 .or. &
- maxval(ibool_crust_mantle(:,:,:,:)) /= NGLOB_CRUST_MANTLE) &
- call exit_MPI(myrank,'incorrect global numbering: iboolmax does not equal nglob in crust and mantle')
-
- deallocate(dummy_idoubling)
-
- end subroutine read_mesh_databases_CM_adios
-
-!
!-------------------------------------------------------------------------------------------------
!
- subroutine read_mesh_databases_OC_adios()
-
-! mesh for OUTER CORE region
-
- use specfem_par
- use specfem_par_outercore
- implicit none
-
- ! local parameters
- integer :: nspec_iso,nspec_tiso,nspec_ani,NGLOB_XY_dummy
- logical :: READ_KAPPA_MU,READ_TISO
- integer :: ierr
-
- ! dummy array that does not need to be actually read
- real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,1) :: dummy_array
- real(kind=CUSTOM_REAL), dimension(:), allocatable :: dummy_rmass
-
- logical, dimension(:), allocatable :: dummy_ispec_is_tiso
- integer, dimension(:), allocatable :: dummy_idoubling_outer_core
-
- ! outer core (no anisotropy nor S velocity)
- ! rmass_ocean_load is not used in this routine because it is meaningless in the outer core
- READ_KAPPA_MU = .false.
- READ_TISO = .false.
- nspec_iso = NSPEC_OUTER_CORE
- nspec_tiso = 1
- nspec_ani = 1
-
- ! dummy allocation
- NGLOB_XY_dummy = 1
-
- allocate(dummy_rmass(NGLOB_XY_dummy), &
- dummy_ispec_is_tiso(NSPEC_OUTER_CORE), &
- dummy_idoubling_outer_core(NSPEC_OUTER_CORE), &
- stat=ierr)
- if(ierr /= 0) stop 'error allocating dummy rmass and dummy ispec/idoubling in outer core'
-
- ! allocates mass matrices in this slice (will be fully assembled in the solver)
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
- ! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
- ! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
- ! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
- ! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
- allocate(rmass_outer_core(NGLOB_OUTER_CORE),stat=ierr)
- if(ierr /= 0) stop 'error allocating rmass in outer core'
-
- call read_arrays_solver(IREGION_OUTER_CORE,myrank, &
- NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, &
- nspec_iso,nspec_tiso,nspec_ani, &
- vp_outer_core,dummy_array, &
- xstore_outer_core,ystore_outer_core,zstore_outer_core, &
- xix_outer_core,xiy_outer_core,xiz_outer_core, &
- etax_outer_core,etay_outer_core,etaz_outer_core, &
- gammax_outer_core,gammay_outer_core,gammaz_outer_core, &
- rhostore_outer_core,kappavstore_outer_core,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- ibool_outer_core,dummy_idoubling_outer_core,dummy_ispec_is_tiso, &
- dummy_rmass,dummy_rmass,rmass_outer_core,rmass_ocean_load, &
- READ_KAPPA_MU,READ_TISO, &
- ABSORBING_CONDITIONS,LOCAL_PATH)
-
- deallocate(dummy_idoubling_outer_core,dummy_ispec_is_tiso,dummy_rmass)
-
- ! check that the number of points in this slice is correct
- if(minval(ibool_outer_core(:,:,:,:)) /= 1 .or. &
- maxval(ibool_outer_core(:,:,:,:)) /= NGLOB_OUTER_CORE) &
- call exit_MPI(myrank,'incorrect global numbering: iboolmax does not equal nglob in outer core')
-
- end subroutine read_mesh_databases_OC_adios
-
-!
-!-------------------------------------------------------------------------------------------------
-!
-
- subroutine read_mesh_databases_IC_adios()
-
-! mesh for INNER CORE region
-
- use specfem_par
- use specfem_par_innercore
- implicit none
-
- ! local parameters
- integer :: nspec_iso,nspec_tiso,nspec_ani,NGLOB_XY_dummy
- logical :: READ_KAPPA_MU,READ_TISO
- integer :: ierr
-
- ! dummy array that does not need to be actually read
- real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,1) :: dummy_array
- real(kind=CUSTOM_REAL), dimension(:), allocatable :: dummy_rmass
- logical, dimension(:),allocatable:: dummy_ispec_is_tiso
-
- ! inner core (no anisotropy)
- ! rmass_ocean_load is not used in this routine because it is meaningless in the inner core
- READ_KAPPA_MU = .true. ! (muvstore needed for attenuation)
- READ_TISO = .false.
- nspec_iso = NSPEC_INNER_CORE
- nspec_tiso = 1
- if(ANISOTROPIC_INNER_CORE_VAL) then
- nspec_ani = NSPEC_INNER_CORE
- else
- nspec_ani = 1
- endif
-
- ! dummy allocation
- NGLOB_XY_dummy = 1
-
- allocate(dummy_rmass(NGLOB_XY_dummy), &
- dummy_ispec_is_tiso(NSPEC_INNER_CORE), &
- stat=ierr)
- if(ierr /= 0) stop 'error allocating dummy rmass and dummy ispec in inner core'
-
- ! allocates mass matrices in this slice (will be fully assembled in the solver)
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
- ! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
- ! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
- ! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
- ! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
- allocate(rmass_inner_core(NGLOB_INNER_CORE),stat=ierr)
- if(ierr /= 0) stop 'error allocating rmass in inner core'
-
- call read_arrays_solver(IREGION_INNER_CORE,myrank, &
- NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_dummy, &
- nspec_iso,nspec_tiso,nspec_ani, &
- dummy_array,dummy_array, &
- xstore_inner_core,ystore_inner_core,zstore_inner_core, &
- xix_inner_core,xiy_inner_core,xiz_inner_core, &
- etax_inner_core,etay_inner_core,etaz_inner_core, &
- gammax_inner_core,gammay_inner_core,gammaz_inner_core, &
- rhostore_inner_core,kappavstore_inner_core,muvstore_inner_core, &
- dummy_array,dummy_array,dummy_array, &
- c11store_inner_core,c12store_inner_core,c13store_inner_core, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- dummy_array,dummy_array,c33store_inner_core, &
- dummy_array,dummy_array,dummy_array, &
- c44store_inner_core,dummy_array,dummy_array, &
- dummy_array,dummy_array,dummy_array, &
- ibool_inner_core,idoubling_inner_core,dummy_ispec_is_tiso, &
- dummy_rmass,dummy_rmass,rmass_inner_core,rmass_ocean_load, &
- READ_KAPPA_MU,READ_TISO, &
- ABSORBING_CONDITIONS,LOCAL_PATH)
-
- deallocate(dummy_ispec_is_tiso,dummy_rmass)
-
- ! check that the number of points in this slice is correct
- if(minval(ibool_inner_core(:,:,:,:)) /= 1 .or. maxval(ibool_inner_core(:,:,:,:)) /= NGLOB_INNER_CORE) &
- call exit_MPI(myrank,'incorrect global numbering: iboolmax does not equal nglob in inner core')
-
- end subroutine read_mesh_databases_IC_adios
-
-!
-!-------------------------------------------------------------------------------------------------
-!
-
subroutine read_mesh_databases_coupling_adios()
! to couple mantle with outer core
@@ -562,16 +299,13 @@
!-------------------------------------------------------------------------------
!> \brief Read crust mantle MPI arrays from an ADIOS file.
- subroutine read_mesh_databases_MPI_CM_adios()
+subroutine read_mesh_databases_MPI_CM_adios()
! External imports
use mpi
use adios_read_mod
! Internal imports
use specfem_par
use specfem_par_crustmantle
-
- use specfem_par
- use specfem_par_crustmantle
implicit none
! local parameters
@@ -718,174 +452,317 @@
call MPI_Barrier(comm, ierr)
- end subroutine read_mesh_databases_MPI_CM_adios
+end subroutine read_mesh_databases_MPI_CM_adios
-
-!
-!-------------------------------------------------------------------------------------------------
-!
-
- subroutine read_mesh_databases_MPI_OC_adios()
-
+!-------------------------------------------------------------------------------
+!> \brief Read outer core MPI arrays from an ADIOS file.
+subroutine read_mesh_databases_MPI_OC_adios()
+ use mpi
+ use adios_read_mod
use specfem_par
use specfem_par_outercore
implicit none
! local parameters
- integer :: ierr
+ integer :: sizeprocs, comm, ierr
+ character(len=150) :: file_name
+ integer(kind=8) :: group_size_inc
+ integer :: local_dim, global_dim, offset
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid, sel
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+ integer :: vars_count, attrs_count, current_step, last_step, vsteps
+ character(len=128), dimension(:), allocatable :: adios_names
+ integer(kind=8), dimension(1) :: start, count
- ! crust mantle region
-
! create the name for the database of the current slide and region
- call create_name_database(prname,myrank,IREGION_OUTER_CORE,LOCAL_PATH)
+ call create_name_database_adios(prname, IREGION_OUTER_CORE, LOCAL_PATH)
- open(unit=IIN,file=prname(1:len_trim(prname))//'solver_data_mpi.bin', &
- status='old',action='read',form='unformatted',iostat=ierr)
- if( ierr /= 0 ) call exit_mpi(myrank,'error opening solver_data_mpi.bin')
+ file_name= trim(prname) // "solver_data_mpi.bp"
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+ call adios_read_init_method (ADIOS_READ_METHOD_BP, comm, &
+ "verbose=1", adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_open_file (adios_handle, file_name, 0, comm, ierr)
+ call check_adios_err(myrank,adios_err)
+
! MPI interfaces
- read(IIN) num_interfaces_outer_core
+ call adios_get_scalar(adios_handle, "num_interfaces", &
+ num_interfaces_outer_core, adios_err)
+
allocate(my_neighbours_outer_core(num_interfaces_outer_core), &
nibool_interfaces_outer_core(num_interfaces_outer_core), &
stat=ierr)
- if( ierr /= 0 ) &
- call exit_mpi(myrank,'error allocating array my_neighbours_outer_core etc.')
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array my_neighbours_outer_coreetc.')
- if( num_interfaces_outer_core > 0 ) then
- read(IIN) max_nibool_interfaces_oc
- allocate(ibool_interfaces_outer_core(max_nibool_interfaces_oc,num_interfaces_outer_core), &
- stat=ierr)
- if( ierr /= 0 ) call exit_mpi(myrank,'error allocating array ibool_interfaces_outer_core')
+ if( num_interfaces_outer_core> 0 ) then
+ call adios_get_scalar(adios_handle, "max_nibool_interfaces", &
+ max_nibool_interfaces_oc, adios_err)
+ allocate(ibool_interfaces_outer_core(max_nibool_interfaces_oc, &
+ num_interfaces_outer_core), stat=ierr)
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array ibool_interfaces_outer_core')
- read(IIN) my_neighbours_outer_core
- read(IIN) nibool_interfaces_outer_core
- read(IIN) ibool_interfaces_outer_core
+ local_dim = num_interfaces_outer_core
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "my_neighbours/array", 0, 1, &
+ my_neighbours_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "nibool_interfaces/array", &
+ 0, 1, nibool_interfaces_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = max_nibool_interfaces_oc * num_interfaces_outer_core
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, &
+ "ibool_interfaces/array", 0, 1, &
+ ibool_interfaces_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
else
! dummy array
max_nibool_interfaces_oc = 0
allocate(ibool_interfaces_outer_core(0,0),stat=ierr)
- if( ierr /= 0 ) call exit_mpi(myrank,'error allocating array dummy ibool_interfaces_outer_core')
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array dummy ibool_interfaces_outer_core')
endif
! inner / outer elements
- read(IIN) nspec_inner_outer_core,nspec_outer_outer_core
- read(IIN) num_phase_ispec_outer_core
- if( num_phase_ispec_outer_core < 0 ) &
- call exit_mpi(myrank,'error num_phase_ispec_outer_core is < zero')
+ call adios_get_scalar(adios_handle, "nspec_inner", &
+ nspec_inner_outer_core, adios_err)
+ call adios_get_scalar(adios_handle, "nspec_outer", &
+ nspec_outer_outer_core, adios_err)
+ call adios_get_scalar(adios_handle, "num_phase_ispec", &
+ num_phase_ispec_outer_core, adios_err)
+ if( num_phase_ispec_outer_core< 0 ) &
+ call exit_mpi(myrank,'error num_phase_ispec_outer_core is < zero')
allocate(phase_ispec_inner_outer_core(num_phase_ispec_outer_core,2),&
stat=ierr)
- if( ierr /= 0 ) &
- call exit_mpi(myrank,'error allocating array phase_ispec_inner_outer_core')
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array phase_ispec_inner_outer_core')
- if(num_phase_ispec_outer_core > 0 ) read(IIN) phase_ispec_inner_outer_core
+ if(num_phase_ispec_outer_core> 0 ) then
+ local_dim = num_phase_ispec_outer_core * 2
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, &
+ "phase_ispec_inner/array", 0, 1, &
+ phase_ispec_inner_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
! mesh coloring for GPUs
if( USE_MESH_COLORING_GPU ) then
+ call adios_get_scalar(adios_handle, "num_colors_outer", &
+ num_colors_outer_outer_core, adios_err)
+ call adios_get_scalar(adios_handle, "num_colors_inner", &
+ num_colors_inner_outer_core, adios_err)
! colors
- read(IIN) num_colors_outer_outer_core,num_colors_inner_outer_core
- allocate(num_elem_colors_outer_core(num_colors_outer_outer_core + num_colors_inner_outer_core), &
- stat=ierr)
+ allocate(num_elem_colors_outer_core(num_colors_outer_outer_core+&
+ num_colors_inner_outer_core), stat=ierr)
if( ierr /= 0 ) &
call exit_mpi(myrank,'error allocating num_elem_colors_outer_core array')
- read(IIN) num_elem_colors_outer_core
+ local_dim = num_colors_outer_outer_core+ num_colors_inner_outer_core
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, &
+ "num_elem_colors/array", 0, 1, &
+ num_elem_colors_outer_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
else
! allocates dummy arrays
num_colors_outer_outer_core = 0
num_colors_inner_outer_core = 0
- allocate(num_elem_colors_outer_core(num_colors_outer_outer_core + num_colors_inner_outer_core), &
- stat=ierr)
+ allocate(num_elem_colors_outer_core(num_colors_outer_outer_core+ &
+ num_colors_inner_outer_core), stat=ierr)
if( ierr /= 0 ) &
- call exit_mpi(myrank,'error allocating num_elem_colors_outer_core array')
+ call exit_mpi(myrank, &
+ 'error allocating num_elem_colors_outer_core array')
endif
+ ! Close ADIOS handler to the restart file.
+ call adios_selection_delete(sel)
+ call adios_read_close(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_finalize_method(ADIOS_READ_METHOD_BP, adios_err)
+ call check_adios_err(myrank,adios_err)
- close(IIN)
+ call MPI_Barrier(comm, ierr)
- end subroutine read_mesh_databases_MPI_OC_adios
+end subroutine read_mesh_databases_MPI_OC_adios
-!
-!-------------------------------------------------------------------------------------------------
-!
- subroutine read_mesh_databases_MPI_IC_adios()
+!-------------------------------------------------------------------------------
+!> \brief Read outer core MPI arrays from an ADIOS file.
+subroutine read_mesh_databases_MPI_IC_adios()
+ use mpi
+ use adios_read_mod
use specfem_par
use specfem_par_innercore
implicit none
! local parameters
- integer :: ierr
+ integer :: sizeprocs, comm, ierr
+ character(len=150) :: file_name
+ integer(kind=8) :: group_size_inc
+ integer :: local_dim, global_dim, offset
+ ! ADIOS variables
+ integer :: adios_err
+ integer(kind=8) :: adios_group, adios_handle, varid, sel
+ integer(kind=8) :: adios_groupsize, adios_totalsize
+ integer :: vars_count, attrs_count, current_step, last_step, vsteps
+ character(len=128), dimension(:), allocatable :: adios_names
+ integer(kind=8), dimension(1) :: start, count
- ! crust mantle region
-
! create the name for the database of the current slide and region
- call create_name_database(prname,myrank,IREGION_INNER_CORE,LOCAL_PATH)
+ call create_name_database_adios(prname, IREGION_INNER_CORE, LOCAL_PATH)
- open(unit=IIN,file=prname(1:len_trim(prname))//'solver_data_mpi.bin', &
- status='old',action='read',form='unformatted',iostat=ierr)
- if( ierr /= 0 ) call exit_mpi(myrank,'error opening solver_data_mpi.bin')
+ file_name= trim(prname) // "solver_data_mpi.bp"
+ call MPI_Comm_dup (MPI_COMM_WORLD, comm, ierr)
+ call adios_read_init_method (ADIOS_READ_METHOD_BP, comm, &
+ "verbose=1", adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_open_file (adios_handle, file_name, 0, comm, ierr)
+ call check_adios_err(myrank,adios_err)
+
! MPI interfaces
- read(IIN) num_interfaces_inner_core
+ call adios_get_scalar(adios_handle, "num_interfaces", &
+ num_interfaces_inner_core, adios_err)
+
allocate(my_neighbours_inner_core(num_interfaces_inner_core), &
nibool_interfaces_inner_core(num_interfaces_inner_core), &
stat=ierr)
- if( ierr /= 0 ) &
- call exit_mpi(myrank,'error allocating array my_neighbours_inner_core etc.')
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array my_neighbours_inner_core etc.')
if( num_interfaces_inner_core > 0 ) then
- read(IIN) max_nibool_interfaces_ic
- allocate(ibool_interfaces_inner_core(max_nibool_interfaces_ic,num_interfaces_inner_core), &
- stat=ierr)
- if( ierr /= 0 ) call exit_mpi(myrank,'error allocating array ibool_interfaces_inner_core')
+ call adios_get_scalar(adios_handle, "max_nibool_interfaces", &
+ max_nibool_interfaces_ic, adios_err)
+ allocate(ibool_interfaces_inner_core(max_nibool_interfaces_ic, &
+ num_interfaces_inner_core), stat=ierr)
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array ibool_interfaces_inner_core')
- read(IIN) my_neighbours_inner_core
- read(IIN) nibool_interfaces_inner_core
- read(IIN) ibool_interfaces_inner_core
+ local_dim = num_interfaces_inner_core
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, "my_neighbours/array", 0, 1, &
+ my_neighbours_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_schedule_read(adios_handle, sel, "nibool_interfaces/array", &
+ 0, 1, nibool_interfaces_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ local_dim = max_nibool_interfaces_ic * num_interfaces_inner_core
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, &
+ "ibool_interfaces/array", 0, 1, &
+ ibool_interfaces_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
else
! dummy array
max_nibool_interfaces_ic = 0
allocate(ibool_interfaces_inner_core(0,0),stat=ierr)
- if( ierr /= 0 ) call exit_mpi(myrank,'error allocating array dummy ibool_interfaces_inner_core')
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array dummy ibool_interfaces_inner_core')
endif
! inner / outer elements
- read(IIN) nspec_inner_inner_core,nspec_outer_inner_core
- read(IIN) num_phase_ispec_inner_core
+ call adios_get_scalar(adios_handle, "nspec_inner", &
+ nspec_inner_inner_core, adios_err)
+ call adios_get_scalar(adios_handle, "nspec_outer", &
+ nspec_outer_inner_core, adios_err)
+ call adios_get_scalar(adios_handle, "num_phase_ispec", &
+ num_phase_ispec_inner_core, adios_err)
if( num_phase_ispec_inner_core < 0 ) &
- call exit_mpi(myrank,'error num_phase_ispec_inner_core is < zero')
+ call exit_mpi(myrank,'error num_phase_ispec_inner_core is < zero')
allocate(phase_ispec_inner_inner_core(num_phase_ispec_inner_core,2),&
stat=ierr)
- if( ierr /= 0 ) &
- call exit_mpi(myrank,'error allocating array phase_ispec_inner_inner_core')
+ if( ierr /= 0 ) call exit_mpi(myrank, &
+ 'error allocating array phase_ispec_inner_inner_core')
- if(num_phase_ispec_inner_core > 0 ) read(IIN) phase_ispec_inner_inner_core
+ if(num_phase_ispec_inner_core > 0 ) then
+ local_dim = num_phase_ispec_inner_core * 2
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, &
+ "phase_ispec_inner/array", 0, 1, &
+ phase_ispec_inner_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ endif
+
! mesh coloring for GPUs
if( USE_MESH_COLORING_GPU ) then
+ call adios_get_scalar(adios_handle, "num_colors_outer", &
+ num_colors_outer_inner_core, adios_err)
+ call adios_get_scalar(adios_handle, "num_colors_inner", &
+ num_colors_inner_inner_core, adios_err)
! colors
- read(IIN) num_colors_outer_inner_core,num_colors_inner_inner_core
- allocate(num_elem_colors_inner_core(num_colors_outer_inner_core + num_colors_inner_inner_core), &
- stat=ierr)
+ allocate(num_elem_colors_inner_core(num_colors_outer_inner_core +&
+ num_colors_inner_inner_core), stat=ierr)
if( ierr /= 0 ) &
call exit_mpi(myrank,'error allocating num_elem_colors_inner_core array')
- read(IIN) num_elem_colors_inner_core
+ local_dim = num_colors_outer_inner_core + num_colors_inner_inner_core
+ start(1) = local_dim*myrank; count(1) = local_dim
+ call adios_selection_boundingbox (sel , 1, start, count)
+ call adios_schedule_read(adios_handle, sel, &
+ "num_elem_colors/array", 0, 1, &
+ num_elem_colors_inner_core, adios_err)
+ call check_adios_err(myrank,adios_err)
+
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
else
! allocates dummy arrays
num_colors_outer_inner_core = 0
num_colors_inner_inner_core = 0
- allocate(num_elem_colors_inner_core(num_colors_outer_inner_core + num_colors_inner_inner_core), &
- stat=ierr)
+ allocate(num_elem_colors_inner_core(num_colors_outer_inner_core + &
+ num_colors_inner_inner_core), stat=ierr)
if( ierr /= 0 ) &
- call exit_mpi(myrank,'error allocating num_elem_colors_inner_core array')
+ call exit_mpi(myrank, &
+ 'error allocating num_elem_colors_inner_core array')
endif
+ ! Close ADIOS handler to the restart file.
+ call adios_selection_delete(sel)
+ call adios_read_close(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
+ call adios_read_finalize_method(ADIOS_READ_METHOD_BP, adios_err)
+ call check_adios_err(myrank,adios_err)
- close(IIN)
+ call MPI_Barrier(comm, ierr)
end subroutine read_mesh_databases_MPI_IC_adios
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90 2013-05-06 18:42:19 UTC (rev 21992)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SUNFLOWER_ADIOS/src/specfem3D/specfem3D_par.F90 2013-05-06 18:42:27 UTC (rev 21993)
@@ -307,7 +307,8 @@
! ADIOS
!-----------------------------------------------------------------
- logical :: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS
+ logical :: ADIOS_ENABLED, ADIOS_FOR_FORWARD_ARRAYS, ADIOS_FOR_MPI_ARRAYS, &
+ ADIOS_FOR_ARRAYS_SOLVER
!-----------------------------------------------------------------
! time scheme
More information about the CIG-COMMITS
mailing list