[cig-commits] [commit] devel: fixes solver_data_mpi.bp read in adios. (c8b107d)
cig_noreply at geodynamics.org
cig_noreply at geodynamics.org
Wed Apr 9 08:57:33 PDT 2014
Repository : ssh://geoshell/specfem3d_globe
On branch : devel
Link : https://github.com/geodynamics/specfem3d_globe/compare/64e1b38f0c5ebb4056cce0b15d41c0b9f94ab6e5...099a4d330d5b173b21e51ad441f9f429e5d37842
>---------------------------------------------------------------
commit c8b107d84f1d7041155157c85fe1cfb5ab43ea55
Author: Matthieu Lefebvre <ml15 at princeton.edu>
Date: Wed Apr 9 11:47:47 2014 -0400
fixes solver_data_mpi.bp read in adios.
>---------------------------------------------------------------
c8b107d84f1d7041155157c85fe1cfb5ab43ea55
src/meshfem3D/save_arrays_solver_adios.F90 | 29 ++++--
src/specfem3D/read_mesh_databases_adios.f90 | 144 ++++++++++++++++------------
2 files changed, 107 insertions(+), 66 deletions(-)
diff --git a/src/meshfem3D/save_arrays_solver_adios.F90 b/src/meshfem3D/save_arrays_solver_adios.F90
index a53abda..00872c0 100644
--- a/src/meshfem3D/save_arrays_solver_adios.F90
+++ b/src/meshfem3D/save_arrays_solver_adios.F90
@@ -1439,9 +1439,17 @@ subroutine save_mpi_arrays_adios(myrank,iregion_code,LOCAL_PATH, &
num_phase_ispec, adios_err)
if(num_phase_ispec > 0 ) then
- local_dim = num_phase_ispec_wmax * 2
- call write_adios_global_1d_array(handle, myrank, sizeprocs, local_dim, &
- trim(region_name) // STRINGIFY_VAR(phase_ispec_inner))
+ !local_dim = num_phase_ispec_wmax * 2
+ !call write_adios_global_1d_array(handle, myrank, sizeprocs, local_dim, &
+ !trim(region_name) // STRINGIFY_VAR(phase_ispec_inner))
+ call adios_write(handle, trim(region_name) // "phase_ispec_inner/local_dim", &
+ 2 * num_phase_ispec, adios_err)
+ call adios_write(handle, trim(region_name) // "phase_ispec_inner/global_dim", &
+ 2 * num_phase_ispec_wmax * sizeprocs, adios_err)
+ call adios_write(handle, trim(region_name) // "phase_ispec_inner/offset", &
+ 2 * num_phase_ispec_wmax * myrank, adios_err)
+ call adios_write(handle, trim(region_name) // "phase_ispec_inner/array", &
+ phase_ispec_inner, adios_err)
endif
! mesh coloring
@@ -1451,9 +1459,18 @@ subroutine save_mpi_arrays_adios(myrank,iregion_code,LOCAL_PATH, &
call adios_write(handle, trim(region_name) // "num_colors_inner", &
nspec_inner, adios_err)
- local_dim = num_colors_outer_wmax + num_colors_inner_wmax
- call write_adios_global_1d_array(handle, myrank, sizeprocs, local_dim, &
- trim(region_name) // STRINGIFY_VAR(num_elem_colors))
+ !local_dim = num_colors_outer_wmax + num_colors_inner_wmax
+ !call write_adios_global_1d_array(handle, myrank, sizeprocs, local_dim, &
+ !trim(region_name) // STRINGIFY_VAR(num_elem_colors))
+
+ call adios_write(handle, trim(region_name) // "num_elem_colors/local_dim", &
+ (num_colors_inner + num_colors_outer), adios_err)
+ call adios_write(handle, trim(region_name) // "num_elem_colors/global_dim", &
+ (num_colors_inner_wmax + num_colors_outer_wmax)* sizeprocs, adios_err)
+ call adios_write(handle, trim(region_name) // "num_elem_colors/offset", &
+ (num_colors_inner_wmax + num_colors_outer_wmax)* myrank, adios_err)
+ call adios_write(handle, trim(region_name) // "num_elem_colors/array", &
+ num_elem_colors, adios_err)
endif
!--- Reset the path to zero and perform the actual write to disk
diff --git a/src/specfem3D/read_mesh_databases_adios.f90 b/src/specfem3D/read_mesh_databases_adios.f90
index 050113d..c18d15d 100644
--- a/src/specfem3D/read_mesh_databases_adios.f90
+++ b/src/specfem3D/read_mesh_databases_adios.f90
@@ -737,9 +737,9 @@ subroutine read_mesh_databases_MPI_CM_adios()
integer(kind=8) :: adios_handle, sel
integer(kind=8), dimension(1) :: start, count
- integer :: local_dim_my_neighbours, local_dim_nibool_interfaces, &
- local_dim_ibool_interfaces, local_dim_phase_ispec_inner, &
- local_dim_num_elem_colors
+ integer :: offset_my_neighbours, offset_nibool_interfaces, &
+ offset_ibool_interfaces, offset_phase_ispec_inner, &
+ offset_num_elem_colors
character(len=128) :: region_name
@@ -773,25 +773,33 @@ subroutine read_mesh_databases_MPI_CM_adios()
call adios_perform_reads(adios_handle, adios_err)
call check_adios_err(myrank,adios_err)
- !----------------------------------------.
- ! Get local_dim to avoid buffer overflow |
- !----------------------------------------'
+ !--------------------------------------.
+ ! Get offsets to avoid buffer overflow |
+ !--------------------------------------'
+ call adios_selection_writeblock(sel, myrank)
if( num_interfaces_crust_mantle > 0 ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "my_neighbours/local_dim", &
- local_dim_my_neighbours, adios_err)
- call adios_get_scalar(adios_handle, trim(region_name) // "nibool_interfaces/local_dim", &
- local_dim_nibool_interfaces, adios_err)
- call adios_get_scalar(adios_handle, trim(region_name) // "ibool_interfaces/local_dim", &
- local_dim_ibool_interfaces, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "my_neighbours/offset", &
+ 0, 1, offset_my_neighbours, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "nibool_interfaces/offset", &
+ 0, 1, offset_nibool_interfaces, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "ibool_interfaces/offset", &
+ 0, 1, offset_ibool_interfaces, adios_err)
endif
if(num_phase_ispec_crust_mantle > 0 ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "phase_ispec_inner/local_dim", &
- local_dim_phase_ispec_inner, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "phase_ispec_inner/offset", &
+ 0, 1, offset_phase_ispec_inner, adios_err)
endif
if( USE_MESH_COLORING_GPU ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "num_elem_colors/local_dim", &
- local_dim_num_elem_colors, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "num_elem_colors/offset", &
+ 0, 1, offset_num_elem_colors, adios_err)
endif
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
allocate(my_neighbours_crust_mantle(num_interfaces_crust_mantle), &
nibool_interfaces_crust_mantle(num_interfaces_crust_mantle), &
@@ -811,7 +819,7 @@ subroutine read_mesh_databases_MPI_CM_adios()
if( ierr /= 0 ) call exit_mpi(myrank, &
'error allocating array ibool_interfaces_crust_mantle')
- start(1) = local_dim_my_neighbours * myrank
+ start(1) = offset_my_neighbours
count(1) = num_interfaces_crust_mantle
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // "my_neighbours/array", 0, 1, &
@@ -824,7 +832,7 @@ subroutine read_mesh_databases_MPI_CM_adios()
call adios_perform_reads(adios_handle, adios_err)
call check_adios_err(myrank,adios_err)
- start(1) = local_dim_ibool_interfaces * myrank
+ start(1) = offset_ibool_interfaces
count(1) = max_nibool_interfaces_cm * num_interfaces_crust_mantle
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -853,7 +861,7 @@ subroutine read_mesh_databases_MPI_CM_adios()
'error allocating array phase_ispec_inner_crust_mantle')
if(num_phase_ispec_crust_mantle > 0 ) then
- start(1) = local_dim_phase_ispec_inner * myrank
+ start(1) = offset_phase_ispec_inner
count(1) = num_phase_ispec_crust_mantle * 2
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -883,7 +891,7 @@ subroutine read_mesh_databases_MPI_CM_adios()
if( ierr /= 0 ) &
call exit_mpi(myrank,'error allocating num_elem_colors_crust_mantle array')
- start(1) = local_dim_num_elem_colors * myrank
+ start(1) = offset_num_elem_colors
count(1)= num_colors_outer_crust_mantle + num_colors_inner_crust_mantle
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -934,9 +942,9 @@ subroutine read_mesh_databases_MPI_OC_adios()
integer(kind=8) :: adios_handle, sel
integer(kind=8), dimension(1) :: start, count
- integer :: local_dim_my_neighbours, local_dim_nibool_interfaces, &
- local_dim_ibool_interfaces, local_dim_phase_ispec_inner, &
- local_dim_num_elem_colors
+ integer :: offset_my_neighbours, offset_nibool_interfaces, &
+ offset_ibool_interfaces, offset_phase_ispec_inner, &
+ offset_num_elem_colors
character(len=128) :: region_name
@@ -975,25 +983,33 @@ subroutine read_mesh_databases_MPI_OC_adios()
if( ierr /= 0 ) call exit_mpi(myrank, &
'error allocating array my_neighbours_outer_coreetc.')
- !----------------------------------------.
- ! Get local_dim to avoid buffer overflow |
- !----------------------------------------'
+ !--------------------------------------.
+ ! Get offsets to avoid buffer overflow |
+ !--------------------------------------'
+ call adios_selection_writeblock(sel, myrank)
if( num_interfaces_outer_core > 0 ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "my_neighbours/local_dim", &
- local_dim_my_neighbours, adios_err)
- call adios_get_scalar(adios_handle, trim(region_name) // "nibool_interfaces/local_dim", &
- local_dim_nibool_interfaces, adios_err)
- call adios_get_scalar(adios_handle, trim(region_name) // "ibool_interfaces/local_dim", &
- local_dim_ibool_interfaces, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "my_neighbours/offset", &
+ 0, 1, offset_my_neighbours, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "nibool_interfaces/offset", &
+ 0, 1, offset_nibool_interfaces, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "ibool_interfaces/offset", &
+ 0, 1, offset_ibool_interfaces, adios_err)
endif
if(num_phase_ispec_outer_core > 0 ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "phase_ispec_inner/local_dim", &
- local_dim_phase_ispec_inner, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "phase_ispec_inner/offset", &
+ 0, 1, offset_phase_ispec_inner, adios_err)
endif
if( USE_MESH_COLORING_GPU ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "num_elem_colors/local_dim", &
- local_dim_num_elem_colors, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "num_elem_colors/offset", &
+ 0, 1, offset_num_elem_colors, adios_err)
endif
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
if( num_interfaces_outer_core> 0 ) then
call adios_selection_writeblock(sel, myrank)
@@ -1007,7 +1023,7 @@ subroutine read_mesh_databases_MPI_OC_adios()
if( ierr /= 0 ) call exit_mpi(myrank, &
'error allocating array ibool_interfaces_outer_core')
- start(1) = local_dim_my_neighbours * myrank
+ start(1) = offset_my_neighbours
count(1) = num_interfaces_outer_core
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // "my_neighbours/array", 0, 1, &
@@ -1020,7 +1036,7 @@ subroutine read_mesh_databases_MPI_OC_adios()
call adios_perform_reads(adios_handle, adios_err)
call check_adios_err(myrank,adios_err)
- start(1) = local_dim_ibool_interfaces * myrank
+ start(1) = offset_ibool_interfaces
count(1) = max_nibool_interfaces_oc * num_interfaces_outer_core
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -1047,7 +1063,7 @@ subroutine read_mesh_databases_MPI_OC_adios()
'error allocating array phase_ispec_inner_outer_core')
if(num_phase_ispec_outer_core> 0 ) then
- start(1) = local_dim_phase_ispec_inner * myrank
+ start(1) = offset_phase_ispec_inner
count(1) = num_phase_ispec_outer_core * 2
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -1076,7 +1092,7 @@ subroutine read_mesh_databases_MPI_OC_adios()
if( ierr /= 0 ) &
call exit_mpi(myrank,'error allocating num_elem_colors_outer_core array')
- start(1) = local_dim_num_elem_colors * myrank
+ start(1) = offset_num_elem_colors
count(1)= num_colors_outer_outer_core + num_colors_inner_outer_core
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -1127,9 +1143,9 @@ subroutine read_mesh_databases_MPI_IC_adios()
integer(kind=8) :: adios_handle, sel
integer(kind=8), dimension(1) :: start, count
- integer :: local_dim_my_neighbours, local_dim_nibool_interfaces, &
- local_dim_ibool_interfaces, local_dim_phase_ispec_inner, &
- local_dim_num_elem_colors
+ integer :: offset_my_neighbours, offset_nibool_interfaces, &
+ offset_ibool_interfaces, offset_phase_ispec_inner, &
+ offset_num_elem_colors
character(len=128) :: region_name
@@ -1164,25 +1180,33 @@ subroutine read_mesh_databases_MPI_IC_adios()
call adios_perform_reads(adios_handle, adios_err)
call check_adios_err(myrank,adios_err)
- !----------------------------------------.
- ! Get local_dim to avoid buffer overflow |
- !----------------------------------------'
+ !--------------------------------------.
+ ! Get offsets to avoid buffer overflow |
+ !--------------------------------------'
+ call adios_selection_writeblock(sel, myrank)
if( num_interfaces_inner_core > 0 ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "my_neighbours/local_dim", &
- local_dim_my_neighbours, adios_err)
- call adios_get_scalar(adios_handle, trim(region_name) // "nibool_interfaces/local_dim", &
- local_dim_nibool_interfaces, adios_err)
- call adios_get_scalar(adios_handle, trim(region_name) // "ibool_interfaces/local_dim", &
- local_dim_ibool_interfaces, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "my_neighbours/offset", &
+ 0, 1, offset_my_neighbours, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "nibool_interfaces/offset", &
+ 0, 1, offset_nibool_interfaces, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "ibool_interfaces/offset", &
+ 0, 1, offset_ibool_interfaces, adios_err)
endif
if(num_phase_ispec_inner_core > 0 ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "phase_ispec_inner/local_dim", &
- local_dim_phase_ispec_inner, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "phase_ispec_inner/offset", &
+ 0, 1, offset_phase_ispec_inner, adios_err)
endif
if( USE_MESH_COLORING_GPU ) then
- call adios_get_scalar(adios_handle, trim(region_name) // "num_elem_colors/local_dim", &
- local_dim_num_elem_colors, adios_err)
+ call adios_schedule_read(adios_handle, sel, &
+ trim(region_name) // "num_elem_colors/offset", &
+ 0, 1, offset_num_elem_colors, adios_err)
endif
+ call adios_perform_reads(adios_handle, adios_err)
+ call check_adios_err(myrank,adios_err)
allocate(my_neighbours_inner_core(num_interfaces_inner_core), &
nibool_interfaces_inner_core(num_interfaces_inner_core), &
@@ -1203,7 +1227,7 @@ subroutine read_mesh_databases_MPI_IC_adios()
if( ierr /= 0 ) call exit_mpi(myrank, &
'error allocating array ibool_interfaces_inner_core')
- start(1) = local_dim_my_neighbours * myrank
+ start(1) = offset_my_neighbours
count(1) = num_interfaces_inner_core
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // "my_neighbours/array", 0, 1, &
@@ -1216,7 +1240,7 @@ subroutine read_mesh_databases_MPI_IC_adios()
call adios_perform_reads(adios_handle, adios_err)
call check_adios_err(myrank,adios_err)
- start(1) = local_dim_ibool_interfaces * myrank
+ start(1) = offset_ibool_interfaces
count(1) = max_nibool_interfaces_ic * num_interfaces_inner_core
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -1243,7 +1267,7 @@ subroutine read_mesh_databases_MPI_IC_adios()
'error allocating array phase_ispec_inner_inner_core')
if(num_phase_ispec_inner_core > 0 ) then
- start(1) = local_dim_phase_ispec_inner * myrank
+ start(1) = offset_phase_ispec_inner
count(1) = num_phase_ispec_inner_core * 2
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, trim(region_name) // &
@@ -1272,7 +1296,7 @@ subroutine read_mesh_databases_MPI_IC_adios()
if( ierr /= 0 ) &
call exit_mpi(myrank,'error allocating num_elem_colors_inner_core array')
- start(1) = local_dim_num_elem_colors * myrank
+ start(1) = offset_num_elem_colors
count(1)= num_colors_outer_inner_core + num_colors_inner_inner_core
call adios_selection_boundingbox (sel , 1, start, count)
call adios_schedule_read(adios_handle, sel, &
More information about the CIG-COMMITS
mailing list