[cig-commits] [commit] master: adding an option to write the snapshots directly in the chunking (2b399bb)
cig_noreply at geodynamics.org
cig_noreply at geodynamics.org
Fri Oct 17 05:30:14 PDT 2014
Repository : https://github.com/geodynamics/axisem
On branch : master
Link : https://github.com/geodynamics/axisem/compare/607f803cf074063627513d235f9ed0837fc1dd44...b6457db24acdde4a4e1c08935ae1b22adf87f5bf
>---------------------------------------------------------------
commit 2b399bb768395705d4ded2ce92dddf04bfb36dd6
Author: martinvandriel <vandriel at erdw.ethz.ch>
Date: Thu Oct 16 21:14:38 2014 +0200
adding an option to write the snapshots directly in the chunking
needed for kerner and instaseis
>---------------------------------------------------------------
2b399bb768395705d4ded2ce92dddf04bfb36dd6
SOLVER/inparam_advanced.TEMPLATE | 12 +++++++++++-
SOLVER/nc_routines.F90 | 20 ++++++++++++++++++--
SOLVER/parameters.F90 | 7 ++++++-
3 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/SOLVER/inparam_advanced.TEMPLATE b/SOLVER/inparam_advanced.TEMPLATE
index c00368b..61faf0a 100644
--- a/SOLVER/inparam_advanced.TEMPLATE
+++ b/SOLVER/inparam_advanced.TEMPLATE
@@ -63,11 +63,21 @@ CHECKPOINTING true
# Size of buffer for wavefield dumps. This is only used for the kernel wavefield
# output. Determines, how many time steps should be held in memory by each
# processor. Increase, if disk access is too frequent, however, for short period
-# meshes (<10s), setting a value above 256 can require >>10GB of memory per CPU.
+# meshes (<10s) running on few cores, setting a value above 256 can require >>10GB of
+# memory per CPU. When using CHUNK_TIME_TRACES and many cores, a value larger
+# then the number of strain samples might be appropriate.
# Default value: 128
NETCDF_DUMP_BUFFER 128
+# Chunking of the kernel wavefields: Default is snapshots to write to contiguos
+# regions on the drive. When running on many cores and using collective IO, it
+# might be useful to write time traces directly and avoid the rechunking
+# (fieldtransfom) usually done in postprocessing.
+# Default: false
+CHUNK_TIME_TRACES false
+
# Level of compression of wavefield data (0: off, 9:highest, default: 5)
+# deflation can not be used with collective IO
DEFLATE_LEVEL 5
####################### Wavefield output #######################################
diff --git a/SOLVER/nc_routines.F90 b/SOLVER/nc_routines.F90
index 420d341..36356b3 100644
--- a/SOLVER/nc_routines.F90
+++ b/SOLVER/nc_routines.F90
@@ -126,6 +126,10 @@ module nc_routines
!> How many snaps should be buffered in RAM?
integer :: nc_dumpbuffersize
+ !> chunking
+ logical :: nc_chunk_time_traces
+ integer, parameter :: disk_block_size = 8192
+
public :: nc_dump_strain, nc_dump_rec, nc_dump_surface
public :: nc_dump_field_solid, nc_dump_field_fluid
public :: nc_define_outputfile, nc_finish_prepare, nc_end_output
@@ -135,7 +139,7 @@ module nc_routines
public :: nc_dump_elastic_parameters
public :: nc_dump_stf, nc_rec_checkpoint
- public :: nc_dumpbuffersize
+ public :: nc_dumpbuffersize, nc_chunk_time_traces
public :: set_npoints
contains
@@ -956,6 +960,10 @@ subroutine nc_define_outputfile(nrec, rec_names, rec_th, rec_th_req, rec_ph, rec
integer :: nc_mesh_elem_dimid, nc_mesh_npol_dimid
integer :: nc_mesh_cntrlpts_dimid
+ integer :: chunk_pt, chunk_time
+ !< Contains chunk size in GLL points. Should be system
+ !! int(disk_block_size / nsnap)
+
if ((mynum == 0) .and. (verbose > 1)) then
write(6,*)
write(6,*) '************************************************************************'
@@ -1392,6 +1400,14 @@ subroutine nc_define_outputfile(nrec, rec_names, rec_th, rec_th_req, rec_ph, rec
varid = nc_mesh_glj_varid) )
endif
+ if (nc_chunk_time_traces) then
+ chunk_pt = max(1, disk_block_size / nstrain)
+ chunk_time = nstrain
+ else
+ chunk_pt = npoints_global
+ chunk_time = 1
+ endif
+
do ivar=1, nvar/2 ! The big snapshot variables for the kerner.
call check( nf90_def_var(ncid = ncid_snapout, &
@@ -1399,7 +1415,7 @@ subroutine nc_define_outputfile(nrec, rec_names, rec_th, rec_th_req, rec_ph, rec
xtype = NF90_FLOAT, &
dimids = [nc_pt_dimid, nc_snap_dimid],&
varid = nc_field_varid(ivar), &
- chunksizes = [npoints_global, 1] ))
+ chunksizes = [chunk_pt, chunk_time] ))
call check( nf90_def_var_fill(ncid = ncid_snapout, &
varid = nc_field_varid(ivar), &
diff --git a/SOLVER/parameters.F90 b/SOLVER/parameters.F90
index 4b66ecc..5fc257b 100644
--- a/SOLVER/parameters.F90
+++ b/SOLVER/parameters.F90
@@ -355,7 +355,7 @@ end subroutine
!> Read file inparam_advanced
subroutine read_inparam_advanced
- use nc_routines, only: nc_dumpbuffersize
+ use nc_routines, only: nc_dumpbuffersize, nc_chunk_time_traces
use data_mesh, only: naxel, meshname, do_mesh_tests
use commun, only: broadcast_int, broadcast_log, broadcast_char, broadcast_dble
@@ -400,6 +400,7 @@ subroutine read_inparam_advanced
use_netcdf = .false.
checkpointing = .false.
nc_dumpbuffersize = 128
+ nc_chunk_time_traces = .false.
! xdmf stuff
i_n_xdmf = -1
@@ -520,6 +521,9 @@ subroutine read_inparam_advanced
case('NETCDF_DUMP_BUFFER')
read(keyvalue, *) nc_dumpbuffersize
+ case('CHUNK_TIME_TRACES')
+ read(keyvalue, *) nc_chunk_time_traces
+
case('DEFLATE_LEVEL')
read(keyvalue,*) deflate_level
@@ -611,6 +615,7 @@ subroutine read_inparam_advanced
call broadcast_log(use_netcdf, 0)
call broadcast_log(checkpointing, 0)
call broadcast_int(nc_dumpbuffersize, 0)
+ call broadcast_log(nc_chunk_time_traces, 0)
call broadcast_dble(xdmf_rmin, 0)
call broadcast_dble(xdmf_rmax, 0)
More information about the CIG-COMMITS
mailing list