[cig-commits] [commit] devel: done adding support for new option NUMBER_OF_SIMULTANEOUS_RUNS. Also fixed a dependency issue that was in the code when compiling without MPI. (b4aa078)
cig_noreply at geodynamics.org
cig_noreply at geodynamics.org
Sun Jul 20 11:01:53 PDT 2014
Repository : https://github.com/geodynamics/specfem3d
On branch : devel
Link : https://github.com/geodynamics/specfem3d/compare/b6ea28a44e0ddab4032121e0cd5d44c1f01f1ada...b4aa0782b0d2e703525e73f7817868cef1a17192
>---------------------------------------------------------------
commit b4aa0782b0d2e703525e73f7817868cef1a17192
Author: Dimitri Komatitsch <komatitsch at lma.cnrs-mrs.fr>
Date: Sun Jul 20 18:41:30 2014 +0200
done adding support for new option NUMBER_OF_SIMULTANEOUS_RUNS. Also fixed a dependency issue that was in the code when compiling without MPI.
>---------------------------------------------------------------
b4aa0782b0d2e703525e73f7817868cef1a17192
Makefile.in | 4 ++
doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.pdf | Bin 12647247 -> 12647371 bytes
doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.tex | 12 ++---
setup/constants.h.in | 38 +++++++++-------
src/auxiliaries/model_update.f90 | 21 +++++----
src/auxiliaries/rules.mk | 12 ++++-
src/check_mesh_quality_CUBIT_Abaqus/rules.mk | 3 +-
src/decompose_mesh/rules.mk | 5 ++-
src/generate_databases/rules.mk | 1 +
src/meshfem3D/rules.mk | 1 +
src/shared/constants_mod.f90 | 7 +++
src/shared/get_cmt.f90 | 9 +++-
src/shared/get_force.f90 | 10 ++++-
src/shared/parallel.f90 | 59 ++++++++++++++++---------
src/shared/read_parameter_file.f90 | 42 +++++++++++++++++-
src/shared/read_value_parameters.f90 | 12 ++++-
src/shared/rules.mk | 4 +-
src/shared/serial.f90 | 47 ++++++++++++++++++--
src/specfem3D/finalize_simulation.f90 | 3 +-
src/specfem3D/initialize_simulation.f90 | 22 +++++----
src/specfem3D/prepare_timerun.F90 | 1 +
src/specfem3D/rules.mk | 7 +--
src/specfem3D/setup_sources_receivers.f90 | 24 ++++++++++
src/specfem3D/specfem3D_par.f90 | 1 +
src/specfem3D/write_output_SU.f90 | 7 +--
src/specfem3D/write_seismograms.f90 | 39 ++++------------
26 files changed, 273 insertions(+), 118 deletions(-)
diff --git a/Makefile.in b/Makefile.in
index 65f7944..c199f71 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -254,10 +254,14 @@ ifdef CLEAN
clean:
@echo "cleaning by CLEAN defined"
-rm -f $(foreach dir, $(CLEAN), $($(dir)_OBJECTS) $($(dir)_MODULES) $($(dir)_SHARED_OBJECTS) $($(dir)_TARGETS))
+ -rm -f ${E}/*__genmod.*
+ -rm -f ${O}/*__genmod.*
else
clean:
@echo "cleaning by CLEAN not defined"
-rm -f $(foreach dir, $(SUBDIRS), $($(dir)_OBJECTS) $($(dir)_MODULES) $($(dir)_TARGETS))
+ -rm -f ${E}/*__genmod.*
+ -rm -f ${O}/*__genmod.*
endif
realclean: clean
diff --git a/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.pdf b/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.pdf
index 3a28034..c56baf9 100644
Binary files a/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.pdf and b/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.pdf differ
diff --git a/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.tex b/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.tex
index d31d151..5304d98 100644
--- a/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.tex
+++ b/doc/USER_MANUAL/manual_SPECFEM3D_Cartesian.tex
@@ -2164,18 +2164,20 @@ We have also added the ability to run several calculations (several earthquakes)
in an embarrassingly-parallel fashion from within the same run;
this can be useful when using a very large supercomputer to compute
many earthquakes in a catalog, in which case it can be better from
-a batch job submission point of few to start fewer and much larger jobs,
+a batch job submission point of view to start fewer and much larger jobs,
each of them computing several earthquakes in parallel.
To turn that option on, set parameter NUMBER\_OF\_SIMULTANEOUS\_RUNS
to a value greater than 1 in file setup/constants.h.in before
configuring and compiling the code.
When that option is on, of course the number of processor cores used to start
the code in the batch system must be a multiple of NUMBER\_OF\_SIMULTANEOUS\_RUNS,
-all the individual runs must use the same number of processor cores
-(which as usual is NPROC in the input file DATA/Par\_file,
+all the individual runs must use the same number of processor cores,
+which as usual is NPROC in the input file DATA/Par\_file,
and thus the total number of processor cores to request from the batch system
-should be NUMBER\_OF\_SIMULTANEOUS\_RUNS * NPROC.
-All the runs to perform must be placed in directories called run0001, run0002, run0003 and so on (with exactly four digits).
+should be NUMBER\_OF\_SIMULTANEOUS\_RUNS $\times$ NPROC.
+All the runs to perform must be placed in directories called run0001, run0002, run0003 and so on (with exactly four digits)
+and you must create a link from the root directory of the code
+to the first copy of the executable programs by typing " ln -s run0001/bin bin ".
\chapter{\label{cha:Fault-Kinematics-Dynamics}Kinematic and dynamic fault
diff --git a/setup/constants.h.in b/setup/constants.h.in
index ec4369f..d907208 100644
--- a/setup/constants.h.in
+++ b/setup/constants.h.in
@@ -51,7 +51,7 @@
!! DK DK in an embarrassingly-parallel fashion from within the same run;
!! DK DK this can be useful when using a very large supercomputer to compute
!! DK DK many earthquakes in a catalog, in which case it can be better from
-!! DK DK a batch job submission point of few to start fewer and much larger jobs,
+!! DK DK a batch job submission point of view to start fewer and much larger jobs,
!! DK DK each of them computing several earthquakes in parallel.
!! DK DK To turn that option on, set parameter NUMBER_OF_SIMULTANEOUS_RUNS
!! DK DK to a value greater than 1 in file setup/constants.h.in before
@@ -59,15 +59,30 @@
!! DK DK To implement that, we create NUMBER_OF_SIMULTANEOUS_RUNS MPI sub-communicators,
!! DK DK each of them being labeled "my_local_mpi_comm_world", and we use them
!! DK DK in all the routines in "src/shared/parallel.f90", except in MPI_ABORT() because in that case
-!! DK DK we need to keep the entire run.
+!! DK DK we need to kill the entire run.
!! DK DK When that option is on, of course the number of processor cores used to start
!! DK DK the code in the batch system must be a multiple of NUMBER_OF_SIMULTANEOUS_RUNS,
-!! DK DK all the individual runs must use the same number of processor cores
-!! DK DK (which as usual is NPROC in the input file DATA/Par_file,
+!! DK DK all the individual runs must use the same number of processor cores,
+!! DK DK which as usual is NPROC in the input file DATA/Par_file,
!! DK DK and thus the total number of processor cores to request from the batch system
!! DK DK should be NUMBER_OF_SIMULTANEOUS_RUNS * NPROC.
!! DK DK All the runs to perform must be placed in directories called run0001, run0002, run0003 and so on
-!! DK DK (with exactly four digits).
+!! DK DK (with exactly four digits) and you must create a link from the root directory of the code
+!! DK DK to the first copy of the executable programs by typing " ln -s run0001/bin bin ".
+!!
+!! DK DK Imagine you have 10 independent calculations to do, each of them on 100 cores; you have three options:
+!!
+!! DK DK 1/ submit 10 jobs to the batch system
+!!
+!! DK DK 2/ submit a single job on 1000 cores to the batch, and in that script create a sub-array of jobs to start 10 jobs,
+!! DK DK each running on 100 cores (see e.g. http://www.schedmd.com/slurmdocs/job_array.html )
+!!
+!! DK DK 3/ submit a single job on 1000 cores to the batch, start SPECFEM3D on 1000 cores, create 10 sub-communicators,
+!! DK DK cd into one of 10 subdirectories (called e.g. run0001, run0002,... run0010) depending on the sub-communicator
+!! DK DK your MPI rank belongs to, and run normally on 100 cores using that sub-communicator.
+!!
+!! DK DK The option below implements 3/.
+!!
integer, parameter :: NUMBER_OF_SIMULTANEOUS_RUNS = 1
! set to .false. if running on a Beowulf-type machine with local disks
@@ -217,9 +232,6 @@
! decide if master process writes all the seismograms or if all processes do it in parallel
logical, parameter :: WRITE_SEISMOGRAMS_BY_MASTER = .false.
-! use directory OUTPUT_FILES/ for seismogram output
- logical,parameter :: USE_OUTPUT_FILES_PATH = .true.
-
!!-----------------------------------------------------------
!!
!! C-PML absorbing boundary conditions
@@ -249,8 +261,7 @@
! paths for inputs and outputs files
character(len=*), parameter :: IN_DATA_FILES_PATH = './DATA/'
character(len=*), parameter :: MF_IN_DATA_FILES_PATH = './DATA/meshfem3D_files/'
- character(len=*), parameter :: OUTPUT_FILES_PATH = './OUTPUT_FILES/'
-
+ character(len=*), parameter :: OUTPUT_FILES_PATH_BASE = './OUTPUT_FILES/'
! ---------------------------------------------------------------------------------------
! LQY -- Following 3 variables stays here temporarily,
@@ -266,13 +277,8 @@
! to be used in the backward reconstruction in the presence of attenuation
integer, parameter :: NSTEP_Q_SAVE = 50 ! depending on stability of reconstruction, up to 200
-! the scratch disk to save the state variables saved in the forward
-! simulation, this can be a global scratch disk in case you run out of
-! space on the local scratch disk, e.g. '/ibrixfs1/scratch/lqy/DATABASES_MPI_Q/'
- character(len=MAX_STRING_LEN), parameter :: LOCAL_PATH_Q = './OUTPUT_FILES/DATABASES_MPI'
-
!------------------------------------------------------
-! nlegoff -- Variables that should be read/computed elsewhere.
+! Nicolas Legoff -- Variables that should be read/computed elsewhere.
! Temporarily declared here.
!------------------------------------------------------
diff --git a/src/auxiliaries/model_update.f90 b/src/auxiliaries/model_update.f90
index 0079091..fa504b6 100644
--- a/src/auxiliaries/model_update.f90
+++ b/src/auxiliaries/model_update.f90
@@ -25,29 +25,26 @@
!
!=====================================================================
-
program model_update
use specfem_par
use specfem_par_elastic
use specfem_par_acoustic
use specfem_par_poroelastic
+
implicit none
! ======================================================
! USER PARAMETERS
! directory where the mesh files for the NEW model will be written
- character(len=MAX_STRING_LEN), parameter :: &
- LOCAL_PATH_NEW = trim(LOCAL_PATH_Q)//'/mesh_files_m01'
+ character(len=MAX_STRING_LEN) :: LOCAL_PATH_NEW
! directory where the output files of model_update will be written
- character(len=*), parameter :: &
- OUTPUT_MODEL_UPD = trim(OUTPUT_FILES_PATH)//'/OUTPUT_FILES_MODEL_UPD'
+ character(len=MAX_STRING_LEN) :: OUTPUT_MODEL_UPD
! directory where the summed and smoothed input kernels are linked
- character(len=*), parameter :: &
- INPUT_KERNELS = trim(LOCAL_PATH_Q)//'/sum_smooth_kern'
+ character(len=MAX_STRING_LEN) :: INPUT_KERNELS
! by default, this algorithm uses (bulk,bulk_beta,rho) kernels to update vp,vs,rho
! if you prefer using (alpha,beta,rho) kernels, set this flag to true
@@ -148,13 +145,21 @@ program model_update
! subjective step length to multiply to the gradient
! e.g. step_fac = 0.03
-
call get_command_argument(1,s_step_fac)
if (trim(s_step_fac) == '') then
call exit_MPI(myrank,'Usage: add_model step_factor')
endif
+ ! directory where the mesh files for the NEW model will be written
+ LOCAL_PATH_NEW = trim(OUTPUT_FILES_PATH)//'/mesh_files_m01'
+
+ ! directory where the output files of model_update will be written
+ OUTPUT_MODEL_UPD = trim(OUTPUT_FILES_PATH)//'/OUTPUT_FILES_MODEL_UPD'
+
+ ! directory where the summed and smoothed input kernels are linked
+ INPUT_KERNELS = trim(OUTPUT_FILES_PATH)//'/sum_smooth_kern'
+
! read in parameter information
read(s_step_fac,*) step_fac
if( abs(step_fac) < 1.e-10) then
diff --git a/src/auxiliaries/rules.mk b/src/auxiliaries/rules.mk
index 9598fd6..92c12ac 100644
--- a/src/auxiliaries/rules.mk
+++ b/src/auxiliaries/rules.mk
@@ -54,6 +54,7 @@ auxiliaries_OBJECTS = \
# These files come from the shared directory
auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/check_mesh_resolution.shared.o \
$O/create_name_database.shared.o \
$O/exit_mpi.shared.o \
@@ -85,6 +86,7 @@ model_upd_auxiliaries_OBJECTS = \
$(EMPTY_MACRO)
model_upd_auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/check_mesh_resolution.shared.o \
$O/create_name_database.shared.o \
$O/exit_mpi.shared.o \
@@ -106,6 +108,7 @@ adios_model_upd_auxiliaries_OBJECTS= \
$O/read_forward_arrays_adios.spec_adios.o
adios_model_upd_auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/adios_manager.shared_adios.o \
$O/adios_helpers_definitions.shared_adios_module.o \
$O/adios_helpers_writers.shared_adios_module.o \
@@ -138,6 +141,7 @@ sum_kernels_auxiliaries_OBJECTS = \
$(EMPTY_MACRO)
sum_kernels_auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/exit_mpi.shared.o \
$O/param_reader.cc.o \
$O/read_parameter_file.shared.o \
@@ -157,6 +161,7 @@ smooth_vol_data_auxiliaries_OBJECTS = \
$(EMPTY_MACRO)
smooth_vol_data_auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/exit_mpi.shared.o \
$O/gll_library.shared.o \
$O/param_reader.cc.o \
@@ -176,6 +181,7 @@ combine_surf_data_auxiliaries_OBJECTS = \
$(EMPTY_MACRO)
combine_surf_data_auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/param_reader.cc.o \
$O/write_c_binary.cc.o \
$(EMPTY_MACRO)
@@ -192,6 +198,7 @@ combine_vol_data_auxiliaries_OBJECTS = \
$(EMPTY_MACRO)
combine_vol_data_auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/read_parameter_file.shared.o \
$O/read_value_parameters.shared.o \
$O/param_reader.cc.o \
@@ -223,6 +230,7 @@ create_movie_shakemap_AVS_DX_GMT_auxiliaries_OBJECTS = \
$(EMPTY_MACRO)
create_movie_shakemap_AVS_DX_GMT_auxiliaries_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/get_global.shared.o \
$O/param_reader.cc.o \
$O/read_parameter_file.shared.o \
@@ -280,10 +288,10 @@ $E/xcreate_movie_shakemap_AVS_DX_GMT: $(create_movie_shakemap_AVS_DX_GMT_auxilia
$E/xmodel_update: $(model_upd_auxiliaries_OBJECTS) $(model_upd_auxiliaries_SHARED_OBJECTS) $(COND_MPI_OBJECTS)
${FCLINK} -o $@ $+ $(MPILIBS)
-$E/xsmooth_vol_data: $(smooth_vol_data_auxiliaries_OBJECTS) $(smooth_vol_data_auxiliaries_SHARED_OBJECTS) $(COND_MPI_OBJECTS)
+$E/xsmooth_vol_data: $(smooth_vol_data_auxiliaries_OBJECTS) $(smooth_vol_data_auxiliaries_SHARED_OBJECTS) $(COND_MPI_OBJECTS) $O/unused_mod.shared_module.o
${FCLINK} -o $@ $+ $(MPILIBS)
-$E/xsum_kernels: $(sum_kernels_auxiliaries_OBJECTS) $(sum_kernels_auxiliaries_SHARED_OBJECTS) $(COND_MPI_OBJECTS)
+$E/xsum_kernels: $(sum_kernels_auxiliaries_OBJECTS) $(sum_kernels_auxiliaries_SHARED_OBJECTS) $(COND_MPI_OBJECTS) $O/unused_mod.shared_module.o
${FCLINK} -o $@ $+ $(MPILIBS)
diff --git a/src/check_mesh_quality_CUBIT_Abaqus/rules.mk b/src/check_mesh_quality_CUBIT_Abaqus/rules.mk
index 2d01c91..05320af 100644
--- a/src/check_mesh_quality_CUBIT_Abaqus/rules.mk
+++ b/src/check_mesh_quality_CUBIT_Abaqus/rules.mk
@@ -45,6 +45,7 @@ check_mesh_quality_CUBIT_Abaqus_TARGETS = \
check_mesh_quality_CUBIT_Abaqus_OBJECTS = \
$O/check_mesh_quality_CUBIT_Abaqus.check.o \
$O/convert_skewness_to_angle.check.o \
+ $O/unused_mod.shared_module.o \
$O/multiply_CUBIT_Abaqus_mesh_by_1000.check.o \
$(EMPTY_MACRO)
@@ -86,7 +87,7 @@ $E/xmultiply_CUBIT_Abaqus_mesh_by_1000: $O/multiply_CUBIT_Abaqus_mesh_by_1000.ch
#### rule to build each .o file below
####
-$O/%.check.o: $S/%.f90 $O/constants_mod.shared_module.o
+$O/%.check.o: $S/%.f90 $O/constants_mod.shared_module.o $O/unused_mod.shared_module.o
${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $<
diff --git a/src/decompose_mesh/rules.mk b/src/decompose_mesh/rules.mk
index 4c82f24..4bc8f6d 100644
--- a/src/decompose_mesh/rules.mk
+++ b/src/decompose_mesh/rules.mk
@@ -55,6 +55,7 @@ decompose_mesh_MODULES = \
$(EMPTY_MACRO)
decompose_mesh_SHARED_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/param_reader.cc.o \
$O/read_parameter_file.shared.o \
$O/read_value_parameters.shared.o \
@@ -99,8 +100,8 @@ endif
### Module dependencies
###
-$O/decompose_mesh.dec.o: $O/part_decompose_mesh.dec.o $O/fault_scotch.dec.o ${SCOTCH_DIR}/include/scotchf.h
-$O/program_decompose_mesh.dec.o: $O/decompose_mesh.dec.o
+$O/decompose_mesh.dec.o: $O/part_decompose_mesh.dec.o $O/fault_scotch.dec.o ${SCOTCH_DIR}/include/scotchf.h $O/constants_mod.shared_module.o
+$O/program_decompose_mesh.dec.o: $O/decompose_mesh.dec.o $O/constants_mod.shared_module.o
#######################################
diff --git a/src/generate_databases/rules.mk b/src/generate_databases/rules.mk
index 73b2df2..69ca870 100644
--- a/src/generate_databases/rules.mk
+++ b/src/generate_databases/rules.mk
@@ -88,6 +88,7 @@ generate_databases_MODULES = \
generate_databases_SHARED_OBJECTS = \
$O/assemble_MPI_scalar.shared.o \
+ $O/constants_mod.shared_module.o \
$O/check_mesh_resolution.shared.o \
$O/create_name_database.shared.o \
$O/create_serial_name_database.shared.o \
diff --git a/src/meshfem3D/rules.mk b/src/meshfem3D/rules.mk
index bb5fc83..b42ebea 100644
--- a/src/meshfem3D/rules.mk
+++ b/src/meshfem3D/rules.mk
@@ -67,6 +67,7 @@ meshfem3D_MODULES = \
meshfem3D_SHARED_OBJECTS = \
$O/create_name_database.shared.o \
+ $O/constants_mod.shared_module.o \
$O/exit_mpi.shared.o \
$O/get_global.shared.o \
$O/hex_nodes.shared.o \
diff --git a/src/shared/constants_mod.f90 b/src/shared/constants_mod.f90
index 1cc6cb2..99ff0de 100644
--- a/src/shared/constants_mod.f90
+++ b/src/shared/constants_mod.f90
@@ -31,5 +31,12 @@ module constants
include "constants.h"
+ ! a negative initial value is a convention that indicates that groups (i.e. sub-communicators, one per run) are off by default
+ integer :: mygroup = -1
+
+ ! create a copy of the original output file path, to which we may add a "run0001/", "run0002/", "run0003/" prefix later
+ ! if NUMBER_OF_SIMULTANEOUS_RUNS > 1
+ character(len=MAX_STRING_LEN) :: OUTPUT_FILES_PATH = OUTPUT_FILES_PATH_BASE
+
end module constants
diff --git a/src/shared/get_cmt.f90 b/src/shared/get_cmt.f90
index 6172cb2..73b0ca7 100644
--- a/src/shared/get_cmt.f90
+++ b/src/shared/get_cmt.f90
@@ -47,7 +47,7 @@
integer mo,da,julian_day,isource
double precision t_shift(NSOURCES)
character(len=5) datasource
- character(len=MAX_STRING_LEN) :: string, CMTSOLUTION
+ character(len=MAX_STRING_LEN) :: string, CMTSOLUTION,path_to_add
! initializes
lat(:) = 0.d0
@@ -67,6 +67,13 @@
!---- read hypocenter info
!
CMTSOLUTION = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'CMTSOLUTION'
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ CMTSOLUTION = path_to_add(1:len_trim(path_to_add))//CMTSOLUTION(1:len_trim(CMTSOLUTION))
+ endif
open(unit=1,file=CMTSOLUTION,status='old',action='read')
diff --git a/src/shared/get_force.f90 b/src/shared/get_force.f90
index 2f8bc57..b673138 100644
--- a/src/shared/get_force.f90
+++ b/src/shared/get_force.f90
@@ -48,7 +48,7 @@
double precision t_shift(NSOURCES)
double precision length
character(len=7) dummy
- character(len=MAX_STRING_LEN) :: string, FORCESOLUTION
+ character(len=MAX_STRING_LEN) :: string, FORCESOLUTION,path_to_add
! initializes
lat(:) = 0.d0
@@ -61,10 +61,18 @@
comp_dir_vect_source_E(:) = 0.d0
comp_dir_vect_source_N(:) = 0.d0
comp_dir_vect_source_Z_UP(:) = 0.d0
+
!
!---- read info
!
FORCESOLUTION = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'FORCESOLUTION'
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ FORCESOLUTION = path_to_add(1:len_trim(path_to_add))//FORCESOLUTION(1:len_trim(FORCESOLUTION))
+ endif
open(unit=1,file=trim(FORCESOLUTION),status='old',action='read')
diff --git a/src/shared/parallel.f90 b/src/shared/parallel.f90
index 832839d..97350b6 100644
--- a/src/shared/parallel.f90
+++ b/src/shared/parallel.f90
@@ -30,23 +30,24 @@
!! DK DK in an embarrassingly-parallel fashion from within the same run;
!! DK DK this can be useful when using a very large supercomputer to compute
!! DK DK many earthquakes in a catalog, in which case it can be better from
-!! DK DK a batch job submission point of few to start fewer and much larger jobs,
+!! DK DK a batch job submission point of view to start fewer and much larger jobs,
!! DK DK each of them computing several earthquakes in parallel.
!! DK DK To turn that option on, set parameter NUMBER_OF_SIMULTANEOUS_RUNS
!! DK DK to a value greater than 1 in file setup/constants.h.in before
!! DK DK configuring and compiling the code.
!! DK DK To implement that, we create NUMBER_OF_SIMULTANEOUS_RUNS MPI sub-communicators,
!! DK DK each of them being labeled "my_local_mpi_comm_world", and we use them
-!! DK DK in all the routines below, except in MPI_ABORT() because in that case
-!! DK DK we need to keep the entire run.
+!! DK DK in all the routines in "src/shared/parallel.f90", except in MPI_ABORT() because in that case
+!! DK DK we need to kill the entire run.
!! DK DK When that option is on, of course the number of processor cores used to start
!! DK DK the code in the batch system must be a multiple of NUMBER_OF_SIMULTANEOUS_RUNS,
-!! DK DK all the individual runs must use the same number of processor cores
-!! DK DK (which as usual is NPROC in the input file DATA/Par_file,
+!! DK DK all the individual runs must use the same number of processor cores,
+!! DK DK which as usual is NPROC in the input file DATA/Par_file,
!! DK DK and thus the total number of processor cores to request from the batch system
!! DK DK should be NUMBER_OF_SIMULTANEOUS_RUNS * NPROC.
!! DK DK All the runs to perform must be placed in directories called run0001, run0002, run0003 and so on
-!! DK DK (with exactly four digits).
+!! DK DK (with exactly four digits) and you must create a link from the root directory of the code
+!! DK DK to the first copy of the executable programs by typing " ln -s run0001/bin bin ".
module my_mpi
@@ -1199,21 +1200,39 @@ end module my_mpi
implicit none
-! integer,intent(out) :: comm
-! integer :: ier
+ integer :: sizeval,myrank,ier,key
-! call MPI_COMM_DUP(my_local_mpi_comm_world,comm,ier)
-! if( ier /= 0 ) stop 'error duplicating my_local_mpi_comm_world communicator'
+ character(len=MAX_STRING_LEN) :: path_to_add
+
+ if(NUMBER_OF_SIMULTANEOUS_RUNS <= 0) stop 'NUMBER_OF_SIMULTANEOUS_RUNS <= 0 makes no sense'
+
+ call MPI_COMM_SIZE(MPI_COMM_WORLD,sizeval,ier)
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mod(sizeval,NUMBER_OF_SIMULTANEOUS_RUNS) /= 0) &
+ stop 'the number of MPI processes is not a multiple of NUMBER_OF_SIMULTANEOUS_RUNS'
+
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. IMAIN == ISTANDARD_OUTPUT) &
+ stop 'must not have IMAIN == ISTANDARD_OUTPUT when NUMBER_OF_SIMULTANEOUS_RUNS > 1 otherwise output to screen is mingled'
if(NUMBER_OF_SIMULTANEOUS_RUNS == 1) then
+
my_local_mpi_comm_world = MPI_COMM_WORLD
- else if(NUMBER_OF_SIMULTANEOUS_RUNS <= 0) then
- stop 'NUMBER_OF_SIMULTANEOUS_RUNS <= 0 makes no sense'
+
else
- !! DK DK temporary hack
- my_local_mpi_comm_world = MPI_COMM_WORLD
- stop 'NUMBER_OF_SIMULTANEOUS_RUNS > 1 not implemented yet'
- ! we should call MPI_COMM_SPLIT() here
+
+ call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ier)
+! create the different groups of processes, one for each independent run
+ mygroup = mod(myrank,NUMBER_OF_SIMULTANEOUS_RUNS)
+ key = myrank
+ if(mygroup < 0 .or. mygroup > NUMBER_OF_SIMULTANEOUS_RUNS-1) stop 'invalid value of mygroup'
+
+! build the sub-communicators
+ call MPI_COMM_SPLIT(MPI_COMM_WORLD, mygroup, key, my_local_mpi_comm_world, ier)
+ if(ier /= 0) stop 'error while trying to create the sub-communicators'
+
+! add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ OUTPUT_FILES_PATH = path_to_add(1:len_trim(path_to_add))//OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH))
+
endif
end subroutine world_split
@@ -1230,11 +1249,9 @@ end module my_mpi
implicit none
- if(NUMBER_OF_SIMULTANEOUS_RUNS > 1) then
- !! DK DK temporary hack
- stop 'NUMBER_OF_SIMULTANEOUS_RUNS > 1 not implemented yet'
- ! we should call MPI_COMM_FREE() here
- endif
+ integer :: ier
+
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1) call MPI_COMM_FREE(my_local_mpi_comm_world,ier)
end subroutine world_unsplit
diff --git a/src/shared/read_parameter_file.f90 b/src/shared/read_parameter_file.f90
index 5ee1325..4fea2a8 100644
--- a/src/shared/read_parameter_file.f90
+++ b/src/shared/read_parameter_file.f90
@@ -54,7 +54,7 @@
logical USE_FORCE_POINT_SOURCE,STACEY_INSTEAD_OF_FREE_SURFACE,USE_RICKER_TIME_FUNCTION
logical PML_CONDITIONS,PML_INSTEAD_OF_FREE_SURFACE,FULL_ATTENUATION_SOLID,COUPLE_WITH_DSM
- character(len=MAX_STRING_LEN) :: LOCAL_PATH,TOMOGRAPHY_PATH,CMTSOLUTION,FORCESOLUTION,TRACTION_PATH
+ character(len=MAX_STRING_LEN) :: LOCAL_PATH,TOMOGRAPHY_PATH,CMTSOLUTION,FORCESOLUTION,TRACTION_PATH,path_to_add
! local variables
integer ::ios,icounter,isource,idummy,nproc_eta_old,nproc_xi_old
@@ -120,8 +120,17 @@
if (ierr /= 0) return
call read_value_logical(ANISOTROPY, 'ANISOTROPY', ierr)
if (ierr /= 0) return
+
call read_value_string(TOMOGRAPHY_PATH, 'TOMOGRAPHY_PATH', ierr)
if (ierr /= 0) return
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ TOMOGRAPHY_PATH = path_to_add(1:len_trim(path_to_add))//TOMOGRAPHY_PATH(1:len_trim(TOMOGRAPHY_PATH))
+ endif
+
call read_value_logical(USE_OLSEN_ATTENUATION, 'USE_OLSEN_ATTENUATION', ierr)
if (ierr /= 0) return
call read_value_double_precision(OLSEN_ATTENUATION_RATIO, 'OLSEN_ATTENUATION_RATIO', ierr)
@@ -154,8 +163,17 @@
if (ierr /= 0) return
call read_value_logical(SAVE_MESH_FILES, 'SAVE_MESH_FILES', ierr)
if (ierr /= 0) return
+
call read_value_string(LOCAL_PATH, 'LOCAL_PATH', ierr)
if (ierr /= 0) return
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ LOCAL_PATH = path_to_add(1:len_trim(path_to_add))//LOCAL_PATH(1:len_trim(LOCAL_PATH))
+ endif
+
call read_value_integer(NTSTEP_BETWEEN_OUTPUT_INFO, 'NTSTEP_BETWEEN_OUTPUT_INFO', ierr)
if (ierr /= 0) return
call read_value_integer(NTSTEP_BETWEEN_OUTPUT_SEISMOS, 'NTSTEP_BETWEEN_OUTPUT_SEISMOS', ierr)
@@ -170,8 +188,16 @@
if (ierr /= 0) return
call read_value_logical(COUPLE_WITH_DSM, 'COUPLE_WITH_DSM', ierr)
if (ierr /= 0) return
+
call read_value_string(TRACTION_PATH, 'TRACTION_PATH', ierr)
if (ierr /= 0) return
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ TRACTION_PATH = path_to_add(1:len_trim(path_to_add))//TRACTION_PATH(1:len_trim(TRACTION_PATH))
+ endif
! close parameter file
call close_parameter_file()
@@ -211,6 +237,13 @@
! compute the total number of sources in the FORCESOLUTION file
! there are NLINES_PER_FORCESOLUTION_SOURCE lines per source in that file
FORCESOLUTION = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'FORCESOLUTION'
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ FORCESOLUTION = path_to_add(1:len_trim(path_to_add))//FORCESOLUTION(1:len_trim(FORCESOLUTION))
+ endif
open(unit=21,file=trim(FORCESOLUTION),iostat=ios,status='old',action='read')
if (ios /= 0) stop 'error opening FORCESOLUTION file'
@@ -232,6 +265,13 @@
! compute the total number of sources in the CMTSOLUTION file
! there are NLINES_PER_CMTSOLUTION_SOURCE lines per source in that file
CMTSOLUTION = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'CMTSOLUTION'
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ CMTSOLUTION = path_to_add(1:len_trim(path_to_add))//CMTSOLUTION(1:len_trim(CMTSOLUTION))
+ endif
open(unit=21,file=trim(CMTSOLUTION),iostat=ios,status='old',action='read')
if (ios /= 0) stop 'error opening CMTSOLUTION file'
diff --git a/src/shared/read_value_parameters.f90 b/src/shared/read_value_parameters.f90
index 7546967..d67bd17 100644
--- a/src/shared/read_value_parameters.f90
+++ b/src/shared/read_value_parameters.f90
@@ -101,13 +101,21 @@
subroutine open_parameter_file(ierr)
- use constants, only: MAX_STRING_LEN,IN_DATA_FILES_PATH
+ use constants, only: MAX_STRING_LEN,IN_DATA_FILES_PATH,NUMBER_OF_SIMULTANEOUS_RUNS,mygroup
implicit none
integer ierr
- character(len=MAX_STRING_LEN) :: filename
+ character(len=MAX_STRING_LEN) :: filename,path_to_add
+
filename = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'Par_file'
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ filename = path_to_add(1:len_trim(path_to_add))//filename(1:len_trim(filename))
+ endif
call param_open(filename, len(filename), ierr)
if (ierr /= 0) then
diff --git a/src/shared/rules.mk b/src/shared/rules.mk
index 75e9d26..896e64e 100644
--- a/src/shared/rules.mk
+++ b/src/shared/rules.mk
@@ -136,10 +136,10 @@ $O/unused_mod.shared_module.o: $O/constants_mod.shared_module.o
$O/%.shared_module.o: $S/%.f90 ${SETUP}/constants.h
${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $<
-$O/%.shared.o: $S/%.f90 $O/constants_mod.shared_module.o
+$O/%.shared.o: $S/%.f90 $O/constants_mod.shared_module.o $O/unused_mod.shared_module.o
${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $<
-$O/%.shared.o: $S/%.F90 $O/constants_mod.shared_module.o
+$O/%.shared.o: $S/%.F90 $O/constants_mod.shared_module.o $O/unused_mod.shared_module.o
${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $<
$O/%.sharedmpi.o: $S/%.f90 ${SETUP}/constants.h
diff --git a/src/shared/serial.f90 b/src/shared/serial.f90
index 5f30ece..6bd82e9 100644
--- a/src/shared/serial.f90
+++ b/src/shared/serial.f90
@@ -29,7 +29,6 @@
!---- Stubs for parallel routines. Used by the serial version.
!----
-
subroutine stop_all()
stop 'error, program ended in exit_MPI'
end subroutine stop_all
@@ -236,8 +235,14 @@
!----
!
-
subroutine init()
+
+ use constants, only: NUMBER_OF_SIMULTANEOUS_RUNS
+
+ if(NUMBER_OF_SIMULTANEOUS_RUNS <= 0) stop 'NUMBER_OF_SIMULTANEOUS_RUNS <= 0 makes no sense'
+
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1) stop 'serial runs require NUMBER_OF_SIMULTANEOUS_RUNS == 1'
+
end subroutine init
!
@@ -247,7 +252,6 @@
subroutine finalize()
end subroutine finalize
-
!
!----
!
@@ -489,6 +493,21 @@
!----
!
+ subroutine sum_all_1Darray_dp(sendbuf, recvbuf, nx)
+
+ implicit none
+
+ integer :: nx
+ double precision, dimension(nx) :: sendbuf, recvbuf
+
+ recvbuf = sendbuf
+
+ end subroutine sum_all_1Darray_dp
+
+!
+!----
+!
+
subroutine sum_all_all_cr(sendbuf, recvbuf)
use constants
@@ -909,3 +928,25 @@
end subroutine world_duplicate
+!
+!----
+!
+
+ subroutine world_split()
+
+ use constants
+
+ implicit none
+
+ mygroup = 0
+
+ end subroutine world_split
+
+!
+!----
+!
+
+ subroutine world_unsplit()
+
+ end subroutine world_unsplit
+
diff --git a/src/specfem3D/finalize_simulation.f90 b/src/specfem3D/finalize_simulation.f90
index 5c1c0ef..d880c43 100644
--- a/src/specfem3D/finalize_simulation.f90
+++ b/src/specfem3D/finalize_simulation.f90
@@ -128,8 +128,7 @@
if (nrec_local > 0) then
if (.not. (SIMULATION_TYPE == 1 .or. SIMULATION_TYPE == 3)) then
! seismograms
- call write_adj_seismograms2_to_file(myrank,seismograms_eps,number_receiver_global, &
- nrec_local,it,DT,NSTEP,t0,LOCAL_PATH)
+ call write_adj_seismograms2_to_file(myrank,seismograms_eps,number_receiver_global,nrec_local,it,DT,NSTEP,t0)
! source gradients (for sources in elastic domains)
do irec_local = 1, nrec_local
diff --git a/src/specfem3D/initialize_simulation.f90 b/src/specfem3D/initialize_simulation.f90
index 81a8267..b0746cc 100644
--- a/src/specfem3D/initialize_simulation.f90
+++ b/src/specfem3D/initialize_simulation.f90
@@ -396,12 +396,11 @@
implicit none
! check simulation parameters
- if (SIMULATION_TYPE /= 1 .and. NSOURCES > 1000) &
- call exit_mpi(myrank, 'for adjoint simulations, NSOURCES <= 1000')
+ if (SIMULATION_TYPE /= 1 .and. NSOURCES > 1000) call exit_mpi(myrank, 'for adjoint simulations, NSOURCES <= 1000')
! snapshot file names: ADJOINT attenuation
if (ATTENUATION .and. ((SIMULATION_TYPE == 1 .and. SAVE_FORWARD) .or. SIMULATION_TYPE == 3)) &
- call create_name_database(prname_Q,myrank,LOCAL_PATH_Q)
+ call create_name_database(prname_Q,myrank,OUTPUT_FILES_PATH)
! number of elements and points for adjoint arrays
if( SIMULATION_TYPE == 3 ) then
@@ -448,19 +447,18 @@
endif
! check for GPU runs
- if( NGLLX /= 5 .or. NGLLY /= 5 .or. NGLLZ /= 5 ) &
- stop 'GPU mode can only be used if NGLLX == NGLLY == NGLLZ == 5'
- if( CUSTOM_REAL /= 4 ) &
- stop 'GPU mode runs only with CUSTOM_REAL == 4'
- if( SAVE_MOHO_MESH ) &
- stop 'GPU mode does not support SAVE_MOHO_MESH yet'
+ if( NGLLX /= 5 .or. NGLLY /= 5 .or. NGLLZ /= 5 ) stop 'GPU mode can only be used if NGLLX == NGLLY == NGLLZ == 5'
+
+ if( CUSTOM_REAL /= 4 ) stop 'GPU mode runs only with CUSTOM_REAL == 4'
+
+ if( SAVE_MOHO_MESH ) stop 'GPU mode does not support SAVE_MOHO_MESH yet'
+
if( ATTENUATION ) then
if( N_SLS /= 3 ) &
stop 'GPU mode does not support N_SLS /= 3 yet'
endif
- if( POROELASTIC_SIMULATION ) then
- stop 'poroelastic simulations on GPU not supported yet'
- endif
+
+ if( POROELASTIC_SIMULATION ) stop 'poroelastic simulations on GPU not supported yet'
! initializes GPU and outputs info to files for all processes
call initialize_cuda_device(myrank,ncuda_devices)
diff --git a/src/specfem3D/prepare_timerun.F90 b/src/specfem3D/prepare_timerun.F90
index 4ee5a6c..5d99e41 100644
--- a/src/specfem3D/prepare_timerun.F90
+++ b/src/specfem3D/prepare_timerun.F90
@@ -1462,6 +1462,7 @@
subroutine compute_Roland_Sylvain_integr()
use constants
+ use unused_mod
use specfem_par
diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk
index a1243e0..891370c 100644
--- a/src/specfem3D/rules.mk
+++ b/src/specfem3D/rules.mk
@@ -43,6 +43,7 @@ specfem3D_TARGETS = \
specfem3D_OBJECTS = \
+ $O/constants_mod.shared_module.o \
$O/assemble_MPI_vector.spec.o \
$O/check_stability.spec.o \
$O/fault_solver_common.spec.o \
@@ -297,7 +298,7 @@ endif
### Module dependencies
###
-$O/specfem3D_par.spec.o: $O/constants_mod.shared_module.o
+$O/specfem3D_par.spec.o: $O/constants_mod.shared_module.o $O/unused_mod.shared_module.o
$O/compute_stacey_acoustic.spec.o: $O/constants_mod.shared_module.o
$O/compute_stacey_poroelastic.spec.o: $O/constants_mod.shared_module.o
$O/locate_receivers.spec.o: $O/constants_mod.shared_module.o
@@ -335,7 +336,7 @@ $O/model_update.spec.o: $O/specfem3D_par.spec.o
$O/pml_par.spec.o: $O/specfem3D_par.spec.o
$O/setup_GLL_points.spec.o: $O/specfem3D_par.spec.o
$O/setup_movie_meshes.spec.o: $O/specfem3D_par.spec.o
-$O/setup_sources_receivers.spec.o: $O/specfem3D_par.spec.o
+$O/setup_sources_receivers.spec.o: $O/specfem3D_par.spec.o $O/constants_mod.shared_module.o
$O/specfem3D.spec.o: $O/specfem3D_par.spec.o
$O/save_adjoint_kernels.spec.o: $O/specfem3D_par.spec.o
$O/save_external_bin_m_up.spec.o: $O/specfem3D_par.spec.o
@@ -362,7 +363,7 @@ $O/compute_forces_viscoelastic_Dev.spec.o: $O/specfem3D_par.spec.o $O/pml_par.sp
$O/compute_forces_viscoelastic_noDev.spec.o: $O/specfem3D_par.spec.o $O/pml_par.spec.o $O/fault_solver_dynamic.spec.o
$O/compute_forces_viscoelastic_calling_routine.spec.o: $O/specfem3D_par.spec.o $O/pml_par.spec.o $O/fault_solver_dynamic.spec.o $O/fault_solver_kinematic.spec.o
$O/iterate_time.spec.o: $O/specfem3D_par.spec.o $O/gravity_perturbation.spec.o
-$O/prepare_timerun.spec.o: $O/specfem3D_par.spec.o $O/pml_par.spec.o $O/fault_solver_dynamic.spec.o $O/fault_solver_kinematic.spec.o $O/gravity_perturbation.spec.o
+$O/prepare_timerun.spec.o: $O/specfem3D_par.spec.o $O/pml_par.spec.o $O/fault_solver_dynamic.spec.o $O/fault_solver_kinematic.spec.o $O/gravity_perturbation.spec.o $O/unused_mod.shared_module.o
## adios
$O/read_forward_arrays_adios.spec_adios.o: $O/specfem3D_par.spec.o $O/pml_par.spec.o
diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90
index 5d5c9db..44e0184 100644
--- a/src/specfem3D/setup_sources_receivers.f90
+++ b/src/specfem3D/setup_sources_receivers.f90
@@ -30,6 +30,7 @@
subroutine setup_sources_receivers()
use specfem_par
+
implicit none
! locates sources and determines simulation start time t0
@@ -333,14 +334,27 @@
use specfem_par
use specfem_par_acoustic
+
implicit none
integer :: irec,isource,ier
+ character(len=MAX_STRING_LEN) :: path_to_add
+
! reads in station file
if (SIMULATION_TYPE == 1) then
rec_filename = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'STATIONS'
filtered_rec_filename = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'STATIONS_FILTERED'
+
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ rec_filename = path_to_add(1:len_trim(path_to_add))//rec_filename(1:len_trim(rec_filename))
+ filtered_rec_filename = path_to_add(1:len_trim(path_to_add))//filtered_rec_filename(1:len_trim(filtered_rec_filename))
+ endif
+
call station_filter(SUPPRESS_UTM_PROJECTION,UTM_PROJECTION_ZONE,myrank,rec_filename,filtered_rec_filename,nrec, &
LATITUDE_MIN, LATITUDE_MAX, LONGITUDE_MIN, LONGITUDE_MAX)
@@ -350,6 +364,16 @@
else
rec_filename = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'STATIONS_ADJOINT'
filtered_rec_filename = IN_DATA_FILES_PATH(1:len_trim(IN_DATA_FILES_PATH))//'STATIONS_ADJOINT_FILTERED'
+
+! see if we are running several independent runs in parallel
+! if so, add the right directory for that run (group numbers start at zero, but directory names start at run0001, thus we add one)
+! a negative value for "mygroup" is a convention that indicates that groups (i.e. sub-communicators, one per run) are off
+ if(NUMBER_OF_SIMULTANEOUS_RUNS > 1 .and. mygroup >= 0) then
+ write(path_to_add,"('run',i4.4,'/')") mygroup + 1
+ rec_filename = path_to_add(1:len_trim(path_to_add))//rec_filename(1:len_trim(rec_filename))
+ filtered_rec_filename = path_to_add(1:len_trim(path_to_add))//filtered_rec_filename(1:len_trim(filtered_rec_filename))
+ endif
+
call station_filter(SUPPRESS_UTM_PROJECTION,UTM_PROJECTION_ZONE,myrank,rec_filename,filtered_rec_filename,nrec, &
LATITUDE_MIN, LATITUDE_MAX, LONGITUDE_MIN, LONGITUDE_MAX)
if (nrec < 1) call exit_MPI(myrank, 'adjoint simulation needs at least one receiver')
diff --git a/src/specfem3D/specfem3D_par.f90 b/src/specfem3D/specfem3D_par.f90
index 23dff01..133455b 100644
--- a/src/specfem3D/specfem3D_par.f90
+++ b/src/specfem3D/specfem3D_par.f90
@@ -32,6 +32,7 @@ module specfem_par
! main parameter module for specfem simulations
use constants
+ use unused_mod
implicit none
diff --git a/src/specfem3D/write_output_SU.f90 b/src/specfem3D/write_output_SU.f90
index 29faf3a..107b717 100644
--- a/src/specfem3D/write_output_SU.f90
+++ b/src/specfem3D/write_output_SU.f90
@@ -66,12 +66,7 @@
close(IIN_SU1)
! directory to store seismograms
- if( USE_OUTPUT_FILES_PATH ) then
- final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
- else
- ! create full final local path
- final_LOCAL_PATH = LOCAL_PATH(1:len_trim(LOCAL_PATH)) // '/'
- endif
+ final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
write(procname,"(i4)") myrank
procname = adjustl(procname)
diff --git a/src/specfem3D/write_seismograms.f90 b/src/specfem3D/write_seismograms.f90
index abe6bff..ee4fbb4 100644
--- a/src/specfem3D/write_seismograms.f90
+++ b/src/specfem3D/write_seismograms.f90
@@ -25,7 +25,6 @@
!
!=====================================================================
-
subroutine write_seismograms()
! writes the seismograms with time shift
@@ -296,7 +295,7 @@
call write_seismograms_to_file(seismograms_a,3)
else
call write_adj_seismograms_to_file(myrank,seismograms_d,number_receiver_global, &
- nrec_local,it,DT,NSTEP,t0,LOCAL_PATH,1)
+ nrec_local,it,DT,NSTEP,t0,1)
endif
endif
@@ -372,7 +371,7 @@
call write_one_seismogram(one_seismogram,irec, &
station_name,network_name,nrec, &
DT,t0,it,NSTEP,SIMULATION_TYPE, &
- myrank,irecord,component,LOCAL_PATH)
+ myrank,irecord,component)
enddo ! nrec_local
@@ -436,7 +435,7 @@
call write_one_seismogram(one_seismogram,irec, &
station_name,network_name,nrec, &
DT,t0,it,NSTEP,SIMULATION_TYPE, &
- myrank,irecord,component,LOCAL_PATH)
+ myrank,irecord,component)
enddo ! nrec_local_received
endif ! if(nrec_local_received > 0 )
@@ -478,7 +477,7 @@
subroutine write_one_seismogram(one_seismogram,irec, &
station_name,network_name,nrec, &
DT,t0,it,NSTEP,SIMULATION_TYPE, &
- myrank,irecord,component,LOCAL_PATH)
+ myrank,irecord,component)
use constants
@@ -494,7 +493,6 @@
character(len=MAX_LENGTH_STATION_NAME), dimension(nrec) :: station_name
character(len=MAX_LENGTH_NETWORK_NAME), dimension(nrec) :: network_name
character(len=1) component
- character(len=MAX_STRING_LEN) :: LOCAL_PATH
! local parameters
integer iorientation
@@ -524,12 +522,7 @@
network_name(irec)(1:length_network_name),channel,component
! directory to store seismograms
- if( USE_OUTPUT_FILES_PATH ) then
- final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
- else
- ! create full final local path
- final_LOCAL_PATH = LOCAL_PATH(1:len_trim(LOCAL_PATH)) // '/'
- endif
+ final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
! ASCII output format
call write_output_ASCII(one_seismogram, &
@@ -545,7 +538,7 @@
! write adjoint seismograms (displacement) to text files
subroutine write_adj_seismograms_to_file(myrank,seismograms,number_receiver_global, &
- nrec_local,it,DT,NSTEP,t0,LOCAL_PATH,istore)
+ nrec_local,it,DT,NSTEP,t0,istore)
use constants
@@ -555,7 +548,6 @@
integer, dimension(nrec_local) :: number_receiver_global
real(kind=CUSTOM_REAL), dimension(NDIM,nrec_local,NSTEP) :: seismograms
double precision t0,DT
- character(len=MAX_STRING_LEN) :: LOCAL_PATH
integer irec,irec_local
integer iorientation,irecord,isample
@@ -594,13 +586,7 @@
'NT',channel,component
! directory to store seismograms
- if( USE_OUTPUT_FILES_PATH ) then
- final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
- else
- ! create full final local path
- final_LOCAL_PATH = LOCAL_PATH(1:len_trim(LOCAL_PATH)) // '/'
- endif
-
+ final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
! save seismograms in text format with no subsampling.
! Because we do not subsample the output, this can result in large files
@@ -636,8 +622,7 @@
! write adjoint seismograms (strain) to text files
- subroutine write_adj_seismograms2_to_file(myrank,seismograms,number_receiver_global, &
- nrec_local,it,DT,NSTEP,t0,LOCAL_PATH)
+ subroutine write_adj_seismograms2_to_file(myrank,seismograms,number_receiver_global,nrec_local,it,DT,NSTEP,t0)
use constants
@@ -647,7 +632,6 @@
integer, dimension(nrec_local) :: number_receiver_global
real(kind=CUSTOM_REAL), dimension(NDIM,NDIM,nrec_local,NSTEP) :: seismograms
double precision t0,DT
- character(len=MAX_STRING_LEN) :: LOCAL_PATH
integer irec,irec_local
integer idimval,jdimval,irecord,isample
@@ -691,12 +675,7 @@
'NT',chn,component
! directory to store seismograms
- if( USE_OUTPUT_FILES_PATH ) then
- final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
- else
- ! create full final local path
- final_LOCAL_PATH = LOCAL_PATH(1:len_trim(LOCAL_PATH)) // '/'
- endif
+ final_LOCAL_PATH = OUTPUT_FILES_PATH(1:len_trim(OUTPUT_FILES_PATH)) // '/'
! save seismograms in text format with no subsampling.
! Because we do not subsample the output, this can result in large files
More information about the CIG-COMMITS
mailing list