[cig-commits] [commit] devel: updates calls to write_seismograms(), putting runtime check inside the routine; adds some additional synchronization calls to facilitate debugging of simulation setup crashes due to memory issues (245aaba)
cig_noreply at geodynamics.org
cig_noreply at geodynamics.org
Thu Jan 8 11:53:43 PST 2015
Repository : https://github.com/geodynamics/specfem3d_globe
On branch : devel
Link : https://github.com/geodynamics/specfem3d_globe/compare/3ad4ad0dda8bce0265d5fa9ed7567ee338346d81...133892d126073f174444ba2cb859e8cc72f5478e
>---------------------------------------------------------------
commit 245aaba1e4e45c140986669f228c2990dd2cc1af
Author: daniel peter <peterda at ethz.ch>
Date: Wed Jan 7 16:27:27 2015 +0100
updates calls to write_seismograms(), putting runtime check inside the routine; adds some additional synchronization calls to facilitate debugging of simulation setup crashes due to memory issues
>---------------------------------------------------------------
245aaba1e4e45c140986669f228c2990dd2cc1af
src/specfem3D/iterate_time.F90 | 6 +-----
src/specfem3D/iterate_time_undoatt.F90 | 12 ++----------
src/specfem3D/prepare_timerun.f90 | 28 ++++++++++++++++++++++++----
src/specfem3D/read_forward_arrays.f90 | 6 ++++++
src/specfem3D/write_output_ASCII.f90 | 2 +-
src/specfem3D/write_output_ASDF.F90 | 27 +++++++++++++++++++++++++++
src/specfem3D/write_seismograms.f90 | 15 +++++++++++----
7 files changed, 72 insertions(+), 24 deletions(-)
diff --git a/src/specfem3D/iterate_time.F90 b/src/specfem3D/iterate_time.F90
index 911b637..f3895e0 100644
--- a/src/specfem3D/iterate_time.F90
+++ b/src/specfem3D/iterate_time.F90
@@ -238,11 +238,7 @@
endif ! kernel simulations
! write the seismograms with time shift
- if (nrec_local > 0 .or. ( WRITE_SEISMOGRAMS_BY_MASTER .and. myrank == 0 ) .or. OUTPUT_SEISMOS_ASDF) then
- ! note: ASDF uses adios that defines the MPI communicator group that the solver is
- ! run with. this means every processor in the group is needed for write_seismograms
- call write_seismograms()
- endif
+ call write_seismograms()
! outputs movie files
call write_movie_output()
diff --git a/src/specfem3D/iterate_time_undoatt.F90 b/src/specfem3D/iterate_time_undoatt.F90
index f80aae7..4897527 100644
--- a/src/specfem3D/iterate_time_undoatt.F90
+++ b/src/specfem3D/iterate_time_undoatt.F90
@@ -185,11 +185,7 @@
enddo ! istage
! write the seismograms with time shift
- if (nrec_local > 0 .or. ( WRITE_SEISMOGRAMS_BY_MASTER .and. myrank == 0 ) .or. OUTPUT_SEISMOS_ASDF) then
- ! note: ASDF uses adios that defines the MPI communicator group that the solver is
- ! run with. this means every processor in the group is needed for write_seismograms
- call write_seismograms()
- endif
+ call write_seismograms()
! outputs movie files
call write_movie_output()
@@ -326,11 +322,7 @@
enddo ! istage
! write the seismograms with time shift
- if (nrec_local > 0 .or. ( WRITE_SEISMOGRAMS_BY_MASTER .and. myrank == 0 ) .or. OUTPUT_SEISMOS_ASDF) then
- ! note: ASDF uses adios that defines the MPI communicator group that the solver is
- ! run with. this means every processor in the group is needed for write_seismograms
- call write_seismograms()
- endif
+ call write_seismograms()
! kernel computation
! adjoint simulations: kernels
diff --git a/src/specfem3D/prepare_timerun.f90 b/src/specfem3D/prepare_timerun.f90
index d6757e6..e594190 100644
--- a/src/specfem3D/prepare_timerun.f90
+++ b/src/specfem3D/prepare_timerun.f90
@@ -206,6 +206,9 @@
call flush_IMAIN()
endif
+ ! synchronizes processes
+ call synchronize_all()
+
end subroutine prepare_timerun_user_output
!
@@ -322,6 +325,9 @@
! outer core
rmass_outer_core = 1._CUSTOM_REAL / rmass_outer_core
+ ! synchronizes processes
+ call synchronize_all()
+
end subroutine prepare_timerun_mass_matrices
!
@@ -711,6 +717,7 @@
b_two_omega_earth = two_omega_earth
endif
+ ! synchronizes processes
call synchronize_all()
end subroutine prepare_timerun_constants
@@ -800,6 +807,7 @@
endif
+ ! synchronizes processes
call synchronize_all()
end subroutine prepare_timerun_gravity
@@ -1004,6 +1012,7 @@
write(IMAIN,*) "preparing wavefields"
call flush_IMAIN()
endif
+ call synchronize_all()
! put negligible initial value to avoid very slow underflow trapping
if (FIX_UNDERFLOW_PROBLEM) then
@@ -1418,6 +1427,7 @@
endif
+ ! synchronizes processes
call synchronize_all()
end subroutine prepare_timerun_init_wavefield
@@ -1709,6 +1719,9 @@
endif
endif
+ ! synchronizes processes
+ call synchronize_all()
+
end subroutine prepare_timerun_stacey
!
@@ -1726,12 +1739,13 @@
! NOISE TOMOGRAPHY
if (NOISE_TOMOGRAPHY /= 0) then
-
+ ! user info
if (myrank == 0) then
write(IMAIN,*) "preparing noise arrays"
call flush_IMAIN()
endif
+ ! allocates noise arrays
allocate(noise_sourcearray(NDIM,NGLLX,NGLLY,NGLLZ,NSTEP), &
normal_x_noise(nmovie_points), &
normal_y_noise(nmovie_points), &
@@ -1740,6 +1754,7 @@
noise_surface_movie(NDIM,NGLLX,NGLLY,NSPEC_TOP),stat=ier)
if (ier /= 0 ) call exit_MPI(myrank,'Error allocating noise arrays')
+ ! initializes
noise_sourcearray(:,:,:,:,:) = 0._CUSTOM_REAL
normal_x_noise(:) = 0._CUSTOM_REAL
normal_y_noise(:) = 0._CUSTOM_REAL
@@ -1747,14 +1762,16 @@
mask_noise(:) = 0._CUSTOM_REAL
noise_surface_movie(:,:,:,:) = 0._CUSTOM_REAL
+ ! gets noise parameters
call read_parameters_noise()
+ ! checks noise setup
call check_parameters_noise()
-
- call synchronize_all()
-
endif
+ ! synchronizes processes
+ call synchronize_all()
+
end subroutine prepare_timerun_noise
!
@@ -1785,6 +1802,7 @@
real(kind=CUSTOM_REAL) :: dummy
! user output
+ call synchronize_all()
if (myrank == 0) then
write(IMAIN,*) "preparing fields and constants on GPU devices:"
call flush_IMAIN()
@@ -2342,6 +2360,8 @@
endif
call flush_IMAIN()
endif
+
+ ! synchronizes processes
call synchronize_all()
contains
diff --git a/src/specfem3D/read_forward_arrays.f90 b/src/specfem3D/read_forward_arrays.f90
index f99be89..733580a 100644
--- a/src/specfem3D/read_forward_arrays.f90
+++ b/src/specfem3D/read_forward_arrays.f90
@@ -64,6 +64,12 @@
! read files back from local disk or MT tape system if restart file
if (NUMBER_OF_THIS_RUN > 1) then
+ ! user output
+ if (myrank == 0) then
+ write(IMAIN,*) 'reading startup file for run = ',NUMBER_OF_THIS_RUN
+ call flush_IMAIN()
+ endif
+
if (ADIOS_FOR_FORWARD_ARRAYS) then
call read_intermediate_forward_arrays_adios()
else
diff --git a/src/specfem3D/write_output_ASCII.f90 b/src/specfem3D/write_output_ASCII.f90
index 2dfba24..1522ab2 100644
--- a/src/specfem3D/write_output_ASCII.f90
+++ b/src/specfem3D/write_output_ASCII.f90
@@ -77,7 +77,7 @@
open(unit=IOUT,file=trim(OUTPUT_FILES)//trim(sisname_2), &
status='old',position='append',action='write',iostat=ier)
endif
- if (ier /= 0 ) call exit_mpi(myrank,'Error opening file:'//trim(OUTPUT_FILES)//trim(sisname_2))
+ if (ier /= 0) call exit_mpi(myrank,'Error opening file:'//trim(OUTPUT_FILES)//trim(sisname_2))
endif
! subtract half duration of the source to make sure travel time is correct
diff --git a/src/specfem3D/write_output_ASDF.F90 b/src/specfem3D/write_output_ASDF.F90
index 0eb3ad5..e717af6 100644
--- a/src/specfem3D/write_output_ASDF.F90
+++ b/src/specfem3D/write_output_ASDF.F90
@@ -118,6 +118,9 @@ subroutine init_asdf_data(asdf_container,total_seismos_local)
end subroutine init_asdf_data
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Stores the records into the ASDF structure
!! \param asdf_container The ASDF data structure
@@ -218,6 +221,9 @@ subroutine store_asdf_data(asdf_container, seismogram_tmp, irec_local, &
end subroutine store_asdf_data
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Closes the ASDF data structure by deallocating all arrays
!! \param asdf_container The ASDF data structure
@@ -267,6 +273,9 @@ subroutine close_asdf_data(asdf_container, total_seismos_local)
end subroutine close_asdf_data
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Writes the ASDF data structure to the file
!! \param asdf_container The ASDF data structure
@@ -306,6 +315,9 @@ subroutine write_asdf(asdf_container)
end subroutine write_asdf
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Writes the ASDF data structure to asdf_fn using parallel write
!! \param asdf_fn The file name for ASDF
@@ -356,6 +368,9 @@ subroutine write_asdf_data(asdf_fn, asdf_container, adios_group, rank, nproc, co
end subroutine write_asdf_data
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Defines the ASDF structure using adios
!! \param adios_group The adios group
@@ -510,6 +525,9 @@ subroutine define_asdf_data (adios_group, my_group_size, asdf_container, rank, n
end subroutine define_asdf_data
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Writes the ASDF data structure to the adios arrays
!! \param asdf_container The ASDF data structure
@@ -754,6 +772,9 @@ subroutine write_asdf_data_sub(asdf_container, adios_handle, rank, nproc)
end subroutine write_asdf_data_sub
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Gets offset values for arrays
!! \param local_dim The local dimension on the processor
@@ -803,6 +824,9 @@ subroutine gather_offset_info(local_dim, global_dim, offset, rank, nproc)
end subroutine gather_offset_info
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Gets total length of strings from each processor
!! \param local_dim The local dimension on the processor
@@ -838,6 +862,9 @@ subroutine gather_string_total_length(local_dim, global_dim, rank, nproc)
end subroutine gather_string_total_length
+!
+!-------------------------------------------------------------------------------------------------
+!
!> Gets offset values for strings
!! \param local_dim The local dimension on the processor
diff --git a/src/specfem3D/write_seismograms.f90 b/src/specfem3D/write_seismograms.f90
index a993091..2c6fbb2 100644
--- a/src/specfem3D/write_seismograms.f90
+++ b/src/specfem3D/write_seismograms.f90
@@ -36,7 +36,12 @@ contains
implicit none
- ! note: this routine gets called if (nrec_local > 0 .or. ( WRITE_SEISMOGRAMS_BY_MASTER .and. myrank == 0 ) )
+ ! checks if anything to do
+ ! note: ASDF uses adios that defines the MPI communicator group that the solver is
+ ! run with. this means every processor in the group is needed for write_seismograms
+ if (.not. (nrec_local > 0 .or. ( WRITE_SEISMOGRAMS_BY_MASTER .and. myrank == 0 ) .or. OUTPUT_SEISMOS_ASDF)) then
+ return
+ endif
! update position in seismograms
seismo_current = seismo_current + 1
@@ -206,7 +211,7 @@ contains
! get global number of that receiver
irec = number_receiver_global(irec_local)
- one_seismogram = seismograms(:,irec_local,:)
+ one_seismogram(:,:) = seismograms(:,irec_local,:)
! write this seismogram
if (OUTPUT_SEISMOS_ASDF) then
@@ -245,7 +250,8 @@ contains
write_time_begin = wtime()
- if (myrank == 0) then ! on the master, gather all the seismograms
+ if (myrank == 0) then
+ ! on the master, gather all the seismograms
! create one large file instead of one small file per station to avoid file system overload
if (OUTPUT_SEISMOS_ASCII_TEXT .and. SAVE_ALL_SEISMOS_IN_ONE_FILE) then
@@ -335,7 +341,8 @@ contains
! create one large file instead of one small file per station to avoid file system overload
if (SAVE_ALL_SEISMOS_IN_ONE_FILE) close(IOUT)
- else ! on the nodes, send the seismograms to the master
+ else
+ ! on the nodes, send the seismograms to the master
receiver = 0
! only sends if this slice contains receiver stations
if (nrec_local > 0) then
More information about the CIG-COMMITS
mailing list