[cig-commits] r14502 - in seismo/3D/ADJOINT_TOMO/iterate_adj/pangu: model_pert model_vp_vs model_vp_vs/src scripts smooth smooth_all smooth_all/SERIAL_RUN smooth_all/TO_COMBINE smooth_all/TO_COMBINE/DONE subspace_hessian subspace_update sum_kernel sum_kernel/src
carltape at geodynamics.org
carltape at geodynamics.org
Fri Mar 27 17:09:48 PDT 2009
Author: carltape
Date: 2009-03-27 17:09:46 -0700 (Fri, 27 Mar 2009)
New Revision: 14502
Added:
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/README
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/OUTPUT_FILES/
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/check_dir.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/go1.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/run1.lsf
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/setup_smooth_all.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/DONE/
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/DONE/finish_smooth_all.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/go_process.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_001
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_launch_001.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/setup_batch.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/check_dir.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels_weight.f90
Removed:
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/DONE/
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/FAILED/
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/finish_smooth_all.pl
Modified:
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/run.lsf
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/setup_model_pert.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/run.lsf
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/setup_model_vp_vs.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/src/add_model
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/src/add_model.f90
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/README
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_SEM_dir.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_kernel_dir.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_forward_dir.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_kernel_dir.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth/go.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/run_dir.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/setup_smooth_all.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/prepare_run.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/run.lsf
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/README
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/make_mesh.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/prepare_run.pl
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/run.lsf
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/run.lsf
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/compile.bash
seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels.f90
Log:
Updates to numerous codes and scripts pertaining to a source-subspace-based iterative tomographic inversion. Added options for thresholding updated model based on the highest and lowest resolvable wavespeeds in the mesh. Additional options for running processing scripts using parallel batch mode, to avoid running on the head node whenever possible.
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/run.lsf
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/run.lsf 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/run.lsf 2009-03-28 00:09:46 UTC (rev 14502)
@@ -2,5 +2,6 @@
mkdir -p OUTPUT_FILES
rm OUTPUT_FILES/*
date
-bsub -q test -W 30 -n 168 < go.bash
+bsub -q normal -W 30 -n 168 < go.bash
+#bsub -q test -W 30 -n 168 < go.bash
date
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/setup_model_pert.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/setup_model_pert.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_pert/setup_model_pert.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -1,7 +1,7 @@
#!/usr/bin/perl -w
#-----------------------------------
-# Carl Tape, 17-June-2008
+# Carl Tape, 30-Jan-2009
# setup_model_pert.pl
#
# Start with one vp and vs model, and then add a perturbation to new vp and vs models.
@@ -9,8 +9,11 @@
# Removed linked files in INPUT_MODEL prior to running.
#
# EXAMPLE:
-# setup_model_pert.pl 168 m12 m00 vs_m12 vs_m00 vs_m12_m00
-# setup_model_pert.pl 168 m12 m11 vs_m12 vs_m11 vs_m12_m11
+# setup_model_pert.pl 168 m16 m00 vs_m16 vs_m00 vs_m16_m00
+# setup_model_pert.pl 168 m16 m15 vs_m16 vs_m15 vs_m16_m15
+# setup_model_pert.pl 168 m16 m00 vp_m16 vp_m00 vp_m16_m00
+# setup_model_pert.pl 168 m16 m00 vb_m16 vb_m00 vb_m16_m00
+# setup_model_pert.pl 168 m16 m00 poisson_m16 poisson_m00 poisson_m16_m00
#
# setup_model_pert.pl 168 m09 m00 vp_m09 vp_m00 vp_m09_m00
# setup_model_pert.pl 168 m09 m08 vp_m09 vp_m08 vp_m09_m08
@@ -19,7 +22,7 @@
if (@ARGV < 6) {die("Usage: setup_model_pert.pl nproc mtag1 mtag2 ftag1 ftag2 ftag3\n")}
($nproc,$mtag1,$mtag2,$ftag1,$ftag2,$ftag3) = @ARGV;
-$basedir = "/ibrixfs1/scratch/carltape/KERNELS_MODELS/models";
+$basedir = "/ibrixfs1/home/carltape/ADJOINT_TOMO_EXTRA/KERNELS_MODELS/models";
if(not -e $basedir) {die("basedir $basedir does not exist");}
# write out tags to be read in by model_pert.f90
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/run.lsf
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/run.lsf 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/run.lsf 2009-03-28 00:09:46 UTC (rev 14502)
@@ -2,5 +2,6 @@
mkdir -p OUTPUT_FILES
rm OUTPUT_FILES/*
date
-bsub -q test -W 30 -n 168 < go.bash
+#bsub -q test -W 30 -n 168 < go.bash
+bsub -q normal -W 30 -n 168 < go.bash
date
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/setup_model_vp_vs.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/setup_model_vp_vs.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/setup_model_vp_vs.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -1,12 +1,17 @@
#!/usr/bin/perl -w
#-----------------------------------
-# Carl Tape, 01-October-2008
+# Carl Tape, 01-Feb-2009
# setup_model_vp_vs.pl
#
-# Start with one vp and vs model, and then add a perturbation and comput new vp and vs models.
+# Start with one vp and vs model, and then add a perturbation and compute new vp and vs models.
+# Note the minmax wavespeed specifications within the fortran code.
#
# EXAMPLE:
+# setup_model_vp_vs.pl 168 m15 dm15 vp_m15 vs_m15 dm_kappa_kernel_smooth_p090 dm_mu_kernel_smooth_p090
+# setup_model_vp_vs.pl 168 m14 dm14 vp_m14 vs_m14 dm_kappa_kernel_smooth_p080 dm_mu_kernel_smooth_p080
+# setup_model_vp_vs.pl 168 m13 dm13 vp_m13 vs_m13 dm_kappa_kernel_smooth_p078 dm_mu_kernel_smooth_p078
+# setup_model_vp_vs.pl 168 m12 dm12 vp_m12 vs_m12 dm_kappa_kernel_smooth_p084 dm_mu_kernel_smooth_p084
# setup_model_vp_vs.pl 168 m11 dm11 vp_m11 vs_m11 dm_kappa_kernel_smooth_p040 dm_mu_kernel_smooth_p040
# setup_model_vp_vs.pl 168 m10 dm10 vp_m10 vs_m10 dm_kappa_kernel_smooth_p040 dm_mu_kernel_smooth_p040
# setup_model_vp_vs.pl 168 m09 dm09 vp_m09 vs_m09 dm_kappa_kernel_smooth_p041 dm_mu_kernel_smooth_p041
@@ -18,14 +23,14 @@
# setup_model_vp_vs.pl 168 m03 dm03_smooth vp_m03 vs_m03 dm_kappa_kernel_smooth_p025_smooth dm_mu_kernel_smooth_p025_smooth
# setup_model_vp_vs.pl 168 m02 dm02_smooth vp_m02 vs_m02 dm_kappa_kernel_smooth_p031_smooth dm_mu_kernel_smooth_p031_smooth
# setup_model_vp_vs.pl 168 m01 dm01_smooth vp_m01 vs_m01 dm_kappa_kernel_smooth_p060_smooth dm_mu_kernel_smooth_p060_smooth
-# setup_model_vp_vs.pl 168 m00 dm00_smooth vp vs dm_kappa_kernel_smooth_p068_smooth dm_mu_kernel_smooth_p068_smooth
+# setup_model_vp_vs.pl 168 m00 dm00_smooth vp_m00 vs_m00 dm_kappa_kernel_smooth_p068_smooth dm_mu_kernel_smooth_p068_smooth
#
#-----------------------------------
if (@ARGV < 7) {die("Usage: setup_model_vp_vs.pl xxx\n")}
($nproc,$mtag1,$mtag2,$ftag1,$ftag2,$ftag3,$ftag4) = @ARGV;
-$basedir = "/ibrixfs1/scratch/carltape/KERNELS_MODELS/models";
+$basedir = "/ibrixfs1/home/carltape/ADJOINT_TOMO_EXTRA/KERNELS_MODELS/models";
if(not -e $basedir) {die("basedir $basedir does not exist");}
@ftags = ($ftag1,$ftag2,$ftag3,$ftag4);
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/src/add_model
===================================================================
(Binary files differ)
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/src/add_model.f90
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/src/add_model.f90 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/model_vp_vs/src/add_model.f90 2009-03-28 00:09:46 UTC (rev 14502)
@@ -9,13 +9,23 @@
! ======================================================
integer, parameter :: NSPEC=NSPEC_AB
+ logical, parameter :: MINMAX_THRESHOLD_OLD = .false. ! threshold the old model ("current model")
+ logical, parameter :: MINMAX_THRESHOLD_NEW = .true. ! threshold the new model
character(len=150) :: sline, m_file, fname
character(len=150) :: ftag_file_list, ftag_list(10)
- real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: model_vp, model_vs, model_dC, model_dB, total_model
+ real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: model_vp, model_vs, model_vp_new, model_vs_new
+ real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: model_dC, model_dB, total_model
real(kind=CUSTOM_REAL) :: alpha0, beta0, bulk0, step_fac
+
+ real(kind=CUSTOM_REAL) :: VS_MIN, VS_MAX, VP_MIN, VP_MAX
+ real(kind=CUSTOM_REAL) :: vsmin_before, vsmax_before, vpmin_before, vpmax_before
+ real(kind=CUSTOM_REAL) :: vsmin_after, vsmax_after, vpmin_after, vpmax_after
+ real(kind=CUSTOM_REAL) :: vsmin_new_before, vsmax_new_before, vpmin_new_before, vpmax_new_before
+ real(kind=CUSTOM_REAL) :: vsmin_new_after, vsmax_new_after, vpmin_new_after, vpmax_new_after
+
integer :: nfile, myrank, sizeprocs, ier
- integer :: i, j, k,ispec, iglob, ishell, n, it, j1, ib, npts_sem, ios
+ integer :: i,j,k,ispec,iglob, ishell, n, it, j1, ib, npts_sem, ios
! ============ program starts here =====================
! initialize the MPI communicator and start the NPROCTOT MPI processes
@@ -23,27 +33,41 @@
call MPI_COMM_SIZE(MPI_COMM_WORLD,sizeprocs,ier)
call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ier)
- alpha0 = 5000.0
- beta0 = 3000.0
- bulk0 = sqrt( alpha0**2 - (4.0/3.0)*beta0**2 )
-
- ! subjective step to multiply to the model update; in general,
- ! the computed model update is too strong
+ ! KEY: subjective multiplication factor for the model update
! SOCAL: dm00 1.0, dm01 1.0, dm02 0.125, dm03 0.08, dm04 0.05, dm05 0.08
- ! dm06 0.09, dm07 0.05, dm08 0.035, dm09 0.04, dm10 0.04, dm11 0.03
- step_fac = 0.03
+ ! dm06 0.09, dm07 0.05, dm08 0.035, dm09 0.04, dm10 0.04
+ ! dm11 0.03, dm12 0.05, dm13 0.03, dm14 0.03, dm15 0.05
+ step_fac = 0.05
- open(19,file='wavespeed_reference',status='unknown')
- write(19,'(3e24.12)') alpha0, beta0, bulk0
- close(19)
-
open(20,file='step_fac',status='unknown')
write(20,'(1e24.12)') step_fac
close(20)
+ if(MINMAX_THRESHOLD_OLD .or. MINMAX_THRESHOLD_NEW) then
+ ! minmax wavespeed values for southern california simulations
+ VS_MIN = 600.0
+ VS_MAX = 4700.0
+ VP_MIN = 1500.0
+ VP_MAX = 8200.0
+
+ if (myrank == 0) then
+ open(19,file='VS_VP_MINMAX',status='unknown')
+ write(19,'(4e24.12)') VS_MIN, VS_MAX, VP_MIN, VP_MAX
+ close(19)
+ endif
+ endif
+
+!!$ alpha0 = 5000.0
+!!$ beta0 = 3000.0
+!!$ bulk0 = sqrt( alpha0**2 - (4.0/3.0)*beta0**2 )
+!!$
+!!$ open(19,file='wavespeed_reference',status='unknown')
+!!$ write(19,'(3e24.12)') alpha0, beta0, bulk0
+!!$ close(19)
+
!-----------------------------------------------------
- ! read in list of file tags
+ ! read in list of file tags showing what model iteration you are on
nfile=0
open(unit=20, file='INPUT/ftags', status='old',iostat=ios)
if (ios /= 0) then
@@ -90,43 +114,184 @@
model_dB = step_fac * model_dB
!-----------------------------------------------------
- ! compute current model in terms of B = ln(beta/beta0) and C = log(c/c0)
- total_model = 0.
- total_model = log( model_vs(:,:,:,1:nspec) / beta0 )
- fname = 'Beta'
+ ! compute minmax values of current model
+ ! NOTE: mpi_reduce operates on the values from all procs,
+ ! but the reduced value only exists on the root proc.
+ call mpi_reduce(minval(model_vs(:,:,:,1:nspec)), vsmin_before, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vs(:,:,:,1:nspec)), vsmax_before, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(minval(model_vp(:,:,:,1:nspec)), vpmin_before, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vp(:,:,:,1:nspec)), vpmax_before, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+ if (myrank == 0) then
+ open(19,file='vs_vp_minmax_before',status='unknown')
+ write(19,'(4e24.12)') vsmin_before, vsmax_before, vpmin_before, vpmax_before
+ close(19)
+ endif
+
+ ! threshold current model and write out the modified version
+ if(MINMAX_THRESHOLD_OLD) then
+ do ispec=1,NSPEC
+ do k=1,NGLLZ
+ do j=1,NGLLY
+ do i=1,NGLLX
+ if(model_vs(i,j,k,ispec) < VS_MIN) model_vs(i,j,k,ispec) = VS_MIN
+ if(model_vs(i,j,k,ispec) > VS_MAX) model_vs(i,j,k,ispec) = VS_MAX
+ if(model_vp(i,j,k,ispec) < VP_MIN) model_vp(i,j,k,ispec) = VP_MIN
+ if(model_vp(i,j,k,ispec) > VP_MAX) model_vp(i,j,k,ispec) = VP_MAX
+ enddo
+ enddo
+ enddo
+ enddo
+
+ ! compute minmax values of the thresholded current model
+ call mpi_reduce(minval(model_vs(:,:,:,1:nspec)), vsmin_after, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vs(:,:,:,1:nspec)), vsmax_after, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(minval(model_vp(:,:,:,1:nspec)), vpmin_after, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vp(:,:,:,1:nspec)), vpmax_after, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+
+ if (myrank == 0) then
+ open(19,file='vs_vp_minmax_after',status='unknown')
+ write(19,'(4e24.12)') vsmin_after, vsmax_after, vpmin_after, vpmax_after
+ close(19)
+ endif
+
+ fname = 'vs'
+ write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
+ open(12,file=trim(m_file),form='unformatted')
+ write(12) model_vs(:,:,:,1:nspec)
+ close(12)
+
+ fname = 'vp'
+ write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
+ open(12,file=trim(m_file),form='unformatted')
+ write(12) model_vp(:,:,:,1:nspec)
+ close(12)
+
+ endif
+
+!!$ !-----------------------------------------------------
+!!$ ! compute current model in terms of B = ln(beta/beta0) and C = log(c/c0)
+!!$
+!!$ total_model = 0.
+!!$ total_model = log( model_vs(:,:,:,1:nspec) / beta0 )
+!!$ fname = 'Beta'
+!!$ write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
+!!$ open(12,file=trim(m_file),form='unformatted')
+!!$ write(12) total_model(:,:,:,1:nspec)
+!!$ close(12)
+!!$
+!!$ total_model = 0.
+!!$ total_model = 0.5 * log( (model_vp**2 - (4.0/3.0)*model_vs**2) / (alpha0**2 - (4.0/3.0)*beta0**2) )
+!!$ fname = 'Bulk'
+!!$ write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
+!!$ open(12,file=trim(m_file),form='unformatted')
+!!$ write(12) total_model(:,:,:,1:nspec)
+!!$ close(12)
+
+ !-----------------------------------------------------
+
+ ! S wavespeed model
+ model_vs_new = 0.
+ model_vs_new = model_vs * exp( model_dB )
+
+ ! P wavespeed model
+ model_vp_new = 0.
+ model_vp_new = sqrt( (4.0/3.0)* model_vs**2 * exp( 2.0*model_dB ) + &
+ (model_vp**2 - (4.0/3.0)* model_vs**2) * exp( 2.0*model_dC ) )
+
+ ! compute minmax values of new model
+ call mpi_reduce(minval(model_vs_new(:,:,:,1:nspec)), vsmin_new_before, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vs_new(:,:,:,1:nspec)), vsmax_new_before, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(minval(model_vp_new(:,:,:,1:nspec)), vpmin_new_before, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vp_new(:,:,:,1:nspec)), vpmax_new_before, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+
+ if (myrank == 0) then
+ open(19,file='vs_vp_new_minmax_before',status='unknown')
+ write(19,'(4e24.12)') vsmin_new_before, vsmax_new_before, vpmin_new_before, vpmax_new_before
+ close(19)
+ endif
+
+ !-----------------------------------------------------
+ ! threshold model according to minmax values specified above
+
+ if(MINMAX_THRESHOLD_NEW) then
+ do ispec=1,NSPEC
+ do k=1,NGLLZ
+ do j=1,NGLLY
+ do i=1,NGLLX
+ if(model_vs_new(i,j,k,ispec) < VS_MIN) model_vs_new(i,j,k,ispec) = VS_MIN
+ if(model_vs_new(i,j,k,ispec) > VS_MAX) model_vs_new(i,j,k,ispec) = VS_MAX
+ if(model_vp_new(i,j,k,ispec) < VP_MIN) model_vp_new(i,j,k,ispec) = VP_MIN
+ if(model_vp_new(i,j,k,ispec) > VP_MAX) model_vp_new(i,j,k,ispec) = VP_MAX
+ enddo
+ enddo
+ enddo
+ enddo
+ endif
+
+ !-----------------------------------------------------
+ ! write out new models and their global minmax values
+
+ ! compute minmax values of new model
+ call mpi_reduce(minval(model_vs_new(:,:,:,1:nspec)), vsmin_new_after, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vs_new(:,:,:,1:nspec)), vsmax_new_after, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(minval(model_vp_new(:,:,:,1:nspec)), vpmin_new_after, 1, CUSTOM_MPI_TYPE, MPI_MIN, 0, MPI_COMM_WORLD,ier)
+ call mpi_reduce(maxval(model_vp_new(:,:,:,1:nspec)), vpmax_new_after, 1, CUSTOM_MPI_TYPE, MPI_MAX, 0, MPI_COMM_WORLD,ier)
+
+ ! this should only be different if using MINMAX_THRESHOLD_NEW
+ if (myrank == 0) then
+ open(19,file='vs_vp_new_minmax_after',status='unknown')
+ write(19,'(4e24.12)') vsmin_new_after, vsmax_new_after, vpmin_new_after, vpmax_new_after
+ close(19)
+ endif
+
+ fname = 'vs_new'
write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
open(12,file=trim(m_file),form='unformatted')
- write(12) total_model(:,:,:,1:nspec)
+ write(12) model_vs_new(:,:,:,1:nspec)
close(12)
- total_model = 0.
- total_model = 0.5 * log( (model_vp**2 - (4.0/3.0)*model_vs**2) / (alpha0**2 - (4.0/3.0)*beta0**2) )
- fname = 'Bulk'
+ fname = 'vp_new'
write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
open(12,file=trim(m_file),form='unformatted')
- write(12) total_model(:,:,:,1:nspec)
+ write(12) model_vp_new(:,:,:,1:nspec)
close(12)
!-----------------------------------------------------
- ! compute new model in terms of alpha and beta (see Carl's Latex notes)
+ ! compute Poisson's ratio and bulk wavespeed of the current model and new model
- ! S wavespeed model
+ ! Poisson's ratio of current model
total_model = 0.
- total_model = model_vs * exp( model_dB )
+ total_model = ( model_vp**2 - 2.0*model_vs**2 ) / ( 2.0*model_vp**2 - 2.0*model_vs**2 )
+ fname = 'poisson'
+ write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
+ open(12,file=trim(m_file),form='unformatted')
+ write(12) total_model(:,:,:,1:nspec)
+ close(12)
- fname = 'vs_new'
+ ! Poisson's ratio of new model
+ total_model = 0.
+ total_model = ( model_vp_new**2 - 2.0*model_vs_new**2 ) / &
+ ( 2.0*model_vp_new**2 - 2.0*model_vs_new**2 )
+ fname = 'poisson_new'
write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
open(12,file=trim(m_file),form='unformatted')
write(12) total_model(:,:,:,1:nspec)
close(12)
- ! P wavespeed model
+ ! bulk wavespeed of current model
total_model = 0.
- total_model = sqrt( (4.0/3.0)* model_vs**2 * exp( 2.0*model_dB ) + &
- (model_vp**2 - (4.0/3.0)* model_vs**2) * exp( 2.0*model_dC ) )
+ total_model = sqrt( model_vp**2 - (4.0/3.0)*model_vs**2 )
+ fname = 'vb'
+ write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
+ open(12,file=trim(m_file),form='unformatted')
+ write(12) total_model(:,:,:,1:nspec)
+ close(12)
- fname = 'vp_new'
+ ! bulk wavespeed of new model
+ total_model = 0.
+ total_model = sqrt( model_vp_new**2 - (4.0/3.0)*model_vs_new**2 )
+ fname = 'vb_new'
write(m_file,'(a,i6.6,a)') 'OUTPUT_MODEL/proc',myrank,'_'//trim(fname)//'.bin'
open(12,file=trim(m_file),form='unformatted')
write(12) total_model(:,:,:,1:nspec)
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/README
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/README 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/README 2009-03-28 00:09:46 UTC (rev 14502)
@@ -2,7 +2,7 @@
/ADJOINT_TOMO/iterate_adj/pangu/scripts/README
-These scripts are used in setting up the forward simulations and the kernel simulations.
+These scripts are used in setting up the forward simulations and the kernel simulations. They are included as a guide only, and will need to be modified for a particular setup.
setup_forward_dir.pl
setup_kernel_dir.pl
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_SEM_dir.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_SEM_dir.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_SEM_dir.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -26,8 +26,9 @@
if (-e $odir) {
die("--> $odir already exists\n");
} else {
- `mkdir $odir`;
- `cp $eid/SEM/* $odir`;
+ #`mkdir $odir`;
+ #`cp $eid/SEM/* $odir`;
+ `cp -r $eid/SEM $odir`;
}
}
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_kernel_dir.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_kernel_dir.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/copy_kernel_dir.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -3,7 +3,7 @@
#-------------------------
# copies kernel output files from cluster to disk
#
-# EXAMPLE: copy_kernel_dir.pl m04 all
+# EXAMPLE: copy_kernel_dir.pl m14 all
#-------------------------
if (@ARGV < 2) {die("Usage: copy_kernel_dir.pl smodel ftag\n")}
@@ -36,14 +36,19 @@
if (-e ${dir_sem}) {die("dir_sem ${dir_sem} already exists\n")}
if (-e ${dir_mesh}) {die("dir_mesh ${dir_mesh} already exists\n")}
- # copy files over
- `cp -r $eid/OUTPUT_FILES ${dir_output}`;
- `sleep 5s`;
- `cp -r $eid/SEM/MESH_${ftag} ${dir_mesh}`;
- `sleep 5s`;
+ # copy files from cluster to raid disk
+ #`cp -r $eid/SEM/MESH_${ftag} ${dir_mesh}`;
+ `mkdir $dir_mesh`; `sleep 2s`;
+ `mkdir ${dir_mesh}/collect`; `sleep 2s`;
+ `cp $eid/SEM/MESH_${ftag}/*mesh ${dir_mesh}`; # mesh
+ `cp $eid/SEM/MESH_${ftag}/collect/*kernel* ${dir_mesh}/collect`; # only get the kernel files
+ `sleep 2s`;
+
`cp -r $eid/SEM/SEM_${ftag} ${dir_sem}`;
- `sleep 5s`;
-
+ `sleep 2s`;
+ `cp -r $eid/SEM/OUTPUT_${ftag} ${dir_output}`;
+ `sleep 2s`;
+ `cp $eid/OUTPUT_FILES/*.o ${dir_output}`; # output file from run
}
#=================================================================
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_forward_dir.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_forward_dir.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_forward_dir.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -1,7 +1,7 @@
#!/usr/bin/perl -w
#
-# EXAMPLE: setup_forward_dir.pl m12 1 300
+# EXAMPLE: setup_forward_dir.pl m16 1 10
#
if (@ARGV < 3) {die("Usage: setup_forward_dir.pl smodel imin imax\n")}
@@ -14,7 +14,7 @@
$scripts_dir = "/ibrixfs1/home/carltape/BASIN_FORWARD/Scripts";
# directory containing CMTSOLUTION files for all events
-$cmt_dir = "/ibrixfs1/home/carltape/CMT/CMT_files_pre_inverted";
+$cmt_dir = "/ibrixfs1/home/carltape/CMT/CMT_files";
# directory containing finished synthetics
$done_dir = "/net/sierra/raid1/carltape/socal/socal_3D/SYN/model_${smodel}";
@@ -45,7 +45,7 @@
# loop over all events
for ($i = $imin; $i <= $imax; $i++) {
- ($index,$eid,$sdur_round,$dur) = split(" ",$lines[$i-1]);
+ ($index,$eid,$sdur_round,$npt0,$hdur0,$mag0) = split(" ",$lines[$i-1]);
print "\n $i : $eid";
# find the matching EID in the subset list
@@ -63,9 +63,10 @@
} else {
$dir1 = $eid;
- $dir2 = "done_dirs/${eid}";
+ $dir2 = "DONE/${eid}";
+ $dir3 = "DONE_SINGLE/${eid}";
- if ( (-e "${done_dir}/$eid") || (-e $dir1 || -e $dir2 ) ) {
+ if ( (-e "${done_dir}/$eid" || -e $dir3) || (-e $dir1 || -e $dir2 ) ) {
print "\n --> event has already been setup or done";
} else {
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_kernel_dir.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_kernel_dir.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/scripts/setup_kernel_dir.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -13,7 +13,7 @@
#
# EXAMPLE:
# setup_kernel_dir.pl m00 all 10
-# setup_kernel_dir.pl m01 all 10
+# setup_kernel_dir.pl m12 all 10
#
#-----------------------------------
@@ -24,7 +24,7 @@
$scripts_dir = "/ibrixfs1/home/carltape/BASIN_KERNEL/Scripts";
# directory containing SEM-inverted CMTSOLUTION files
-$cmt_dir = "/ibrixfs1/home/carltape/CMT/CMT_files_pre_inverted";
+$cmt_dir = "/ibrixfs1/home/carltape/CMT/CMT_files";
# directory containing window files, measurement files, adjoint sources, etc
$run_dir = "/net/sierra/raid1/carltape/socal/socal_3D/RUNS";
@@ -36,7 +36,7 @@
# read in list of kernels that will be used for this model update
$ker_list = "/net/sierra/raid1/carltape/results/EID_LISTS/kernels_run_${smodel}";
-if (not -f ${ker_list}) {print "\n check if ker_list ${ker_list} exists\n";}
+if (not -f ${ker_list}) {die("check if ker_list ${ker_list} exists\n");}
open(IN,"${ker_list}"); @keids = <IN>; close(IN);
$nker = @keids;
@@ -51,7 +51,7 @@
}
$imin = 1; $imax = $neid; # default
-#$imin = 1; $imax = 10;
+#$imin = 1; $imax = 40;
#$imin = 19; $imax = $imin;
#===================================
@@ -82,7 +82,7 @@
print "--> cmtfile does not exist\n";
} else {
- if ( -e $eid || -e "done_dirs/${eid}" ) {
+ if ( -e $eid || (-e "done_dirs/${eid}" || -e "PROCESS/${eid}") ) {
print "--> kernel simulation has been setup or has been done\n";
} else {
@@ -107,17 +107,25 @@
if ($nstat_adj < $nstat) {
print "--> fewer than $nstat stations\n";
} else {
- print "--> LET'S RUN THIS EVENT!\n";
+ print "--> RUN THIS EVENT\n";
#----------------------------
# replicate the master directory
`${scripts_dir}/copy_basin_sem_dir.bash ${master_dir} $eid`;
+ # modify the EID label for the post-processing script
+ # eid=9818433
+ $processfile = "$eid/go_process.bash";
+ $ptemp = "$eid/go_process_temp.bash";
+ `sed '/^eid=/s/^.*\$/eid=$eid/' $processfile > $ptemp`;
+ `mv $ptemp $processfile`;
+ `chmod +x $processfile`;
+
# copy in a new CMTSOLUTION
$cmtfile0 = "$cmtfile";
$cmtfile1 = "$eid/DATA/CMTSOLUTION";
- `cp $cmtfile0 $cmtfile1`;
+ `cp $cmtfile0 $cmtfile1`;
# change the half-duration
#$hdur_st = sprintf("%8.4f",$hdur);
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth/go.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth/go.bash 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth/go.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -25,12 +25,12 @@
mpirun.lsf --gm-no-shmem --gm-copy-env smooth_sem_fun 14 12 0 2 6 1 kappa_kernel inout_smooth topo
sleep 5s
-xcombine_vol_data slice_file mu_kernel_smooth topo inout_smooth . 0
-sleep 5s
-mv mu_kernel_smooth.mesh mu_kernel_smooth_h006km_v001km.mesh
-sleep 5s
+#xcombine_vol_data slice_file mu_kernel_smooth topo inout_smooth . 0
+#sleep 5s
+#mv mu_kernel_smooth.mesh mu_kernel_smooth_h006km_v001km.mesh
+#sleep 5s
-xcombine_vol_data slice_file kappa_kernel_smooth topo inout_smooth . 0
-sleep 5s
-mv kappa_kernel_smooth.mesh kappa_kernel_smooth_h006km_v001km.mesh
-sleep 5s
+#xcombine_vol_data slice_file kappa_kernel_smooth topo inout_smooth . 0
+#sleep 5s
+#mv kappa_kernel_smooth.mesh kappa_kernel_smooth_h006km_v001km.mesh
+#sleep 5s
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/README
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/README (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/README 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,9 @@
+Carl Tape, 27-March-2009
+
+The smoothing operation -- as written -- requires the same number of processors as the wavefield simulations, so it is run in parallel on the cluster.
+
+We use two different scripts for performing the smoothing. From smooth_all, you can setup the directories and launch them one at a time using run_dir.bash. Or you can launch sets of them in serial from the directory SERIAL_RUN.
+
+As it is set up here, the jobs will NOT perform the operation to combine the binary files to create a low-res mesh file to view using vtk. The Caltech cluster overseers discourage such jobs from running on the head nodes, so we have a "parallel batch" script to launch many single-processor jobs onto the compute nodes, rather than the head nodes. This is done in TO_COMBINE.
+
+---------------------------------
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/check_dir.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/check_dir.bash (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/check_dir.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+for eid in `ls -1 -d [0-9]*`; do
+
+ echo "============ $eid ============"
+ ls -1 $eid/inout_smooth/*mu_kernel_smooth.bin | wc
+ ls -1 $eid/inout_smooth/*kappa_kernel_smooth.bin | wc
+
+done
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/check_dir.bash
___________________________________________________________________
Name: svn:executable
+ *
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/go1.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/go1.bash (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/go1.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,46 @@
+#!/bin/bash -v
+#BSUB -o OUTPUT_FILES/%J.o
+#BSUB -a mpich_gm
+#BSUB -J smooth_model_1
+
+current_pwd=$PWD
+
+echo "$LSB_MCPU_HOSTS" > OUTPUT_FILES/lsf_machines
+echo "$LSB_JOBID" > OUTPUT_FILES/jobid
+remap_lsf_machines.pl OUTPUT_FILES/lsf_machines >OUTPUT_FILES/machines
+
+# number of slices 14x12, 0 stands for basin code (1 would be 1-chunk global,
+# and 6 would be 6-chunk global)
+# element size on the surface would be 2 km,
+# the smoothing 'radius' sigma = 5 km
+
+# the neigboring points within 3*sigma+element_size are used for the smoothing
+# the 3D Gaussian function is defined as
+# G(x,y,z) = 1/(sqrt(2*pi)*sigma)**3 * exp[-r**2/(2*sigma**2)]
+# which is parallel to the 1D Gaussian function
+
+# the last two entries are the directories for (1) the unsmoothed and smoothed files and (2) topology
+
+for eid in `ls -1 -d [0-9]*`; do
+
+cd $current_pwd
+
+cd $eid
+
+mpirun.lsf --gm-no-shmem --gm-copy-env smooth_sem_fun 14 12 0 2 6 1 mu_kernel inout_smooth topo
+sleep 5s
+
+mpirun.lsf --gm-no-shmem --gm-copy-env smooth_sem_fun 14 12 0 2 6 1 kappa_kernel inout_smooth topo
+sleep 5s
+
+#xcombine_vol_data slice_file mu_kernel_smooth topo inout_smooth . 0
+#sleep 5s
+#mv mu_kernel_smooth.mesh mu_kernel_smooth_h006km_v001km.mesh
+#sleep 5s
+
+#xcombine_vol_data slice_file kappa_kernel_smooth topo inout_smooth . 0
+#sleep 5s
+#mv kappa_kernel_smooth.mesh kappa_kernel_smooth_h006km_v001km.mesh
+#sleep 5s
+
+done
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/go1.bash
___________________________________________________________________
Name: svn:executable
+ *
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/run1.lsf
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/run1.lsf (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/run1.lsf 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,6 @@
+#!/bin/bash
+mkdir -p OUTPUT_FILES
+rm OUTPUT_FILES/*
+date
+bsub -q normal -W 1440 -n 168 < go1.bash
+date
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/run1.lsf
___________________________________________________________________
Name: svn:executable
+ *
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/setup_smooth_all.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/setup_smooth_all.pl (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/setup_smooth_all.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,111 @@
+#!/usr/bin/perl -w
+
+#-----------------------------------
+# Carl Tape, 22-Feb-2009
+# setup_smooth_all.pl
+#
+# This script copies directories with binary files of a specified parameter
+# from one place to a local directory on pangu.
+#
+#
+# EXAMPLE:
+# setup_smooth_all.pl m16 168 mu_kernel ; setup_smooth_all.pl m16 168 kappa_kernel
+#
+#-----------------------------------
+
+if (@ARGV < 3) {die("Usage: setup_smooth_all.pl smodel nproc ftag\n")}
+($smodel,$nproc,$ftag) = @ARGV;
+
+$basedir = "/ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work";
+$edir = "$basedir/KERNELS_MODELS/event_kernels/kernel_${smodel}";
+$edir_smooth = "$basedir/KERNELS_MODELS/event_kernels_smooth/kernel_${smodel}";
+$dir_done = "/net/sierra/raid1/carltape/socal/socal_3D/RUNS/SMOOTH_EVENT_KERNELS/$smodel";
+
+$masterdir = "$basedir/smooth";
+
+if(not -e $edir) {die("edir $edir does not exist");}
+if(not -e $edir_smooth) {`mkdir $edir_smooth`;}
+if(not -e $masterdir) {die("masterdir $masterdir does not exist");}
+if(not -e ${dir_done}) {die("dir_done $dir_done does not exist");}
+
+# read in the list of kernels that you want to sum
+$kernel_list = "$basedir/KERNELS_MODELS/kernels_run";
+if (not -e ${kernel_list}) {die("check if kernel_list $kernel_list exist or not\n")}
+open(IN,"${kernel_list}"); @eids = <IN>; close(IN);
+$neid = @eids;
+
+print "number of event kernels : $neid\n";
+
+$imin = 1; $imax = $neid; # default
+#$imin = 1; $imax = 20;
+#$imin = 10; $imax = $imin;
+
+for ($i = $imin; $i <= $imax; $i++) {
+
+ $eid = $eids[$i-1]; chomp($eid);
+ print "======== $eid -- $i out of $imax ========\n";
+ $edir_in = "$edir/$eid";
+ ${efiles_in} = "${edir_in}/*${ftag}.bin";
+ $nfiles_in = `ls -1 $efiles_in | wc | awk '{print \$1}'`; chomp($nfiles_in);
+ print "nfiles_in = ${nfiles_in}\n";
+
+ if (${nfiles_in} != $nproc ) {
+ #die("no $nproc files in ${efiles_in}");
+ print "no $nproc files in ${efiles_in}\n";
+
+ } else {
+
+ $edir_done = "${dir_done}/$eid";
+ $edir_done1 = "${basedir}/smooth_all/TO_COMBINE/${eid}";
+ $edir_done2 = "${basedir}/smooth_all/TO_COMBINE/DONE/${eid}";
+ #if ( (-e ${edir_done}) || ( (-e ${eid_done1}) || (-e ${eid_done2}) ) {
+ if ( (-e ${edir_done}) || (-e ${eid_done1}) ) {
+ print "--> kernel is already smoothed\n";
+
+ } else {
+
+ $edir_copy = "$eid";
+ $edir_out = "${edir_copy}/inout_smooth";
+ $efiles_out = "${edir_out}/*${ftag}.bin";
+ $nfiles_out = `ls -1 ${efiles_out} | wc | awk '{print \$1}'`; chomp($nfiles_out);
+ print "nfiles_out = ${nfiles_out}\n";
+
+ if ( ${nfiles_out} == $nproc ) {
+ print "--> $nfiles_out bin files exist -- ready to go\n";
+
+ } else {
+
+ `mkdir -p $edir_smooth/$eid`;
+ `mkdir -p $edir_copy`;
+ `mkdir -p $eid/OUTPUT_FILES`;
+ `mkdir -p $eid/inout_smooth`;
+
+ $sfiles = "${edir_out}/*${ftag}_smooth.bin";
+ $nsfiles = `ls -1 $sfiles | wc | awk '{print \$1}'`; chomp($nsfiles);
+ if ( $nsfiles == $nproc ) {
+ print "--> $nsfiles bin files already exist -- no need to run\n";
+
+ } else {
+ # link input event kernels
+ `ln -s $efiles_in $edir_out`;
+
+ # copy run scripts and link executable file
+ `cp $masterdir/run.lsf $edir_copy`;
+ `cp $masterdir/go.bash $edir_copy`;
+ `ln -s $masterdir/topo $edir_copy`;
+ `ln -s $masterdir/smooth_sem_fun $edir_copy`;
+ `ln -s $masterdir/xcombine_vol_data $edir_copy`;
+ `ln -s $masterdir/slice_file $edir_copy`;
+ `ln -s $masterdir/combine.bash $edir_copy`;
+
+ # change the label for the run
+ `sed "/smooth_model/s/smooth_model/${eid}_smooth/" ${edir_copy}/go.bash > ${edir_copy}/run.tmp`;
+ `mv -f ${edir_copy}/run.tmp ${edir_copy}/go.bash`;
+ }
+ }
+ }
+ }
+
+}
+
+#=================================================================
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/SERIAL_RUN/setup_smooth_all.pl
___________________________________________________________________
Name: svn:executable
+ *
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/DONE/finish_smooth_all.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/DONE/finish_smooth_all.pl (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/DONE/finish_smooth_all.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,175 @@
+#!/usr/bin/perl -w
+
+#-----------------------------------
+# Carl Tape, 29-Jan-2009
+# finish_smooth_all.pl
+#
+# This script copies directories with binary files of a specified parameter
+# from one place to a local directory on pangu.
+#
+# EXAMPLE:
+# finish_smooth_all.pl m15 168 mu_kernel 6 1
+# finish_smooth_all.pl m15 168 kappa_kernel 6 1
+#
+#-----------------------------------
+
+if (@ARGV < 5) {die("Usage: setup_smooth_all.pl xxx\n")}
+($smodel,$nproc,$ftag,$hsmooth,$vsmooth) = @ARGV;
+
+$pwd = $ENV{PWD};
+
+$odir0 = "/net/sierra/raid1/carltape/socal/socal_3D/RUNS/SMOOTH_EVENT_KERNELS";
+$odir1 = "$odir0/$smodel";
+
+#$basedir = "..";
+$basedir = "/ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work";
+$pdir0 = "$basedir/KERNELS_MODELS/event_kernels_smooth/kernel_${smodel}";
+$sdir0 = "$basedir/smooth_all/TO_COMBINE/DONE";
+
+if(not -e $pdir0) {die("pdir0 $pdir0 does not exist");}
+if(not -e $sdir0) {die("sdir0 $sdir0 does not exist");}
+
+# read in the list of kernels that you want to sum
+$kernel_list = "$basedir/KERNELS_MODELS/kernels_run";
+if (not -e ${kernel_list}) {die("check if kernel_list $kernel_list exist or not\n")}
+open(IN,"${kernel_list}"); @eids = <IN>; close(IN);
+$neid = @eids;
+
+# label denoting horizontal and vertical smoothing
+$stg = sprintf("h%3.3ikm_v%3.3ikm",$hsmooth,$vsmooth);
+
+print "number of event kernels : $neid\n";
+print "smoothing label : $stg\n";
+
+if (0==1) {
+ for ($i = 1; $i <= $neid; $i++) {
+ $eid = $eids[$i-1]; chomp($eid);
+ print "$i, $eid\n";
+ }
+ die("testing");
+}
+
+#===================================
+
+$imin = 1; $imax = $neid; # default
+#$imin = 1; $imax = 10;
+#$imin = 1; $imax = $imin;
+
+for ($i = $imin; $i <= $imax; $i++) {
+
+ $eid = $eids[$i-1]; chomp($eid);
+ print "======== $eid -- $i to $imax ========\n";
+ $pdir = "$pdir0/$eid";
+ $sdir = "$sdir0/$eid";
+ $odir = "$odir1/$eid";
+
+ # (1) check if job is done
+ # (2) check if mesh files are off pangu
+ # (3) check if mesh files are in smooth_all
+
+ $mfiles = "${ftag}_smooth_${stg}.mesh";
+ $bfiles = "*${ftag}_smooth.bin";
+
+ $mfile_out = `ls -1 $odir/$mfiles | wc | awk '{print \$1}'`; chomp($mfile_out);
+ #$bfile_out = `ls -1 $pdir/$bfiles | wc | awk '{print \$1}'`; chomp($bfile_out);
+
+ $ofile = `ls -1 $eid/OUTPUT_FILES/*.o`; chomp($ofile);
+
+ if ( 0==1 ) {
+ #if ( not -f $ofile ) {
+ print "--> job has not been run, or is running, or has finished\n";
+
+ } else {
+
+ if ( ${mfile_out} == 1 ) {
+ print "--> ${mfile_out} mesh file exists -- $odir/$mfiles\n";
+
+ # check that there are 168 files there
+ $ncheck = `ls -1 $pdir/$bfiles | wc | awk '{print \$1}'`; chomp($ncheck);
+ if ($ncheck != $nproc) {
+ die("ncheck ($ncheck) not equal to nproc ($nproc)");
+ }
+
+ } else {
+
+ # low-res mesh files
+ $mfiles2 = "$sdir/$mfiles";
+ $mfile_out2 = `ls -1 $mfiles2 | wc | awk '{print \$1}'`; chomp($mfile_out2);
+
+ # high-res binary files
+ $bfiles2 = "$sdir/inout_smooth/$bfiles";
+ $bfile_out2 = `ls -1 $bfiles2 | wc | awk '{print \$1}'`; chomp($bfile_out2);
+
+ if ( (${mfile_out2} == 1) && (${bfile_out2} == $nproc) ) {
+ # (1) copy the mesh files off pangu
+ # (2) copy the bin files to event_kernels_smooth
+ # NOTE: cp is a safer operation than mv -- especially for the sensitive IBRIX system
+ print "--> copying the mesh and binary files\n";
+ `mkdir -p $odir`;
+ `cp $mfiles2 $odir`;
+ `sleep 3s`;
+ `mkdir -p $pdir`;
+ if (0==1) {
+ `cp $bfiles2 $pdir`;
+ } else {
+ `mv $bfiles2 $pdir`;
+ }
+ `sleep 3s`;
+
+ # check that all the binary files made it
+ # (And copy any that were left behind.)
+ # (This absurd section of code is due to the faulty filesystem IBRIX at Caltech.)
+ $bfiles3 = "$pdir/$bfiles";
+ $bfile_out3 = `ls -1 $bfiles3 | wc | awk '{print \$1}'`; chomp($bfile_out3);
+ if ( ${bfile_out3} != $nproc ) {
+ #die("only ${bfile_out3} out of $nproc made it into $pdir");
+ print "only ${bfile_out3} out of $nproc made it into $pdir\n";
+ @leftfiles = glob("$bfiles2");
+ print "leftfiles are @leftfiles\n";
+ `cp @leftfiles $pdir`;
+ `sleep 3s`;
+
+ # check the numbers again
+ $bfiles3 = "$pdir/$bfiles";
+ $bfile_out3 = `ls -1 $bfiles3 | wc | awk '{print \$1}'`; chomp($bfile_out3);
+ if ( ${bfile_out3} != $nproc ) {
+ die("only ${bfile_out3} out of $nproc made it into $pdir");
+ }
+ }
+
+ } else {
+ if( ${mfile_out2} == 1 ) {
+ die("mesh file was made but only ${bfile_out2} out of $nproc binary files");
+ }
+ if ( ${bfile_out2} == $nproc ) {
+ die("$nproc binary files but no $mesh file\n");
+ }
+ if ( (${mfile_out2} == 0) && (${bfile_out2} == 0) ) {
+ print "--> this event has not been setup yet\n";
+ }
+ }
+ }
+ }
+}
+
+#================================================
+
+#sub cd_directory {
+# my($new) = @_;
+# my $old = `pwd`;
+# chomp($old);
+# check_directory($new);
+# #print "$prog: cd $new\n";
+# chdir $new;
+# return($old);
+#}
+
+#sub check_directory {
+# if(! -e $_[0] ) {
+# print "Directory not found: $_[0]\n";
+# exit(-1);
+# }
+#}
+
+#=================================================================
+
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/DONE/finish_smooth_all.pl
___________________________________________________________________
Name: svn:executable
+ *
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/go_process.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/go_process.bash (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/go_process.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,20 @@
+#!/bin/bash -v
+
+dirbase=$PWD
+
+# modified in the setup script
+eid=9114763
+
+current_pwd=$dirbase/$eid
+
+cd $current_pwd
+
+$current_pwd/xcombine_vol_data slice_file mu_kernel_smooth topo inout_smooth . 0
+sleep 5s
+mv mu_kernel_smooth.mesh mu_kernel_smooth_h006km_v001km.mesh
+sleep 5s
+
+$current_pwd/xcombine_vol_data slice_file kappa_kernel_smooth topo inout_smooth . 0
+sleep 5s
+mv kappa_kernel_smooth.mesh kappa_kernel_smooth_h006km_v001km.mesh
+sleep 5s
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_001
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_001 (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_001 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,15 @@
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14178188/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14178212/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14178236/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14178248/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14179292/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14179736/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14239184/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14255632/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14263544/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14263712/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./14263768/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./9627557/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./9774569/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./9826789/go_process.bash
+cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./9944301/go_process.bash
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_launch_001.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_launch_001.bash (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_launch_001.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,2 @@
+#!/bin/bash
+bsub -W 60 -n 15 -o ./%J.out -a mpich_gm -q normal -J parallel_batch.py mpirun.lsf `which mpipython.exe` /usr/bin/parallel_batch.py run_file_001
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/run_file_launch_001.bash
___________________________________________________________________
Name: svn:executable
+ *
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/setup_batch.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/setup_batch.pl (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/setup_batch.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,126 @@
+#!/usr/bin/perl -w
+
+#-------------------------
+# Setup post-processing for parallel batch launch.
+#
+# EXAMPLE:
+# setup_batch.pl 1/100 10 1 # check that all files are there
+# setup_batch.pl 1/100 10 0 # prepare parallel batch scripts
+#-------------------------
+
+if (@ARGV < 3) {die("Usage: setup_batch.pl ncore\n")}
+($inds,$ncore,$icheck) = @ARGV;
+
+$pwd = $ENV{PWD};
+
+#$queue = "test"; $nminutes = 60;
+$queue = "normal"; $nminutes = 60;
+$nmesh0 = 2;
+$nbin0 = 336;
+
+`rm *.out`;
+
+ at dirs = `ls -1 -d [1-9]*`;
+$neid = @dirs;
+
+($imin,$imax) = split("/",$inds);
+if($imin < 1) {$imin = 1}
+if($imin > $neid) {$imin = $neid}
+if($imax > $neid) {$imax = $neid}
+
+#print "\n -- @dirs --\n";
+
+#$neid = 1;
+
+if (0==1) {
+ $k = 0; $p = 0;
+ for ($i = 1; $i <= $neid; $i++) {
+ $k = $k+1;
+ if ($k % $ncore == 1) {
+ $p = $p+1;
+ }
+ print "i = $i, k = $k, p = $p\n";
+ }
+ die("TESTING");
+}
+
+#----------------------
+
+$k = 0; $p = 0;
+for ($i = $imin; $i <= $imax; $i++) {
+
+ cd_directory($pwd);
+ $eid = $dirs[$i-1]; chomp($eid);
+ print "==== $eid =====\n";
+
+ if ($icheck == 0) {
+
+ $k = $k+1;
+ if ( ($k % $ncore == 1) || ($imin == $imax) ) {
+ $p = $p+1;
+ $rfile = sprintf("run_file_%3.3i",$p);
+ open(OUT,">$rfile");
+
+ # launch command -- NEEDS TO BE MODIFIED FOR THE LAST BATCH
+ $efile = sprintf("run_file_launch_%3.3i.bash",$p);
+ open(RUN,">$efile");
+ print RUN "#!/bin/bash\n";
+ print RUN "bsub -W $nminutes -n $ncore -o ./%J.out -a mpich_gm -q $queue -J parallel_batch.py mpirun.lsf `which mpipython.exe` /usr/bin/parallel_batch.py $rfile\n";
+ close(RUN);
+ `chmod +x $efile`;
+ }
+
+ # each event gets its own line
+ print OUT "cd /ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu_work/smooth_all/TO_COMBINE ; ./${eid}/go_process.bash\n";
+
+ } else {
+
+ # check for mesh files
+ @meshfiles = glob("$eid/*.mesh");
+ $nmesh = @meshfiles;
+
+ if ($nmesh == $nmesh0) {
+ print "--> $nmesh mesh files already exist\n";
+ } else {
+ print "$nmesh mesh files\n";
+ @sfiles = `ls -1 -d $eid/inout_smooth/*smooth.bin`;
+ $nbin = @sfiles;
+ if ($nbin != $nbin0) {
+ print "--> re-run the smoothing, since you have $nbin not $nbin0 files\n";
+ } else {
+ print "--> ready to set this up\n";
+ `cp go_process.bash $eid`;
+ #`chmod +x $eid/go_process.bash`;
+
+ # modify the EID label for the post-processing script
+ # eid=9818433
+ $processfile = "$eid/go_process.bash";
+ $ptemp = "$eid/go_process_temp.bash";
+ `sed '/^eid=/s/^.*\$/eid=$eid/' $processfile > $ptemp`;
+ `mv $ptemp $processfile`;
+ `chmod +x $processfile`;
+ }
+ }
+ } # icheck
+}
+
+#================================================
+
+sub cd_directory {
+ my($new) = @_;
+ my $old = `pwd`;
+ chomp($old);
+ check_directory($new);
+ #print "$prog: cd $new\n";
+ chdir $new;
+ return($old);
+}
+
+sub check_directory {
+ if(! -e $_[0] ) {
+ print "Directory not found: $_[0]\n";
+ exit(-1);
+ }
+}
+
+#=================================================================
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/TO_COMBINE/setup_batch.pl
___________________________________________________________________
Name: svn:executable
+ *
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/check_dir.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/check_dir.bash (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/check_dir.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+for eid in `ls -1 -d [0-9]*`; do
+
+ echo "============ $eid ============"
+ ls -1 $eid/inout_smooth/*mu_kernel_smooth.bin | wc
+ ls -1 $eid/inout_smooth/*kappa_kernel_smooth.bin | wc
+
+done
Property changes on: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/check_dir.bash
___________________________________________________________________
Name: svn:executable
+ *
Deleted: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/finish_smooth_all.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/finish_smooth_all.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/finish_smooth_all.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -1,141 +0,0 @@
-#!/usr/bin/perl -w
-
-#-----------------------------------
-# Carl Tape, 12-July-2008
-# finish_smooth_all.pl
-#
-# This script copies directories with binary files of a specified parameter
-# from one place to a local directory on pangu.
-#
-# EXAMPLE:
-# finish_smooth_all.pl m07 168 mu_kernel 6 1
-# finish_smooth_all.pl m07 168 kappa_kernel 6 1
-#
-#-----------------------------------
-
-if (@ARGV < 5) {die("Usage: setup_smooth_all.pl xxx\n")}
-($smodel,$nproc,$ftag,$hsmooth,$vsmooth) = @ARGV;
-
-$pwd = $ENV{PWD};
-
-$odir0 = "/net/sierra/raid1/carltape/socal/socal_3D/RUNS/SMOOTH_EVENT_KERNELS";
-$odir1 = "$odir0/$smodel";
-
-$basedir = "..";
-#$basedir = "/ibrixfs1/home/carltape/ADJOINT_TOMO/iterate_adj/pangu";
-$pdir0 = "$basedir/KERNELS_MODELS/event_kernels_smooth/kernel_${smodel}";
-$sdir0 = "$basedir/smooth_all";
-
-if(not -e $pdir0) {die("pdir0 $pdir0 does not exist");}
-if(not -e $sdir0) {die("sdir0 $sdir0 does not exist");}
-
-# read in the list of kernels that you want to sum
-$kernel_list = "$basedir/KERNELS_MODELS/kernels_run";
-if (not -e ${kernel_list}) {die("check if kernel_list $kernel_list exist or not\n")}
-open(IN,"${kernel_list}"); @eids = <IN>; close(IN);
-$neid = @eids;
-
-# label denoting horizontal and vertical smoothing
-$stg = sprintf("h%3.3ikm_v%3.3ikm",$hsmooth,$vsmooth);
-
-print "number of event kernels : $neid\n";
-print "smoothing label : $stg\n";
-
-if (0==1) {
- for ($i = 1; $i <= $neid; $i++) {
- $eid = $eids[$i-1]; chomp($eid);
- print "$i, $eid\n";
- }
- die("testing");
-}
-
-#===================================
-
-$imin = 1; $imax = $neid; # default
-#$imin = 16; $imax = 50;
-#$imin = 14; $imax = $imin;
-
-for ($i = $imin; $i <= $imax; $i++) {
-
- $eid = $eids[$i-1]; chomp($eid);
- print "======== $eid -- $i to $imax ========\n";
- $pdir = "$pdir0/$eid";
- $sdir = "$sdir0/$eid";
- $odir = "$odir1/$eid";
-
- # (1) check if job is done
- # (2) check if mesh files are off pangu
- # (3) check if mesh files are in smooth_all
-
- $mfiles = "${ftag}_smooth_${stg}.mesh";
- $bfiles = "*${ftag}_smooth.bin";
-
- $mfile_out = `ls -1 $odir/$mfiles | wc | awk '{print \$1}'`; chomp($mfile_out);
- #$bfile_out = `ls -1 $pdir/$bfiles | wc | awk '{print \$1}'`; chomp($bfile_out);
-
- $ofile = `ls -1 $eid/OUTPUT_FILES/*.o`; chomp($ofile);
-
- if ( not -f $ofile ) {
- print "--> job has not been run, or is running, or has finished\n";
-
- } else {
-
- if ( ${mfile_out} == 1 ) {
- print "--> ${mfile_out} mesh file exists -- $odir/$mfiles\n";
-
- # check that there are 168 files there
- $ncheck = `ls -1 $pdir/$bfiles | wc | awk '{print \$1}'`; chomp($ncheck);
- if ($ncheck != $nproc) {
- die("ncheck ($ncheck) not equal to nproc ($nproc)");
- }
-
- } else {
-
- $mfiles2 = "$sdir/$mfiles";
- $mfile_out2 = `ls -1 $mfiles2 | wc | awk '{print \$1}'`; chomp($mfile_out2);
- $bfiles2 = "$sdir/inout_smooth/$bfiles";
- $bfile_out2 = `ls -1 $bfiles2 | wc | awk '{print \$1}'`; chomp($bfile_out2);
-
- if ( ${mfile_out2} == 1 ) {
- # (1) move the files off pangu
- # (2) move the bin files to event_kernels_smooth
- print "--> moving the mesh and binary files\n";
- `mkdir -p $odir`;
- `mv $mfiles2 $odir`;
- `sleep 3s`;
- `mkdir -p $pdir`;
- `mv $bfiles2 $pdir`;
- `sleep 3s`;
-
- } else {
- if ( ${bfile_out2} == $nproc ) {
- die("no mesh files were made\n");
- } else {
- print "--> this event has not been setup yet\n";
- }
- }
- }
- }
-}
-
-#================================================
-
-#sub cd_directory {
-# my($new) = @_;
-# my $old = `pwd`;
-# chomp($old);
-# check_directory($new);
-# #print "$prog: cd $new\n";
-# chdir $new;
-# return($old);
-#}
-
-#sub check_directory {
-# if(! -e $_[0] ) {
-# print "Directory not found: $_[0]\n";
-# exit(-1);
-# }
-#}
-
-#=================================================================
-
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/run_dir.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/run_dir.bash 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/run_dir.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -2,16 +2,16 @@
run_pwd=$PWD
-#sleep 60m
+#sleep 120m
for eid in `ls -1 -d [0-9]*`;do
-#for eid in `ls -1 -d 14* 10006857 10992159 12659440 91* 96* 98* 99* 9753489 9753497 9753949 9755013`; do
+#for eid in `ls -1 -d 13935988 9703873 9818433 9735129 9105672 10223765`; do
cd $run_pwd
cd $eid
echo "==== $eid ====="
run.lsf
- sleep 10m
+ sleep 5s
done
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/setup_smooth_all.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/setup_smooth_all.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/smooth_all/setup_smooth_all.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -1,7 +1,7 @@
#!/usr/bin/perl -w
#-----------------------------------
-# Carl Tape, 12-Oct-2008
+# Carl Tape, 29-Jan-2009
# setup_smooth_all.pl
#
# This script copies directories with binary files of a specified parameter
@@ -9,7 +9,7 @@
#
#
# EXAMPLE:
-# setup_smooth_all.pl m09 168 mu_kernel ; setup_smooth_all.pl m09 168 kappa_kernel
+# setup_smooth_all.pl m14 168 mu_kernel ; setup_smooth_all.pl m14 168 kappa_kernel
#
#-----------------------------------
@@ -56,7 +56,7 @@
} else {
$edir_done = "${dir_done}/$eid";
- if (-e ${edir_done}) {
+ if ( (-e "${dir_done}/$eid") || ( (-e "TO_COMBINE/$eid") || (-e "TO_COMBINE/DONE/$eid") ) ) {
print "--> kernel is already smoothed\n";
} else {
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/prepare_run.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/prepare_run.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/prepare_run.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -8,7 +8,7 @@
# from a set of event kernels.
#
# EXAMPLE:
-# prepare_run.pl m04 1
+# prepare_run.pl m14 1
#
#-----------------------------------
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/run.lsf
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/run.lsf 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_hessian/run.lsf 2009-03-28 00:09:46 UTC (rev 14502)
@@ -3,6 +3,7 @@
mkdir -p OUTPUT_FILES
rm OUTPUT_FILES/*
date
-bsub -q test -W 30 -n 168 < go.bash
+bsub -q normal -W 30 -n 168 < go.bash
+#bsub -q test -W 30 -n 168 < go.bash
#bsub -q normal -W 300 -n 168 < go.bash
date
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/README
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/README 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/README 2009-03-28 00:09:46 UTC (rev 14502)
@@ -1,11 +1,12 @@
-Carl Tape, 17-June-2008
+Carl Tape, 27-March-2009
Directions:
(1) Copy output file directory ADJOINT_TOMO/iterate_adj/matlab/OUTPUT into INPUT
(2) Run prepare_run.pl and check that the kernels in INPUT_KERNELS are properly linked
(3) run.lsf
-(4) make_mesh.pl
-(5) copy INPUT/pmax and .mesh files off cluster
-(6) off cluster, make .vtu files, then figures
+(4) make_mesh.pl to generate the parallel batch script run_file_launch.bash
+(5) run_file_launch.bash
+(6) copy INPUT/pmax and .mesh files off cluster
+(7) off cluster, make .vtu files, then figures
--------------------------
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/make_mesh.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/make_mesh.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/make_mesh.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -12,6 +12,7 @@
$idir = "INPUT/mu_run";
$odir = "OUTPUT_MODEL";
+$pwd = $ENV{PWD};
# read the label that designates the normalization in the data covariance
open(IN,"$idir/dcov_tag"); $ftag = <IN>; chomp($ftag); close(IN);
@@ -27,6 +28,22 @@
#===================================
+# one-line run file
+$rfile = "run_file";
+open(OUT,">$rfile");
+
+# launch command for parallel batch
+$nminutes = 30;
+$queue = "normal";
+$efile = "run_file_launch.bash";
+open(RUN,">$efile");
+print RUN "#!/bin/bash\n";
+print RUN "bsub -W $nminutes -n $npmax -o ./%J.out -a mpich_gm -q $queue -J parallel_batch.py mpirun.lsf `which mpipython.exe` /usr/bin/parallel_batch.py $rfile\n";
+`chmod +x $efile`;
+close(RUN);
+
+#------------------------
+
$imin = 1; $imax = $npmax;
#$imin = 2; $imax = $imin;
@@ -42,10 +59,27 @@
print "nfiles ($ifiles) = $nfiles\n";
print "$dmtag\n";
- print "xcombine_vol_data $slicefile $dmtag topo $odir . 0\n";
- `xcombine_vol_data $slicefile $dmtag topo $odir . 0`;
+ # this will run on the head node in serial
+ #print "xcombine_vol_data $slicefile $dmtag topo $odir . 0\n";
+ #`xcombine_vol_data $slicefile $dmtag topo $odir . 0`;
+ #-------------------------------
+ # instead we setup a parallel batch script to run on the cluster nodes
+
+ # commands to make low-res mesh
+ $mfile = sprintf("make_mesh_%2.2i.bash",$i);
+ open(MFILE,">$mfile");
+ print MFILE "#!/bin/bash\n";
+ print MFILE "cd $pwd\n";
+ print MFILE "$pwd/xcombine_vol_data $slicefile $dmtag topo $odir . 0\n";
+ close(MFILE);
+ `chmod +x $mfile`;
+
+ # one-line run file
+ print OUT "cd $pwd ; ./$mfile\n";
}
+close(OUT);
+
#=================================================================
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/prepare_run.pl
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/prepare_run.pl 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/prepare_run.pl 2009-03-28 00:09:46 UTC (rev 14502)
@@ -11,6 +11,7 @@
# Make sure you have both the mu and the kappa update.
#
# EXAMPLE:
+# prepare_run.pl 60/4/80 m14 mu_kernel both 1 window 1
# prepare_run.pl 20/4/60 m10 mu_kernel both 1 window 1
# prepare_run.pl 44/1/44 m10 kappa_kernel both 1 window 1
#
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/run.lsf
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/run.lsf 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/subspace_update/run.lsf 2009-03-28 00:09:46 UTC (rev 14502)
@@ -3,6 +3,6 @@
mkdir -p OUTPUT_FILES
rm OUTPUT_FILES/*
date
-bsub -q test -W 30 -n 168 < go.bash
-#bsub -q normal -W 120 -n 168 < go.bash
+#bsub -q debug-long -W 30 -n 168 < go.bash
+bsub -q normal -W 120 -n 168 < go.bash
date
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/run.lsf
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/run.lsf 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/run.lsf 2009-03-28 00:09:46 UTC (rev 14502)
@@ -3,5 +3,6 @@
mkdir -p OUTPUT_FILES
rm OUTPUT_FILES/*
date
-bsub -q test -W 30 -n 168 < go.bash
+#bsub -q normal -W 30 -n 168 < go.bash
+bsub -q debug -W 30 -n 168 < go.bash
date
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/compile.bash
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/compile.bash 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/compile.bash 2009-03-28 00:09:46 UTC (rev 14502)
@@ -1,5 +1 @@
mpif90 -o sum_kernels -O3 sum_kernels.f90 exit_mpi.f90 gll_library.f90
-
-#mpif90 -o sem_fun_resample -O3 sem_fun_resample.f90 exit_mpi.f90 rthetaphi_xyz.f90
-
-#ifort -o sem_fun_resample -CB sem_fun_resample.f90 exit_mpi.f90 rthetaphi_xyz.f90
Modified: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels.f90
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels.f90 2009-03-27 23:02:56 UTC (rev 14501)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels.f90 2009-03-28 00:09:46 UTC (rev 14502)
@@ -49,7 +49,9 @@
read(12) kernel(:,:,:,1:nspec)
close(12)
+ !total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + abs( kernel(:,:,:,1:nspec) )
total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + kernel(:,:,:,1:nspec)
+
enddo
if(myrank==1) write(*,*) 'writing out summed kernel for mu'
write(k_file,'(a,i6.6,a)') 'OUTPUT_SUM/proc',myrank,'_'//trim(kernel_name)//'.bin'
@@ -69,7 +71,9 @@
read(12) kernel(:,:,:,1:nspec)
close(12)
+ !total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + abs( kernel(:,:,:,1:nspec) )
total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + kernel(:,:,:,1:nspec)
+
enddo
if(myrank==1) write(*,*) 'writing out summed kernel for kappa'
write(k_file,'(a,i6.6,a)') 'OUTPUT_SUM/proc',myrank,'_'//trim(kernel_name)//'.bin'
Added: seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels_weight.f90
===================================================================
--- seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels_weight.f90 (rev 0)
+++ seismo/3D/ADJOINT_TOMO/iterate_adj/pangu/sum_kernel/src/sum_kernels_weight.f90 2009-03-28 00:09:46 UTC (rev 14502)
@@ -0,0 +1,120 @@
+program sum_kernels
+
+ implicit none
+ include 'mpif.h'
+ include 'constants.h'
+ include 'precision.h'
+ include 'values_from_mesher.h'
+
+ ! ======================================================
+
+ integer, parameter :: NSPEC=NSPEC_AB
+
+ character(len=150) :: kernel_file_list, kernel_list(1000), sline, k_file, kernel_name
+
+ character(len=150) :: nwin_file_list
+ integer :: nwin_list(1000), nwin
+
+ real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: kernel, total_kernel
+ integer :: iker, nker, myrank, sizeprocs, ier
+ integer :: i, j, k,ispec, iglob, ishell, n, it, j1, ib, npts_sem, ios
+
+ ! ============ program starts here =====================
+ ! initialize the MPI communicator and start the NPROCTOT MPI processes
+ call MPI_INIT(ier)
+ call MPI_COMM_SIZE(MPI_COMM_WORLD,sizeprocs,ier)
+ call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ier)
+
+ !kernel_file_list='kernels_run'
+ kernel_file_list='m16_window_picks_eids'
+ nwin_file_list='m16_window_picks_nwin'
+
+ ! read in list of kernels
+ nker=0
+ open(unit = 20, file = trim(kernel_file_list), status = 'old',iostat = ios)
+ if (ios /= 0) then
+ print *,'Error opening ',trim(kernel_file_list)
+ stop
+ endif
+ do while (1 == 1)
+ read(20,'(a)',iostat=ios) sline
+ if (ios /= 0) exit
+ nker=nker+1
+ kernel_list(nker) = sline
+ enddo
+ close(20)
+
+ ! read in corresponding list of window picks for each event kernel
+ ! --> this is relevant depending on your choice of weighting for the data covariance
+ open(unit = 20, file = trim(nwin_file_list), status = 'old',iostat = ios)
+ if (ios /= 0) then
+ print *,'Error opening ',trim(nwin_file_list)
+ stop
+ endif
+ do iker = 1,nker
+ read(20,'(i)',iostat=ios) nwin
+ if (ios /= 0) exit
+ nwin_list(iker) = nwin
+ enddo
+ close(20)
+
+ !do iker = 1, nker
+ ! if(myrank==1) write(*,*) 'Kernel, nwin: ', trim(kernel_list(iker)), ' out of ', nwin_list(iker)
+ !enddo
+
+ !------------------------------------------------------
+
+ !kernel_name = 'mu_kernel'
+ kernel_name = 'mu_kernel_smooth'
+
+ total_kernel=0.
+ do iker = 1, nker
+ if(myrank==1) write(*,*) 'reading in event kernel for mu: ', iker, ' out of ', nker
+ write(k_file,'(a,i6.6,a)') 'INPUT_KERNELS/'//trim(kernel_list(iker))//'/proc',myrank,'_'//trim(kernel_name)//'.bin'
+
+ open(12,file=trim(k_file),status='old',form='unformatted')
+ read(12) kernel(:,:,:,1:nspec)
+ close(12)
+
+ !total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + kernel(:,:,:,1:nspec)
+ !total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + abs( kernel(:,:,:,1:nspec) )
+ total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + 1.0/dble(nwin_list(iker)) * abs( kernel(:,:,:,1:nspec) )
+
+ enddo
+ if(myrank==1) write(*,*) 'writing out summed kernel for mu'
+ write(k_file,'(a,i6.6,a)') 'OUTPUT_SUM/proc',myrank,'_'//trim(kernel_name)//'.bin'
+ open(12,file=trim(k_file),form='unformatted')
+ write(12) total_kernel(:,:,:,1:nspec)
+ close(12)
+
+ !kernel_name = 'kappa_kernel'
+ kernel_name = 'kappa_kernel_smooth'
+
+ total_kernel=0.
+ do iker = 1, nker
+ if(myrank==1) write(*,*) 'reading in event kernel for kappa: ', iker, ' out of ', nker
+ write(k_file,'(a,i6.6,a)') 'INPUT_KERNELS/'//trim(kernel_list(iker))//'/proc',myrank,'_'//trim(kernel_name)//'.bin'
+
+ open(12,file=trim(k_file),status='old',form='unformatted')
+ read(12) kernel(:,:,:,1:nspec)
+ close(12)
+
+ !total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + kernel(:,:,:,1:nspec)
+ !total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + abs( kernel(:,:,:,1:nspec) )
+ total_kernel(:,:,:,1:nspec) = total_kernel(:,:,:,1:nspec) + 1.0/dble(nwin_list(iker)) * abs( kernel(:,:,:,1:nspec) )
+
+ enddo
+ if(myrank==1) write(*,*) 'writing out summed kernel for kappa'
+ write(k_file,'(a,i6.6,a)') 'OUTPUT_SUM/proc',myrank,'_'//trim(kernel_name)//'.bin'
+ open(12,file=trim(k_file),form='unformatted')
+ write(12) total_kernel(:,:,:,1:nspec)
+ close(12)
+
+ if(myrank==1) write(*,*) 'done writing all kernels, now finishing...'
+
+ ! stop all the MPI processes, and exit
+ call MPI_FINALIZE(ier)
+
+end program sum_kernels
+
+
More information about the CIG-COMMITS
mailing list