[cig-commits] r19784 - in seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER: DATA EXAMPLES/global_PREM_kernels/amplitude/DATA EXAMPLES/global_PREM_kernels/traveltime/DATA EXAMPLES/global_s362ani/DATA EXAMPLES/global_s362ani_small/DATA EXAMPLES/noise_examples/global_long EXAMPLES/noise_examples/global_short EXAMPLES/noise_examples/regional EXAMPLES/noise_examples/test_global EXAMPLES/noise_examples/test_regional EXAMPLES/regional_Greece_small/DATA EXAMPLES/regional_MiddleEast/DATA src/auxiliaries src/create_header_file src/cuda src/meshfem3D src/shared src/specfem3D

danielpeter at geodynamics.org danielpeter at geodynamics.org
Thu Mar 15 08:23:18 PDT 2012


Author: danielpeter
Date: 2012-03-15 08:23:17 -0700 (Thu, 15 Mar 2012)
New Revision: 19784

Modified:
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/DATA/Par_file
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani/DATA/Par_file
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_small/DATA/Par_file
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_3_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_3_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_3_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_3_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_attenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_3_noattenuation
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_Greece_small/DATA/Par_file
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_MiddleEast/DATA/Par_file
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_1D.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_2D.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_corners_chunks.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_faces_chunks.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_AVS_DX.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_AVS_DX.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_GMT_global.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/create_header_file/create_header_file.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_outer_core_cuda.cu
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/mesh_constants_cuda.h
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/transfer_fields_cuda.cu
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_gapp2.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/broadcast_compute_parameters.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/model_topo_bathy.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_parameter_file.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/noise_tomography.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_forward_arrays.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_forward_arrays.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_kernels.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_GLL_points.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_sources_receivers.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_output.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_volume.f90
   seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90
Log:
adds another local path parameter to separate mesh files from temporary storage file; updates locate receivers routine; updates read topo file; updates check norm routines on cuda

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/DATA/Par_file	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/DATA/Par_file	2012-03-15 15:23:17 UTC (rev 19784)
@@ -81,7 +81,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
+# mesh files
 LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 50

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,7 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /tigress-hsm/bozdag/SEM_NM_AMP
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 1000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,7 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /tigress-hsm/bozdag/SEM_NM_AMP
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 1000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani/DATA/Par_file	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani/DATA/Par_file	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,7 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
+# mesh files
 LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 1000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_small/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_small/DATA/Par_file	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_small/DATA/Par_file	2012-03-15 15:23:17 UTC (rev 19784)
@@ -81,7 +81,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 1000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_1_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_2_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_3_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_3_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_long/Par_file_NOISE_3_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_1_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_2_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_3_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_3_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/global_short/Par_file_NOISE_3_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_1_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_2_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_3_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_3_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/regional/Par_file_NOISE_3_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_1_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_2_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_3_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_3_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_global/Par_file_NOISE_3_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_1_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_attenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_attenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_attenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_2_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_3_noattenuation
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_3_noattenuation	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/noise_examples/test_regional/Par_file_NOISE_3_noattenuation	2012-03-15 15:23:17 UTC (rev 19784)
@@ -87,8 +87,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
-LOCAL_PATH                      = /scratch/yangl/DATABASES_MPI
-#/scratch/yangl/DATABASES_MPI
+# mesh files
+LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 10000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_Greece_small/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_Greece_small/DATA/Par_file	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_Greece_small/DATA/Par_file	2012-03-15 15:23:17 UTC (rev 19784)
@@ -81,7 +81,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
+# mesh files
 LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 50

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_MiddleEast/DATA/Par_file
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_MiddleEast/DATA/Par_file	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/regional_MiddleEast/DATA/Par_file	2012-03-15 15:23:17 UTC (rev 19784)
@@ -81,7 +81,10 @@
 NUMBER_OF_THIS_RUN              = 1
 
 # path to store the local database files on each node
+# mesh files
 LOCAL_PATH                      = ./DATABASES_MPI
+# temporary wavefield/kernel/movie files
+LOCAL_TMP_PATH                  = ./DATABASES_MPI
 
 # interval at which we output time step info and max of norm of displacement
 NTSTEP_BETWEEN_OUTPUT_INFO      = 1000

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_1D.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_1D.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_1D.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -79,7 +79,7 @@
           ROTATE_SEISMOGRAMS_RT,HONOR_1D_SPHERICAL_MOHO,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -136,7 +136,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC, &
          NSPEC2D_XI, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_2D.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_2D.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_2D.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -91,7 +91,7 @@
   logical, dimension(MAX_NUMBER_OF_MESH_LAYERS) :: this_region_has_a_doubling
   double precision, dimension(MAX_NUMBER_OF_MESH_LAYERS) :: rmins,rmaxs
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -139,7 +139,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,&
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC, &
          NSPEC2D_XI, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_corners_chunks.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_corners_chunks.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_corners_chunks.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -76,7 +76,7 @@
           ROTATE_SEISMOGRAMS_RT,HONOR_1D_SPHERICAL_MOHO,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -128,7 +128,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC, &
          NSPEC2D_XI, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_faces_chunks.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_faces_chunks.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/check_buffers_faces_chunks.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -80,7 +80,7 @@
           ROTATE_SEISMOGRAMS_RT,HONOR_1D_SPHERICAL_MOHO,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -133,7 +133,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC,NSPEC2D_XI,NSPEC2D_ETA, &
          NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_AVS_DX.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_AVS_DX.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_AVS_DX.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -149,7 +149,7 @@
           ROTATE_SEISMOGRAMS_RT,HONOR_1D_SPHERICAL_MOHO,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -212,7 +212,9 @@
                  MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
                  PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
                  ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-                 INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+                 INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+                 LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+                 SIMULATION_TYPE,SAVE_FORWARD, &
                  NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
                  NSPEC_COMP,NSPEC2D_XI,NSPEC2D_ETA,NSPEC2DMAX_XMIN_XMAX,&
                  NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_AVS_DX.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_AVS_DX.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_AVS_DX.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -856,7 +856,8 @@
                NGLOB2DMAX_XMIN_XMAX,NGLOB2DMAX_YMIN_YMAX, &
                NGLOB
 
-  character(len=150) LOCAL_PATH,MODEL
+  character(len=150) LOCAL_PATH,LOCAL_TMP_PATH,MODEL
+
   logical :: CUT_SUPERBRICK_XI,CUT_SUPERBRICK_ETA
   integer, dimension(NB_SQUARE_CORNERS,NB_CUT_CASE) :: DIFF_NSPEC1D_RADIAL
   integer, dimension(NB_SQUARE_EDGES_ONEDIR,NB_CUT_CASE) :: DIFF_NSPEC2D_XI,DIFF_NSPEC2D_ETA
@@ -885,7 +886,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC,NSPEC2D_XI,NSPEC2D_ETA, &
          NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_GMT_global.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_GMT_global.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/create_movie_GMT_global.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -125,7 +125,7 @@
           ROTATE_SEISMOGRAMS_RT,HONOR_1D_SPHERICAL_MOHO,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) LOCAL_PATH,MODEL
+  character(len=150) LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -188,7 +188,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC,NSPEC2D_XI,NSPEC2D_ETA, &
          NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/create_header_file/create_header_file.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/create_header_file/create_header_file.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/create_header_file/create_header_file.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -60,7 +60,7 @@
           ROTATE_SEISMOGRAMS_RT,HONOR_1D_SPHERICAL_MOHO,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) LOCAL_PATH,MODEL
+  character(len=150) LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -128,7 +128,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC,NSPEC2D_XI,NSPEC2D_ETA, &
          NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu	2012-03-15 15:23:17 UTC (rev 19784)
@@ -669,7 +669,7 @@
 
   // load shared mem
   unsigned int tid = threadIdx.x;
-  unsigned int i = blockIdx.x*blockDim.x + threadIdx.x;
+  unsigned int i = threadIdx.x + blockIdx.x*blockDim.x + blockIdx.y*gridDim.x*blockDim.x;
 
   // loads absolute values into shared memory
   sdata[tid] = (i < size) ? fabs(array[i]) : 0.0 ;
@@ -747,11 +747,19 @@
 
   // outer core
   int size = mp->NGLOB_OUTER_CORE;
-  int num_blocks_x = (int) ceil(size/blocksize);
 
-  h_max = (realw*) calloc(num_blocks_x,sizeof(realw));
-  cudaMalloc((void**)&d_max,num_blocks_x*sizeof(realw));
-  dim3 grid(num_blocks_x,1);
+  int size_padded = ((int)ceil(((double)size)/((double)blocksize)))*blocksize;
+  int num_blocks_x = size_padded/blocksize;
+  int num_blocks_y = 1;
+  while(num_blocks_x > 65535) {
+    num_blocks_x = (int) ceil(num_blocks_x*0.5f);
+    num_blocks_y = num_blocks_y*2;
+  }
+
+  h_max = (realw*) calloc(num_blocks_x*num_blocks_y,sizeof(realw));
+  cudaMalloc((void**)&d_max,num_blocks_x*num_blocks_y*sizeof(realw));
+
+  dim3 grid(num_blocks_x,num_blocks_y);
   dim3 threads(blocksize,1,1);
 
   if(*SIMULATION_TYPE == 1 ){
@@ -760,11 +768,12 @@
     get_maximum_scalar_kernel<<<grid,threads>>>(mp->d_b_displ_outer_core,size,d_max);
   }
 
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),222);
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),222);
 
   // determines max for all blocks
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
 
@@ -830,7 +839,7 @@
 
   // load shared mem
   unsigned int tid = threadIdx.x;
-  unsigned int i = blockIdx.x*blockDim.x + threadIdx.x;
+  unsigned int i = threadIdx.x + blockIdx.x*blockDim.x + blockIdx.y*gridDim.x*blockDim.x;
 
   // loads values into shared memory: assume array is a vector array
   sdata[tid] = (i < size) ? sqrt(array[i*3]*array[i*3]
@@ -871,9 +880,10 @@
 
   realw max,max_crust_mantle,max_inner_core;
   realw *d_max;
-  int num_blocks_x,size;
+  int num_blocks_x,num_blocks_y;
+  int size,size_padded;
+  dim3 grid,threads;
 
-
   // launch simple reduction kernel
   realw* h_max;
   int blocksize = 256;
@@ -882,24 +892,33 @@
   max = 0;
   size = mp->NGLOB_CRUST_MANTLE;
 
-  num_blocks_x = (int) ceil(size/blocksize);
-  h_max = (realw*) calloc(num_blocks_x,sizeof(realw));
-  cudaMalloc((void**)&d_max,num_blocks_x*sizeof(realw));
+  size_padded = ((int)ceil(((double)size)/((double)blocksize)))*blocksize;
+  num_blocks_x = size_padded/blocksize;
+  num_blocks_y = 1;
+  while(num_blocks_x > 65535) {
+    num_blocks_x = (int) ceil(num_blocks_x*0.5f);
+    num_blocks_y = num_blocks_y*2;
+  }
 
-  dim3 grid1(num_blocks_x,1);
-  dim3 threads1(blocksize,1,1);
+  h_max = (realw*) calloc(num_blocks_x*num_blocks_y,sizeof(realw));
+  cudaMalloc((void**)&d_max,num_blocks_x*num_blocks_y*sizeof(realw));
+
+  grid = dim3(num_blocks_x,num_blocks_y);
+  threads = dim3(blocksize,1,1);
+
   if(*SIMULATION_TYPE == 1 ){
-    get_maximum_vector_kernel<<<grid1,threads1>>>(mp->d_displ_crust_mantle,size,d_max);
+    get_maximum_vector_kernel<<<grid,threads>>>(mp->d_displ_crust_mantle,size,d_max);
   }else if(*SIMULATION_TYPE == 3 ){
-    get_maximum_vector_kernel<<<grid1,threads1>>>(mp->d_b_displ_crust_mantle,size,d_max);
+    get_maximum_vector_kernel<<<grid,threads>>>(mp->d_b_displ_crust_mantle,size,d_max);
   }
 
   // copies to CPU
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),222);
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),222);
 
   // determines max for all blocks
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   max_crust_mantle = max;
@@ -911,24 +930,33 @@
   max = 0;
   size = mp->NGLOB_INNER_CORE;
 
-  num_blocks_x = (int) ceil(size/blocksize);
-  h_max = (realw*) calloc(num_blocks_x,sizeof(realw));
-  cudaMalloc((void**)&d_max,num_blocks_x*sizeof(realw));
+  size_padded = ((int)ceil(((double)size)/((double)blocksize)))*blocksize;
+  num_blocks_x = size_padded/blocksize;
+  num_blocks_y = 1;
+  while(num_blocks_x > 65535) {
+    num_blocks_x = (int) ceil(num_blocks_x*0.5f);
+    num_blocks_y = num_blocks_y*2;
+  }
 
-  dim3 grid2(num_blocks_x,1);
-  dim3 threads2(blocksize,1,1);
+  h_max = (realw*) calloc(num_blocks_x*num_blocks_y,sizeof(realw));
+  cudaMalloc((void**)&d_max,num_blocks_x*num_blocks_y*sizeof(realw));
+
+  grid = dim3(num_blocks_x,num_blocks_y);
+  threads = dim3(blocksize,1,1);
+
   if(*SIMULATION_TYPE == 1 ){
-    get_maximum_vector_kernel<<<grid2,threads2>>>(mp->d_displ_inner_core,size,d_max);
+    get_maximum_vector_kernel<<<grid,threads>>>(mp->d_displ_inner_core,size,d_max);
   }else if(*SIMULATION_TYPE == 3 ){
-    get_maximum_vector_kernel<<<grid2,threads2>>>(mp->d_b_displ_inner_core,size,d_max);
+    get_maximum_vector_kernel<<<grid,threads>>>(mp->d_b_displ_inner_core,size,d_max);
   }
 
   // copies to CPU
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),222);
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),222);
 
   // determines max for all blocks
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   max_inner_core = max;
@@ -951,8 +979,8 @@
 
 extern "C"
 void FC_FUNC_(check_norm_strain_from_device,
-              CHECK_NORM_STRAIN_FROM_DEVICE)(realw* norm_strain,
-                                             realw* norm_strain2,
+              CHECK_NORM_STRAIN_FROM_DEVICE)(realw* strain_norm,
+                                             realw* strain_norm2,
                                              long* Mesh_pointer_f) {
 
   TRACE("check_norm_strain_from_device");
@@ -962,83 +990,127 @@
 
   realw max,max_eps;
   realw *d_max;
-  int num_blocks_x,size;
+  int num_blocks_x,num_blocks_y;
+  int size,size_padded;
+  dim3 grid;
+  dim3 threads;
 
-
   // launch simple reduction kernel
   realw* h_max;
   int blocksize = 256;
 
   // crust_mantle strain arrays
-  size = NGLL3*(mp->NSPEC_CRUST_MANTLE);
+  size = NGLL3*(mp->NSPEC_CRUST_MANTLE_STRAIN_ONLY);
 
-  num_blocks_x = (int) ceil(size/blocksize);
-  h_max = (realw*) calloc(num_blocks_x,sizeof(realw));
-  cudaMalloc((void**)&d_max,num_blocks_x*sizeof(realw));
+  size_padded = ((int)ceil(((double)size)/((double)blocksize)))*blocksize;
+  num_blocks_x = size_padded/blocksize;
+  num_blocks_y = 1;
+  while(num_blocks_x > 65535) {
+    num_blocks_x = (int) ceil(num_blocks_x*0.5f);
+    num_blocks_y = num_blocks_y*2;
+  }
 
-  dim3 grid(num_blocks_x,1);
-  dim3 threads(blocksize,1,1);
+  h_max = (realw*) calloc(num_blocks_x*num_blocks_y,sizeof(realw));
+  cudaMalloc((void**)&d_max,num_blocks_x*num_blocks_y*sizeof(realw));
 
+  grid = dim3(num_blocks_x,num_blocks_y);
+  threads = dim3(blocksize,1,1);
+
+  max = 0.0f;
+
   // determines max for: eps_trace_over_3_crust_mantle
   get_maximum_scalar_kernel<<<grid,threads>>>(mp->d_eps_trace_over_3_crust_mantle,size,d_max);
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),221);
+
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),221);
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   // strain trace maximum
-  *norm_strain = max;
+  *strain_norm = max;
 
+  // frees arrays
+  cudaFree(d_max);
+  free(h_max);
+
   // initializes
+  // crust_mantle arrays
+  size = NGLL3*(mp->NSPEC_CRUST_MANTLE);
+
+  size_padded = ((int)ceil(((double)size)/((double)blocksize)))*blocksize;
+  num_blocks_x = size_padded/blocksize;
+  num_blocks_y = 1;
+  while(num_blocks_x > 65535) {
+    num_blocks_x = (int) ceil(num_blocks_x*0.5f);
+    num_blocks_y = num_blocks_y*2;
+  }
+
+  h_max = (realw*) calloc(num_blocks_x*num_blocks_y,sizeof(realw));
+  cudaMalloc((void**)&d_max,num_blocks_x*num_blocks_y*sizeof(realw));
+
+  grid = dim3(num_blocks_x,num_blocks_y);
+  threads = dim3(blocksize,1,1);
+
   max_eps = 0.0f;
 
   // determines max for: epsilondev_xx_crust_mantle
   get_maximum_scalar_kernel<<<grid,threads>>>(mp->d_epsilondev_xx_crust_mantle,size,d_max);
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),222);
+
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),222);
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   max_eps = MAX(max_eps,max);
 
   // determines max for: epsilondev_yy_crust_mantle
   get_maximum_scalar_kernel<<<grid,threads>>>(mp->d_epsilondev_yy_crust_mantle,size,d_max);
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),223);
+
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),223);
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   max_eps = MAX(max_eps,max);
 
   // determines max for: epsilondev_xy_crust_mantle
   get_maximum_scalar_kernel<<<grid,threads>>>(mp->d_epsilondev_xy_crust_mantle,size,d_max);
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),224);
+
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),224);
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   max_eps = MAX(max_eps,max);
 
   // determines max for: epsilondev_xz_crust_mantle
   get_maximum_scalar_kernel<<<grid,threads>>>(mp->d_epsilondev_xz_crust_mantle,size,d_max);
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),225);
+
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),225);
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   max_eps = MAX(max_eps,max);
 
   // determines max for: epsilondev_yz_crust_mantle
   get_maximum_scalar_kernel<<<grid,threads>>>(mp->d_epsilondev_yz_crust_mantle,size,d_max);
-  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*sizeof(realw),cudaMemcpyDeviceToHost),226);
+
+  print_CUDA_error_if_any(cudaMemcpy(h_max,d_max,num_blocks_x*num_blocks_y*sizeof(realw),
+                                     cudaMemcpyDeviceToHost),226);
   max = h_max[0];
-  for(int i=1;i<num_blocks_x;i++) {
+  for(int i=1;i<num_blocks_x*num_blocks_y;i++) {
     if( max < h_max[i]) max = h_max[i];
   }
   max_eps = MAX(max_eps,max);
 
   // strain maximum
-  *norm_strain2 = max_eps;
+  *strain_norm2 = max_eps;
 
   // frees arrays
   cudaFree(d_max);

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu	2012-03-15 15:23:17 UTC (rev 19784)
@@ -44,18 +44,18 @@
 /* ----------------------------------------------------------------------------------------------- */
 
 __global__ void compute_coupling_fluid_CMB_kernel(realw* displ_crust_mantle,
-						  realw* accel_outer_core,
-						  int* ibool_crust_mantle,
-						  int* ibelm_bottom_crust_mantle,
-						  realw* normal_top_outer_core,
-						  realw* jacobian2D_top_outer_core,
-						  realw* wgllwgll_xy,
-						  int* ibool_outer_core,
-						  int* ibelm_top_outer_core,
-						  int NSPEC2D_TOP_OC) {
+              realw* accel_outer_core,
+              int* ibool_crust_mantle,
+              int* ibelm_bottom_crust_mantle,
+              realw* normal_top_outer_core,
+              realw* jacobian2D_top_outer_core,
+              realw* wgllwgll_xy,
+              int* ibool_outer_core,
+              int* ibelm_top_outer_core,
+              int NSPEC2D_TOP_OC) {
 
   int i = threadIdx.x;
-  int j = threadIdx.y;    
+  int j = threadIdx.y;
   int iface = blockIdx.x + gridDim.x*blockIdx.y;
 
   int k,k_corresp,iglob_cm,iglob_oc,ispec,ispec_selected;
@@ -74,8 +74,8 @@
 
     // get displacement on the solid side using pointwise matching
     ispec_selected = ibelm_bottom_crust_mantle[iface] - 1;
-	
-    // get global point number 
+
+    // get global point number
     // corresponding points are located at the bottom of the mantle
     k_corresp = 0;
     iglob_cm = ibool_crust_mantle[INDEX4(NGLLX,NGLLX,NGLLX,i,j,k_corresp,ispec_selected)] - 1;
@@ -102,24 +102,24 @@
 
     // update fluid acceleration/pressure
     atomicAdd(&accel_outer_core[iglob_oc],+weight*displ_n);
-    
+
   // }
   }
 }
 
 __global__ void compute_coupling_fluid_ICB_kernel(realw* displ_inner_core,
-						  realw* accel_outer_core,
-						  int* ibool_inner_core,
-						  int* ibelm_top_inner_core,
-						  realw* normal_bottom_outer_core,
-						  realw* jacobian2D_bottom_outer_core,
-						  realw* wgllwgll_xy,
-						  int* ibool_outer_core,
-						  int* ibelm_bottom_outer_core,
-						  int NSPEC2D_BOTTOM_OC) {
+              realw* accel_outer_core,
+              int* ibool_inner_core,
+              int* ibelm_top_inner_core,
+              realw* normal_bottom_outer_core,
+              realw* jacobian2D_bottom_outer_core,
+              realw* wgllwgll_xy,
+              int* ibool_outer_core,
+              int* ibelm_bottom_outer_core,
+              int NSPEC2D_BOTTOM_OC) {
 
   int i = threadIdx.x;
-  int j = threadIdx.y;    
+  int j = threadIdx.y;
   int iface = blockIdx.x + gridDim.x*blockIdx.y;
 
   int k,k_corresp,iglob_ic,iglob_oc,ispec,ispec_selected;
@@ -138,8 +138,8 @@
 
     // get displacement on the solid side using pointwise matching
     ispec_selected = ibelm_top_inner_core[iface] - 1;
-	
-    // get global point number 			       
+
+    // get global point number
     // corresponding points are located at the bottom of the mantle
     k_corresp = NGLLX - 1;
     iglob_ic = ibool_inner_core[INDEX4(NGLLX,NGLLX,NGLLX,i,j,k_corresp,ispec_selected)] - 1;
@@ -194,28 +194,28 @@
 
   // launches GPU kernel
   compute_coupling_fluid_CMB_kernel<<<grid,threads>>>(mp->d_displ_crust_mantle,
-						      mp->d_accel_outer_core,
-						      mp->d_ibool_crust_mantle,
-						      mp->d_ibelm_bottom_crust_mantle,
-						      mp->d_normal_top_outer_core,
-						      mp->d_jacobian2D_top_outer_core,
-						      mp->d_wgllwgll_xy,
-						      mp->d_ibool_outer_core,
-						      mp->d_ibelm_top_outer_core,
-						      mp->nspec2D_top_outer_core);
+                  mp->d_accel_outer_core,
+                  mp->d_ibool_crust_mantle,
+                  mp->d_ibelm_bottom_crust_mantle,
+                  mp->d_normal_top_outer_core,
+                  mp->d_jacobian2D_top_outer_core,
+                  mp->d_wgllwgll_xy,
+                  mp->d_ibool_outer_core,
+                  mp->d_ibelm_top_outer_core,
+                  mp->nspec2D_top_outer_core);
 
   // adjoint simulations
   if ( mp->simulation_type == 3 ){
     compute_coupling_fluid_CMB_kernel<<<grid,threads>>>(mp->d_b_displ_crust_mantle,
-							mp->d_b_accel_outer_core,
-							mp->d_ibool_crust_mantle,
-							mp->d_ibelm_bottom_crust_mantle,
-							mp->d_normal_top_outer_core,
-							mp->d_jacobian2D_top_outer_core,
-							mp->d_wgllwgll_xy,
-							mp->d_ibool_outer_core,
-							mp->d_ibelm_top_outer_core,
-							mp->nspec2D_top_outer_core);
+              mp->d_b_accel_outer_core,
+              mp->d_ibool_crust_mantle,
+              mp->d_ibelm_bottom_crust_mantle,
+              mp->d_normal_top_outer_core,
+              mp->d_jacobian2D_top_outer_core,
+              mp->d_wgllwgll_xy,
+              mp->d_ibool_outer_core,
+              mp->d_ibelm_top_outer_core,
+              mp->nspec2D_top_outer_core);
   }
 
 #ifdef ENABLE_VERY_SLOW_ERROR_CHECKING
@@ -248,28 +248,28 @@
 
   // launches GPU kernel
   compute_coupling_fluid_ICB_kernel<<<grid,threads>>>(mp->d_displ_inner_core,
-						      mp->d_accel_outer_core,
-						      mp->d_ibool_inner_core,
-						      mp->d_ibelm_top_inner_core,
-						      mp->d_normal_bottom_outer_core,
-						      mp->d_jacobian2D_bottom_outer_core,
-						      mp->d_wgllwgll_xy,
-						      mp->d_ibool_outer_core,
-						      mp->d_ibelm_bottom_outer_core,
-						      mp->nspec2D_bottom_outer_core);
+                  mp->d_accel_outer_core,
+                  mp->d_ibool_inner_core,
+                  mp->d_ibelm_top_inner_core,
+                  mp->d_normal_bottom_outer_core,
+                  mp->d_jacobian2D_bottom_outer_core,
+                  mp->d_wgllwgll_xy,
+                  mp->d_ibool_outer_core,
+                  mp->d_ibelm_bottom_outer_core,
+                  mp->nspec2D_bottom_outer_core);
 
   // adjoint simulations
   if ( mp->simulation_type == 3 ){
     compute_coupling_fluid_ICB_kernel<<<grid,threads>>>(mp->d_b_displ_inner_core,
-							mp->d_b_accel_outer_core,
-							mp->d_ibool_inner_core,
-							mp->d_ibelm_top_inner_core,
-							mp->d_normal_bottom_outer_core,
-							mp->d_jacobian2D_bottom_outer_core,
-							mp->d_wgllwgll_xy,
-							mp->d_ibool_outer_core,
-							mp->d_ibelm_bottom_outer_core,
-							mp->nspec2D_bottom_outer_core);
+              mp->d_b_accel_outer_core,
+              mp->d_ibool_inner_core,
+              mp->d_ibelm_top_inner_core,
+              mp->d_normal_bottom_outer_core,
+              mp->d_jacobian2D_bottom_outer_core,
+              mp->d_wgllwgll_xy,
+              mp->d_ibool_outer_core,
+              mp->d_ibelm_bottom_outer_core,
+              mp->nspec2D_bottom_outer_core);
   }
 #ifdef ENABLE_VERY_SLOW_ERROR_CHECKING
   //double end_time = get_time();
@@ -285,22 +285,22 @@
 /* ----------------------------------------------------------------------------------------------- */
 
 __global__ void compute_coupling_CMB_fluid_kernel(realw* displ_crust_mantle,
-						  realw* accel_crust_mantle,
-						  realw* accel_outer_core,
-						  int* ibool_crust_mantle,
-						  int* ibelm_bottom_crust_mantle,
-						  realw* normal_top_outer_core,
-						  realw* jacobian2D_top_outer_core,
-						  realw* wgllwgll_xy,
-						  int* ibool_outer_core,
-						  int* ibelm_top_outer_core,
-						  double RHO_TOP_OC,
-						  realw minus_g_cmb,
-						  int GRAVITY_VAL,
-						  int NSPEC_BOTTOM_CM) {
+              realw* accel_crust_mantle,
+              realw* accel_outer_core,
+              int* ibool_crust_mantle,
+              int* ibelm_bottom_crust_mantle,
+              realw* normal_top_outer_core,
+              realw* jacobian2D_top_outer_core,
+              realw* wgllwgll_xy,
+              int* ibool_outer_core,
+              int* ibelm_top_outer_core,
+              double RHO_TOP_OC,
+              realw minus_g_cmb,
+              int GRAVITY_VAL,
+              int NSPEC_BOTTOM_CM) {
 
   int i = threadIdx.x;
-  int j = threadIdx.y;    
+  int j = threadIdx.y;
   int iface = blockIdx.x + gridDim.x*blockIdx.y;
 
   int k,k_corresp,iglob_cm,iglob_oc,ispec,ispec_selected;
@@ -325,7 +325,7 @@
     ny = normal_top_outer_core[INDEX4(NDIM,NGLLX,NGLLX,1,i,j,iface)]; // (2,i,j,iface)
     nz = normal_top_outer_core[INDEX4(NDIM,NGLLX,NGLLX,2,i,j,iface)]; // (3,i,j,iface)
 
-    // get global point number 
+    // get global point number
     // corresponding points are located at the top of the outer core
     k_corresp = NGLLX - 1;
     iglob_oc = ibool_outer_core[INDEX4(NGLLX,NGLLX,NGLLX,i,j,k_corresp,ispec_selected)] - 1;
@@ -333,14 +333,14 @@
 
     // compute pressure, taking gravity into account
     if( GRAVITY_VAL ){
-      pressure = RHO_TOP_OC * ( - accel_outer_core[iglob_oc] + minus_g_cmb 
-				 * (displ_crust_mantle[iglob_cm*3]*nx 
-				    + displ_crust_mantle[iglob_cm*3+1]*ny
-				    + displ_crust_mantle[iglob_cm*3+2]*nz) );
+      pressure = RHO_TOP_OC * ( - accel_outer_core[iglob_oc] + minus_g_cmb
+         * (displ_crust_mantle[iglob_cm*3]*nx
+            + displ_crust_mantle[iglob_cm*3+1]*ny
+            + displ_crust_mantle[iglob_cm*3+2]*nz) );
     }else{
       pressure = - RHO_TOP_OC * accel_outer_core[iglob_oc];
     }
-      
+
     // formulation with generalized potential: gets associated, weighted jacobian
     weight = jacobian2D_top_outer_core[INDEX3(NGLLX,NGLLX,i,j,iface)]*wgllwgll_xy[INDEX2(NGLLX,i,j)];
 
@@ -354,22 +354,22 @@
 }
 
 __global__ void compute_coupling_ICB_fluid_kernel(realw* displ_inner_core,
-						  realw* accel_inner_core,
-						  realw* accel_outer_core,
-						  int* ibool_inner_core,
-						  int* ibelm_top_inner_core,
-						  realw* normal_bottom_outer_core,
-						  realw* jacobian2D_bottom_outer_core,
-						  realw* wgllwgll_xy,
-						  int* ibool_outer_core,
-						  int* ibelm_bottom_outer_core,
-						  double RHO_BOTTOM_OC,
-						  realw minus_g_icb,
-						  int GRAVITY_VAL,
-						  int NSPEC_TOP_IC) {
+              realw* accel_inner_core,
+              realw* accel_outer_core,
+              int* ibool_inner_core,
+              int* ibelm_top_inner_core,
+              realw* normal_bottom_outer_core,
+              realw* jacobian2D_bottom_outer_core,
+              realw* wgllwgll_xy,
+              int* ibool_outer_core,
+              int* ibelm_bottom_outer_core,
+              double RHO_BOTTOM_OC,
+              realw minus_g_icb,
+              int GRAVITY_VAL,
+              int NSPEC_TOP_IC) {
 
   int i = threadIdx.x;
-  int j = threadIdx.y;    
+  int j = threadIdx.y;
   int iface = blockIdx.x + gridDim.x*blockIdx.y;
 
   int k,k_corresp,iglob_ic,iglob_oc,ispec,ispec_selected;
@@ -394,7 +394,7 @@
     ny = normal_bottom_outer_core[INDEX4(NDIM,NGLLX,NGLLX,1,i,j,iface)]; // (2,i,j,iface)
     nz = normal_bottom_outer_core[INDEX4(NDIM,NGLLX,NGLLX,2,i,j,iface)]; // (3,i,j,iface)
 
-    // get global point number 
+    // get global point number
     // corresponding points are located at the bottom of the outer core
     k_corresp = 0;
     iglob_oc = ibool_outer_core[INDEX4(NGLLX,NGLLX,NGLLX,i,j,k_corresp,ispec_selected)] - 1;
@@ -403,13 +403,13 @@
     // compute pressure, taking gravity into account
     if( GRAVITY_VAL ){
       pressure = RHO_BOTTOM_OC * ( - accel_outer_core[iglob_oc] + minus_g_icb
-				 * (displ_inner_core[iglob_ic*3]*nx 
-				    + displ_inner_core[iglob_ic*3+1]*ny
-				    + displ_inner_core[iglob_ic*3+2]*nz) );
+         * (displ_inner_core[iglob_ic*3]*nx
+            + displ_inner_core[iglob_ic*3+1]*ny
+            + displ_inner_core[iglob_ic*3+2]*nz) );
     }else{
       pressure = - RHO_BOTTOM_OC * accel_outer_core[iglob_oc];
     }
-      
+
     // formulation with generalized potential: gets associated, weighted jacobian
     weight = jacobian2D_bottom_outer_core[INDEX3(NGLLX,NGLLX,i,j,iface)]*wgllwgll_xy[INDEX2(NGLLX,i,j)];
 
@@ -427,9 +427,9 @@
 extern "C"
 void FC_FUNC_(compute_coupling_cmb_fluid_cuda,
               COMPUTE_COUPLING_CMB_FLUID_CUDA)(long* Mesh_pointer_f,
-					       double RHO_TOP_OC,
-					       realw minus_g_cmb,
-					       int GRAVITY_VAL) {
+                 double RHO_TOP_OC,
+                 realw minus_g_cmb,
+                 int GRAVITY_VAL) {
 
   TRACE("compute_coupling_cmb_fluid_cuda");
   //double start_time = get_time();
@@ -448,36 +448,36 @@
 
   // launches GPU kernel
   compute_coupling_CMB_fluid_kernel<<<grid,threads>>>(mp->d_displ_crust_mantle,
-						      mp->d_accel_crust_mantle,
-						      mp->d_accel_outer_core,
-						      mp->d_ibool_crust_mantle,
-						      mp->d_ibelm_bottom_crust_mantle,
-						      mp->d_normal_top_outer_core,
-						      mp->d_jacobian2D_top_outer_core,
-						      mp->d_wgllwgll_xy,
-						      mp->d_ibool_outer_core,
-						      mp->d_ibelm_top_outer_core,
-						      RHO_TOP_OC,
-						      minus_g_cmb,
-						      GRAVITY_VAL,
-						      mp->nspec2D_bottom_crust_mantle);
+                  mp->d_accel_crust_mantle,
+                  mp->d_accel_outer_core,
+                  mp->d_ibool_crust_mantle,
+                  mp->d_ibelm_bottom_crust_mantle,
+                  mp->d_normal_top_outer_core,
+                  mp->d_jacobian2D_top_outer_core,
+                  mp->d_wgllwgll_xy,
+                  mp->d_ibool_outer_core,
+                  mp->d_ibelm_top_outer_core,
+                  RHO_TOP_OC,
+                  minus_g_cmb,
+                  GRAVITY_VAL,
+                  mp->nspec2D_bottom_crust_mantle);
 
   //  adjoint simulations
   if ( mp->simulation_type == 3 ){
     compute_coupling_CMB_fluid_kernel<<<grid,threads>>>(mp->d_b_displ_crust_mantle,
-							mp->d_b_accel_crust_mantle,
-							mp->d_b_accel_outer_core,
-							mp->d_ibool_crust_mantle,
-							mp->d_ibelm_bottom_crust_mantle,
-							mp->d_normal_top_outer_core,
-							mp->d_jacobian2D_top_outer_core,
-							mp->d_wgllwgll_xy,
-							mp->d_ibool_outer_core,
-							mp->d_ibelm_top_outer_core,
-							RHO_TOP_OC,
-							minus_g_cmb,
-							GRAVITY_VAL,
-							mp->nspec2D_bottom_crust_mantle);
+              mp->d_b_accel_crust_mantle,
+              mp->d_b_accel_outer_core,
+              mp->d_ibool_crust_mantle,
+              mp->d_ibelm_bottom_crust_mantle,
+              mp->d_normal_top_outer_core,
+              mp->d_jacobian2D_top_outer_core,
+              mp->d_wgllwgll_xy,
+              mp->d_ibool_outer_core,
+              mp->d_ibelm_top_outer_core,
+              RHO_TOP_OC,
+              minus_g_cmb,
+              GRAVITY_VAL,
+              mp->nspec2D_bottom_crust_mantle);
   }
 
 #ifdef ENABLE_VERY_SLOW_ERROR_CHECKING
@@ -492,9 +492,9 @@
 extern "C"
 void FC_FUNC_(compute_coupling_icb_fluid_cuda,
               COMPUTE_COUPLING_ICB_FLUID_CUDA)(long* Mesh_pointer_f,
-					       double RHO_BOTTOM_OC,
-					       realw minus_g_icb,
-					       int GRAVITY_VAL) {
+                 double RHO_BOTTOM_OC,
+                 realw minus_g_icb,
+                 int GRAVITY_VAL) {
 
   TRACE("compute_coupling_icb_fluid_cuda");
   //double start_time = get_time();
@@ -513,36 +513,36 @@
 
   // launches GPU kernel
   compute_coupling_ICB_fluid_kernel<<<grid,threads>>>(mp->d_displ_inner_core,
-						      mp->d_accel_inner_core,
-						      mp->d_accel_outer_core,
-						      mp->d_ibool_inner_core,
-						      mp->d_ibelm_top_inner_core,
-						      mp->d_normal_bottom_outer_core,
-						      mp->d_jacobian2D_bottom_outer_core,
-						      mp->d_wgllwgll_xy,
-						      mp->d_ibool_outer_core,
-						      mp->d_ibelm_bottom_outer_core,
-						      RHO_BOTTOM_OC,
-						      minus_g_icb,
-						      GRAVITY_VAL,
-						      mp->nspec2D_top_inner_core);
+                  mp->d_accel_inner_core,
+                  mp->d_accel_outer_core,
+                  mp->d_ibool_inner_core,
+                  mp->d_ibelm_top_inner_core,
+                  mp->d_normal_bottom_outer_core,
+                  mp->d_jacobian2D_bottom_outer_core,
+                  mp->d_wgllwgll_xy,
+                  mp->d_ibool_outer_core,
+                  mp->d_ibelm_bottom_outer_core,
+                  RHO_BOTTOM_OC,
+                  minus_g_icb,
+                  GRAVITY_VAL,
+                  mp->nspec2D_top_inner_core);
 
   //  adjoint simulations
   if ( mp->simulation_type == 3 ){
     compute_coupling_ICB_fluid_kernel<<<grid,threads>>>(mp->d_b_displ_inner_core,
-							mp->d_b_accel_inner_core,
-							mp->d_b_accel_outer_core,
-							mp->d_ibool_inner_core,
-							mp->d_ibelm_top_inner_core,
-							mp->d_normal_bottom_outer_core,
-							mp->d_jacobian2D_bottom_outer_core,
-							mp->d_wgllwgll_xy,
-							mp->d_ibool_outer_core,
-							mp->d_ibelm_bottom_outer_core,
-							RHO_BOTTOM_OC,
-							minus_g_icb,
-							GRAVITY_VAL,
-							mp->nspec2D_top_inner_core);
+              mp->d_b_accel_inner_core,
+              mp->d_b_accel_outer_core,
+              mp->d_ibool_inner_core,
+              mp->d_ibelm_top_inner_core,
+              mp->d_normal_bottom_outer_core,
+              mp->d_jacobian2D_bottom_outer_core,
+              mp->d_wgllwgll_xy,
+              mp->d_ibool_outer_core,
+              mp->d_ibelm_bottom_outer_core,
+              RHO_BOTTOM_OC,
+              minus_g_icb,
+              GRAVITY_VAL,
+              mp->nspec2D_top_inner_core);
   }
 
 #ifdef ENABLE_VERY_SLOW_ERROR_CHECKING
@@ -560,13 +560,13 @@
 
 
 __global__ void compute_coupling_ocean_cuda_kernel(realw* accel_crust_mantle,
-						   realw* rmass_crust_mantle,
-						   realw* rmass_ocean_load,
-						   realw* normal_top_crust_mantle,
-						   int* ibool_crust_mantle,
-						   int* ibelm_top_crust_mantle,
-						   int* updated_dof_ocean_load,
-						   int NSPEC2D_TOP_CM) {
+               realw* rmass_crust_mantle,
+               realw* rmass_ocean_load,
+               realw* normal_top_crust_mantle,
+               int* ibool_crust_mantle,
+               int* ibelm_top_crust_mantle,
+               int* updated_dof_ocean_load,
+               int NSPEC2D_TOP_CM) {
 
   int i = threadIdx.x;
   int j = threadIdx.y;
@@ -592,13 +592,13 @@
 
     // daniel: TODO - there might be better ways to implement a mutex like below,
     //            and find a workaround to not use the temporary update array.
-    //            As a reminder, atomicExch atomically sets the value of the memory 
+    //            As a reminder, atomicExch atomically sets the value of the memory
     //            location stored in address to val and returns the old value.
     //            0 indicates that we still have to do this point
 
     if( atomicExch(&updated_dof_ocean_load[iglob],1) == 0){
 
-      // get normal on the CMB 
+      // get normal on the CMB
       nx = normal_top_crust_mantle[INDEX4(NDIM,NGLLX,NGLLX,0,i,j,iface)]; // (1,i,j,iface)
       ny = normal_top_crust_mantle[INDEX4(NDIM,NGLLX,NGLLX,1,i,j,iface)]; // (2,i,j,iface)
       nz = normal_top_crust_mantle[INDEX4(NDIM,NGLLX,NGLLX,2,i,j,iface)]; // (3,i,j,iface)
@@ -637,7 +637,7 @@
 
   dim3 grid(num_blocks_x,num_blocks_y);
   dim3 threads(5,5,1);
-               
+
   // initializes temporary array to zero
   print_CUDA_error_if_any(cudaMemset(mp->d_updated_dof_ocean_load,0,sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),88501);
 
@@ -648,11 +648,11 @@
   compute_coupling_ocean_cuda_kernel<<<grid,threads>>>(mp->d_accel_crust_mantle,
                                                        mp->d_rmass_crust_mantle,
                                                        mp->d_rmass_ocean_load,
-						       mp->d_normal_top_crust_mantle,
-						       mp->d_ibool_crust_mantle,
-						       mp->d_ibelm_top_crust_mantle,
-						       mp->d_updated_dof_ocean_load,
-						       mp->nspec2D_top_crust_mantle);
+                   mp->d_normal_top_crust_mantle,
+                   mp->d_ibool_crust_mantle,
+                   mp->d_ibelm_top_crust_mantle,
+                   mp->d_updated_dof_ocean_load,
+                   mp->nspec2D_top_crust_mantle);
 
   // for backward/reconstructed potentials
   if( mp->simulation_type == 3 ) {
@@ -661,13 +661,13 @@
                                        sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),88502);
 
     compute_coupling_ocean_cuda_kernel<<<grid,threads>>>(mp->d_b_accel_crust_mantle,
-							 mp->d_rmass_crust_mantle,
-							 mp->d_rmass_ocean_load,
-							 mp->d_normal_top_crust_mantle,
-							 mp->d_ibool_crust_mantle,
-							 mp->d_ibelm_top_crust_mantle,
-							 mp->d_updated_dof_ocean_load,
-							 mp->nspec2D_top_crust_mantle);
+               mp->d_rmass_crust_mantle,
+               mp->d_rmass_ocean_load,
+               mp->d_normal_top_crust_mantle,
+               mp->d_ibool_crust_mantle,
+               mp->d_ibelm_top_crust_mantle,
+               mp->d_updated_dof_ocean_load,
+               mp->nspec2D_top_crust_mantle);
   }
 
 #ifdef ENABLE_VERY_SLOW_ERROR_CHECKING

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu	2012-03-15 15:23:17 UTC (rev 19784)
@@ -928,7 +928,8 @@
                                           realw* d_minus_gravity_table,
                                           realw* d_minus_deriv_gravity_table,
                                           realw* d_density_table,
-                                          realw* wgll_cube){
+                                          realw* wgll_cube,
+                                          int NSPEC_CRUST_MANTLE_STRAIN_ONLY){
 
   /* int bx = blockIdx.y*blockDim.x+blockIdx.x; //possible bug in original code*/
   int bx = blockIdx.y*gridDim.x+blockIdx.x;
@@ -1165,7 +1166,9 @@
         epsilondev_xz_loc = 0.5f * duzdxl_plus_duxdzl;
         epsilondev_yz_loc = 0.5f * duzdyl_plus_duydzl;
 
-        if(SIMULATION_TYPE == 3) {
+        if(NSPEC_CRUST_MANTLE_STRAIN_ONLY == 1) {
+          epsilon_trace_over_3[tx] = templ;
+        }else{
           epsilon_trace_over_3[tx + working_element*NGLL3] = templ;
         }
       }
@@ -1563,7 +1566,8 @@
                                   mp->d_minus_gravity_table,
                                   mp->d_minus_deriv_gravity_table,
                                   mp->d_density_table,
-                                  mp->d_wgll_cube);
+                                  mp->d_wgll_cube,
+                                  mp->NSPEC_CRUST_MANTLE_STRAIN_ONLY);
 
 
   if(mp->simulation_type == 3) {
@@ -1606,7 +1610,8 @@
                                      mp->d_minus_gravity_table,
                                      mp->d_minus_deriv_gravity_table,
                                      mp->d_density_table,
-                                     mp->d_wgll_cube);
+                                     mp->d_wgll_cube,
+                                     mp->NSPEC_CRUST_MANTLE_STRAIN_ONLY);
   }
 
   // cudaEventRecord( stop, 0 );

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu	2012-03-15 15:23:17 UTC (rev 19784)
@@ -940,7 +940,7 @@
   // cudaEventRecord( start, 0 );
 
   Kernel_2_inner_core_impl<<<grid,threads>>>(nb_blocks_to_compute,
-                                  mp->NGLOB_AB,
+                                  mp->NGLOB_INNER_CORE,
                                   d_ibool,
                                   d_idoubling,
                                   mp->d_phase_ispec_inner_inner_core,
@@ -983,7 +983,7 @@
 
   if(mp->simulation_type == 3) {
     Kernel_2_inner_core_impl<<< grid,threads>>>(nb_blocks_to_compute,
-                                     mp->NGLOB_AB,
+                                     mp->NGLOB_INNER_CORE,
                                      d_ibool,
                                      d_idoubling,
                                      mp->d_phase_ispec_inner_inner_core,

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_outer_core_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_outer_core_cuda.cu	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_outer_core_cuda.cu	2012-03-15 15:23:17 UTC (rev 19784)
@@ -605,9 +605,9 @@
       nb_colors = mp->num_colors_outer_outer_core;
       istart = 0;
 
-      // array offsets (acoustic elements start after elastic ones)
-      color_offset = mp->nspec_elastic * NGLL3_PADDED;
-      color_offset_nonpadded = mp->nspec_elastic * NGLL3;
+      // array offsets
+      color_offset = 0;
+      color_offset_nonpadded = 0;
     }else{
       // inner element colors (start after outer elements)
       nb_colors = mp->num_colors_outer_outer_core + mp->num_colors_inner_outer_core;

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/mesh_constants_cuda.h
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/mesh_constants_cuda.h	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/mesh_constants_cuda.h	2012-03-15 15:23:17 UTC (rev 19784)
@@ -169,6 +169,7 @@
   // ------------------------------------------------------------------ //
   int NSPEC_CRUST_MANTLE;
   int NGLOB_CRUST_MANTLE;
+  int NSPEC_CRUST_MANTLE_STRAIN_ONLY;
 
   // interpolators
   realw* d_xix_crust_mantle; realw* d_xiy_crust_mantle; realw* d_xiz_crust_mantle;
@@ -607,205 +608,10 @@
   realw* d_normal_z_noise;
   realw* d_mask_noise;
   realw* d_jacobian2D_top_crust_mantle;
+
   // noise sensitivity kernel
   realw* d_Sigma_kl;
 
-
-// ------------------------------------------------------------------ //
-//daniel: TODO - former code...
-
-  // mesh resolution
-  int NSPEC_AB;
-  int NGLOB_AB;
-
-  // interpolators
-  realw* d_xix; realw* d_xiy; realw* d_xiz;
-  realw* d_etax; realw* d_etay; realw* d_etaz;
-  realw* d_gammax; realw* d_gammay; realw* d_gammaz;
-
-  // model parameters
-  realw* d_kappav; realw* d_muv;
-
-  // global indexing
-  int* d_ibool;
-
-  // inner / outer elements
-  int* d_ispec_is_inner;
-
-  // mpi buffers
-  int num_interfaces_ext_mesh;
-  int max_nibool_interfaces_ext_mesh;
-
-  // ------------------------------------------------------------------ //
-  // elastic wavefield parameters
-  // ------------------------------------------------------------------ //
-
-  // displacement, velocity, acceleration
-  realw* d_displ; realw* d_veloc; realw* d_accel;
-  // backward/reconstructed elastic wavefield
-  realw* d_b_displ; realw* d_b_veloc; realw* d_b_accel;
-
-  // elastic elements
-  int* d_ispec_is_elastic;
-
-  // elastic domain parameters
-  int* d_phase_ispec_inner_elastic;
-  int num_phase_ispec_elastic;
-
-  // ------------------------------------------------------------------ //
-  // mesh coloring
-  // ------------------------------------------------------------------ //
-  int* h_num_elem_colors_elastic;
-  int num_colors_outer_elastic,num_colors_inner_elastic;
-  int nspec_elastic;
-
-  realw* d_rmass;
-
-  // ------------------------------------------------------------------ //
-  // mpi buffer
-  // ------------------------------------------------------------------ //
-  realw* d_send_accel_buffer;
-
-  // interfaces
-  int* d_nibool_interfaces_ext_mesh;
-  int* d_ibool_interfaces_ext_mesh;
-
-  // ------------------------------------------------------------------ //
-  //used for absorbing stacey boundaries
-  // ------------------------------------------------------------------ //
-  int d_num_abs_boundary_faces;
-  int* d_abs_boundary_ispec;
-  int* d_abs_boundary_ijk;
-  realw* d_abs_boundary_normal;
-  realw* d_abs_boundary_jacobian2Dw;
-
-  realw* d_b_absorb_field;
-  int d_b_reclen_field;
-
-  realw* d_rho_vp;
-  realw* d_rho_vs;
-
-  // surface elements (to save for noise tomography and acoustic simulations)
-  int* d_free_surface_ispec;
-  int* d_free_surface_ijk;
-  int num_free_surface_faces;
-
-  // surface movie elements to save for noise tomography
-  //realw* d_noise_surface_movie;
-
-  // attenuation
-  realw* d_R_xx;
-  realw* d_R_yy;
-  realw* d_R_xy;
-  realw* d_R_xz;
-  realw* d_R_yz;
-
-  realw* d_one_minus_sum_beta;
-  realw* d_factor_common;
-
-  // attenuation & kernel
-  realw* d_epsilondev_xx;
-  realw* d_epsilondev_yy;
-  realw* d_epsilondev_xy;
-  realw* d_epsilondev_xz;
-  realw* d_epsilondev_yz;
-  realw* d_epsilon_trace_over_3;
-
-  // anisotropy
-  realw* d_c11store;
-  realw* d_c12store;
-  realw* d_c13store;
-  realw* d_c14store;
-  realw* d_c15store;
-  realw* d_c16store;
-  realw* d_c22store;
-  realw* d_c23store;
-  realw* d_c24store;
-  realw* d_c25store;
-  realw* d_c26store;
-  realw* d_c33store;
-  realw* d_c34store;
-  realw* d_c35store;
-  realw* d_c36store;
-  realw* d_c44store;
-  realw* d_c45store;
-  realw* d_c46store;
-  realw* d_c55store;
-  realw* d_c56store;
-  realw* d_c66store;
-
-  // noise
-  realw* d_free_surface_jacobian2Dw;
-
-  // attenuation & kernel backward fields
-  realw* d_b_R_xx;
-  realw* d_b_R_yy;
-  realw* d_b_R_xy;
-  realw* d_b_R_xz;
-  realw* d_b_R_yz;
-
-  realw* d_b_epsilondev_xx;
-  realw* d_b_epsilondev_yy;
-  realw* d_b_epsilondev_xy;
-  realw* d_b_epsilondev_xz;
-  realw* d_b_epsilondev_yz;
-  realw* d_b_epsilon_trace_over_3;
-
-  // sensitivity kernels
-  realw* d_rho_kl;
-  realw* d_mu_kl;
-  realw* d_kappa_kl;
-
-  // approximative hessian for preconditioning kernels
-  realw* d_hess_el_kl;
-
-  // ------------------------------------------------------------------ //
-  // acoustic wavefield
-  // ------------------------------------------------------------------ //
-  // potential and first and second time derivative
-  realw* d_potential_acoustic; realw* d_potential_dot_acoustic; realw* d_potential_dot_dot_acoustic;
-  // backward/reconstructed wavefield
-  realw* d_b_potential_acoustic; realw* d_b_potential_dot_acoustic; realw* d_b_potential_dot_dot_acoustic;
-
-  // acoustic domain parameters
-  int* d_ispec_is_acoustic;
-
-  int* d_phase_ispec_inner_acoustic;
-  int num_phase_ispec_acoustic;
-
-  // mesh coloring
-  int* h_num_elem_colors_acoustic;
-  int num_colors_outer_acoustic,num_colors_inner_acoustic;
-  int nspec_acoustic;
-
-  realw* d_rhostore;
-  realw* d_kappastore;
-  realw* d_rmass_acoustic;
-
-  // mpi buffer
-  realw* d_send_potential_dot_dot_buffer;
-
-  realw* d_b_absorb_potential;
-  int d_b_reclen_potential;
-
-  // for writing seismograms
-  realw* d_station_seismo_potential;
-  realw* h_station_seismo_potential;
-
-  // sensitivity kernels
-  realw* d_rho_ac_kl;
-  realw* d_kappa_ac_kl;
-
-  // approximative hessian for preconditioning kernels
-  realw* d_hess_ac_kl;
-
-  // coupling acoustic-elastic
-  int* d_coupling_ac_el_ispec;
-  int* d_coupling_ac_el_ijk;
-  realw* d_coupling_ac_el_normal;
-  realw* d_coupling_ac_el_jacobian2Dw;
-
-
 } Mesh;
 
 

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu	2012-03-15 15:23:17 UTC (rev 19784)
@@ -195,15 +195,16 @@
                                         int* h_number_receiver_global,
                                         int* h_islice_selected_rec,int* h_ispec_selected_rec,
                                         int* nrec,int* nrec_local, int* nadj_rec_local,
-                                        int* NSPEC_CRUST_MANTLE, int* NGLOB_CRUST_MANTLE, 
-					int* NGLOB_CRUST_MANTLE_OCEANS,
+                                        int* NSPEC_CRUST_MANTLE, int* NGLOB_CRUST_MANTLE,
+                                        int* NSPEC_CRUST_MANTLE_STRAIN_ONLY,
+                                        int* NGLOB_CRUST_MANTLE_OCEANS,
                                         int* NSPEC_OUTER_CORE, int* NGLOB_OUTER_CORE,
                                         int* NSPEC_INNER_CORE, int* NGLOB_INNER_CORE,
                                         int* SIMULATION_TYPE,
                                         int* NOISE_TOMOGRAPHY,
                                         int* SAVE_FORWARD_f,
                                         int* ABSORBING_CONDITIONS_f,
-					int* OCEANS_f,
+                                        int* OCEANS_f,
                                         int* GRAVITY_f,
                                         int* ROTATION_f,
                                         int* ATTENUATION_f,
@@ -242,6 +243,7 @@
   // sets global parameters
   mp->NSPEC_CRUST_MANTLE = *NSPEC_CRUST_MANTLE;
   mp->NGLOB_CRUST_MANTLE = *NGLOB_CRUST_MANTLE;
+  mp->NSPEC_CRUST_MANTLE_STRAIN_ONLY = *NSPEC_CRUST_MANTLE_STRAIN_ONLY;
   mp->NGLOB_CRUST_MANTLE_OCEANS = *NGLOB_CRUST_MANTLE_OCEANS;
   mp->NSPEC_OUTER_CORE = *NSPEC_OUTER_CORE;
   mp->NGLOB_OUTER_CORE = *NGLOB_OUTER_CORE;
@@ -677,7 +679,7 @@
                                             ) {
 
   TRACE("prepare_fields_strain_device");
-  int R_size;
+  int R_size,size_strain_only;
 
   Mesh* mp = (Mesh*)(*Mesh_pointer_f);
 
@@ -708,10 +710,13 @@
   print_CUDA_error_if_any(cudaMemcpy(mp->d_epsilondev_yz_crust_mantle,epsilondev_yz_crust_mantle,
                                      R_size*sizeof(realw),cudaMemcpyHostToDevice),4433);
 
+  // strain
+  size_strain_only = NGLL3*(mp->NSPEC_CRUST_MANTLE_STRAIN_ONLY);
   print_CUDA_error_if_any(cudaMalloc((void**) &mp->d_eps_trace_over_3_crust_mantle,
-                                       R_size*sizeof(realw)),4401);
+                                      size_strain_only*sizeof(realw)),4401);
   print_CUDA_error_if_any(cudaMemcpy(mp->d_eps_trace_over_3_crust_mantle,eps_trace_over_3_crust_mantle,
-                                       R_size*sizeof(realw),cudaMemcpyHostToDevice),4402);
+                                      size_strain_only*sizeof(realw),cudaMemcpyHostToDevice),4402);
+
   // backward/reconstructed fields
   if( mp->simulation_type == 3 ){
     print_CUDA_error_if_any(cudaMalloc((void**) &mp->d_b_epsilondev_xx_crust_mantle,
@@ -1339,34 +1344,34 @@
 extern "C"
 void FC_FUNC_(prepare_crust_mantle_device,
               PREPARE_CRUST_MANTLE_DEVICE)(long* Mesh_pointer_f,
-					   realw* h_xix, realw* h_xiy, realw* h_xiz,
-					   realw* h_etax, realw* h_etay, realw* h_etaz,
-					   realw* h_gammax, realw* h_gammay, realw* h_gammaz,
-					   realw* h_rho,
-					   realw* h_kappav, realw* h_muv,
-					   realw* h_kappah, realw* h_muh,
-					   realw* h_eta_aniso,
-					   realw* h_rmass,
-					   realw* h_normal_top_crust_mantle,
-					   int* h_ibelm_top_crust_mantle,
-					   int* h_ibelm_bottom_crust_mantle,
-					   int* h_ibool,
-					   realw* h_xstore, realw* h_ystore, realw* h_zstore,
-					   int* h_ispec_is_tiso,
-					   realw *c11store,realw *c12store,realw *c13store,
-					   realw *c14store,realw *c15store,realw *c16store,
-					   realw *c22store,realw *c23store,realw *c24store,
-					   realw *c25store,realw *c26store,realw *c33store,
-					   realw *c34store,realw *c35store,realw *c36store,
-					   realw *c44store,realw *c45store,realw *c46store,
-					   realw *c55store,realw *c56store,realw *c66store,
-					   int* num_phase_ispec,
-					   int* phase_ispec_inner,
-					   int* nspec_outer,
-					   int* nspec_inner,
-					   int* NSPEC2D_TOP_CM,
-					   int* NSPEC2D_BOTTOM_CM
-					   ) {
+             realw* h_xix, realw* h_xiy, realw* h_xiz,
+             realw* h_etax, realw* h_etay, realw* h_etaz,
+             realw* h_gammax, realw* h_gammay, realw* h_gammaz,
+             realw* h_rho,
+             realw* h_kappav, realw* h_muv,
+             realw* h_kappah, realw* h_muh,
+             realw* h_eta_aniso,
+             realw* h_rmass,
+             realw* h_normal_top_crust_mantle,
+             int* h_ibelm_top_crust_mantle,
+             int* h_ibelm_bottom_crust_mantle,
+             int* h_ibool,
+             realw* h_xstore, realw* h_ystore, realw* h_zstore,
+             int* h_ispec_is_tiso,
+             realw *c11store,realw *c12store,realw *c13store,
+             realw *c14store,realw *c15store,realw *c16store,
+             realw *c22store,realw *c23store,realw *c24store,
+             realw *c25store,realw *c26store,realw *c33store,
+             realw *c34store,realw *c35store,realw *c36store,
+             realw *c44store,realw *c45store,realw *c46store,
+             realw *c55store,realw *c56store,realw *c66store,
+             int* num_phase_ispec,
+             int* phase_ispec_inner,
+             int* nspec_outer,
+             int* nspec_inner,
+             int* NSPEC2D_TOP_CM,
+             int* NSPEC2D_BOTTOM_CM
+             ) {
 
   TRACE("prepare_crust_mantle_device");
 
@@ -1580,7 +1585,7 @@
   // CMB/ocean coupling
   mp->nspec2D_top_crust_mantle = *NSPEC2D_TOP_CM;
   mp->nspec2D_bottom_crust_mantle = *NSPEC2D_BOTTOM_CM;
-  int size_tcm = NGLL2*(mp->nspec2D_top_crust_mantle); 
+  int size_tcm = NGLL2*(mp->nspec2D_top_crust_mantle);
 
   print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_normal_top_crust_mantle),sizeof(realw)*NDIM*size_tcm),40020);
   print_CUDA_error_if_any(cudaMemcpy(mp->d_normal_top_crust_mantle,h_normal_top_crust_mantle,sizeof(realw)*NDIM*size_tcm,cudaMemcpyHostToDevice),40030);
@@ -1662,20 +1667,20 @@
                                          realw* h_gammax, realw* h_gammay, realw* h_gammaz,
                                          realw* h_rho, realw* h_kappav,
                                          realw* h_rmass,
-					 realw* h_normal_top_outer_core,
-					 realw* h_normal_bottom_outer_core,
-					 realw* h_jacobian2D_top_outer_core,
-					 realw* h_jacobian2D_bottom_outer_core,
-					 int* h_ibelm_top_outer_core,
-					 int* h_ibelm_bottom_outer_core,
+           realw* h_normal_top_outer_core,
+           realw* h_normal_bottom_outer_core,
+           realw* h_jacobian2D_top_outer_core,
+           realw* h_jacobian2D_bottom_outer_core,
+           int* h_ibelm_top_outer_core,
+           int* h_ibelm_bottom_outer_core,
                                          int* h_ibool,
                                          realw* h_xstore, realw* h_ystore, realw* h_zstore,
                                          int* num_phase_ispec,
                                          int* phase_ispec_inner,
                                          int* nspec_outer,
                                          int* nspec_inner,
-					 int* NSPEC2D_TOP_OC, 
-					 int* NSPEC2D_BOTTOM_OC
+           int* NSPEC2D_TOP_OC,
+           int* NSPEC2D_BOTTOM_OC
                                          ) {
 
   TRACE("prepare_outer_core_device");
@@ -1829,22 +1834,22 @@
 extern "C"
 void FC_FUNC_(prepare_inner_core_device,
               PREPARE_INNER_CORE_DEVICE)(long* Mesh_pointer_f,
-					 realw* h_xix, realw* h_xiy, realw* h_xiz,
-					 realw* h_etax, realw* h_etay, realw* h_etaz,
-					 realw* h_gammax, realw* h_gammay, realw* h_gammaz,
-					 realw* h_rho, realw* h_kappav, realw* h_muv,
-					 realw* h_rmass,
-					 int* h_ibelm_top_inner_core,
-					 int* h_ibool,
-					 realw* h_xstore, realw* h_ystore, realw* h_zstore,
-					 realw *c11store,realw *c12store,realw *c13store,
-					 realw *c33store,realw *c44store,
-					 int* h_idoubling_inner_core,
-					 int* num_phase_ispec,
-					 int* phase_ispec_inner,
-					 int* nspec_outer,
-					 int* nspec_inner,
-					 int* NSPEC2D_TOP_IC) {
+           realw* h_xix, realw* h_xiy, realw* h_xiz,
+           realw* h_etax, realw* h_etay, realw* h_etaz,
+           realw* h_gammax, realw* h_gammay, realw* h_gammaz,
+           realw* h_rho, realw* h_kappav, realw* h_muv,
+           realw* h_rmass,
+           int* h_ibelm_top_inner_core,
+           int* h_ibool,
+           realw* h_xstore, realw* h_ystore, realw* h_zstore,
+           realw *c11store,realw *c12store,realw *c13store,
+           realw *c33store,realw *c44store,
+           int* h_idoubling_inner_core,
+           int* num_phase_ispec,
+           int* phase_ispec_inner,
+           int* nspec_outer,
+           int* nspec_inner,
+           int* NSPEC2D_TOP_IC) {
 
   TRACE("prepare_inner_core_device");
 
@@ -1976,7 +1981,7 @@
 
   print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_ibelm_top_inner_core),sizeof(int)*(mp->nspec2D_top_inner_core)),40021);
   print_CUDA_error_if_any(cudaMemcpy(mp->d_ibelm_top_inner_core,h_ibelm_top_inner_core,sizeof(int)*(mp->nspec2D_top_inner_core),cudaMemcpyHostToDevice),40031);
- 
+
   // wavefield
   int size = NDIM * mp->NGLOB_INNER_CORE;
   print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_displ_inner_core),sizeof(realw)*size),4001);
@@ -2028,7 +2033,7 @@
 extern "C"
 void FC_FUNC_(prepare_oceans_device,
               PREPARE_OCEANS_DEVICE)(long* Mesh_pointer_f,
-				     realw* h_rmass_ocean_load) {
+             realw* h_rmass_ocean_load) {
 
   TRACE("prepare_oceans_device");
 
@@ -2036,15 +2041,15 @@
 
   // mass matrix
   print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_rmass_ocean_load),
-				     sizeof(realw)*mp->NGLOB_CRUST_MANTLE_OCEANS),4501);
+             sizeof(realw)*mp->NGLOB_CRUST_MANTLE_OCEANS),4501);
   print_CUDA_error_if_any(cudaMemcpy(mp->d_rmass_ocean_load,h_rmass_ocean_load,
-				     sizeof(realw)*mp->NGLOB_CRUST_MANTLE_OCEANS,cudaMemcpyHostToDevice),4502);
+             sizeof(realw)*mp->NGLOB_CRUST_MANTLE_OCEANS,cudaMemcpyHostToDevice),4502);
 
   // temporary global array: used to synchronize updates on global accel array
   print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_updated_dof_ocean_load),
-				     sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),4502);
-  
+             sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),4502);
 
+
 #ifdef ENABLE_VERY_SLOW_ERROR_CHECKING
   exit_on_cuda_error("prepare_oceans_device");
 #endif

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c	2012-03-15 15:23:17 UTC (rev 19784)
@@ -121,8 +121,8 @@
                                               int* SIMULATION_TYPE) {} 
 
 void FC_FUNC_(check_norm_strain_from_device,
-              CHECK_NORM_STRAIN_FROM_DEVICE)(realw* norm_strain,
-                                             realw* norm_strain2,
+              CHECK_NORM_STRAIN_FROM_DEVICE)(realw* strain_norm,
+                                             realw* strain_norm2,
                                              long* Mesh_pointer_f) {} 
 
 
@@ -352,14 +352,15 @@
                                         int* h_islice_selected_rec,int* h_ispec_selected_rec,
                                         int* nrec,int* nrec_local, int* nadj_rec_local,
                                         int* NSPEC_CRUST_MANTLE, int* NGLOB_CRUST_MANTLE,
-          int* NGLOB_CRUST_MANTLE_OCEANS,
+                                        int* NSPEC_CRUST_MANTLE_STRAIN_ONLY,
+                                        int* NGLOB_CRUST_MANTLE_OCEANS,
                                         int* NSPEC_OUTER_CORE, int* NGLOB_OUTER_CORE,
                                         int* NSPEC_INNER_CORE, int* NGLOB_INNER_CORE,
                                         int* SIMULATION_TYPE,
                                         int* NOISE_TOMOGRAPHY,
                                         int* SAVE_FORWARD_f,
                                         int* ABSORBING_CONDITIONS_f,
-          int* OCEANS_f,
+                                        int* OCEANS_f,
                                         int* GRAVITY_f,
                                         int* ROTATION_f,
                                         int* ATTENUATION_f,

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/transfer_fields_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/transfer_fields_cuda.cu	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/transfer_fields_cuda.cu	2012-03-15 15:23:17 UTC (rev 19784)
@@ -228,7 +228,7 @@
               TRANSFER_B_FIELDS_IC_FROM_DEVICE)(int* size, realw* b_displ, realw* b_veloc, realw* b_accel,
                                                 long* Mesh_pointer_f) {
   TRACE("transfer_fields_b_ic_from_device");
-  
+
   Mesh* mp = (Mesh*)(*Mesh_pointer_f); //get mesh pointer out of fortran integer container
 
   print_CUDA_error_if_any(cudaMemcpy(b_displ,mp->d_b_displ_inner_core,sizeof(realw)*(*size),cudaMemcpyDeviceToHost),40006);
@@ -244,9 +244,9 @@
 void FC_FUNC_(transfer_b_fields_oc_from_device,
               TRANSFER_B_FIELDS_OC_FROM_DEVICE)(int* size, realw* b_displ, realw* b_veloc, realw* b_accel,
                                                 long* Mesh_pointer_f) {
-  
+
   TRACE("transfer_b_fields_oc_from_device");
-  
+
   Mesh* mp = (Mesh*)(*Mesh_pointer_f); //get mesh pointer out of fortran integer container
 
   print_CUDA_error_if_any(cudaMemcpy(b_displ,mp->d_b_displ_outer_core,sizeof(realw)*(*size),cudaMemcpyDeviceToHost),40006);

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -45,9 +45,7 @@
 
   implicit none
 
-  !include "constants.h"
-
-! correct number of spectral elements in each block depending on chunk type
+  ! correct number of spectral elements in each block depending on chunk type
   integer ispec,nspec,nspec_stacey
 
   logical ABSORBING_CONDITIONS,ACTUALLY_STORE_ARRAYS

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -343,7 +343,7 @@
           ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) :: OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters deduced from parameters read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
@@ -506,7 +506,9 @@
           MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
           PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
           ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-          INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+          INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+          LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+          SIMULATION_TYPE,SAVE_FORWARD, &
           NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
           NSPEC,NSPEC2D_XI,NSPEC2D_ETA,NSPEC2DMAX_XMIN_XMAX, &
           NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
@@ -545,7 +547,7 @@
                 SAVE_ALL_SEISMOS_IN_ONE_FILE,MOVIE_COARSE,OUTPUT_SEISMOS_ASCII_TEXT, &
                 OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY, &
                 ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MASTER,USE_BINARY_FOR_LARGE_FILE, &
-                LOCAL_PATH,MODEL, &
+                LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
                 NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
                 NSPEC,NSPEC2D_XI,NSPEC2D_ETA, &
                 NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
@@ -596,7 +598,8 @@
   ! distributes 3D models
   call meshfem3D_models_broadcast(myrank,NSPEC, &
                                 MIN_ATTENUATION_PERIOD,MAX_ATTENUATION_PERIOD,&
-                                R80,R220,R670,RCMB,RICB)
+                                R80,R220,R670,RCMB,RICB, &
+                                LOCAL_PATH)
 
 
   if(myrank == 0 ) then

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -442,7 +442,8 @@
 
   subroutine meshfem3D_models_broadcast(myrank,NSPEC, &
                         MIN_ATTENUATION_PERIOD,MAX_ATTENUATION_PERIOD,&
-                        R80,R220,R670,RCMB,RICB)
+                        R80,R220,R670,RCMB,RICB, &
+                        LOCAL_PATH)
 
 ! preparing model parameter coefficients on all processes
 
@@ -460,6 +461,8 @@
 
   double precision R80,R220,R670,RCMB,RICB
 
+  character(len=150) :: LOCAL_PATH
+
 !---
 !
 ! ADD YOUR MODEL HERE
@@ -563,7 +566,7 @@
 
   ! read topography and bathymetry file
   if(TOPOGRAPHY .or. OCEANS) &
-    call model_topo_bathy_broadcast(myrank,ibathy_topo)
+    call model_topo_bathy_broadcast(myrank,ibathy_topo,LOCAL_PATH)
 
   ! re-defines/initializes models 1066a and ak135 and ref
   ! ( with possible external crustal model: if CRUSTAL is set to true

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_gapp2.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_gapp2.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_gapp2.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -36,7 +36,7 @@
 
   module gapp2_mantle_model_constants
     ! data file resolution
-    integer, parameter :: ma=228,mo=576,mr=32,mr1=64
+    integer, parameter :: ma=288,mo=576,mr=32,mr1=64
     integer no,na,nnr,nr1
     real dela,delo
     ! allocatable model arrays
@@ -95,7 +95,7 @@
 
   implicit none
   include "constants.h"
-  integer i,ir,ia,io
+  integer i,ir,ia,io,ier
   character(len=150) GAPP2
 
 !...........................................input data
@@ -104,21 +104,65 @@
   call get_value_string(GAPP2, 'model.GAPP2', 'DATA/3dvpGAP_P2')
 
   ! reads in GAP-P2 model from Obayashi
-  open(unit=10,file=GAPP2,status='old',action='read')
+  open(unit=10,file=GAPP2,status='old',action='read',iostat=ier)
+  if( ier /= 0 ) call exit_MPI(0,'error opening file for GAPP2 model')
 
-  read(10,'(3i4,2f10.6)') no,na,nnr,dela,delo
-  read(10,'(34f8.2)') (dep(i),i=0,nnr)
+  read(10,*) no,na,nnr,dela,delo
+
+  ! user output
+  write(IMAIN,*)
+  write(IMAIN,*) "model GAPP2: "
+  write(IMAIN,*) "  dimensions no = ",no
+  write(IMAIN,*) "             na,nnr = ",na,nnr
+  write(IMAIN,*) "             dela,delon = ",dela,delo
+
+  ! checks bounds
+  if( nnr /= mr .or. no /= mo .or. na /= ma ) then
+    print*,'error GAPP2 model bounds: '
+    print*,'  file dimensions: nnr,no,na = ',nnr,no,na
+    print*,'  module dimensions: mr,mo,ma = ',mr,mo,ma
+    close(10)
+    call exit_MPI(0,'please check GAPP2 model dimensions, and update model_gapp2.f90')
+  endif
+
+  read(10,*) (dep(i),i=0,nnr)
   read(10,*) nr1
-  read(10,'(67f8.2)') (dep1(i),i=0,nr1)
-  read(10,'(67f8.3)') (vp1(i),i=0,nr1)
+
+  ! checks bounds
+  write(IMAIN,*) "             nr1 = ",nr1
+  if( nr1 /= mr1 ) then
+    print*,'error GAPP2 model bounds: '
+    print*,'  file dimensions: nr1 = ',nr1
+    print*,'  module dimensions: mr1 = ',mr1
+    close(10)
+    call exit_MPI(0,'please check GAPP2 model dimensions, and update model_gapp2.f90')
+  endif
+
+  read(10,*) (dep1(i),i=0,nr1)
+  read(10,*) (vp1(i),i=0,nr1)
+
+  ! reads vp
   do ir=1,nnr
     do ia=1,na
-      read(10,'(256f7.3)') (vp3(ia,io,ir),io=1,no)
+      ! reads file 2 lines for all no values
+      read(10,'(288f7.3)') (vp3(ia,io,ir),io=1,no)
+
+      !read(10,*,iostat=ier) (vp3(ia,io,ir),io=1,no/2)
+      !read(10,*,iostat=ier) (vp3(ia,io,ir),io=no/2,no)
+
+      if( ier /= 0 ) then
+        print*,'error GAPP2 read: ia,ir = ',ia,ir
+        call exit_MPI(0,'error GAPP2 read')
+      endif
     enddo
   enddo
-  write(6,*) vp3(1,1,1),vp3(na,no,nnr)
   close(10)
 
+  ! user output
+  write(IMAIN,*) '  check vp3: ',vp3(1,1,1),vp3(na,no,nnr)
+  write(IMAIN,*) '  check vp3: min/max = ',minval(vp3),maxval(vp3)
+  write(IMAIN,*)
+
   end subroutine read_mantle_gapmodel
 
 !

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/broadcast_compute_parameters.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/broadcast_compute_parameters.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/broadcast_compute_parameters.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -44,7 +44,7 @@
                 SAVE_ALL_SEISMOS_IN_ONE_FILE,MOVIE_COARSE,OUTPUT_SEISMOS_ASCII_TEXT, &
                 OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY, &
                 ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MASTER,USE_BINARY_FOR_LARGE_FILE, &
-                LOCAL_PATH,MODEL, &
+                LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
                 NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
                 NSPEC,NSPEC2D_XI,NSPEC2D_ETA, &
                 NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
@@ -90,7 +90,7 @@
           OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY,&
           ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MASTER,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) LOCAL_PATH,MODEL
+  character(len=150) LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
   ! parameters to be computed based upon parameters above read from file
   integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA
@@ -174,6 +174,7 @@
 
   ! broadcasts non-single value parameters
   call MPI_BCAST(LOCAL_PATH,150,MPI_CHARACTER,0,MPI_COMM_WORLD,ier)
+  call MPI_BCAST(LOCAL_TMP_PATH,150,MPI_CHARACTER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(MODEL,150,MPI_CHARACTER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(ner,MAX_NUMBER_OF_MESH_LAYERS,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(ratio_sampling_array,MAX_NUMBER_OF_MESH_LAYERS,MPI_INTEGER,0,MPI_COMM_WORLD,ier)

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/model_topo_bathy.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/model_topo_bathy.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/model_topo_bathy.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -34,7 +34,7 @@
 !--------------------------------------------------------------------------------------------------
 
 
-  subroutine model_topo_bathy_broadcast(myrank,ibathy_topo)
+  subroutine model_topo_bathy_broadcast(myrank,ibathy_topo,LOCAL_PATH)
 
 ! standard routine to setup model
 
@@ -44,13 +44,20 @@
   ! standard include of the MPI library
   include 'mpif.h'
 
+  integer :: myrank
+
   ! bathymetry and topography: use integer array to store values
   integer, dimension(NX_BATHY,NY_BATHY) :: ibathy_topo
 
-  integer :: myrank
+  character(len=150) :: LOCAL_PATH
+
+  ! local parameters
   integer :: ier
 
-  if(myrank == 0) call read_topo_bathy_file(ibathy_topo)
+  if(myrank == 0) then
+    call read_topo_bathy_file(ibathy_topo)
+    call save_topo_bathy_database(ibathy_topo,LOCAL_PATH)
+  endif
 
   ! broadcast the information read on the master to the nodes
   call MPI_BCAST(ibathy_topo,NX_BATHY*NY_BATHY,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
@@ -69,12 +76,13 @@
 
   include "constants.h"
 
-  character(len=150) topo_bathy_file
-
 ! use integer array to store values
   integer, dimension(NX_BATHY,NY_BATHY) :: ibathy_topo
 
-  integer itopo_x,itopo_y,ier
+  ! local parameters
+  real :: val
+  integer :: itopo_x,itopo_y,ier
+  character(len=150) :: topo_bathy_file
 
   call get_value_string(topo_bathy_file, 'model.topoBathy.PATHNAME_TOPO_FILE', PATHNAME_TOPO_FILE)
 
@@ -84,15 +92,26 @@
     print*,'error opening:',trim(topo_bathy_file)
     call exit_mpi(0,'error opening topography data file')
   endif
+
   ! reads in topography array
   do itopo_y=1,NY_BATHY
     do itopo_x=1,NX_BATHY
-      read(13,*) ibathy_topo(itopo_x,itopo_y)
+      read(13,*,iostat=ier) val
+
+      ! checks
+      if( ier /= 0 ) then
+        print*,'error read topo_bathy: ix,iy = ',itopo_x,itopo_y,val
+        print*,'topo_bathy dimension: nx,ny = ',NX_BATHY,NY_BATHY
+        call exit_mpi(0,'error reading topo_bathy file')
+      endif
+
+      ! converts to integer
+      ibathy_topo(itopo_x,itopo_y) = val
+
     enddo
   enddo
   close(13)
 
-
   ! note: we check the limits after reading in the data. this seems to perform sligthly faster
   !          however, reading ETOPO1.xyz will take ~ 2m 1.2s for a single process
 
@@ -114,6 +133,9 @@
 
   endif
 
+  ! user output
+  write(IMAIN,*) "topography/bathymetry: min/max = ",minval(ibathy_topo),maxval(ibathy_topo)
+
   end subroutine read_topo_bathy_file
 
 
@@ -121,6 +143,83 @@
 !-------------------------------------------------------------------------------------------------
 !
 
+  subroutine save_topo_bathy_database(ibathy_topo,LOCAL_PATH)
+
+  implicit none
+
+  include "constants.h"
+
+  ! use integer array to store values
+  integer, dimension(NX_BATHY,NY_BATHY) :: ibathy_topo
+  character(len=150) :: LOCAL_PATH
+
+  ! local parameters
+  character(len=150) :: prname
+  integer :: ier
+
+  ! create the name for the database of the current slide and region
+  ! only master needs to save this
+  call create_name_database(prname,0,IREGION_CRUST_MANTLE,LOCAL_PATH)
+
+  ! saves topography and bathymetry file for solver
+
+  open(unit=27,file=prname(1:len_trim(prname))//'topo.bin', &
+        status='unknown',form='unformatted',action='write',iostat=ier)
+  if( ier /= 0 ) then
+    print*,'TOPOGRAPHY problem:'
+    print*,'error creating file: ',prname(1:len_trim(prname))//'topo.bin'
+    print*,'please check if path exists and rerun mesher'
+    call exit_mpi(0,'error opening file for database topo')
+  endif
+
+  write(27) ibathy_topo
+
+  close(27)
+
+  end subroutine save_topo_bathy_database
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+  subroutine read_topo_bathy_database(ibathy_topo,LOCAL_PATH)
+
+  implicit none
+
+  include "constants.h"
+
+  ! use integer array to store values
+  integer, dimension(NX_BATHY,NY_BATHY) :: ibathy_topo
+  character(len=150) :: LOCAL_PATH
+
+  ! local parameters
+  character(len=150) :: prname
+  integer :: ier
+
+  ! create the name for the database of the current slide and region
+  ! only master needs to save this
+  call create_name_database(prname,0,IREGION_CRUST_MANTLE,LOCAL_PATH)
+
+  ! reads topography and bathymetry file from saved database file
+  open(unit=27,file=prname(1:len_trim(prname))//'topo.bin', &
+        status='unknown',form='unformatted',action='read',iostat=ier)
+  if( ier /= 0 ) then
+    print*,'TOPOGRAPHY problem:'
+    print*,'error opening file: ',prname(1:len_trim(prname))//'topo.bin'
+    print*,'please check if file exists and rerun solver'
+    call exit_mpi(0,'error opening file for database topo')
+  endif
+
+  read(27) ibathy_topo
+
+  close(27)
+
+  end subroutine read_topo_bathy_database
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
   subroutine get_topo_bathy(xlat,xlon,value,ibathy_topo)
 
 !
@@ -189,7 +288,3 @@
   endif
 
   end subroutine get_topo_bathy
-
-! -------------------------------------------
-
-

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -44,7 +44,9 @@
                         MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
                         PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
                         ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-                        INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+                        INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+                        LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+                        SIMULATION_TYPE,SAVE_FORWARD, &
                         NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
                         NSPEC,NSPEC2D_XI,NSPEC2D_ETA,NSPEC2DMAX_XMIN_XMAX, &
                         NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
@@ -85,7 +87,7 @@
          ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MASTER,&
          SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! parameters to be computed based upon parameters above read from file
   integer NSTEP,MIN_ATTENUATION_PERIOD,MAX_ATTENUATION_PERIOD,NER_CRUST, &
@@ -149,7 +151,8 @@
 
 
   ! reads in Par_file values
-  call read_parameter_file(OUTPUT_FILES,LOCAL_PATH,MODEL, &
+  call read_parameter_file(OUTPUT_FILES, &
+                          LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
                           NTSTEP_BETWEEN_OUTPUT_SEISMOS,NTSTEP_BETWEEN_READ_ADJSRC,NTSTEP_BETWEEN_FRAMES, &
                           NTSTEP_BETWEEN_OUTPUT_INFO,NUMBER_OF_RUNS, &
                           NUMBER_OF_THIS_RUN,NCHUNKS,SIMULATION_TYPE, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_parameter_file.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_parameter_file.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_parameter_file.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -25,7 +25,8 @@
 !
 !=====================================================================
 
-  subroutine read_parameter_file(OUTPUT_FILES,LOCAL_PATH,MODEL, &
+  subroutine read_parameter_file(OUTPUT_FILES, &
+                                LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
                                 NTSTEP_BETWEEN_OUTPUT_SEISMOS,NTSTEP_BETWEEN_READ_ADJSRC,NTSTEP_BETWEEN_FRAMES, &
                                 NTSTEP_BETWEEN_OUTPUT_INFO,NUMBER_OF_RUNS, &
                                 NUMBER_OF_THIS_RUN,NCHUNKS,SIMULATION_TYPE, &
@@ -68,7 +69,7 @@
          ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MASTER,&
          SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
 
 ! local variables
   integer, external :: err_occurred
@@ -162,6 +163,8 @@
   if(err_occurred() /= 0) stop 'an error occurred while reading the parameter file: NUMBER_OF_THIS_RUN'
   call read_value_string(LOCAL_PATH, 'LOCAL_PATH')
   if(err_occurred() /= 0) stop 'an error occurred while reading the parameter file: LOCAL_PATH'
+  call read_value_string(LOCAL_TMP_PATH, 'LOCAL_TMP_PATH')
+  if(err_occurred() /= 0) stop 'an error occurred while reading the parameter file: LOCAL_TMP_PATH'
   call read_value_integer(NTSTEP_BETWEEN_OUTPUT_INFO, 'solver.NTSTEP_BETWEEN_OUTPUT_INFO')
   if(err_occurred() /= 0) stop 'an error occurred while reading the parameter file: NTSTEP_BETWEEN_OUTPUT_INFO'
   call read_value_integer(NTSTEP_BETWEEN_OUTPUT_SEISMOS, 'solver.NTSTEP_BETWEEN_OUTPUT_SEISMOS')

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -109,7 +109,7 @@
                                       b_div_displ_outer_core,phase_is_inner)
         else
           call compute_forces_outer_core(b_time,b_deltat,b_two_omega_earth, &
-                                      NSPEC_OUTER_CORE_ROT_ADJOINT,NGLOB_OUTER_CORE_ADJOINT, &  
+                                      NSPEC_OUTER_CORE_ROT_ADJOINT,NGLOB_OUTER_CORE_ADJOINT, &
                                       b_A_array_rotation,b_B_array_rotation, &
                                       b_displ_outer_core,b_accel_outer_core, &
                                       b_div_displ_outer_core,phase_is_inner)
@@ -134,7 +134,7 @@
        ! ****************************************************
        ! only for elements in first matching layer in the fluid
        if( .not. GPU_MODE ) then
-          ! on CPU    
+          ! on CPU
           !---
           !--- couple with mantle at the top of the outer core
           !---
@@ -299,7 +299,7 @@
   implicit none
 
   integer :: NGLOB
-  
+
   ! velocity potential
   real(kind=CUSTOM_REAL), dimension(NGLOB) :: veloc_outer_core,accel_outer_core
 

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -28,7 +28,7 @@
   subroutine compute_forces_crust_mantle(NSPEC,NGLOB,NSPEC_ATT, &
                                         displ_crust_mantle,accel_crust_mantle, &
                                         phase_is_inner, &
-                                        R_xx,R_yy,R_xy,R_xz,R_yz, &                                        
+                                        R_xx,R_yy,R_xy,R_xz,R_yz, &
                                         epsilondev_xx,epsilondev_yy,epsilondev_xy, &
                                         epsilondev_xz,epsilondev_yz, &
                                         epsilon_trace_over_3, &
@@ -96,7 +96,7 @@
   real(kind=CUSTOM_REAL), dimension(N_SLS,NGLLX,NGLLY,NGLLZ,NSPEC_ATT) :: &
     R_xx,R_yy,R_xy,R_xz,R_yz
 
-  
+
 !  real(kind=CUSTOM_REAL), dimension(5,NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: epsilondev
   real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: &
     epsilondev_xx,epsilondev_yy,epsilondev_xy,epsilondev_xz,epsilondev_yz

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -37,7 +37,7 @@
   subroutine compute_forces_crust_mantle_Dev( NSPEC,NGLOB,NSPEC_ATT, &
                                               displ_crust_mantle,accel_crust_mantle, &
                                               phase_is_inner, &
-                                              R_xx,R_yy,R_xy,R_xz,R_yz, &                                              
+                                              R_xx,R_yy,R_xy,R_xz,R_yz, &
                                               epsilondev_xx,epsilondev_yy,epsilondev_xy, &
                                               epsilondev_xz,epsilondev_yz, &
                                               epsilon_trace_over_3, &
@@ -94,7 +94,7 @@
   implicit none
 
   integer :: NSPEC,NGLOB,NSPEC_ATT
-  
+
   ! displacement and acceleration
   real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB) :: displ_crust_mantle,accel_crust_mantle
 
@@ -105,7 +105,7 @@
 !  real(kind=CUSTOM_REAL), dimension(5,N_SLS,NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ATTENUAT) :: R_memory
   real(kind=CUSTOM_REAL), dimension(N_SLS,NGLLX,NGLLY,NGLLZ,NSPEC_ATT) :: &
     R_xx,R_yy,R_xy,R_xz,R_yz
-  
+
 !  real(kind=CUSTOM_REAL), dimension(5,NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: epsilondev
   real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: &
     epsilondev_xx,epsilondev_yy,epsilondev_xy,epsilondev_xz,epsilondev_yz

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -77,11 +77,11 @@
         ! uses Deville (2002) optimizations
         ! crust/mantle region
         call compute_forces_crust_mantle_Dev( NSPEC_CRUST_MANTLE_STR_OR_ATT,NGLOB_CRUST_MANTLE, &
-                                      NSPEC_CRUST_MANTLE_ATTENUAT, &      
+                                      NSPEC_CRUST_MANTLE_ATTENUAT, &
                                       displ_crust_mantle,accel_crust_mantle, &
                                       phase_is_inner, &
                                       R_xx_crust_mantle,R_yy_crust_mantle,R_xy_crust_mantle, &
-                                      R_xz_crust_mantle,R_yz_crust_mantle, &                                      
+                                      R_xz_crust_mantle,R_yz_crust_mantle, &
                                       epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, &
                                       epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, &
                                       eps_trace_over_3_crust_mantle, &
@@ -106,7 +106,7 @@
         ! no Deville optimization
         ! crust/mantle region
         call compute_forces_crust_mantle(  NSPEC_CRUST_MANTLE_STR_OR_ATT,NGLOB_CRUST_MANTLE, &
-                                      NSPEC_CRUST_MANTLE_ATTENUAT, &              
+                                      NSPEC_CRUST_MANTLE_ATTENUAT, &
                                       displ_crust_mantle,accel_crust_mantle, &
                                       phase_is_inner, &
                                       R_xx_crust_mantle,R_yy_crust_mantle,R_xy_crust_mantle, &
@@ -261,7 +261,7 @@
        ! ****************************************************
        ! only for elements in first matching layer in the solid
        if( .not. GPU_MODE ) then
-          ! on CPU   
+          ! on CPU
           !---
           !--- couple with outer core at the bottom of the mantle
           !---
@@ -548,7 +548,7 @@
   implicit none
 
   integer :: NGLOB
-  
+
   ! velocity & acceleration
   ! crust/mantle region
   real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB) :: veloc_crust_mantle,accel_crust_mantle
@@ -628,7 +628,7 @@
   implicit none
 
   integer :: NGLOB_CM,NGLOB_IC
-  
+
   ! acceleration & velocity
   ! crust/mantle region
   real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CM) :: veloc_crust_mantle,accel_crust_mantle

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -28,7 +28,7 @@
   subroutine compute_forces_inner_core( NSPEC,NGLOB,NSPEC_ATT, &
                                         displ_inner_core,accel_inner_core, &
                                         phase_is_inner, &
-                                        R_xx,R_yy,R_xy,R_xz,R_yz, &                                        
+                                        R_xx,R_yy,R_xy,R_xz,R_yz, &
                                         epsilondev_xx,epsilondev_yy,epsilondev_xy, &
                                         epsilondev_xz,epsilondev_yz, &
                                         epsilon_trace_over_3,&
@@ -92,7 +92,7 @@
 !  real(kind=CUSTOM_REAL), dimension(5,N_SLS,NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE_ATTENUATION) :: R_memory
   real(kind=CUSTOM_REAL), dimension(N_SLS,NGLLX,NGLLY,NGLLZ,NSPEC_ATT) :: &
     R_xx,R_yy,R_xy,R_xz,R_yz
-  
+
 !  real(kind=CUSTOM_REAL), dimension(5,NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: epsilondev
   real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: &
     epsilondev_xx,epsilondev_yy,epsilondev_xy,epsilondev_xz,epsilondev_yz

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -37,7 +37,7 @@
   subroutine compute_forces_inner_core_Dev( NSPEC,NGLOB,NSPEC_ATT, &
                                             displ_inner_core,accel_inner_core, &
                                             phase_is_inner, &
-                                            R_xx,R_yy,R_xy,R_xz,R_yz, &                                            
+                                            R_xx,R_yy,R_xy,R_xz,R_yz, &
                                             epsilondev_xx,epsilondev_yy,epsilondev_xy, &
                                             epsilondev_xz,epsilondev_yz, &
                                             epsilon_trace_over_3,&
@@ -103,7 +103,7 @@
 !  real(kind=CUSTOM_REAL), dimension(5,N_SLS,NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE_ATTENUATION) :: R_memory
   real(kind=CUSTOM_REAL), dimension(N_SLS,NGLLX,NGLLY,NGLLZ,NSPEC_ATT) :: &
     R_xx,R_yy,R_xy,R_xz,R_yz
-  
+
 !  real(kind=CUSTOM_REAL), dimension(5,NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: epsilondev
   real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: &
     epsilondev_xx,epsilondev_yy,epsilondev_xy,epsilondev_xz,epsilondev_yz

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -103,72 +103,46 @@
     call close_file_abs(9)
   endif
 
-  ! synchronize all processes
-  call sync_all()
-
   ! save files to local disk or tape system if restart file
   call save_forward_arrays()
 
-  ! synchronize all processes
-  call sync_all()
-
   ! dump kernel arrays
   if (SIMULATION_TYPE == 3) then
     ! crust mantle
-    call save_kernels_crust_mantle(myrank,scale_t,scale_displ, &
-                  cijkl_kl_crust_mantle,rho_kl_crust_mantle, &
-                  alpha_kl_crust_mantle,beta_kl_crust_mantle, &
-                  ystore_crust_mantle,zstore_crust_mantle, &
-                  rhostore_crust_mantle,muvstore_crust_mantle, &
-                  kappavstore_crust_mantle,ibool_crust_mantle, &
-                  kappahstore_crust_mantle,muhstore_crust_mantle, &
-                  eta_anisostore_crust_mantle,ispec_is_tiso_crust_mantle, &
-              ! --idoubling_crust_mantle, &
-                  LOCAL_PATH)
+    call save_kernels_crust_mantle()
 
     ! noise strength kernel
     if (NOISE_TOMOGRAPHY == 3) then
-       call save_kernels_strength_noise(myrank,LOCAL_PATH,Sigma_kl_crust_mantle)
+       call save_kernels_strength_noise()
     endif
 
     ! outer core
-    call save_kernels_outer_core(myrank,scale_t,scale_displ, &
-                        rho_kl_outer_core,alpha_kl_outer_core, &
-                        rhostore_outer_core,kappavstore_outer_core, &
-                        deviatoric_outercore,nspec_beta_kl_outer_core,beta_kl_outer_core, &
-                        LOCAL_PATH)
+    call save_kernels_outer_core()
 
     ! inner core
-    call save_kernels_inner_core(myrank,scale_t,scale_displ, &
-                          rho_kl_inner_core,beta_kl_inner_core,alpha_kl_inner_core, &
-                          rhostore_inner_core,muvstore_inner_core,kappavstore_inner_core, &
-                          LOCAL_PATH)
+    call save_kernels_inner_core()
 
     ! boundary kernel
     if (SAVE_BOUNDARY_MESH) then
-      call save_kernels_boundary_kl(myrank,scale_t,scale_displ, &
-                                   moho_kl,d400_kl,d670_kl,cmb_kl,icb_kl, &
-                                   LOCAL_PATH,HONOR_1D_SPHERICAL_MOHO)
+      call save_kernels_boundary_kl()
     endif
 
     ! approximate hessian
     if( APPROXIMATE_HESS_KL ) then
-      call save_kernels_hessian(myrank,scale_t,scale_displ, &
-                               hess_kl_crust_mantle,LOCAL_PATH)
+      call save_kernels_hessian()
     endif
   endif
 
   ! save source derivatives for adjoint simulations
   if (SIMULATION_TYPE == 2 .and. nrec_local > 0) then
-    call save_kernels_source_derivatives(nrec_local,NSOURCES,scale_displ,scale_t, &
-                                nu_source,moment_der,sloc_der,stshift_der,shdur_der,number_receiver_global)
+    call save_kernels_source_derivatives()
   endif
 
   ! frees dynamically allocated memory
   ! mpi buffers
   deallocate(buffer_send_faces, &
-            buffer_received_faces, &
-            b_buffer_send_faces, &
+            buffer_received_faces)
+  deallocate(b_buffer_send_faces, &
             b_buffer_received_faces)
 
   ! central cube buffers
@@ -182,22 +156,11 @@
 
   ! sources
   deallocate(islice_selected_source, &
-          ispec_selected_source, &
-          Mxx, &
-          Myy, &
-          Mzz, &
-          Mxy, &
-          Mxz, &
-          Myz, &
-          xi_source, &
-          eta_source, &
-          gamma_source, &
-          tshift_cmt, &
-          hdur, &
-          hdur_gaussian, &
-          theta_source, &
-          phi_source, &
-          nu_source)
+            ispec_selected_source, &
+            Mxx,Myy,Mzz,Mxy,Mxz,Myz)
+  deallocate(tshift_cmt,hdur,hdur_gaussian)
+  deallocate(nu_source)
+
   if (SIMULATION_TYPE == 1  .or. SIMULATION_TYPE == 3) deallocate(sourcearrays)
   if (SIMULATION_TYPE == 2 .or. SIMULATION_TYPE == 3) then
     deallocate(iadj_vec)
@@ -208,19 +171,12 @@
   endif
 
   ! receivers
-  deallocate(islice_selected_rec, &
-          ispec_selected_rec, &
-          xi_receiver, &
-          eta_receiver, &
-          gamma_receiver, &
-          station_name, &
-          network_name, &
-          stlat, &
-          stlon, &
-          stele, &
-          stbur, &
-          nu, &
-          number_receiver_global)
+  deallocate(islice_selected_rec,ispec_selected_rec, &
+            xi_receiver,eta_receiver,gamma_receiver)
+  deallocate(station_name,network_name, &
+            stlat,stlon,stele,stbur)
+  deallocate(nu,number_receiver_global)
+
   if( nrec_local > 0 ) then
     deallocate(hxir_store, &
               hetar_store, &
@@ -240,19 +196,11 @@
 
   ! movies
   if(MOVIE_SURFACE .or. NOISE_TOMOGRAPHY /= 0 ) then
-    deallocate(store_val_x, &
-              store_val_y, &
-              store_val_z, &
-              store_val_ux, &
-              store_val_uy, &
-              store_val_uz)
+    deallocate(store_val_x,store_val_y,store_val_z, &
+              store_val_ux,store_val_uy,store_val_uz)
     if (MOVIE_SURFACE) then
-      deallocate(store_val_x_all, &
-            store_val_y_all, &
-            store_val_z_all, &
-            store_val_ux_all, &
-            store_val_uy_all, &
-            store_val_uz_all)
+      deallocate(store_val_x_all,store_val_y_all,store_val_z_all, &
+            store_val_ux_all,store_val_uy_all,store_val_uz_all)
     endif
   endif
   if(MOVIE_VOLUME) then
@@ -262,11 +210,8 @@
   ! noise simulations
   if ( NOISE_TOMOGRAPHY /= 0 ) then
     deallocate(noise_sourcearray, &
-            normal_x_noise, &
-            normal_y_noise, &
-            normal_z_noise, &
-            mask_noise, &
-            noise_surface_movie)
+              normal_x_noise,normal_y_noise,normal_z_noise, &
+              mask_noise,noise_surface_movie)
   endif
 
   ! close the main output file

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -82,7 +82,9 @@
          MOVIE_VOLUME,MOVIE_COARSE,ATTENUATION_3D,RECEIVERS_CAN_BE_BURIED, &
          PRINT_SOURCE_TIME_FUNCTION,SAVE_MESH_FILES, &
          ATTENUATION,REFERENCE_1D_MODEL,THREE_D_MODEL,ABSORBING_CONDITIONS, &
-         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE,LOCAL_PATH,MODEL,SIMULATION_TYPE,SAVE_FORWARD, &
+         INCLUDE_CENTRAL_CUBE,INFLATE_CENTRAL_CUBE, &
+         LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
+         SIMULATION_TYPE,SAVE_FORWARD, &
          NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
          NSPEC_computed,NSPEC2D_XI,NSPEC2D_ETA,NSPEC2DMAX_XMIN_XMAX, &
          NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
@@ -127,7 +129,7 @@
                 SAVE_ALL_SEISMOS_IN_ONE_FILE,MOVIE_COARSE,OUTPUT_SEISMOS_ASCII_TEXT, &
                 OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY, &
                 ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MASTER,USE_BINARY_FOR_LARGE_FILE, &
-                LOCAL_PATH,MODEL, &
+                LOCAL_PATH,LOCAL_TMP_PATH,MODEL, &
                 NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
                 NSPEC_computed,NSPEC2D_XI,NSPEC2D_ETA, &
                 NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
@@ -141,7 +143,7 @@
                 ISOTROPIC_3D_MANTLE,ANISOTROPIC_3D_MANTLE,HETEROGEN_3D_MANTLE, &
                 ATTENUATION,ATTENUATION_3D,ANISOTROPIC_INNER_CORE,NOISE_TOMOGRAPHY)
 
-  ! broadcasts GPU_MODE
+  ! broadcasts optional GPU_MODE
   call broadcast_gpu_parameters(myrank,GPU_MODE)
 
   ! get the base pathname for output files

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -34,19 +34,19 @@
                              nrec,islice_selected_rec,ispec_selected_rec, &
                              xi_receiver,eta_receiver,gamma_receiver,station_name,network_name, &
                              stlat,stlon,stele,stbur,nu, &
-                             yr,jda,ho,mi,sec,NPROCTOT,ELLIPTICITY,TOPOGRAPHY, &
+                             yr,jda,ho,mi,sec,&
+                             ELLIPTICITY,TOPOGRAPHY, &
                              theta_source,phi_source,rspl,espl,espl2,nspl, &
                              ibathy_topo,RECEIVERS_CAN_BE_BURIED,NCHUNKS)
 
+  use constants
   implicit none
 
-! standard include of the MPI library
+  ! standard include of the MPI library
   include 'mpif.h'
-
-  include "constants.h"
   include "precision.h"
 
-  integer NPROCTOT,NCHUNKS
+  integer NCHUNKS
 
   logical ELLIPTICITY,TOPOGRAPHY,RECEIVERS_CAN_BE_BURIED
 
@@ -144,17 +144,19 @@
   double precision, dimension(nrec) :: stlat,stlon,stele,stbur
   double precision, allocatable, dimension(:,:) :: xi_receiver_all,eta_receiver_all,gamma_receiver_all
 
+  double precision typical_size
+  logical located_target
+
   character(len=150) OUTPUT_FILES
   character(len=2) bic
 
-! **************
+  ! get MPI starting time
+  time_start = MPI_WTIME()
 
-! make sure we clean the array before the gather
+  ! make sure we clean the array before the gather
   ispec_selected_rec(:) = 0
 
-! get MPI starting time
-  time_start = MPI_WTIME()
-
+  ! user output
   if(myrank == 0) then
     write(IMAIN,*)
     write(IMAIN,*) '********************'
@@ -163,9 +165,15 @@
     write(IMAIN,*)
   endif
 
-! define topology of the control element
+  ! define topology of the control element
   call hex_nodes(iaddx,iaddy,iaddr)
 
+  ! compute typical size of elements at the surface
+  typical_size = TWO_PI * R_UNIT_SPHERE / (4.*NEX_XI_VAL)
+
+  ! use 10 times the distance as a criterion for source detection
+  typical_size = 10. * typical_size
+
   if(myrank == 0) then
     write(IMAIN,*)
     write(IMAIN,*) '****************************'
@@ -174,7 +182,7 @@
     write(IMAIN,*)
   endif
 
-! allocate memory for arrays using number of stations
+  ! allocate memory for arrays using number of stations
   allocate(epidist(nrec), &
           ix_initial_guess(nrec), &
           iy_initial_guess(nrec), &
@@ -186,14 +194,14 @@
           y_found(nrec), &
           z_found(nrec), &
           final_distance(nrec), &
-          ispec_selected_rec_all(nrec,0:NPROCTOT-1), &
-          xi_receiver_all(nrec,0:NPROCTOT-1), &
-          eta_receiver_all(nrec,0:NPROCTOT-1), &
-          gamma_receiver_all(nrec,0:NPROCTOT-1), &
-          x_found_all(nrec,0:NPROCTOT-1), &
-          y_found_all(nrec,0:NPROCTOT-1), &
-          z_found_all(nrec,0:NPROCTOT-1), &
-          final_distance_all(nrec,0:NPROCTOT-1),stat=ier)
+          ispec_selected_rec_all(nrec,0:NPROCTOT_VAL-1), &
+          xi_receiver_all(nrec,0:NPROCTOT_VAL-1), &
+          eta_receiver_all(nrec,0:NPROCTOT_VAL-1), &
+          gamma_receiver_all(nrec,0:NPROCTOT_VAL-1), &
+          x_found_all(nrec,0:NPROCTOT_VAL-1), &
+          y_found_all(nrec,0:NPROCTOT_VAL-1), &
+          z_found_all(nrec,0:NPROCTOT_VAL-1), &
+          final_distance_all(nrec,0:NPROCTOT_VAL-1),stat=ier)
   if( ier /= 0 ) call exit_MPI(myrank,'error allocating temporary receiver arrays')
 
   ! read that STATIONS file on the master
@@ -218,9 +226,7 @@
 
   endif
 
-
-
-! broadcast the information read on the master to the nodes
+  ! broadcast the information read on the master to the nodes
   call MPI_BCAST(station_name,nrec*MAX_LENGTH_STATION_NAME,MPI_CHARACTER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(network_name,nrec*MAX_LENGTH_NETWORK_NAME,MPI_CHARACTER,0,MPI_COMM_WORLD,ier)
 
@@ -229,13 +235,13 @@
   call MPI_BCAST(stele,nrec,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(stbur,nrec,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
 
-! loop on all the stations to locate them in the mesh
+  ! loop on all the stations to locate them in the mesh
   do irec=1,nrec
 
-! set distance to huge initial value
+    ! set distance to huge initial value
     distmin = HUGEVAL
 
-! convert geographic latitude stlat (degrees) to geocentric colatitude theta (radians)
+    ! convert geographic latitude stlat (degrees) to geocentric colatitude theta (radians)
     if(ASSUME_PERFECT_SPHERE) then
       theta = PI/2.0d0 - stlat(irec)*PI/180.0d0
     else
@@ -245,28 +251,28 @@
     phi = stlon(irec)*PI/180.0d0
     call reduce(theta,phi)
 
-! compute epicentral distance
+    ! compute epicentral distance
     epidist(irec) = acos(cos(theta)*cos(theta_source) + &
               sin(theta)*sin(theta_source)*cos(phi-phi_source))*180.0d0/PI
 
-! print some information about stations
+    ! print some information about stations
     if(myrank == 0) &
       write(IMAIN,*) 'Station #',irec,': ',station_name(irec)(1:len_trim(station_name(irec))), &
                        '.',network_name(irec)(1:len_trim(network_name(irec))), &
                        '    epicentral distance:  ',sngl(epidist(irec)),' degrees'
 
-! record three components for each station
+    ! record three components for each station
     do iorientation = 1,3
 
-!     North
+      !     North
       if(iorientation == 1) then
         stazi = 0.d0
         stdip = 0.d0
-!     East
+      !     East
       else if(iorientation == 2) then
         stazi = 90.d0
         stdip = 0.d0
-!     Vertical
+      !     Vertical
       else if(iorientation == 3) then
         stazi = 0.d0
         stdip = - 90.d0
@@ -274,20 +280,20 @@
         call exit_MPI(myrank,'incorrect orientation')
       endif
 
-!     get the orientation of the seismometer
+      !     get the orientation of the seismometer
       thetan=(90.0d0+stdip)*PI/180.0d0
       phin=stazi*PI/180.0d0
 
-! we use the same convention as in Harvard normal modes for the orientation
+      ! we use the same convention as in Harvard normal modes for the orientation
 
-!     vertical component
+      !     vertical component
       n(1) = cos(thetan)
-!     N-S component
+      !     N-S component
       n(2) = - sin(thetan)*cos(phin)
-!     E-W component
+      !     E-W component
       n(3) = sin(thetan)*sin(phin)
 
-!     get the Cartesian components of n in the model: nu
+      !     get the Cartesian components of n in the model: nu
       sint = sin(theta)
       cost = cos(theta)
       sinp = sin(phi)
@@ -298,23 +304,27 @@
 
     enddo
 
-!     ellipticity
-    r0=1.0d0
+    ! normalized receiver radius
+    r0 = R_UNIT_SPHERE
+
+    ! finds elevation of receiver
+    if(TOPOGRAPHY) then
+       call get_topo_bathy(stlat(irec),stlon(irec),elevation,ibathy_topo)
+       r0 = r0 + elevation/R_EARTH
+    endif
+
+    !     ellipticity
     if(ELLIPTICITY) then
-      if(TOPOGRAPHY) then
-         call get_topo_bathy(stlat(irec),stlon(irec),elevation,ibathy_topo)
-         r0 = r0 + elevation/R_EARTH
-      endif
       cost=cos(theta)
       p20=0.5d0*(3.0d0*cost*cost-1.0d0)
       call spline_evaluation(rspl,espl,espl2,nspl,r0,ell)
       r0=r0*(1.0d0-(2.0d0/3.0d0)*ell*p20)
     endif
 
-! subtract station burial depth (in meters)
+    ! subtract station burial depth (in meters)
     r0 = r0 - stbur(irec)/R_EARTH
 
-! compute the Cartesian position of the receiver
+    ! compute the Cartesian position of the receiver
     x_target(irec) = r0*sin(theta)*cos(phi)
     y_target(irec) = r0*sin(theta)*sin(phi)
     z_target(irec) = r0*cos(theta)
@@ -322,13 +332,20 @@
     ! would write out desired target locations of receivers
     !if (myrank == 0) write(IOVTK,*) sngl(x_target(irec)), sngl(y_target(irec)), sngl(z_target(irec))
 
-! examine top of the elements only (receivers always at the surface)
-!      k = NGLLZ
+    ! flag to check that we located at least one target element
+    located_target = .false.
 
     do ispec=1,nspec
 
-! loop only on points inside the element
-! exclude edges to ensure this point is not shared with other elements
+      ! exclude elements that are too far from target
+      iglob = ibool(1,1,1,ispec)
+      dist = dsqrt((x_target(irec) - dble(xstore(iglob)))**2 &
+                 + (y_target(irec) - dble(ystore(iglob)))**2 &
+                 + (z_target(irec) - dble(zstore(iglob)))**2)
+      if(USE_DISTANCE_CRITERION .and. dist > typical_size) cycle
+
+      ! loop only on points inside the element
+      ! exclude edges to ensure this point is not shared with other elements
       do k=2,NGLLZ-1
         do j=2,NGLLY-1
           do i=2,NGLLX-1
@@ -338,26 +355,36 @@
                         +(y_target(irec)-dble(ystore(iglob)))**2 &
                         +(z_target(irec)-dble(zstore(iglob)))**2)
 
-!           keep this point if it is closer to the receiver
+            !  keep this point if it is closer to the receiver
             if(dist < distmin) then
               distmin = dist
               ispec_selected_rec(irec) = ispec
               ix_initial_guess(irec) = i
               iy_initial_guess(irec) = j
               iz_initial_guess(irec) = k
+              located_target = .true.
             endif
 
           enddo
         enddo
       enddo
 
-! end of loop on all the spectral elements in current slice
+    ! end of loop on all the spectral elements in current slice
     enddo
 
-! end of loop on all the stations
+    ! if we have not located a target element, the receiver is not in this slice
+    ! therefore use first element only for fictitious iterative search
+    if(.not. located_target) then
+      ispec_selected_rec(irec)=1
+      ix_initial_guess(irec) = 2
+      iy_initial_guess(irec) = 2
+      iz_initial_guess(irec) = 2
+    endif
+
+  ! end of loop on all the stations
   enddo
 
-! create RECORDHEADER file with usual format for normal-mode codes
+  ! create RECORDHEADER file with usual format for normal-mode codes
   if(myrank == 0) then
 
     ! get the base pathname for output files
@@ -430,12 +457,12 @@
 
     ispec_iterate = ispec_selected_rec(irec)
 
-! use initial guess in xi and eta
+    ! use initial guess in xi and eta
     xi = xigll(ix_initial_guess(irec))
     eta = yigll(iy_initial_guess(irec))
     gamma = zigll(iz_initial_guess(irec))
 
-! define coordinates of the control points of the element
+    ! define coordinates of the control points of the element
 
     do ia=1,NGNOD
 
@@ -476,37 +503,37 @@
 
     enddo
 
-! iterate to solve the non linear system
+    ! iterate to solve the non linear system
     do iter_loop = 1,NUM_ITER
 
-! impose receiver exactly at the surface
+      ! impose receiver exactly at the surface
       if(.not. RECEIVERS_CAN_BE_BURIED) gamma = 1.d0
 
-! recompute jacobian for the new point
+      ! recompute jacobian for the new point
       call recompute_jacobian(xelm,yelm,zelm,xi,eta,gamma,x,y,z, &
            xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz)
 
-! compute distance to target location
+      ! compute distance to target location
       dx = - (x - x_target(irec))
       dy = - (y - y_target(irec))
       dz = - (z - z_target(irec))
 
-! compute increments
-! gamma does not change since we know the receiver is exactly on the surface
+      ! compute increments
+      ! gamma does not change since we know the receiver is exactly on the surface
       dxi  = xix*dx + xiy*dy + xiz*dz
       deta = etax*dx + etay*dy + etaz*dz
       if(RECEIVERS_CAN_BE_BURIED) dgamma = gammax*dx + gammay*dy + gammaz*dz
 
-! update values
+      ! update values
       xi = xi + dxi
       eta = eta + deta
       if(RECEIVERS_CAN_BE_BURIED) gamma = gamma + dgamma
 
-! impose that we stay in that element
-! (useful if user gives a receiver outside the mesh for instance)
-! we can go slightly outside the [1,1] segment since with finite elements
-! the polynomial solution is defined everywhere
-! can be useful for convergence of iterative scheme with distorted elements
+      ! impose that we stay in that element
+      ! (useful if user gives a receiver outside the mesh for instance)
+      ! we can go slightly outside the [1,1] segment since with finite elements
+      ! the polynomial solution is defined everywhere
+      ! can be useful for convergence of iterative scheme with distorted elements
       if (xi > 1.10d0) xi = 1.10d0
       if (xi < -1.10d0) xi = -1.10d0
       if (eta > 1.10d0) eta = 1.10d0
@@ -514,17 +541,17 @@
       if (gamma > 1.10d0) gamma = 1.10d0
       if (gamma < -1.10d0) gamma = -1.10d0
 
-! end of non linear iterations
+    ! end of non linear iterations
     enddo
 
-! impose receiver exactly at the surface after final iteration
+    ! impose receiver exactly at the surface after final iteration
     if(.not. RECEIVERS_CAN_BE_BURIED) gamma = 1.d0
 
-! compute final coordinates of point found
+    ! compute final coordinates of point found
     call recompute_jacobian(xelm,yelm,zelm,xi,eta,gamma,x,y,z, &
          xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz)
 
-! store xi,eta and x,y,z of point found
+    ! store xi,eta and x,y,z of point found
     xi_receiver(irec) = xi
     eta_receiver(irec) = eta
     gamma_receiver(irec) = gamma
@@ -567,7 +594,7 @@
     islice_selected_rec(:) = -1
     do irec = 1,nrec
       distmin = HUGEVAL
-      do iprocloop = 0,NPROCTOT-1
+      do iprocloop = 0,NPROCTOT_VAL-1
         if(final_distance_all(irec,iprocloop) < distmin) then
           distmin = final_distance_all(irec,iprocloop)
           islice_selected_rec(irec) = iprocloop
@@ -588,7 +615,7 @@
 
       if(final_distance(irec) == HUGEVAL) call exit_MPI(myrank,'error locating receiver')
 
-      if(DISPLAY_DETAILS_STATIONS) then
+      if(DISPLAY_DETAILS_STATIONS .or. final_distance(irec) > 0.01d0 ) then
         write(IMAIN,*)
         write(IMAIN,*) 'station # ',irec,'    ',station_name(irec),network_name(irec)
         write(IMAIN,*) '     original latitude: ',sngl(stlat(irec))

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -31,24 +31,22 @@
 
   subroutine locate_sources(NSOURCES,myrank,nspec,nglob,ibool,&
                  xstore,ystore,zstore,xigll,yigll,zigll, &
-                 NPROCTOT,ELLIPTICITY,TOPOGRAPHY, &
+                 ELLIPTICITY,TOPOGRAPHY, &
                  sec,tshift_cmt,min_tshift_cmt_original,yr,jda,ho,mi,theta_source,phi_source, &
                  NSTEP,DT,hdur,Mxx,Myy,Mzz,Mxy,Mxz,Myz, &
                  islice_selected_source,ispec_selected_source, &
                  xi_source,eta_source,gamma_source, nu_source, &
-                 rspl,espl,espl2,nspl,ibathy_topo,NEX_XI,PRINT_SOURCE_TIME_FUNCTION, &
-                 LOCAL_PATH,SIMULATION_TYPE)
+                 rspl,espl,espl2,nspl,ibathy_topo,PRINT_SOURCE_TIME_FUNCTION, &
+                 LOCAL_TMP_PATH,SIMULATION_TYPE)
 
+  use constants
   implicit none
 
-! standard include of the MPI library
+  ! standard include of the MPI library
   include 'mpif.h'
-
-  include "constants.h"
   include "precision.h"
 
-  integer NPROCTOT
-  integer NSTEP,NSOURCES,NEX_XI
+  integer NSTEP,NSOURCES
 
   logical ELLIPTICITY,TOPOGRAPHY,PRINT_SOURCE_TIME_FUNCTION
 
@@ -86,7 +84,7 @@
   ! use integer array to store values
   integer, dimension(NX_BATHY,NY_BATHY) :: ibathy_topo
 
-  character(len=150) :: LOCAL_PATH
+  character(len=150) :: LOCAL_TMP_PATH
   integer :: SIMULATION_TYPE
 
 ! local parameters
@@ -104,11 +102,11 @@
   double precision dist,typical_size
   double precision xi,eta,gamma,dx,dy,dz,dxi,deta
 
-! topology of the control points of the surface element
+  ! topology of the control points of the surface element
   integer iax,iay,iaz
   integer iaddx(NGNOD),iaddy(NGNOD),iaddr(NGNOD)
 
-! coordinates of the control points of the surface element
+  ! coordinates of the control points of the surface element
   double precision xelm(NGNOD),yelm(NGNOD),zelm(NGNOD)
 
   integer iter_loop
@@ -156,13 +154,13 @@
 
   logical located_target
 
-! for calculation of source time function and spectrum
+  ! for calculation of source time function and spectrum
   integer it,iom
   double precision time_source,om
   double precision, external :: comp_source_time_function,comp_source_spectrum
   double precision, external :: comp_source_time_function_rickr
 
-! number of points to plot the source time function and spectrum
+  ! number of points to plot the source time function and spectrum
   integer, parameter :: NSAMP_PLOT_SOURCE = 1000
 
   integer iorientation
@@ -171,22 +169,23 @@
   double precision :: f0,t0_ricker
   double precision t_cmt_used(NSOURCES)
 
-! mask source region (mask values are between 0 and 1, with 0 around sources)
+  ! mask source region (mask values are between 0 and 1, with 0 around sources)
   real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: mask_source
 
-! **************
+  ! get MPI starting time for all sources
+  time_start = MPI_WTIME()
 
-! make sure we clean the future final array
+  ! make sure we clean the future final array
   ispec_selected_source(:) = 0
 
-! get the base pathname for output files
+  ! get the base pathname for output files
   call get_value_string(OUTPUT_FILES, 'OUTPUT_FILES', 'OUTPUT_FILES')
 
-! read all the sources
+  ! read all the sources
   if(myrank == 0) call get_cmt(yr,jda,ho,mi,sec,tshift_cmt,hdur,lat,long,depth,moment_tensor, &
                               DT,NSOURCES,min_tshift_cmt_original)
 
-! broadcast the information read on the master to the nodes
+  ! broadcast the information read on the master to the nodes
   call MPI_BCAST(yr,1,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(jda,1,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(ho,1,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
@@ -203,612 +202,616 @@
   call MPI_BCAST(moment_tensor,6*NSOURCES,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(min_tshift_cmt_original,1,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
 
-! define topology of the control element
+  ! define topology of the control element
   call hex_nodes(iaddx,iaddy,iaddr)
 
-! initializes source mask
+  ! compute typical size of elements at the surface
+  typical_size = TWO_PI * R_UNIT_SPHERE / (4.*NEX_XI_VAL)
+
+  ! use 10 times the distance as a criterion for source detection
+  typical_size = 10. * typical_size
+
+  ! initializes source mask
   if( SAVE_SOURCE_MASK .and. SIMULATION_TYPE == 3 ) then
     allocate( mask_source(NGLLX,NGLLY,NGLLZ,NSPEC) )
     mask_source(:,:,:,:) = 1.0_CUSTOM_REAL
   endif
 
-! get MPI starting time for all sources
-  time_start = MPI_WTIME()
+  ! loop on all the sources
+  ! gather source information in subsets to reduce memory requirements
 
-! loop on all the sources
-! gather source information in subsets to reduce memory requirements
-
-! loop over subsets of sources
+  ! loop over subsets of sources
   do isources_already_done = 0, NSOURCES, NSOURCES_SUBSET_MAX
 
-! the size of the subset can be the maximum size, or less (if we are in the last subset,
-! or if there are fewer sources than the maximum size of a subset)
-  NSOURCES_SUBSET_current_size = min(NSOURCES_SUBSET_MAX, NSOURCES - isources_already_done)
+    ! the size of the subset can be the maximum size, or less (if we are in the last subset,
+    ! or if there are fewer sources than the maximum size of a subset)
+    NSOURCES_SUBSET_current_size = min(NSOURCES_SUBSET_MAX, NSOURCES - isources_already_done)
 
-! allocate arrays specific to each subset
-  allocate(final_distance_source_subset(NSOURCES_SUBSET_current_size), &
-    ispec_selected_source_subset(NSOURCES_SUBSET_current_size), &
-    ispec_selected_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    xi_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    eta_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    gamma_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    final_distance_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    x_found_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    y_found_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    z_found_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT-1), &
-    xi_source_subset(NSOURCES_SUBSET_current_size), &
-    eta_source_subset(NSOURCES_SUBSET_current_size), &
-    gamma_source_subset(NSOURCES_SUBSET_current_size), &
-    x_found_source(NSOURCES_SUBSET_current_size), &
-    y_found_source(NSOURCES_SUBSET_current_size), &
-    z_found_source(NSOURCES_SUBSET_current_size),stat=ier)
-  if( ier /= 0 ) call exit_MPI(myrank,'error allocating temporary source arrays')
+    ! allocate arrays specific to each subset
+    allocate(final_distance_source_subset(NSOURCES_SUBSET_current_size), &
+            ispec_selected_source_subset(NSOURCES_SUBSET_current_size), &
+            ispec_selected_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            xi_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            eta_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            gamma_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            final_distance_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            x_found_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            y_found_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            z_found_source_all(NSOURCES_SUBSET_current_size,0:NPROCTOT_VAL-1), &
+            xi_source_subset(NSOURCES_SUBSET_current_size), &
+            eta_source_subset(NSOURCES_SUBSET_current_size), &
+            gamma_source_subset(NSOURCES_SUBSET_current_size), &
+            x_found_source(NSOURCES_SUBSET_current_size), &
+            y_found_source(NSOURCES_SUBSET_current_size), &
+            z_found_source(NSOURCES_SUBSET_current_size),stat=ier)
+    if( ier /= 0 ) call exit_MPI(myrank,'error allocating temporary source arrays')
 
-! make sure we clean the subset array before the gather
-  ispec_selected_source_subset(:) = 0
+    ! make sure we clean the subset array before the gather
+    ispec_selected_source_subset(:) = 0
 
-! loop over sources within this subset
-  do isource_in_this_subset = 1,NSOURCES_SUBSET_current_size
+    ! loop over sources within this subset
+    do isource_in_this_subset = 1,NSOURCES_SUBSET_current_size
 
-! mapping from source number in current subset to real source number in all the subsets
-  isource = isource_in_this_subset + isources_already_done
+      ! mapping from source number in current subset to real source number in all the subsets
+      isource = isource_in_this_subset + isources_already_done
 
-! convert geographic latitude lat (degrees) to geocentric colatitude theta (radians)
-  if(ASSUME_PERFECT_SPHERE) then
-    theta = PI/2.0d0 - lat(isource)*PI/180.0d0
-  else
-    theta = PI/2.0d0 - atan(0.99329534d0*dtan(lat(isource)*PI/180.0d0))
-  endif
+      ! convert geographic latitude lat (degrees) to geocentric colatitude theta (radians)
+      if(ASSUME_PERFECT_SPHERE) then
+        theta = PI/2.0d0 - lat(isource)*PI/180.0d0
+      else
+        theta = PI/2.0d0 - atan(0.99329534d0*dtan(lat(isource)*PI/180.0d0))
+      endif
 
-  phi = long(isource)*PI/180.0d0
-  call reduce(theta,phi)
+      phi = long(isource)*PI/180.0d0
+      call reduce(theta,phi)
 
-! get the moment tensor
-  Mrr = moment_tensor(1,isource)
-  Mtt = moment_tensor(2,isource)
-  Mpp = moment_tensor(3,isource)
-  Mrt = moment_tensor(4,isource)
-  Mrp = moment_tensor(5,isource)
-  Mtp = moment_tensor(6,isource)
+      ! get the moment tensor
+      Mrr = moment_tensor(1,isource)
+      Mtt = moment_tensor(2,isource)
+      Mpp = moment_tensor(3,isource)
+      Mrt = moment_tensor(4,isource)
+      Mrp = moment_tensor(5,isource)
+      Mtp = moment_tensor(6,isource)
 
-! convert from a spherical to a Cartesian representation of the moment tensor
-  st=dsin(theta)
-  ct=dcos(theta)
-  sp=dsin(phi)
-  cp=dcos(phi)
+      ! convert from a spherical to a Cartesian representation of the moment tensor
+      st=dsin(theta)
+      ct=dcos(theta)
+      sp=dsin(phi)
+      cp=dcos(phi)
 
-  Mxx(isource)=st*st*cp*cp*Mrr+ct*ct*cp*cp*Mtt+sp*sp*Mpp &
-      +2.0d0*st*ct*cp*cp*Mrt-2.0d0*st*sp*cp*Mrp-2.0d0*ct*sp*cp*Mtp
-  Myy(isource)=st*st*sp*sp*Mrr+ct*ct*sp*sp*Mtt+cp*cp*Mpp &
-      +2.0d0*st*ct*sp*sp*Mrt+2.0d0*st*sp*cp*Mrp+2.0d0*ct*sp*cp*Mtp
-  Mzz(isource)=ct*ct*Mrr+st*st*Mtt-2.0d0*st*ct*Mrt
-  Mxy(isource)=st*st*sp*cp*Mrr+ct*ct*sp*cp*Mtt-sp*cp*Mpp &
-      +2.0d0*st*ct*sp*cp*Mrt+st*(cp*cp-sp*sp)*Mrp+ct*(cp*cp-sp*sp)*Mtp
-  Mxz(isource)=st*ct*cp*Mrr-st*ct*cp*Mtt &
-      +(ct*ct-st*st)*cp*Mrt-ct*sp*Mrp+st*sp*Mtp
-  Myz(isource)=st*ct*sp*Mrr-st*ct*sp*Mtt &
-      +(ct*ct-st*st)*sp*Mrt+ct*cp*Mrp-st*cp*Mtp
+      Mxx(isource)=st*st*cp*cp*Mrr+ct*ct*cp*cp*Mtt+sp*sp*Mpp &
+          +2.0d0*st*ct*cp*cp*Mrt-2.0d0*st*sp*cp*Mrp-2.0d0*ct*sp*cp*Mtp
+      Myy(isource)=st*st*sp*sp*Mrr+ct*ct*sp*sp*Mtt+cp*cp*Mpp &
+          +2.0d0*st*ct*sp*sp*Mrt+2.0d0*st*sp*cp*Mrp+2.0d0*ct*sp*cp*Mtp
+      Mzz(isource)=ct*ct*Mrr+st*st*Mtt-2.0d0*st*ct*Mrt
+      Mxy(isource)=st*st*sp*cp*Mrr+ct*ct*sp*cp*Mtt-sp*cp*Mpp &
+          +2.0d0*st*ct*sp*cp*Mrt+st*(cp*cp-sp*sp)*Mrp+ct*(cp*cp-sp*sp)*Mtp
+      Mxz(isource)=st*ct*cp*Mrr-st*ct*cp*Mtt &
+          +(ct*ct-st*st)*cp*Mrt-ct*sp*Mrp+st*sp*Mtp
+      Myz(isource)=st*ct*sp*Mrr-st*ct*sp*Mtt &
+          +(ct*ct-st*st)*sp*Mrt+ct*cp*Mrp-st*cp*Mtp
 
-! record three components for each station
-  do iorientation = 1,3
+      ! record three components for each station
+      do iorientation = 1,3
 
-!   North
-    if(iorientation == 1) then
-      stazi = 0.d0
-      stdip = 0.d0
-!   East
-    else if(iorientation == 2) then
-      stazi = 90.d0
-      stdip = 0.d0
-!   Vertical
-    else if(iorientation == 3) then
-      stazi = 0.d0
-      stdip = - 90.d0
-    else
-      call exit_MPI(myrank,'incorrect orientation')
-    endif
+        !   North
+        if(iorientation == 1) then
+          stazi = 0.d0
+          stdip = 0.d0
+        !   East
+        else if(iorientation == 2) then
+          stazi = 90.d0
+          stdip = 0.d0
+        !   Vertical
+        else if(iorientation == 3) then
+          stazi = 0.d0
+          stdip = - 90.d0
+        else
+          call exit_MPI(myrank,'incorrect orientation')
+        endif
 
-!   get the orientation of the seismometer
-    thetan=(90.0d0+stdip)*PI/180.0d0
-    phin=stazi*PI/180.0d0
+        !   get the orientation of the seismometer
+        thetan=(90.0d0+stdip)*PI/180.0d0
+        phin=stazi*PI/180.0d0
 
-! we use the same convention as in Harvard normal modes for the orientation
+        ! we use the same convention as in Harvard normal modes for the orientation
 
-!   vertical component
-    n(1) = dcos(thetan)
-!   N-S component
-    n(2) = - dsin(thetan)*dcos(phin)
-!   E-W component
-    n(3) = dsin(thetan)*dsin(phin)
+        !   vertical component
+        n(1) = dcos(thetan)
+        !   N-S component
+        n(2) = - dsin(thetan)*dcos(phin)
+        !   E-W component
+        n(3) = dsin(thetan)*dsin(phin)
 
-!   get the Cartesian components of n in the model: nu
-    nu_source(iorientation,1,isource) = n(1)*st*cp+n(2)*ct*cp-n(3)*sp
-    nu_source(iorientation,2,isource) = n(1)*st*sp+n(2)*ct*sp+n(3)*cp
-    nu_source(iorientation,3,isource) = n(1)*ct-n(2)*st
+        !   get the Cartesian components of n in the model: nu
+        nu_source(iorientation,1,isource) = n(1)*st*cp+n(2)*ct*cp-n(3)*sp
+        nu_source(iorientation,2,isource) = n(1)*st*sp+n(2)*ct*sp+n(3)*cp
+        nu_source(iorientation,3,isource) = n(1)*ct-n(2)*st
 
-  enddo
+      enddo
 
-! normalized source radius
-  r0 = R_UNIT_SPHERE
+      ! normalized source radius
+      r0 = R_UNIT_SPHERE
 
-  if(ELLIPTICITY) then
-    if(TOPOGRAPHY) then
-      call get_topo_bathy(lat(isource),long(isource),elevation,ibathy_topo)
-      r0 = r0 + elevation/R_EARTH
-    endif
-    dcost = dcos(theta)
-    p20 = 0.5d0*(3.0d0*dcost*dcost-1.0d0)
-    radius = r0 - depth(isource)*1000.0d0/R_EARTH
-    call spline_evaluation(rspl,espl,espl2,nspl,radius,ell)
-    r0 = r0*(1.0d0-(2.0d0/3.0d0)*ell*p20)
-  endif
+      ! finds elevation of position
+      if(TOPOGRAPHY) then
+        call get_topo_bathy(lat(isource),long(isource),elevation,ibathy_topo)
+        r0 = r0 + elevation/R_EARTH
+      endif
 
-! compute the Cartesian position of the source
-  r_target_source = r0 - depth(isource)*1000.0d0/R_EARTH
-  x_target_source = r_target_source*dsin(theta)*dcos(phi)
-  y_target_source = r_target_source*dsin(theta)*dsin(phi)
-  z_target_source = r_target_source*dcos(theta)
+      if(ELLIPTICITY) then
+        dcost = dcos(theta)
+        p20 = 0.5d0*(3.0d0*dcost*dcost-1.0d0)
+        radius = r0 - depth(isource)*1000.0d0/R_EARTH
+        call spline_evaluation(rspl,espl,espl2,nspl,radius,ell)
+        r0 = r0*(1.0d0-(2.0d0/3.0d0)*ell*p20)
+      endif
 
-  ! would only output desired target locations
-  !if(myrank == 0) write(IOVTK,*) sngl(x_target_source),sngl(y_target_source),sngl(z_target_source)
+      ! subtracts source depth (given in km)
+      r_target_source = r0 - depth(isource)*1000.0d0/R_EARTH
 
-! set distance to huge initial value
-  distmin = HUGEVAL
+      ! compute the Cartesian position of the source
+      x_target_source = r_target_source*dsin(theta)*dcos(phi)
+      y_target_source = r_target_source*dsin(theta)*dsin(phi)
+      z_target_source = r_target_source*dcos(theta)
 
-! compute typical size of elements at the surface
-  typical_size = TWO_PI * R_UNIT_SPHERE / (4.*NEX_XI)
+      ! would only output desired target locations
+      !if(myrank == 0) write(IOVTK,*) sngl(x_target_source),sngl(y_target_source),sngl(z_target_source)
 
-! use 10 times the distance as a criterion for source detection
-  typical_size = 10. * typical_size
+      ! set distance to huge initial value
+      distmin = HUGEVAL
 
-! flag to check that we located at least one target element
-  located_target = .false.
+      ! flag to check that we located at least one target element
+      located_target = .false.
 
-  do ispec = 1,nspec
+      do ispec = 1,nspec
 
-    ! exclude elements that are too far from target
-    iglob = ibool(1,1,1,ispec)
-    dist = dsqrt((x_target_source - dble(xstore(iglob)))**2 &
-               + (y_target_source - dble(ystore(iglob)))**2 &
-               + (z_target_source - dble(zstore(iglob)))**2)
-    if(USE_DISTANCE_CRITERION .and. dist > typical_size) cycle
+        ! exclude elements that are too far from target
+        iglob = ibool(1,1,1,ispec)
+        dist = dsqrt((x_target_source - dble(xstore(iglob)))**2 &
+                   + (y_target_source - dble(ystore(iglob)))**2 &
+                   + (z_target_source - dble(zstore(iglob)))**2)
+        if(USE_DISTANCE_CRITERION .and. dist > typical_size) cycle
 
-    located_target = .true.
+        located_target = .true.
 
-    ! define the interval in which we look for points
-    if(USE_FORCE_POINT_SOURCE) then
-      ! force sources will be put on an exact GLL point
-      imin = 1
-      imax = NGLLX
+        ! define the interval in which we look for points
+        if(USE_FORCE_POINT_SOURCE) then
+          ! force sources will be put on an exact GLL point
+          imin = 1
+          imax = NGLLX
 
-      jmin = 1
-      jmax = NGLLY
+          jmin = 1
+          jmax = NGLLY
 
-      kmin = 1
-      kmax = NGLLZ
+          kmin = 1
+          kmax = NGLLZ
 
-    else
-      ! double-couple CMTSOLUTION
-      ! loop only on points inside the element
-      ! exclude edges to ensure this point is not shared with other elements
-      imin = 2
-      imax = NGLLX - 1
+        else
+          ! double-couple CMTSOLUTION
+          ! loop only on points inside the element
+          ! exclude edges to ensure this point is not shared with other elements
+          imin = 2
+          imax = NGLLX - 1
 
-      jmin = 2
-      jmax = NGLLY - 1
+          jmin = 2
+          jmax = NGLLY - 1
 
-      kmin = 2
-      kmax = NGLLZ - 1
-    endif
-    do k = kmin,kmax
-      do j = jmin,jmax
-        do i = imin,imax
+          kmin = 2
+          kmax = NGLLZ - 1
+        endif
+        do k = kmin,kmax
+          do j = jmin,jmax
+            do i = imin,imax
 
-          ! keep this point if it is closer to the receiver
-          iglob = ibool(i,j,k,ispec)
-          dist = dsqrt((x_target_source - dble(xstore(iglob)))**2 &
-                      +(y_target_source - dble(ystore(iglob)))**2 &
-                      +(z_target_source - dble(zstore(iglob)))**2)
-          if(dist < distmin) then
-            distmin = dist
-            ispec_selected_source_subset(isource_in_this_subset) = ispec
-            ix_initial_guess_source = i
-            iy_initial_guess_source = j
-            iz_initial_guess_source = k
-          endif
+              ! keep this point if it is closer to the receiver
+              iglob = ibool(i,j,k,ispec)
+              dist = dsqrt((x_target_source - dble(xstore(iglob)))**2 &
+                          +(y_target_source - dble(ystore(iglob)))**2 &
+                          +(z_target_source - dble(zstore(iglob)))**2)
+              if(dist < distmin) then
+                distmin = dist
+                ispec_selected_source_subset(isource_in_this_subset) = ispec
+                ix_initial_guess_source = i
+                iy_initial_guess_source = j
+                iz_initial_guess_source = k
+              endif
 
+            enddo
+          enddo
         enddo
-      enddo
-    enddo
 
-! calculates a gaussian mask around source point
-    if( SAVE_SOURCE_MASK .and. SIMULATION_TYPE == 3 ) then
-      call calc_mask_source(mask_source,ispec,NSPEC,typical_size, &
-                            x_target_source,y_target_source,z_target_source, &
-                            ibool,xstore,ystore,zstore,NGLOB)
-    endif
+        ! calculates a gaussian mask around source point
+        if( SAVE_SOURCE_MASK .and. SIMULATION_TYPE == 3 ) then
+          call calc_mask_source(mask_source,ispec,NSPEC,typical_size, &
+                                x_target_source,y_target_source,z_target_source, &
+                                ibool,xstore,ystore,zstore,NGLOB)
+        endif
 
-! end of loop on all the elements in current slice
-  enddo
+      ! end of loop on all the elements in current slice
+      enddo
 
-! *******************************************
-! find the best (xi,eta,gamma) for the source
-! *******************************************
+      ! *******************************************
+      ! find the best (xi,eta,gamma) for the source
+      ! *******************************************
 
-  ! if we have not located a target element, the source is not in this slice
-  ! therefore use first element only for fictitious iterative search
-  if(.not. located_target) then
-    ispec_selected_source_subset(isource_in_this_subset)=1
-    ix_initial_guess_source = 2
-    iy_initial_guess_source = 2
-    iz_initial_guess_source = 2
-  endif
+      ! if we have not located a target element, the source is not in this slice
+      ! therefore use first element only for fictitious iterative search
+      if(.not. located_target) then
+        ispec_selected_source_subset(isource_in_this_subset)=1
+        ix_initial_guess_source = 2
+        iy_initial_guess_source = 2
+        iz_initial_guess_source = 2
+      endif
 
-  ! for point sources, the location will be exactly at a GLL point
-  ! otherwise this tries to find best location
-  if( USE_FORCE_POINT_SOURCE ) then
-    ! store xi,eta,gamma and x,y,z of point found
-    ! note: they have range [1.0d0,NGLLX/Y/Z], used for point sources
-    !          see e.g. in compute_add_sources.f90
-    xi_source_subset(isource_in_this_subset) = dble(ix_initial_guess_source)
-    eta_source_subset(isource_in_this_subset) = dble(iy_initial_guess_source)
-    gamma_source_subset(isource_in_this_subset) = dble(iz_initial_guess_source)
+      ! for point sources, the location will be exactly at a GLL point
+      ! otherwise this tries to find best location
+      if( USE_FORCE_POINT_SOURCE ) then
+        ! store xi,eta,gamma and x,y,z of point found
+        ! note: they have range [1.0d0,NGLLX/Y/Z], used for point sources
+        !          see e.g. in compute_add_sources.f90
+        xi_source_subset(isource_in_this_subset) = dble(ix_initial_guess_source)
+        eta_source_subset(isource_in_this_subset) = dble(iy_initial_guess_source)
+        gamma_source_subset(isource_in_this_subset) = dble(iz_initial_guess_source)
 
-    iglob = ibool(ix_initial_guess_source,iy_initial_guess_source, &
-        iz_initial_guess_source,ispec_selected_source_subset(isource_in_this_subset))
-    x_found_source(isource_in_this_subset) = xstore(iglob)
-    y_found_source(isource_in_this_subset) = ystore(iglob)
-    z_found_source(isource_in_this_subset) = zstore(iglob)
+        iglob = ibool(ix_initial_guess_source,iy_initial_guess_source, &
+            iz_initial_guess_source,ispec_selected_source_subset(isource_in_this_subset))
+        x_found_source(isource_in_this_subset) = xstore(iglob)
+        y_found_source(isource_in_this_subset) = ystore(iglob)
+        z_found_source(isource_in_this_subset) = zstore(iglob)
 
-    ! compute final distance between asked and found (converted to km)
-    final_distance_source_subset(isource_in_this_subset) = &
-      dsqrt((x_target_source-x_found_source(isource_in_this_subset))**2 + &
-            (y_target_source-y_found_source(isource_in_this_subset))**2 + &
-            (z_target_source-z_found_source(isource_in_this_subset))**2)*R_EARTH/1000.d0
+        ! compute final distance between asked and found (converted to km)
+        final_distance_source_subset(isource_in_this_subset) = &
+          dsqrt((x_target_source-x_found_source(isource_in_this_subset))**2 + &
+                (y_target_source-y_found_source(isource_in_this_subset))**2 + &
+                (z_target_source-z_found_source(isource_in_this_subset))**2)*R_EARTH/1000.d0
 
-  else
+      else
 
-    ! use initial guess in xi, eta and gamma
-    xi = xigll(ix_initial_guess_source)
-    eta = yigll(iy_initial_guess_source)
-    gamma = zigll(iz_initial_guess_source)
+        ! use initial guess in xi, eta and gamma
+        xi = xigll(ix_initial_guess_source)
+        eta = yigll(iy_initial_guess_source)
+        gamma = zigll(iz_initial_guess_source)
 
-    ! define coordinates of the control points of the element
-    do ia=1,NGNOD
+        ! define coordinates of the control points of the element
+        do ia=1,NGNOD
 
-      if(iaddx(ia) == 0) then
-        iax = 1
-      else if(iaddx(ia) == 1) then
-        iax = (NGLLX+1)/2
-      else if(iaddx(ia) == 2) then
-        iax = NGLLX
-      else
-        call exit_MPI(myrank,'incorrect value of iaddx')
-      endif
+          if(iaddx(ia) == 0) then
+            iax = 1
+          else if(iaddx(ia) == 1) then
+            iax = (NGLLX+1)/2
+          else if(iaddx(ia) == 2) then
+            iax = NGLLX
+          else
+            call exit_MPI(myrank,'incorrect value of iaddx')
+          endif
 
-      if(iaddy(ia) == 0) then
-        iay = 1
-      else if(iaddy(ia) == 1) then
-        iay = (NGLLY+1)/2
-      else if(iaddy(ia) == 2) then
-        iay = NGLLY
-      else
-        call exit_MPI(myrank,'incorrect value of iaddy')
-      endif
+          if(iaddy(ia) == 0) then
+            iay = 1
+          else if(iaddy(ia) == 1) then
+            iay = (NGLLY+1)/2
+          else if(iaddy(ia) == 2) then
+            iay = NGLLY
+          else
+            call exit_MPI(myrank,'incorrect value of iaddy')
+          endif
 
-      if(iaddr(ia) == 0) then
-        iaz = 1
-      else if(iaddr(ia) == 1) then
-        iaz = (NGLLZ+1)/2
-      else if(iaddr(ia) == 2) then
-        iaz = NGLLZ
-      else
-        call exit_MPI(myrank,'incorrect value of iaddr')
-      endif
+          if(iaddr(ia) == 0) then
+            iaz = 1
+          else if(iaddr(ia) == 1) then
+            iaz = (NGLLZ+1)/2
+          else if(iaddr(ia) == 2) then
+            iaz = NGLLZ
+          else
+            call exit_MPI(myrank,'incorrect value of iaddr')
+          endif
 
-      iglob = ibool(iax,iay,iaz,ispec_selected_source_subset(isource_in_this_subset))
-      xelm(ia) = dble(xstore(iglob))
-      yelm(ia) = dble(ystore(iglob))
-      zelm(ia) = dble(zstore(iglob))
+          iglob = ibool(iax,iay,iaz,ispec_selected_source_subset(isource_in_this_subset))
+          xelm(ia) = dble(xstore(iglob))
+          yelm(ia) = dble(ystore(iglob))
+          zelm(ia) = dble(zstore(iglob))
 
-    enddo
+        enddo
 
-    ! iterate to solve the non linear system
-    do iter_loop = 1,NUM_ITER
+        ! iterate to solve the non linear system
+        do iter_loop = 1,NUM_ITER
 
-      ! recompute jacobian for the new point
-      call recompute_jacobian(xelm,yelm,zelm,xi,eta,gamma,x,y,z,xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz)
+          ! recompute jacobian for the new point
+          call recompute_jacobian(xelm,yelm,zelm,xi,eta,gamma,x,y,z,xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz)
 
-      ! compute distance to target location
-      dx = - (x - x_target_source)
-      dy = - (y - y_target_source)
-      dz = - (z - z_target_source)
+          ! compute distance to target location
+          dx = - (x - x_target_source)
+          dy = - (y - y_target_source)
+          dz = - (z - z_target_source)
 
-      ! compute increments
-      dxi  = xix*dx + xiy*dy + xiz*dz
-      deta = etax*dx + etay*dy + etaz*dz
-      dgamma = gammax*dx + gammay*dy + gammaz*dz
+          ! compute increments
+          dxi  = xix*dx + xiy*dy + xiz*dz
+          deta = etax*dx + etay*dy + etaz*dz
+          dgamma = gammax*dx + gammay*dy + gammaz*dz
 
-      ! update values
-      xi = xi + dxi
-      eta = eta + deta
-      gamma = gamma + dgamma
+          ! update values
+          xi = xi + dxi
+          eta = eta + deta
+          gamma = gamma + dgamma
 
-      ! impose that we stay in that element
-      ! (useful if user gives a source outside the mesh for instance)
-      if (xi > 1.d0) xi = 1.d0
-      if (xi < -1.d0) xi = -1.d0
-      if (eta > 1.d0) eta = 1.d0
-      if (eta < -1.d0) eta = -1.d0
-      if (gamma > 1.d0) gamma = 1.d0
-      if (gamma < -1.d0) gamma = -1.d0
+          ! impose that we stay in that element
+          ! (useful if user gives a source outside the mesh for instance)
+          if (xi > 1.d0) xi = 1.d0
+          if (xi < -1.d0) xi = -1.d0
+          if (eta > 1.d0) eta = 1.d0
+          if (eta < -1.d0) eta = -1.d0
+          if (gamma > 1.d0) gamma = 1.d0
+          if (gamma < -1.d0) gamma = -1.d0
 
-    enddo
+        enddo
 
-    ! compute final coordinates of point found
-    call recompute_jacobian(xelm,yelm,zelm,xi,eta,gamma,x,y,z,xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz)
+        ! compute final coordinates of point found
+        call recompute_jacobian(xelm,yelm,zelm,xi,eta,gamma, &
+                               x,y,z,xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz)
 
-    ! store xi,eta,gamma and x,y,z of point found
-    xi_source_subset(isource_in_this_subset) = xi
-    eta_source_subset(isource_in_this_subset) = eta
-    gamma_source_subset(isource_in_this_subset) = gamma
-    x_found_source(isource_in_this_subset) = x
-    y_found_source(isource_in_this_subset) = y
-    z_found_source(isource_in_this_subset) = z
+        ! store xi,eta,gamma and x,y,z of point found
+        xi_source_subset(isource_in_this_subset) = xi
+        eta_source_subset(isource_in_this_subset) = eta
+        gamma_source_subset(isource_in_this_subset) = gamma
+        x_found_source(isource_in_this_subset) = x
+        y_found_source(isource_in_this_subset) = y
+        z_found_source(isource_in_this_subset) = z
 
-    ! compute final distance between asked and found (converted to km)
-    final_distance_source_subset(isource_in_this_subset) = &
-      dsqrt((x_target_source-x_found_source(isource_in_this_subset))**2 + &
-        (y_target_source-y_found_source(isource_in_this_subset))**2 + &
-        (z_target_source-z_found_source(isource_in_this_subset))**2)*R_EARTH/1000.d0
+        ! compute final distance between asked and found (converted to km)
+        final_distance_source_subset(isource_in_this_subset) = &
+          dsqrt((x_target_source-x_found_source(isource_in_this_subset))**2 + &
+            (y_target_source-y_found_source(isource_in_this_subset))**2 + &
+            (z_target_source-z_found_source(isource_in_this_subset))**2)*R_EARTH/1000.d0
 
-  endif ! USE_FORCE_POINT_SOURCE
+      endif ! USE_FORCE_POINT_SOURCE
 
-! end of loop on all the sources
-  enddo
+    ! end of loop on all the sources
+    enddo
 
-! now gather information from all the nodes
-! use -1 as a flag to detect if gather fails for some reason
-  ispec_selected_source_all(:,:) = -1
-  call MPI_GATHER(ispec_selected_source_subset,NSOURCES_SUBSET_current_size,MPI_INTEGER, &
-                  ispec_selected_source_all,NSOURCES_SUBSET_current_size,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(xi_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
-                  xi_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(eta_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
-                  eta_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(gamma_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
-                  gamma_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(final_distance_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
-    final_distance_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(x_found_source,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
-    x_found_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(y_found_source,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
-    y_found_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(z_found_source,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
-    z_found_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
+    ! now gather information from all the nodes
+    ! use -1 as a flag to detect if gather fails for some reason
+    ispec_selected_source_all(:,:) = -1
+    call MPI_GATHER(ispec_selected_source_subset,NSOURCES_SUBSET_current_size,MPI_INTEGER, &
+      ispec_selected_source_all,NSOURCES_SUBSET_current_size,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
+    call MPI_GATHER(xi_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
+      xi_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
+    call MPI_GATHER(eta_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
+      eta_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
+    call MPI_GATHER(gamma_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
+      gamma_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
+    call MPI_GATHER(final_distance_source_subset,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
+      final_distance_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
+    call MPI_GATHER(x_found_source,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
+      x_found_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
+    call MPI_GATHER(y_found_source,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
+      y_found_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
+    call MPI_GATHER(z_found_source,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION, &
+      z_found_source_all,NSOURCES_SUBSET_current_size,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
 
-! this is executed by main process only
-  if(myrank == 0) then
+    ! this is executed by main process only
+    if(myrank == 0) then
 
-    ! check that the gather operation went well
-    if(minval(ispec_selected_source_all) <= 0) call exit_MPI(myrank,'gather operation failed for source')
+      ! check that the gather operation went well
+      if(minval(ispec_selected_source_all) <= 0) call exit_MPI(myrank,'gather operation failed for source')
 
-    ! loop on all the sources within subsets
-    do isource_in_this_subset = 1,NSOURCES_SUBSET_current_size
+      ! loop on all the sources within subsets
+      do isource_in_this_subset = 1,NSOURCES_SUBSET_current_size
 
-      ! mapping from source number in current subset to real source number in all the subsets
-      isource = isources_already_done + isource_in_this_subset
+        ! mapping from source number in current subset to real source number in all the subsets
+        isource = isources_already_done + isource_in_this_subset
 
-      ! loop on all the results to determine the best slice
-      distmin = HUGEVAL
-      do iprocloop = 0,NPROCTOT-1
-        if(final_distance_source_all(isource_in_this_subset,iprocloop) < distmin) then
-          distmin = final_distance_source_all(isource_in_this_subset,iprocloop)
-          islice_selected_source(isource) = iprocloop
-          ispec_selected_source(isource) = ispec_selected_source_all(isource_in_this_subset,iprocloop)
-          xi_source(isource) = xi_source_all(isource_in_this_subset,iprocloop)
-          eta_source(isource) = eta_source_all(isource_in_this_subset,iprocloop)
-          gamma_source(isource) = gamma_source_all(isource_in_this_subset,iprocloop)
-          x_found_source(isource_in_this_subset) = x_found_source_all(isource_in_this_subset,iprocloop)
-          y_found_source(isource_in_this_subset) = y_found_source_all(isource_in_this_subset,iprocloop)
-          z_found_source(isource_in_this_subset) = z_found_source_all(isource_in_this_subset,iprocloop)
-        endif
-      enddo
-      final_distance_source(isource) = distmin
+        ! loop on all the results to determine the best slice
+        distmin = HUGEVAL
+        do iprocloop = 0,NPROCTOT_VAL-1
+          if(final_distance_source_all(isource_in_this_subset,iprocloop) < distmin) then
+            distmin = final_distance_source_all(isource_in_this_subset,iprocloop)
+            islice_selected_source(isource) = iprocloop
+            ispec_selected_source(isource) = ispec_selected_source_all(isource_in_this_subset,iprocloop)
+            xi_source(isource) = xi_source_all(isource_in_this_subset,iprocloop)
+            eta_source(isource) = eta_source_all(isource_in_this_subset,iprocloop)
+            gamma_source(isource) = gamma_source_all(isource_in_this_subset,iprocloop)
+            x_found_source(isource_in_this_subset) = x_found_source_all(isource_in_this_subset,iprocloop)
+            y_found_source(isource_in_this_subset) = y_found_source_all(isource_in_this_subset,iprocloop)
+            z_found_source(isource_in_this_subset) = z_found_source_all(isource_in_this_subset,iprocloop)
+          endif
+        enddo
+        final_distance_source(isource) = distmin
 
-      write(IMAIN,*)
-      write(IMAIN,*) '*************************************'
-      write(IMAIN,*) ' locating source ',isource
-      write(IMAIN,*) '*************************************'
-      write(IMAIN,*)
-      write(IMAIN,*) 'source located in slice ',islice_selected_source(isource_in_this_subset)
-      write(IMAIN,*) '               in element ',ispec_selected_source(isource_in_this_subset)
-      write(IMAIN,*)
-      ! different output for force point sources
-      if(USE_FORCE_POINT_SOURCE) then
-        write(IMAIN,*) '  i index of source in that element: ',nint(xi_source(isource))
-        write(IMAIN,*) '  j index of source in that element: ',nint(eta_source(isource))
-        write(IMAIN,*) '  k index of source in that element: ',nint(gamma_source(isource))
         write(IMAIN,*)
-        write(IMAIN,*) '  component direction: ',COMPONENT_FORCE_SOURCE
+        write(IMAIN,*) '*************************************'
+        write(IMAIN,*) ' locating source ',isource
+        write(IMAIN,*) '*************************************'
         write(IMAIN,*)
-        write(IMAIN,*) '  nu1 = ',nu_source(1,:,isource)
-        write(IMAIN,*) '  nu2 = ',nu_source(2,:,isource)
-        write(IMAIN,*) '  nu3 = ',nu_source(3,:,isource)
+        write(IMAIN,*) 'source located in slice ',islice_selected_source(isource_in_this_subset)
+        write(IMAIN,*) '               in element ',ispec_selected_source(isource_in_this_subset)
         write(IMAIN,*)
-        write(IMAIN,*) '  at (x,y,z) coordinates = ',x_found_source(isource_in_this_subset),&
-          y_found_source(isource_in_this_subset),z_found_source(isource_in_this_subset)
+        ! different output for force point sources
+        if(USE_FORCE_POINT_SOURCE) then
+          write(IMAIN,*) '  i index of source in that element: ',nint(xi_source(isource))
+          write(IMAIN,*) '  j index of source in that element: ',nint(eta_source(isource))
+          write(IMAIN,*) '  k index of source in that element: ',nint(gamma_source(isource))
+          write(IMAIN,*)
+          write(IMAIN,*) '  component direction: ',COMPONENT_FORCE_SOURCE
+          write(IMAIN,*)
+          write(IMAIN,*) '  nu1 = ',nu_source(1,:,isource)
+          write(IMAIN,*) '  nu2 = ',nu_source(2,:,isource)
+          write(IMAIN,*) '  nu3 = ',nu_source(3,:,isource)
+          write(IMAIN,*)
+          write(IMAIN,*) '  at (x,y,z) coordinates = ',x_found_source(isource_in_this_subset),&
+            y_found_source(isource_in_this_subset),z_found_source(isource_in_this_subset)
 
-        ! prints frequency content for point forces
-        f0 = hdur(isource)
-        t0_ricker = 1.2d0/f0
-        write(IMAIN,*) '  using a source of dominant frequency ',f0
-        write(IMAIN,*) '  lambda_S at dominant frequency = ',3000./sqrt(3.)/f0
-        write(IMAIN,*) '  lambda_S at highest significant frequency = ',3000./sqrt(3.)/(2.5*f0)
-        write(IMAIN,*) '  t0_ricker = ',t0_ricker,'tshift_cmt = ',tshift_cmt(isource)
-        write(IMAIN,*)
-        write(IMAIN,*) '  half duration -> frequency: ',hdur(isource),' seconds**(-1)'
-      else
-        write(IMAIN,*) '   xi coordinate of source in that element: ',xi_source(isource)
-        write(IMAIN,*) '  eta coordinate of source in that element: ',eta_source(isource)
-        write(IMAIN,*) 'gamma coordinate of source in that element: ',gamma_source(isource)
-        ! add message if source is a Heaviside
-        if(hdur(isource) <= 5.*DT) then
+          ! prints frequency content for point forces
+          f0 = hdur(isource)
+          t0_ricker = 1.2d0/f0
+          write(IMAIN,*) '  using a source of dominant frequency ',f0
+          write(IMAIN,*) '  lambda_S at dominant frequency = ',3000./sqrt(3.)/f0
+          write(IMAIN,*) '  lambda_S at highest significant frequency = ',3000./sqrt(3.)/(2.5*f0)
+          write(IMAIN,*) '  t0_ricker = ',t0_ricker,'tshift_cmt = ',tshift_cmt(isource)
           write(IMAIN,*)
-          write(IMAIN,*) 'Source time function is a Heaviside, convolve later'
+          write(IMAIN,*) '  half duration -> frequency: ',hdur(isource),' seconds**(-1)'
+        else
+          write(IMAIN,*) '   xi coordinate of source in that element: ',xi_source(isource)
+          write(IMAIN,*) '  eta coordinate of source in that element: ',eta_source(isource)
+          write(IMAIN,*) 'gamma coordinate of source in that element: ',gamma_source(isource)
+          ! add message if source is a Heaviside
+          if(hdur(isource) <= 5.*DT) then
+            write(IMAIN,*)
+            write(IMAIN,*) 'Source time function is a Heaviside, convolve later'
+            write(IMAIN,*)
+          endif
           write(IMAIN,*)
+          write(IMAIN,*) ' half duration: ',hdur(isource),' seconds'
         endif
-        write(IMAIN,*)
-        write(IMAIN,*) ' half duration: ',hdur(isource),' seconds'
-      endif
-      write(IMAIN,*) '    time shift: ',tshift_cmt(isource),' seconds'
+        write(IMAIN,*) '    time shift: ',tshift_cmt(isource),' seconds'
 
-      ! writes out actual source position to vtk file
-      write(IOVTK,*) sngl(x_found_source(isource_in_this_subset)), &
-                    sngl(y_found_source(isource_in_this_subset)), &
-                    sngl(z_found_source(isource_in_this_subset))
+        ! writes out actual source position to vtk file
+        write(IOVTK,*) sngl(x_found_source(isource_in_this_subset)), &
+                      sngl(y_found_source(isource_in_this_subset)), &
+                      sngl(z_found_source(isource_in_this_subset))
 
-      ! get latitude, longitude and depth of the source that will be used
-      call xyz_2_rthetaphi_dble(x_found_source(isource_in_this_subset),y_found_source(isource_in_this_subset), &
-           z_found_source(isource_in_this_subset),r_found_source,theta_source(isource),phi_source(isource))
-      call reduce(theta_source(isource),phi_source(isource))
+        ! get latitude, longitude and depth of the source that will be used
+        call xyz_2_rthetaphi_dble(x_found_source(isource_in_this_subset), &
+                                 y_found_source(isource_in_this_subset), &
+                                 z_found_source(isource_in_this_subset), &
+                                 r_found_source,theta_source(isource),phi_source(isource))
+        call reduce(theta_source(isource),phi_source(isource))
 
-      ! convert geocentric to geographic colatitude
-      colat_source = PI/2.0d0 &
-      - datan(1.006760466d0*dcos(theta_source(isource))/dmax1(TINYVAL,dsin(theta_source(isource))))
-      if(phi_source(isource)>PI) phi_source(isource)=phi_source(isource)-TWO_PI
+        ! convert geocentric to geographic colatitude
+        colat_source = PI/2.0d0 &
+        - datan(1.006760466d0*dcos(theta_source(isource))/dmax1(TINYVAL,dsin(theta_source(isource))))
+        if(phi_source(isource)>PI) phi_source(isource)=phi_source(isource)-TWO_PI
 
-      write(IMAIN,*)
-      write(IMAIN,*) 'original (requested) position of the source:'
-      write(IMAIN,*)
-      write(IMAIN,*) '      latitude: ',lat(isource)
-      write(IMAIN,*) '     longitude: ',long(isource)
-      write(IMAIN,*) '         depth: ',depth(isource),' km'
-      write(IMAIN,*)
+        write(IMAIN,*)
+        write(IMAIN,*) 'original (requested) position of the source:'
+        write(IMAIN,*)
+        write(IMAIN,*) '      latitude: ',lat(isource)
+        write(IMAIN,*) '     longitude: ',long(isource)
+        write(IMAIN,*) '         depth: ',depth(isource),' km'
+        write(IMAIN,*)
 
-      ! compute real position of the source
-      write(IMAIN,*) 'position of the source that will be used:'
-      write(IMAIN,*)
-      write(IMAIN,*) '      latitude: ',(PI/2.0d0-colat_source)*180.0d0/PI
-      write(IMAIN,*) '     longitude: ',phi_source(isource)*180.0d0/PI
-      write(IMAIN,*) '         depth: ',(r0-r_found_source)*R_EARTH/1000.0d0,' km'
-      write(IMAIN,*)
+        ! compute real position of the source
+        write(IMAIN,*) 'position of the source that will be used:'
+        write(IMAIN,*)
+        write(IMAIN,*) '      latitude: ',(PI/2.0d0-colat_source)*180.0d0/PI
+        write(IMAIN,*) '     longitude: ',phi_source(isource)*180.0d0/PI
+        write(IMAIN,*) '         depth: ',(r0-r_found_source)*R_EARTH/1000.0d0,' km'
+        write(IMAIN,*)
 
-      ! display error in location estimate
-      write(IMAIN,*) 'error in location of the source: ',sngl(final_distance_source(isource)),' km'
+        ! display error in location estimate
+        write(IMAIN,*) 'error in location of the source: ',sngl(final_distance_source(isource)),' km'
 
-      ! add warning if estimate is poor
-      ! (usually means source outside the mesh given by the user)
-      if(final_distance_source(isource) > 50.d0) then
-        write(IMAIN,*)
-        write(IMAIN,*) '*****************************************************'
-        write(IMAIN,*) '*****************************************************'
-        write(IMAIN,*) '***** WARNING: source location estimate is poor *****'
-        write(IMAIN,*) '*****************************************************'
-        write(IMAIN,*) '*****************************************************'
-      endif
+        ! add warning if estimate is poor
+        ! (usually means source outside the mesh given by the user)
+        if(final_distance_source(isource) > 50.d0) then
+          write(IMAIN,*)
+          write(IMAIN,*) '*****************************************************'
+          write(IMAIN,*) '*****************************************************'
+          write(IMAIN,*) '***** WARNING: source location estimate is poor *****'
+          write(IMAIN,*) '*****************************************************'
+          write(IMAIN,*) '*****************************************************'
+        endif
 
-      ! print source time function and spectrum
-      if(PRINT_SOURCE_TIME_FUNCTION) then
+        ! print source time function and spectrum
+        if(PRINT_SOURCE_TIME_FUNCTION) then
 
-        write(IMAIN,*)
-        write(IMAIN,*) 'printing the source-time function'
+          write(IMAIN,*)
+          write(IMAIN,*) 'printing the source-time function'
 
-        ! print the source-time function
-        if(NSOURCES == 1) then
-          plot_file = '/plot_source_time_function.txt'
-        else
-         if(isource < 10) then
-            write(plot_file,"('/plot_source_time_function',i1,'.txt')") isource
-          elseif(isource < 100) then
-            write(plot_file,"('/plot_source_time_function',i2,'.txt')") isource
+          ! print the source-time function
+          if(NSOURCES == 1) then
+            plot_file = '/plot_source_time_function.txt'
           else
-            write(plot_file,"('/plot_source_time_function',i3,'.txt')") isource
+           if(isource < 10) then
+              write(plot_file,"('/plot_source_time_function',i1,'.txt')") isource
+            elseif(isource < 100) then
+              write(plot_file,"('/plot_source_time_function',i2,'.txt')") isource
+            else
+              write(plot_file,"('/plot_source_time_function',i3,'.txt')") isource
+            endif
           endif
-        endif
-        open(unit=27,file=trim(OUTPUT_FILES)//plot_file,status='unknown')
+          open(unit=27,file=trim(OUTPUT_FILES)//plot_file,status='unknown')
 
-        scalar_moment = 0.
-        do i = 1,6
-          scalar_moment = scalar_moment + moment_tensor(i,isource)**2
-        enddo
-        scalar_moment = dsqrt(scalar_moment/2.)
+          scalar_moment = 0.
+          do i = 1,6
+            scalar_moment = scalar_moment + moment_tensor(i,isource)**2
+          enddo
+          scalar_moment = dsqrt(scalar_moment/2.)
 
-        ! define t0 as the earliest start time
-        ! note: this calculation here is only used for outputting the plot_source_time_function file
-        !          (see setup_sources_receivers.f90)
-        t0 = - 1.5d0*minval( tshift_cmt(:) - hdur(:) )
-        if( USE_FORCE_POINT_SOURCE ) t0 = - 1.2d0 * minval(tshift_cmt(:) - 1.0d0/hdur(:))
-        t_cmt_used(:) = t_cmt_used(:)
-        if( USER_T0 > 0.d0 ) then
-          if( t0 <= USER_T0 + min_tshift_cmt_original ) then
-            t_cmt_used(:) = tshift_cmt(:) + min_tshift_cmt_original
-            t0 = USER_T0
+          ! define t0 as the earliest start time
+          ! note: this calculation here is only used for outputting the plot_source_time_function file
+          !          (see setup_sources_receivers.f90)
+          t0 = - 1.5d0*minval( tshift_cmt(:) - hdur(:) )
+          if( USE_FORCE_POINT_SOURCE ) t0 = - 1.2d0 * minval(tshift_cmt(:) - 1.0d0/hdur(:))
+          t_cmt_used(:) = t_cmt_used(:)
+          if( USER_T0 > 0.d0 ) then
+            if( t0 <= USER_T0 + min_tshift_cmt_original ) then
+              t_cmt_used(:) = tshift_cmt(:) + min_tshift_cmt_original
+              t0 = USER_T0
+            endif
           endif
-        endif
-        ! convert the half duration for triangle STF to the one for gaussian STF
-        ! note: this calculation here is only used for outputting the plot_source_time_function file
-        !          (see setup_sources_receivers.f90)
-        hdur_gaussian(:) = hdur(:)/SOURCE_DECAY_MIMIC_TRIANGLE
+          ! convert the half duration for triangle STF to the one for gaussian STF
+          ! note: this calculation here is only used for outputting the plot_source_time_function file
+          !          (see setup_sources_receivers.f90)
+          hdur_gaussian(:) = hdur(:)/SOURCE_DECAY_MIMIC_TRIANGLE
 
-        ! writes out source time function to file
-        do it=1,NSTEP
-          time_source = dble(it-1)*DT-t0-t_cmt_used(isource)
-          if( USE_FORCE_POINT_SOURCE ) then
-            ! Ricker source time function
-            f0 = hdur(isource)
-            write(27,*) sngl(dble(it-1)*DT-t0), &
-              sngl(FACTOR_FORCE_SOURCE*comp_source_time_function_rickr(time_source,f0))
-          else
-            ! Gaussian source time function
-            write(27,*) sngl(dble(it-1)*DT-t0), &
-              sngl(scalar_moment*comp_source_time_function(time_source,hdur_gaussian(isource)))
-          endif
-        enddo
-        close(27)
+          ! writes out source time function to file
+          do it=1,NSTEP
+            time_source = dble(it-1)*DT-t0-t_cmt_used(isource)
+            if( USE_FORCE_POINT_SOURCE ) then
+              ! Ricker source time function
+              f0 = hdur(isource)
+              write(27,*) sngl(dble(it-1)*DT-t0), &
+                sngl(FACTOR_FORCE_SOURCE*comp_source_time_function_rickr(time_source,f0))
+            else
+              ! Gaussian source time function
+              write(27,*) sngl(dble(it-1)*DT-t0), &
+                sngl(scalar_moment*comp_source_time_function(time_source,hdur_gaussian(isource)))
+            endif
+          enddo
+          close(27)
 
-        write(IMAIN,*)
-        write(IMAIN,*) 'printing the source spectrum'
+          write(IMAIN,*)
+          write(IMAIN,*) 'printing the source spectrum'
 
-        ! print the spectrum of the derivative of the source from 0 to 1/8 Hz
-        if(NSOURCES == 1) then
-          plot_file = '/plot_source_spectrum.txt'
-        else
-         if(isource < 10) then
-            write(plot_file,"('/plot_source_spectrum',i1,'.txt')") isource
-          elseif(isource < 100) then
-            write(plot_file,"('/plot_source_spectrum',i2,'.txt')") isource
+          ! print the spectrum of the derivative of the source from 0 to 1/8 Hz
+          if(NSOURCES == 1) then
+            plot_file = '/plot_source_spectrum.txt'
           else
-            write(plot_file,"('/plot_source_spectrum',i3,'.txt')") isource
+           if(isource < 10) then
+              write(plot_file,"('/plot_source_spectrum',i1,'.txt')") isource
+            elseif(isource < 100) then
+              write(plot_file,"('/plot_source_spectrum',i2,'.txt')") isource
+            else
+              write(plot_file,"('/plot_source_spectrum',i3,'.txt')") isource
+            endif
           endif
-        endif
-        open(unit=27,file=trim(OUTPUT_FILES)//plot_file,status='unknown')
+          open(unit=27,file=trim(OUTPUT_FILES)//plot_file,status='unknown')
 
-        do iom=1,NSAMP_PLOT_SOURCE
-          om=TWO_PI*(1.0d0/8.0d0)*(iom-1)/dble(NSAMP_PLOT_SOURCE-1)
-          write(27,*) sngl(om/TWO_PI), &
-            sngl(scalar_moment*om*comp_source_spectrum(om,hdur(isource)))
-        enddo
-        close(27)
+          do iom=1,NSAMP_PLOT_SOURCE
+            om=TWO_PI*(1.0d0/8.0d0)*(iom-1)/dble(NSAMP_PLOT_SOURCE-1)
+            write(27,*) sngl(om/TWO_PI), &
+              sngl(scalar_moment*om*comp_source_spectrum(om,hdur(isource)))
+          enddo
+          close(27)
 
-      endif !PRINT_SOURCE_TIME_FUNCTION
+        endif !PRINT_SOURCE_TIME_FUNCTION
 
-    enddo ! end of loop on all the sources within current source subset
+      enddo ! end of loop on all the sources within current source subset
 
-  endif ! end of section executed by main process only
+    endif ! end of section executed by main process only
 
-! deallocate arrays specific to each subset
-  deallocate(final_distance_source_subset)
-  deallocate(ispec_selected_source_subset)
-  deallocate(ispec_selected_source_all)
-  deallocate(xi_source_all,eta_source_all,gamma_source_all,final_distance_source_all)
-  deallocate(x_found_source_all,y_found_source_all,z_found_source_all)
-  deallocate(xi_source_subset,eta_source_subset,gamma_source_subset)
-  deallocate(x_found_source,y_found_source,z_found_source)
+    ! deallocate arrays specific to each subset
+    deallocate(final_distance_source_subset)
+    deallocate(ispec_selected_source_subset)
+    deallocate(ispec_selected_source_all)
+    deallocate(xi_source_all,eta_source_all,gamma_source_all,final_distance_source_all)
+    deallocate(x_found_source_all,y_found_source_all,z_found_source_all)
+    deallocate(xi_source_subset,eta_source_subset,gamma_source_subset)
+    deallocate(x_found_source,y_found_source,z_found_source)
 
   enddo ! end of loop over all source subsets
 
-! display maximum error in location estimate
+  ! display maximum error in location estimate
   if(myrank == 0) then
     write(IMAIN,*)
     write(IMAIN,*) 'maximum error in location of the sources: ',sngl(maxval(final_distance_source)),' km'
@@ -816,14 +819,14 @@
   endif
 
 
-! main process broadcasts the results to all the slices
+  ! main process broadcasts the results to all the slices
   call MPI_BCAST(islice_selected_source,NSOURCES,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(ispec_selected_source,NSOURCES,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(xi_source,NSOURCES,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(eta_source,NSOURCES,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
   call MPI_BCAST(gamma_source,NSOURCES,MPI_DOUBLE_PRECISION,0,MPI_COMM_WORLD,ier)
 
-! elapsed time since beginning of source detection
+  ! elapsed time since beginning of source detection
   if(myrank == 0) then
     tCPU = MPI_WTIME() - time_start
     write(IMAIN,*)
@@ -833,9 +836,9 @@
     write(IMAIN,*)
   endif
 
-! stores source mask
+  ! stores source mask
   if( SAVE_SOURCE_MASK .and. SIMULATION_TYPE == 3 ) then
-    call save_mask_source(myrank,mask_source,NSPEC,LOCAL_PATH)
+    call save_mask_source(myrank,mask_source,NSPEC,LOCAL_TMP_PATH)
     deallocate( mask_source )
   endif
 
@@ -900,7 +903,7 @@
 !-------------------------------------------------------------------------------------------------
 !
 
-  subroutine save_mask_source(myrank,mask_source,NSPEC,LOCAL_PATH)
+  subroutine save_mask_source(myrank,mask_source,NSPEC,LOCAL_TMP_PATH)
 
 ! saves a mask in the crust_mantle region which is 0 around the source locations
 ! and 1 everywhere else
@@ -912,13 +915,14 @@
   integer :: myrank,NSPEC
 
   real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC) :: mask_source
-  character(len=150) :: LOCAL_PATH
+  character(len=150) :: LOCAL_TMP_PATH
 
   ! local parameters
   character(len=150) :: prname
 
   ! stores into file
-  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_PATH)
+  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH)
+
   open(unit=27,file=trim(prname)//'mask_source.bin',status='unknown',form='unformatted',action='write')
   write(27) mask_source
   close(27)

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/noise_tomography.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/noise_tomography.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/noise_tomography.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -73,41 +73,24 @@
 ! subroutine for NOISE TOMOGRAPHY
 ! read parameters
 
-  subroutine read_parameters_noise(myrank,nrec,NSTEP,nmovie_points, &
-                                   islice_selected_rec,xi_receiver,eta_receiver,gamma_receiver,nu, &
-                                   noise_sourcearray,xigll,yigll,zigll,nspec_top, &
-                                   NIT, ibool_crust_mantle, ibelm_top_crust_mantle, &
-                                   xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
-                                   irec_master_noise,normal_x_noise,normal_y_noise,normal_z_noise,mask_noise)
+  subroutine read_parameters_noise()
+
+  use specfem_par
+  use specfem_par_crustmantle
+  use specfem_par_movie
   implicit none
+
   include 'mpif.h'
   include "precision.h"
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-  ! input parameters
-  integer :: myrank, nrec, NSTEP, nmovie_points, nspec_top, NIT
-  integer, dimension(nrec) :: islice_selected_rec
-  integer, dimension(NSPEC2D_TOP_CM) :: ibelm_top_crust_mantle
-  integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle
-  double precision, dimension(nrec)  :: xi_receiver,eta_receiver,gamma_receiver
-  double precision, dimension(NGLLX) :: xigll
-  double precision, dimension(NGLLY) :: yigll
-  double precision, dimension(NGLLZ) :: zigll
-  double precision, dimension(NDIM,NDIM,nrec) :: nu
-  real(kind=CUSTOM_REAL), dimension(NGLOB_CRUST_MANTLE) :: &
-        xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle
-  ! output parameters
-  integer :: irec_master_noise
-  real(kind=CUSTOM_REAL) :: noise_sourcearray(NDIM,NGLLX,NGLLY,NGLLZ,NSTEP)
-  real(kind=CUSTOM_REAL), dimension(nmovie_points) :: normal_x_noise,normal_y_noise,normal_z_noise,mask_noise
+
   ! local parameters
   integer :: ipoin, ispec2D, ispec, i, j, k, iglob, ios, ier
   real(kind=CUSTOM_REAL) :: normal_x_noise_out,normal_y_noise_out,normal_z_noise_out,mask_noise_out
   character(len=150) :: filename
   real(kind=CUSTOM_REAL), dimension(nmovie_points) :: &
-      store_val_x,store_val_y,store_val_z,  store_val_ux,store_val_uy,store_val_uz
+      val_x,val_y,val_z,val_ux,val_uy,val_uz
   real(kind=CUSTOM_REAL), dimension(nmovie_points,0:NPROCTOT_VAL-1) :: &
-      store_val_x_all,store_val_y_all,store_val_z_all, store_val_ux_all,store_val_uy_all,store_val_uz_all
+      val_x_all,val_y_all,val_z_all,val_ux_all,val_uy_all,val_uz_all
 
 
   ! read master receiver ID -- the ID in DATA/STATIONS
@@ -141,7 +124,7 @@
 
   ! noise distribution and noise direction
   ipoin = 0
-  do ispec2D = 1, nspec_top ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
+  do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
     ispec = ibelm_top_crust_mantle(ispec2D)
 
     k = NGLLZ
@@ -167,7 +150,7 @@
 
   !!!BEGIN!!! save mask_noise for check, a file called "mask_noise" is saved in "./OUTPUT_FIELS/"
   ipoin = 0
-  do ispec2D = 1, nspec_top ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
+  do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
     ispec = ibelm_top_crust_mantle(ispec2D)
     k = NGLLZ
     ! loop on all the points inside the element
@@ -175,24 +158,24 @@
       do i = 1,NGLLX,NIT
         ipoin = ipoin + 1
         iglob = ibool_crust_mantle(i,j,k,ispec)
-        store_val_x(ipoin) = xstore_crust_mantle(iglob)
-        store_val_y(ipoin) = ystore_crust_mantle(iglob)
-        store_val_z(ipoin) = zstore_crust_mantle(iglob)
-        store_val_ux(ipoin) = mask_noise(ipoin)
-        store_val_uy(ipoin) = mask_noise(ipoin)
-        store_val_uz(ipoin) = mask_noise(ipoin)
+        val_x(ipoin) = xstore_crust_mantle(iglob)
+        val_y(ipoin) = ystore_crust_mantle(iglob)
+        val_z(ipoin) = zstore_crust_mantle(iglob)
+        val_ux(ipoin) = mask_noise(ipoin)
+        val_uy(ipoin) = mask_noise(ipoin)
+        val_uz(ipoin) = mask_noise(ipoin)
       enddo
     enddo
   enddo
 
   ! gather info on master proc
   ispec = nmovie_points
-  call MPI_GATHER(store_val_x,ispec,CUSTOM_MPI_TYPE,store_val_x_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(store_val_y,ispec,CUSTOM_MPI_TYPE,store_val_y_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(store_val_z,ispec,CUSTOM_MPI_TYPE,store_val_z_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(store_val_ux,ispec,CUSTOM_MPI_TYPE,store_val_ux_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(store_val_uy,ispec,CUSTOM_MPI_TYPE,store_val_uy_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
-  call MPI_GATHER(store_val_uz,ispec,CUSTOM_MPI_TYPE,store_val_uz_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
+  call MPI_GATHER(val_x,ispec,CUSTOM_MPI_TYPE,val_x_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
+  call MPI_GATHER(val_y,ispec,CUSTOM_MPI_TYPE,val_y_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
+  call MPI_GATHER(val_z,ispec,CUSTOM_MPI_TYPE,val_z_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
+  call MPI_GATHER(val_ux,ispec,CUSTOM_MPI_TYPE,val_ux_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
+  call MPI_GATHER(val_uy,ispec,CUSTOM_MPI_TYPE,val_uy_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
+  call MPI_GATHER(val_uz,ispec,CUSTOM_MPI_TYPE,val_uz_all,ispec,CUSTOM_MPI_TYPE,0,MPI_COMM_WORLD,ier)
 
   ! save maks_noise data to disk in home directory
   ! this file can be viewed the same way as surface movie data (xcreate_movie_AVS_DX)
@@ -203,12 +186,12 @@
               status='unknown',form='unformatted',action='write',iostat=ios)
     if( ios /= 0 ) call exit_MPI(myrank,'error opening output file mask_noise')
 
-    write(IOUT_NOISE) store_val_x_all
-    write(IOUT_NOISE) store_val_y_all
-    write(IOUT_NOISE) store_val_z_all
-    write(IOUT_NOISE) store_val_ux_all
-    write(IOUT_NOISE) store_val_uy_all
-    write(IOUT_NOISE) store_val_uz_all
+    write(IOUT_NOISE) val_x_all
+    write(IOUT_NOISE) val_y_all
+    write(IOUT_NOISE) val_z_all
+    write(IOUT_NOISE) val_ux_all
+    write(IOUT_NOISE) val_uy_all
+    write(IOUT_NOISE) val_uz_all
 
     close(IOUT_NOISE)
   endif
@@ -223,19 +206,12 @@
 ! subroutine for NOISE TOMOGRAPHY
 ! check for consistency of the parameters
 
-  subroutine check_parameters_noise(myrank,NOISE_TOMOGRAPHY,SIMULATION_TYPE,SAVE_FORWARD, &
-                                    NUMBER_OF_RUNS, NUMBER_OF_THIS_RUN,ROTATE_SEISMOGRAMS_RT, &
-                                    SAVE_ALL_SEISMOS_IN_ONE_FILE, USE_BINARY_FOR_LARGE_FILE, &
-                                    MOVIE_COARSE,LOCAL_PATH,NSPEC_TOP,NSTEP)
+  subroutine check_parameters_noise()
+
+  use specfem_par
+  use specfem_par_crustmantle
   implicit none
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-  ! input parameters
-  integer :: myrank,NOISE_TOMOGRAPHY,SIMULATION_TYPE,NUMBER_OF_RUNS,NUMBER_OF_THIS_RUN,NSPEC_TOP,NSTEP
-  logical :: SAVE_FORWARD,ROTATE_SEISMOGRAMS_RT,SAVE_ALL_SEISMOS_IN_ONE_FILE, &
-            USE_BINARY_FOR_LARGE_FILE,MOVIE_COARSE
-  character(len=150) :: LOCAL_PATH
-  ! output parameters
+
   ! local parameters
   integer :: reclen,ier
   integer(kind=8) :: filesize
@@ -253,7 +229,7 @@
      WRITE(IOUT_NOISE,*) 'You are running simulations using NOISE TOMOGRAPHY techniques.'
      WRITE(IOUT_NOISE,*) 'Please make sure you understand the procedures before you have a try.'
      WRITE(IOUT_NOISE,*) 'Displacements everywhere at the free surface are saved every timestep,'
-     WRITE(IOUT_NOISE,*) 'so make sure that LOCAL_PATH in DATA/Par_file is not global.'
+     WRITE(IOUT_NOISE,*) 'so make sure that LOCAL_TMP_PATH in DATA/Par_file is not global.'
      WRITE(IOUT_NOISE,*) 'Otherwise the disk storage may be a serious issue, as is the speed of I/O.'
      WRITE(IOUT_NOISE,*) 'Also make sure that NO earthquakes are included,'
      WRITE(IOUT_NOISE,*) 'i.e., set moment tensor to be ZERO in CMTSOLUTION'
@@ -299,14 +275,14 @@
      filesize = filesize*NSTEP
 
      write(outputname,"('/proc',i6.6,'_surface_movie')") myrank
-     if (NOISE_TOMOGRAPHY==1) call open_file_abs_w(9,trim(LOCAL_PATH)//trim(outputname), &
-                                      len_trim(trim(LOCAL_PATH)//trim(outputname)), &
+     if (NOISE_TOMOGRAPHY==1) call open_file_abs_w(9,trim(LOCAL_TMP_PATH)//trim(outputname), &
+                                      len_trim(trim(LOCAL_TMP_PATH)//trim(outputname)), &
                                       filesize)
-     if (NOISE_TOMOGRAPHY==2) call open_file_abs_r(9,trim(LOCAL_PATH)//trim(outputname), &
-                                      len_trim(trim(LOCAL_PATH)//trim(outputname)), &
+     if (NOISE_TOMOGRAPHY==2) call open_file_abs_r(9,trim(LOCAL_TMP_PATH)//trim(outputname), &
+                                      len_trim(trim(LOCAL_TMP_PATH)//trim(outputname)), &
                                       filesize)
-     if (NOISE_TOMOGRAPHY==3) call open_file_abs_r(9,trim(LOCAL_PATH)//trim(outputname), &
-                                      len_trim(trim(LOCAL_PATH)//trim(outputname)), &
+     if (NOISE_TOMOGRAPHY==3) call open_file_abs_r(9,trim(LOCAL_TMP_PATH)//trim(outputname), &
+                                      len_trim(trim(LOCAL_TMP_PATH)//trim(outputname)), &
                                       filesize)
   endif
 
@@ -477,7 +453,7 @@
   ! get coordinates of surface mesh and surface displacement
   if( .not. GPU_MODE ) then
     ! on CPU
-    do ispec2D = 1, nspec_top ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
+    do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
       ispec = ibelm_top_crust_mantle(ispec2D)
       k = NGLLZ
       do j = 1,NGLLY
@@ -493,7 +469,7 @@
   endif
 
   ! save surface motion to disk
-  call write_abs(9,noise_surface_movie,CUSTOM_REAL*NDIM*NGLLX*NGLLY*nspec_top,it)
+  call write_abs(9,noise_surface_movie,CUSTOM_REAL*NDIM*NGLLX*NGLLY*NSPEC_TOP,it)
 
   end subroutine noise_save_surface_movie
 
@@ -533,13 +509,13 @@
 
 
   ! read surface movie
-  call read_abs(9,noise_surface_movie,CUSTOM_REAL*NDIM*NGLLX*NGLLY*nspec_top,it_index)
+  call read_abs(9,noise_surface_movie,CUSTOM_REAL*NDIM*NGLLX*NGLLY*NSPEC_TOP,it_index)
 
   ! get coordinates of surface mesh and surface displacement
   if( .not. GPU_MODE ) then
     ! on CPU
     ipoin = 0
-    do ispec2D = 1, nspec_top ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
+    do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE)
       ispec = ibelm_top_crust_mantle(ispec2D)
 
       k = NGLLZ
@@ -598,25 +574,13 @@
   use specfem_par_crustmantle
 
   implicit none
-!  include "constants.h"
-!  include "OUTPUT_FILES/values_from_mesher.h"
-  ! input parameters
-!  integer :: it,nspec_top,nmovie_points
-!  integer, dimension(NSPEC2D_TOP_CM) :: ibelm_top_crust_mantle
-!  integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle
-!  real(kind=CUSTOM_REAL) :: deltat
-!  real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE_ADJOINT) :: displ_crust_mantle
-!  real(kind=CUSTOM_REAL), dimension(nmovie_points) :: normal_x_noise,normal_y_noise,normal_z_noise
-  ! output parameters
-!  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: &
-!    Sigma_kl_crust_mantle
+
   ! local parameters
   integer :: i,j,k,ispec,iglob,ipoin,ispec2D
   real(kind=CUSTOM_REAL) :: eta
-!  real(kind=CUSTOM_REAL), dimension(NDIM,NGLLX,NGLLY,nspec_top) :: noise_surface_movie
 
   ! read surface movie, needed for Sigma_kl_crust_mantle
-  call read_abs(9,noise_surface_movie,CUSTOM_REAL*NDIM*NGLLX*NGLLY*nspec_top,it)
+  call read_abs(9,noise_surface_movie,CUSTOM_REAL*NDIM*NGLLX*NGLLY*NSPEC_TOP,it)
 
   ! noise source strength kernel
   ! to keep similar structure to other kernels, the source strength kernel is saved as a volumetric kernel
@@ -624,7 +588,7 @@
   if( .not. GPU_MODE ) then
     ! on CPU
     ipoin = 0
-    do ispec2D = 1, nspec_top
+    do ispec2D = 1, NSPEC_TOP
       ispec = ibelm_top_crust_mantle(ispec2D)
 
       k = NGLLZ
@@ -661,20 +625,16 @@
 ! subroutine for NOISE TOMOGRAPHY
 ! step 3: save noise source strength kernel
 
-  subroutine save_kernels_strength_noise(myrank,LOCAL_PATH,Sigma_kl_crust_mantle)
+  subroutine save_kernels_strength_noise()
+
+  use specfem_par
+  use specfem_par_crustmantle
   implicit none
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-  ! input parameters
-  integer myrank
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: Sigma_kl_crust_mantle
-  character(len=150) :: LOCAL_PATH
-  ! output parameters
+
   ! local parameters
-  character(len=150) :: prname
   integer :: ier
 
-  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_PATH)
+  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH)
 
   open(unit=IOUT_NOISE,file=trim(prname)//'Sigma_kernel.bin', &
         status='unknown',form='unformatted',action='write',iostat=ier)

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -79,6 +79,7 @@
   if(GPU_MODE) call prepare_timerun_GPU()
 
   ! user output
+  call sync_all()
   if( myrank == 0 ) then
     ! elapsed time since beginning of mesh generation
     tCPU = MPI_WTIME() - time_start
@@ -426,17 +427,18 @@
   if (NSPEC_CRUST_MANTLE_STRAIN_ONLY /= NSPEC_CRUST_MANTLE) &
     stop 'NSPEC_CRUST_MANTLE_STRAIN_ONLY /= NSPEC_CRUST_MANTLE'
 
-  write(prname,'(a,i6.6,a)') trim(LOCAL_PATH)//'/'//'proc',myrank,'_'
-  call count_points_movie_volume(prname,ibool_crust_mantle, xstore_crust_mantle,ystore_crust_mantle, &
-              zstore_crust_mantle,MOVIE_TOP,MOVIE_BOTTOM,MOVIE_WEST,MOVIE_EAST,MOVIE_NORTH,MOVIE_SOUTH, &
-              MOVIE_COARSE,npoints_3dmovie,nspecel_3dmovie,num_ibool_3dmovie,mask_ibool,mask_3dmovie)
+  call count_points_movie_volume()
+!              LOCAL_TMP_PATH,ibool_crust_mantle, xstore_crust_mantle,ystore_crust_mantle, &
+!              zstore_crust_mantle,MOVIE_TOP,MOVIE_BOTTOM,MOVIE_WEST,MOVIE_EAST,MOVIE_NORTH,MOVIE_SOUTH, &
+!              MOVIE_COARSE,npoints_3dmovie,nspecel_3dmovie,num_ibool_3dmovie,mask_ibool,mask_3dmovie)
 
   allocate(nu_3dmovie(3,3,npoints_3dmovie),stat=ier)
   if( ier /= 0 ) call exit_MPI(myrank,'error allocating nu for 3d movie')
 
-  call write_movie_volume_mesh(npoints_3dmovie,prname,ibool_crust_mantle,xstore_crust_mantle, &
-                         ystore_crust_mantle,zstore_crust_mantle, muvstore_crust_mantle_3dmovie, &
-                         mask_3dmovie,mask_ibool,num_ibool_3dmovie,nu_3dmovie,MOVIE_COARSE)
+  call write_movie_volume_mesh()
+!                         npoints_3dmovie,LOCAL_TMP_PATH,ibool_crust_mantle,xstore_crust_mantle, &
+!                         ystore_crust_mantle,zstore_crust_mantle, muvstore_crust_mantle_3dmovie, &
+!                         mask_3dmovie,mask_ibool,num_ibool_3dmovie,nu_3dmovie,MOVIE_COARSE)
 
   if(myrank == 0) then
     write(IMAIN,*)
@@ -993,14 +995,14 @@
 
   ! NOISE TOMOGRAPHY
   if ( NOISE_TOMOGRAPHY /= 0 ) then
-    nspec_top = NSPEC2D_TOP(IREGION_CRUST_MANTLE)
+    NSPEC_TOP = NSPEC2D_TOP(IREGION_CRUST_MANTLE)
 
     allocate(noise_sourcearray(NDIM,NGLLX,NGLLY,NGLLZ,NSTEP), &
             normal_x_noise(nmovie_points), &
             normal_y_noise(nmovie_points), &
             normal_z_noise(nmovie_points), &
             mask_noise(nmovie_points), &
-            noise_surface_movie(NDIM,NGLLX,NGLLY,nspec_top),stat=ier)
+            noise_surface_movie(NDIM,NGLLX,NGLLY,NSPEC_TOP),stat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error allocating noise arrays')
 
     noise_sourcearray(:,:,:,:,:) = 0._CUSTOM_REAL
@@ -1010,17 +1012,10 @@
     mask_noise(:)                = 0._CUSTOM_REAL
     noise_surface_movie(:,:,:,:) = 0._CUSTOM_REAL
 
-    call read_parameters_noise(myrank,nrec,NSTEP,nmovie_points, &
-                              islice_selected_rec,xi_receiver,eta_receiver,gamma_receiver,nu, &
-                              noise_sourcearray,xigll,yigll,zigll,nspec_top, &
-                              NIT, ibool_crust_mantle, ibelm_top_crust_mantle, &
-                              xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
-                              irec_master_noise,normal_x_noise,normal_y_noise,normal_z_noise,mask_noise)
+    call read_parameters_noise()
 
-    call check_parameters_noise(myrank,NOISE_TOMOGRAPHY,SIMULATION_TYPE,SAVE_FORWARD, &
-                              NUMBER_OF_RUNS, NUMBER_OF_THIS_RUN,ROTATE_SEISMOGRAMS_RT, &
-                              SAVE_ALL_SEISMOS_IN_ONE_FILE, USE_BINARY_FOR_LARGE_FILE, &
-                              MOVIE_COARSE,LOCAL_PATH,nspec_top,NSTEP)
+    call check_parameters_noise()
+
   endif
 
   end subroutine prepare_timerun_noise
@@ -1074,7 +1069,7 @@
                                   number_receiver_global,islice_selected_rec,ispec_selected_rec, &
                                   nrec, nrec_local, nadj_rec_local, &
                                   NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE, &
-                                  NGLOB_CRUST_MANTLE_OCEANS, &
+                                  NSPEC_CRUST_MANTLE_STRAIN_ONLY,NGLOB_CRUST_MANTLE_OCEANS, &
                                   NSPEC_OUTER_CORE,NGLOB_OUTER_CORE, &
                                   NSPEC_INNER_CORE,NGLOB_INNER_CORE, &
                                   SIMULATION_TYPE,NOISE_TOMOGRAPHY, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_forward_arrays.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_forward_arrays.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_forward_arrays.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -72,7 +72,7 @@
   ! read files back from local disk or MT tape system if restart file
   if(NUMBER_OF_THIS_RUN > 1) then
     write(outputname,"('dump_all_arrays',i6.6)") myrank
-    open(unit=55,file=trim(LOCAL_PATH)//'/'//outputname,status='old',action='read',form='unformatted')
+    open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='old',action='read',form='unformatted')
 
     read(55) displ_crust_mantle
     read(55) veloc_crust_mantle
@@ -179,11 +179,11 @@
   character(len=150) outputname
 
   write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin'
-  open(unit=55,file=trim(LOCAL_PATH)//'/'//outputname, &
+  open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname, &
         status='old',action='read',form='unformatted',iostat=ier)
   if( ier /= 0 ) then
     print*,'error: opening proc_****_save_forward_arrays.bin'
-    print*,'path: ',trim(LOCAL_PATH)//'/'//outputname
+    print*,'path: ',trim(LOCAL_TMP_PATH)//'/'//outputname
     call exit_mpi(myrank,'error open file save_forward_arrays.bin')
   endif
 

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -91,7 +91,6 @@
     write(IMAIN,*)
   endif
 
-  
   ! frees temporary allocated arrays
   deallocate(is_on_a_slice_edge_crust_mantle, &
             is_on_a_slice_edge_outer_core, &
@@ -809,7 +808,7 @@
   real(kind=CUSTOM_REAL),dimension(:),allocatable :: test_flag_cc
   integer,dimension(:),allocatable :: dummy_i
   integer :: i,j,k,ispec,iglob
-  
+
   ! estimates initial maximum ibool array
   max_nibool = npoin2D_max_all_CM_IC * NUMFACES_SHARED &
                + non_zero_nb_msgs_theor_in_cube*npoin2D_cube_from_slices
@@ -854,7 +853,7 @@
 
   allocate(dummy_i(NSPEC_CRUST_MANTLE),stat=ier)
   if( ier /= 0 ) call exit_mpi(myrank,'error allocating dummy_i')
-  
+
   ! determines neighbor rank for shared faces
   call rmd_get_MPI_interfaces(myrank,NGLOB_CRUST_MANTLE,NSPEC_CRUST_MANTLE, &
                             test_flag,my_neighbours,nibool_neighbours,ibool_neighbours, &
@@ -866,7 +865,7 @@
 
   deallocate(test_flag)
   deallocate(dummy_i)
-  
+
   ! stores MPI interfaces informations
   allocate(my_neighbours_crust_mantle(num_interfaces_crust_mantle), &
           nibool_interfaces_crust_mantle(num_interfaces_crust_mantle), &
@@ -874,14 +873,14 @@
   if( ier /= 0 ) call exit_mpi(myrank,'error allocating array my_neighbours_crust_mantle etc.')
   my_neighbours_crust_mantle = -1
   nibool_interfaces_crust_mantle = 0
-  
+
   ! copies interfaces arrays
   if( num_interfaces_crust_mantle > 0 ) then
     allocate(ibool_interfaces_crust_mantle(max_nibool_interfaces_crust_mantle,num_interfaces_crust_mantle), &
            stat=ier)
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating array ibool_interfaces_crust_mantle')
     ibool_interfaces_crust_mantle = 0
-    
+
     ! ranks of neighbour processes
     my_neighbours_crust_mantle(:) = my_neighbours(1:num_interfaces_crust_mantle)
     ! number of global ibool entries on each interface
@@ -923,10 +922,10 @@
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating array b_buffer_send_vector_crust_mantle etc.')
   endif
 
-  ! checks with assembly of test fields  
+  ! checks with assembly of test fields
   call rmd_test_MPI_cm()
 
-  
+
 ! outer core region
   if( myrank == 0 ) write(IMAIN,*) 'outer core mpi:'
 
@@ -976,7 +975,7 @@
 
   deallocate(test_flag)
   deallocate(dummy_i)
-  
+
   ! stores MPI interfaces informations
   allocate(my_neighbours_outer_core(num_interfaces_outer_core), &
           nibool_interfaces_outer_core(num_interfaces_outer_core), &
@@ -984,14 +983,14 @@
   if( ier /= 0 ) call exit_mpi(myrank,'error allocating array my_neighbours_outer_core etc.')
   my_neighbours_outer_core = -1
   nibool_interfaces_outer_core = 0
-  
+
   ! copies interfaces arrays
   if( num_interfaces_outer_core > 0 ) then
     allocate(ibool_interfaces_outer_core(max_nibool_interfaces_outer_core,num_interfaces_outer_core), &
            stat=ier)
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating array ibool_interfaces_outer_core')
     ibool_interfaces_outer_core = 0
-    
+
     ! ranks of neighbour processes
     my_neighbours_outer_core(:) = my_neighbours(1:num_interfaces_outer_core)
     ! number of global ibool entries on each interface
@@ -1033,7 +1032,7 @@
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating array b_buffer_send_vector_outer_core etc.')
   endif
 
-  ! checks with assembly of test fields  
+  ! checks with assembly of test fields
   call rmd_test_MPI_oc()
 
 
@@ -1049,15 +1048,15 @@
   do ispec=1,NSPEC_INNER_CORE
     ! suppress fictitious elements in central cube
     if(idoubling_inner_core(ispec) == IFLAG_IN_FICTITIOUS_CUBE) cycle
-    
+
     ! suppress central cube, will be handled afterwards
     if( INCLUDE_CENTRAL_CUBE ) then
       if(idoubling_inner_core(ispec) == IFLAG_MIDDLE_CENTRAL_CUBE .or. &
         idoubling_inner_core(ispec) == IFLAG_BOTTOM_CENTRAL_CUBE .or. &
         idoubling_inner_core(ispec) == IFLAG_TOP_CENTRAL_CUBE) cycle
     endif
-    
-    ! sets flags  
+
+    ! sets flags
     do k = 1,NGLLZ
       do j = 1,NGLLY
         do i = 1,NGLLX
@@ -1094,7 +1093,7 @@
                         test_flag,filename)
 
   ! in sequential order, for testing purpose
-  do i=0,NPROCTOT_VAL - 1  
+  do i=0,NPROCTOT_VAL - 1
     if( myrank == i ) then
 
     ! gets new interfaces for inner_core without central cube yet
@@ -1133,13 +1132,13 @@
     do ispec=1,NSPEC_INNER_CORE
       ! suppress fictitious elements in central cube
       if(idoubling_inner_core(ispec) == IFLAG_IN_FICTITIOUS_CUBE) cycle
-      
+
       ! only takes central cube
       if(idoubling_inner_core(ispec) /= IFLAG_MIDDLE_CENTRAL_CUBE .and. &
         idoubling_inner_core(ispec) /= IFLAG_BOTTOM_CENTRAL_CUBE .and. &
         idoubling_inner_core(ispec) /= IFLAG_TOP_CENTRAL_CUBE) cycle
 
-      ! sets flags    
+      ! sets flags
       do k = 1,NGLLZ
         do j = 1,NGLLY
           do i = 1,NGLLX
@@ -1172,7 +1171,7 @@
                         test_flag_cc,filename)
 
     ! in sequential order, for testing purpose
-    do i=0,NPROCTOT_VAL - 1  
+    do i=0,NPROCTOT_VAL - 1
       if( myrank == i ) then
         ! adds additional inner core points
         call rmd_get_MPI_interfaces(myrank,NGLOB_INNER_CORE,NSPEC_INNER_CORE, &
@@ -1200,7 +1199,7 @@
   endif
 
 !  ! in sequential order, for testing purpose
-!  do i=0,NPROCTOT_VAL - 1  
+!  do i=0,NPROCTOT_VAL - 1
 !    if( myrank == i ) then
 !      ! gets new interfaces for inner_core without central cube yet
 !      ! determines neighbor rank for shared faces
@@ -1215,7 +1214,7 @@
 !    endif
 !    call sync_all()
 !  enddo
-  
+
   deallocate(test_flag)
   call sync_all()
 
@@ -1226,7 +1225,7 @@
   if( ier /= 0 ) call exit_mpi(myrank,'error allocating array my_neighbours_inner_core etc.')
   my_neighbours_inner_core = -1
   nibool_interfaces_inner_core = 0
-  
+
   ! copies interfaces arrays
   if( num_interfaces_inner_core > 0 ) then
     allocate(ibool_interfaces_inner_core(max_nibool_interfaces_inner_core,num_interfaces_inner_core), &
@@ -1259,7 +1258,7 @@
     enddo
   !endif
   call sync_all()
-  
+
   ! checks addressing
   call rmd_test_MPI_neighbours(num_interfaces_inner_core,my_neighbours_inner_core,nibool_interfaces_inner_core)
 
@@ -1280,7 +1279,7 @@
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating array b_buffer_send_vector_inner_core etc.')
   endif
 
-  ! checks with assembly of test fields  
+  ! checks with assembly of test fields
   call rmd_test_MPI_ic()
 
   ! synchronizes MPI processes
@@ -1330,7 +1329,7 @@
   integer,dimension(NSPEC),intent(in) :: idoubling
 
   logical,intent(in) :: INCLUDE_CENTRAL_CUBE
-  
+
   ! local parameters
   integer :: ispec,iglob,j,k
   integer :: iface,iedge,icorner
@@ -1341,7 +1340,7 @@
   logical,dimension(NSPEC) :: work_ispec_is_outer
 
   integer,parameter :: MID = (NGLLX+1)/2
-  
+
   ! initializes
   if( add_central_cube) then
     ! adds points to existing inner_core interfaces
@@ -1474,10 +1473,10 @@
                                         .true.,add_central_cube)
             ! debug
             if( work_test_flag(iglob) < 0 ) then
-              if( IREGION == IREGION_INNER_CORE .and. INCLUDE_CENTRAL_CUBE ) then              
+              if( IREGION == IREGION_INNER_CORE .and. INCLUDE_CENTRAL_CUBE ) then
                 ! we might have missed an interface point on an edge, just re-set to missing value
                 print*,'warning face flag:',myrank,'ispec=',ispec,'rank=',rank
-                print*,'  flag=',work_test_flag(iglob),'iface jk=',iface,j,k,'missed iglob=',iglob                
+                print*,'  flag=',work_test_flag(iglob),'iface jk=',iface,j,k,'missed iglob=',iglob
                 !work_test_flag(iglob) = 0
               else
                 print*,'error face flag:',myrank,'ispec=',ispec,'rank=',rank
@@ -1485,7 +1484,7 @@
                 call exit_mpi(myrank,'error face flag')
               endif
             endif
-            
+
           enddo
         enddo
       endif
@@ -1626,10 +1625,10 @@
 
           ! debug
           if( work_test_flag(iglob) < 0 ) then
-            if( IREGION == IREGION_INNER_CORE .and. INCLUDE_CENTRAL_CUBE ) then              
+            if( IREGION == IREGION_INNER_CORE .and. INCLUDE_CENTRAL_CUBE ) then
               ! we might have missed an interface point on an edge, just re-set to missing value
               print*,'warning edge flag:',myrank,'ispec=',ispec,'rank=',rank
-              print*,'  flag=',work_test_flag(iglob),'iedge jk=',iedge,k,'missed iglob=',iglob                
+              print*,'  flag=',work_test_flag(iglob),'iedge jk=',iedge,k,'missed iglob=',iglob
               !work_test_flag(iglob) = 0
             else
               print*,'error edge flag:',myrank,'ispec=',ispec,'rank=',rank
@@ -1762,7 +1761,7 @@
     print*, '  outer elements: ',npoin
     print*
   endif
-  
+
   ! checks if all points were recognized
   if( minval(work_test_flag) < 0 .or. maxval(work_test_flag) > 0 ) then
     print*,'error mpi interface rank: ',myrank
@@ -1786,11 +1785,11 @@
     do j=1,npoin-1
       if( ibool_neighbours(j,iinterface) == ibool_neighbours(j+1,iinterface) ) then
         if( IREGION == IREGION_INNER_CORE .and. INCLUDE_CENTRAL_CUBE ) then
-          ! missing points might have been counted more than once  
+          ! missing points might have been counted more than once
           if( ibool_neighbours(j,iinterface) > 0 ) then
             print*,'warning mpi interface rank:',myrank
             print*,'  interface: ',my_neighbours(iinterface),'point: ',j,'of',npoin,'iglob=',ibool_neighbours(j,iinterface)
-            ! decrease number of points          
+            ! decrease number of points
             nibool_neighbours(iinterface) = nibool_neighbours(iinterface) - 1
             if( nibool_neighbours(iinterface) <= 0 ) then
               print*,'error zero mpi interface rank:',myrank,'interface=',my_neighbours(iinterface)
@@ -1801,7 +1800,7 @@
               ii = ibool_neighbours(k+1,iinterface)
               ibool_neighbours(k,iinterface) = ii
             enddo
-            ! re-sets values  
+            ! re-sets values
             ibool_neighbours(npoin,iinterface) = 0
             npoin = nibool_neighbours(iinterface)
             max_nibool_interfaces = maxval( nibool_neighbours(1:num_interfaces) )
@@ -1836,16 +1835,16 @@
 
   integer,intent(in) :: iglob,rank,icurrent
   integer,intent(in) :: myrank
-  
+
   integer,intent(in) :: MAX_NEIGHBOURS,max_nibool
   integer, dimension(MAX_NEIGHBOURS),intent(inout) :: nibool_neighbours
   integer, dimension(max_nibool,MAX_NEIGHBOURS),intent(inout) :: ibool_neighbours
 
   integer,intent(in) :: NGLOB
   integer,dimension(NGLOB) :: work_test_flag
-  
+
   logical,intent(in) :: is_face_edge,add_central_cube
-  
+
   ! local parameters
   integer :: i
   logical :: is_done
@@ -1861,8 +1860,8 @@
       exit
     endif
   enddo
-  
-  ! checks if anything to do  
+
+  ! checks if anything to do
   if( is_done ) then
     ! special handling for central cube: removes rank if already added in inner core
     if( add_central_cube ) then
@@ -1871,11 +1870,11 @@
         work_test_flag(iglob) = work_test_flag(iglob) + (rank + 1)
       endif
       ! re-sets flag
-      work_test_flag(iglob) = work_test_flag(iglob) - ( rank + 1 )      
+      work_test_flag(iglob) = work_test_flag(iglob) - ( rank + 1 )
       if( is_face_edge .and. work_test_flag(iglob) < 0 ) then
         ! re-sets to zero if we missed this rank number
         if( work_test_flag(iglob) == - (rank + 1 ) ) work_test_flag(iglob) = 0
-      endif      
+      endif
     endif
     return
   endif
@@ -1884,7 +1883,7 @@
   if( work_test_flag(iglob) <= 0 ) then
     ! we might have missed an interface point on an edge, just re-set to missing value
     print*,'warning flag:',myrank,'rank=',rank,'interface=',icurrent
-    print*,'  flag=',work_test_flag(iglob),'missed iglob=',iglob                
+    print*,'  flag=',work_test_flag(iglob),'missed iglob=',iglob
   endif
   ! we might have missed an interface point on an edge, just re-set to missing value
   if( is_face_edge ) then
@@ -1893,7 +1892,7 @@
       work_test_flag(iglob) = work_test_flag(iglob) + (rank + 1)
     endif
   endif
-  
+
   ! adds point
   ! increases number of total points on this interface
   nibool_neighbours(icurrent) = nibool_neighbours(icurrent) + 1
@@ -1924,10 +1923,10 @@
   implicit none
 
   include 'mpif.h'
-  
+
   integer,intent(in) :: num_interfaces
   integer,dimension(num_interfaces),intent(in) :: my_neighbours,nibool_interfaces
-  
+
   ! local parameters
   integer,dimension(:),allocatable :: dummy_i
   integer,dimension(:,:),allocatable :: test_interfaces
@@ -1935,32 +1934,32 @@
   integer :: msg_status(MPI_STATUS_SIZE)
   integer :: ineighbour,iproc,inum,i,j,ier,ipoints,max_num
   logical :: is_okay
-  
+
   ! checks neighbors
   ! gets maximum interfaces from all processes
   call MPI_REDUCE(num_interfaces,max_num,1,MPI_INTEGER,MPI_MAX,0,MPI_COMM_WORLD,ier)
-  
+
   ! master gathers infos
   if( myrank == 0 ) then
     ! array for gathering infos
-    allocate(test_interfaces(max_num,0:NPROCTOT_VAL),stat=ier)  
+    allocate(test_interfaces(max_num,0:NPROCTOT_VAL),stat=ier)
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating test_interfaces')
     test_interfaces = -1
-  
-    allocate(test_interfaces_nibool(max_num,0:NPROCTOT_VAL),stat=ier)  
+
+    allocate(test_interfaces_nibool(max_num,0:NPROCTOT_VAL),stat=ier)
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating test_interfaces_nibool')
     test_interfaces_nibool = 0
 
     ! used to store number of interfaces per proc
-    allocate(dummy_i(0:NPROCTOT_VAL),stat=ier)  
+    allocate(dummy_i(0:NPROCTOT_VAL),stat=ier)
     if( ier /= 0 ) call exit_mpi(myrank,'error allocating dummy_i for test interfaces')
     dummy_i = 0
-    
+
     ! sets infos for master process
     test_interfaces(1:num_interfaces,0) = my_neighbours(1:num_interfaces)
     test_interfaces_nibool(1:num_interfaces,0) = nibool_interfaces(1:num_interfaces)
     dummy_i(0) = num_interfaces
-    
+
     ! collects from other processes
     do iproc=1,NPROCTOT_VAL-1
       ! gets number of interfaces
@@ -1971,7 +1970,7 @@
                       MPI_INTEGER,iproc,itag,MPI_COMM_WORLD,msg_status,ier)
         call MPI_RECV(test_interfaces_nibool(1:inum,iproc),inum, &
                       MPI_INTEGER,iproc,itag,MPI_COMM_WORLD,msg_status,ier)
-      endif  
+      endif
     enddo
   else
     ! sends infos to master process
@@ -1984,7 +1983,7 @@
     endif
   endif
   call sync_all()
-  
+
   ! checks if addressing is okay
   if( myrank == 0 ) then
     do iproc=0,NPROCTOT_VAL-1
@@ -1997,7 +1996,7 @@
         ipoints = test_interfaces_nibool(i,iproc)
         if( ipoints <= 0 ) then
           print*,'error neighbour points:',iproc,ipoints
-          call exit_mpi(myrank,'error ineighbour points')        
+          call exit_mpi(myrank,'error ineighbour points')
         endif
         ! looks up corresponding entry in neighbour array
         is_okay = .false.
@@ -2029,7 +2028,7 @@
     write(IMAIN,*) '  mpi addressing maximum interfaces:',maxval(dummy_i)
     write(IMAIN,*) '  mpi addressing : all interfaces okay'
     write(IMAIN,*)
-    
+
     deallocate(dummy_i)
     deallocate(test_interfaces)
   endif
@@ -2050,16 +2049,16 @@
   implicit none
 
   include 'mpif.h'
-  
+
   ! local parameters
-  real(kind=CUSTOM_REAL),dimension(:,:),allocatable :: test_flag_vector  
+  real(kind=CUSTOM_REAL),dimension(:,:),allocatable :: test_flag_vector
   integer :: i,j,iglob,ier
   integer :: inum,icount
 
   ! crust mantle
   allocate(test_flag_vector(NDIM,NGLOB_CRUST_MANTLE),stat=ier)
   if( ier /= 0 ) stop 'error allocating array test_flag etc.'
-  
+
   ! points defined by interfaces
   test_flag_vector = 0.0
   do i=1,num_interfaces_crust_mantle
@@ -2069,14 +2068,14 @@
     enddo
   enddo
   i = sum(nibool_interfaces_crust_mantle)
-  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)      
+  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
   i = nint( sum(test_flag_vector) )
-  call MPI_REDUCE(i,icount,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)      
+  call MPI_REDUCE(i,icount,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
   if( myrank == 0 ) then
     write(IMAIN,*) '  total MPI interface points : ',inum
-    write(IMAIN,*) '  unique MPI interface points: ',icount    
-  endif  
-  
+    write(IMAIN,*) '  unique MPI interface points: ',icount
+  endif
+
   ! initializes for assembly
   test_flag_vector = myrank + 1.0_CUSTOM_REAL
   call sync_all()
@@ -2099,7 +2098,7 @@
                               nibool_interfaces_crust_mantle,ibool_interfaces_crust_mantle, &
                               request_send_vector_crust_mantle,request_recv_vector_crust_mantle)
   call sync_all()
-  
+
   ! checks number of interface points
   i = 0
   do iglob=1,NGLOB_CRUST_MANTLE
@@ -2107,21 +2106,21 @@
     if( test_flag_vector(1,iglob) > myrank+1.0+0.5 ) i = i + 1
   enddo
   deallocate(test_flag_vector)
-  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)    
+  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
 
   ! points defined by interfaces
   if( myrank == 0 ) then
     write(IMAIN,*) '  total assembled MPI interface points:',inum
     write(IMAIN,*)
 
-    ! checks  
+    ! checks
     if( inum /= icount ) then
       print*,'error crust mantle : total mpi points:',myrank,'total: ',inum,icount
       call exit_mpi(myrank,'error MPI assembly crust mantle')
     endif
   endif
 
-  call sync_all()  
+  call sync_all()
 
   end subroutine rmd_test_MPI_cm
 
@@ -2138,7 +2137,7 @@
   implicit none
 
   include 'mpif.h'
-  
+
   ! local parameters
   real(kind=CUSTOM_REAL),dimension(:),allocatable :: test_flag
   integer :: i,j,iglob,ier
@@ -2147,7 +2146,7 @@
   ! outer core
   allocate(test_flag(NGLOB_OUTER_CORE),stat=ier)
   if( ier /= 0 ) stop 'error allocating array test_flag etc.'
-  
+
   ! points defined by interfaces
   test_flag = 0.0
   do i=1,num_interfaces_outer_core
@@ -2157,14 +2156,14 @@
     enddo
   enddo
   i = sum(nibool_interfaces_outer_core)
-  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)      
+  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
   i = nint( sum(test_flag) )
-  call MPI_REDUCE(i,icount,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)      
+  call MPI_REDUCE(i,icount,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
   if( myrank == 0 ) then
     write(IMAIN,*) '  total MPI interface points : ',inum
     write(IMAIN,*) '  unique MPI interface points: ',icount
-  endif  
-  
+  endif
+
   ! initialized for assembly
   test_flag = myrank + 1.0_CUSTOM_REAL
   call sync_all()
@@ -2187,17 +2186,17 @@
                                 nibool_interfaces_outer_core,ibool_interfaces_outer_core, &
                                 request_send_scalar_outer_core,request_recv_scalar_outer_core)
   call sync_all()
-  
+
   ! checks number of interface points
   i = 0
   do iglob=1,NGLOB_OUTER_CORE
     ! only counts flags with MPI contributions
     if( test_flag(iglob) > myrank+1.0+0.5 ) i = i + 1
   enddo
-  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)    
+  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
   deallocate(test_flag)
 
-  ! output  
+  ! output
   if( myrank == 0 ) then
     write(IMAIN,*) '  total assembled MPI interface points:',inum
     write(IMAIN,*)
@@ -2208,9 +2207,9 @@
       call exit_mpi(myrank,'error MPI assembly outer_core')
     endif
   endif
-  
-  call sync_all()  
 
+  call sync_all()
+
   end subroutine rmd_test_MPI_oc
 
 
@@ -2227,9 +2226,9 @@
   implicit none
 
   include 'mpif.h'
-  
+
   ! local parameters
-  real(kind=CUSTOM_REAL),dimension(:,:),allocatable :: test_flag_vector  
+  real(kind=CUSTOM_REAL),dimension(:,:),allocatable :: test_flag_vector
   integer :: i,j,iglob,ier
   integer :: inum,icount
 
@@ -2246,14 +2245,14 @@
     enddo
   enddo
   i = sum(nibool_interfaces_inner_core)
-  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)      
+  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
   i = nint( sum(test_flag_vector) )
-  call MPI_REDUCE(i,icount,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)      
+  call MPI_REDUCE(i,icount,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
   if( myrank == 0 ) then
     write(IMAIN,*) '  total MPI interface points : ',inum
-    write(IMAIN,*) '  unique MPI interface points: ',icount    
-  endif  
-  
+    write(IMAIN,*) '  unique MPI interface points: ',icount
+  endif
+
   ! initializes for assembly
   test_flag_vector = myrank + 1.0_CUSTOM_REAL
   call sync_all()
@@ -2276,7 +2275,7 @@
                               nibool_interfaces_inner_core,ibool_interfaces_inner_core, &
                               request_send_vector_inner_core,request_recv_vector_inner_core)
   call sync_all()
-  
+
   ! checks number of interface points
   i = 0
   do iglob=1,NGLOB_INNER_CORE
@@ -2284,7 +2283,7 @@
     if( test_flag_vector(1,iglob) > myrank+1.0+0.5 ) i = i + 1
   enddo
   deallocate(test_flag_vector)
-  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)    
+  call MPI_REDUCE(i,inum,1,MPI_INTEGER,MPI_SUM,0,MPI_COMM_WORLD,ier)
 
   if( myrank == 0 ) then
     write(IMAIN,*) '  total assembled MPI interface points:',inum
@@ -2296,9 +2295,9 @@
       call exit_mpi(myrank,'error MPI assembly inner core')
     endif
   endif
-  
-  call sync_all()  
 
+  call sync_all()
+
   end subroutine rmd_test_MPI_ic
 
 !

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -42,19 +42,23 @@
   time_start = MPI_WTIME()
 
   ! make ellipticity
-  if(ELLIPTICITY_VAL) then
+  if( ELLIPTICITY_VAL ) then
     call make_ellipticity(nspl,rspl,espl,espl2,ONE_CRUST)
   endif
 
   ! read topography and bathymetry file
-  if(myrank == 0 .and. (TOPOGRAPHY .or. OCEANS_VAL)) then
-    call read_topo_bathy_file(ibathy_topo)
+  if( TOPOGRAPHY ) then
+    ! master reads file
+    if(myrank == 0 ) then
+      call read_topo_bathy_database(ibathy_topo,LOCAL_PATH)
+    endif
+
+    ! broadcast the information read on the master to the nodes
+    call MPI_BCAST(ibathy_topo,NX_BATHY*NY_BATHY,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
   endif
 
-  ! broadcast the information read on the master to the nodes
-  call MPI_BCAST(ibathy_topo,NX_BATHY*NY_BATHY,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
-
   ! user output
+  call sync_all()
   if( myrank == 0 .and. (TOPOGRAPHY .or. OCEANS_VAL)) then
     ! elapsed time since beginning of mesh generation
     tCPU = MPI_WTIME() - time_start

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_forward_arrays.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_forward_arrays.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_forward_arrays.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -40,7 +40,7 @@
   ! save files to local disk or tape system if restart file
   if(NUMBER_OF_RUNS > 1 .and. NUMBER_OF_THIS_RUN < NUMBER_OF_RUNS) then
     write(outputname,"('dump_all_arrays',i6.6)") myrank
-    open(unit=55,file=trim(LOCAL_PATH)//'/'//outputname,status='unknown',form='unformatted',action='write')
+    open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='unknown',form='unformatted',action='write')
 
     write(55) displ_crust_mantle
     write(55) veloc_crust_mantle
@@ -85,7 +85,7 @@
   ! save last frame of the forward simulation
   if (SIMULATION_TYPE == 1 .and. SAVE_FORWARD) then
     write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin'
-    open(unit=55,file=trim(LOCAL_PATH)//'/'//outputname,status='unknown',form='unformatted',action='write')
+    open(unit=55,file=trim(LOCAL_TMP_PATH)//'/'//outputname,status='unknown',form='unformatted',action='write')
 
     write(55) displ_crust_mantle
     write(55) veloc_crust_mantle

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_kernels.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_kernels.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/save_kernels.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -26,46 +26,12 @@
 !=====================================================================
 
 
-  subroutine save_kernels_crust_mantle(myrank,scale_t,scale_displ, &
-                  cijkl_kl_crust_mantle,rho_kl_crust_mantle, &
-                  alpha_kl_crust_mantle,beta_kl_crust_mantle, &
-                  ystore_crust_mantle,zstore_crust_mantle, &
-                  rhostore_crust_mantle,muvstore_crust_mantle, &
-                  kappavstore_crust_mantle,ibool_crust_mantle, &
-                  kappahstore_crust_mantle,muhstore_crust_mantle, &
-                  eta_anisostore_crust_mantle,ispec_is_tiso_crust_mantle, &
-                  LOCAL_PATH)
+  subroutine save_kernels_crust_mantle()
 
+  use specfem_par
+  use specfem_par_crustmantle
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-  integer myrank
-
-  double precision :: scale_t,scale_displ
-
-  real(kind=CUSTOM_REAL), dimension(21,NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: &
-    cijkl_kl_crust_mantle
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: &
-    rho_kl_crust_mantle, beta_kl_crust_mantle, alpha_kl_crust_mantle
-
-  real(kind=CUSTOM_REAL), dimension(NGLOB_CRUST_MANTLE) :: &
-        ystore_crust_mantle,zstore_crust_mantle
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPECMAX_ISO_MANTLE) :: &
-        rhostore_crust_mantle,kappavstore_crust_mantle,muvstore_crust_mantle
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPECMAX_TISO_MANTLE) :: &
-        kappahstore_crust_mantle,muhstore_crust_mantle,eta_anisostore_crust_mantle
-
-  logical, dimension(NSPEC_CRUST_MANTLE) :: ispec_is_tiso_crust_mantle
-
-  integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle
-
-  character(len=150) LOCAL_PATH
-
   ! local parameters
   real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: &
     mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle
@@ -73,7 +39,6 @@
   real(kind=CUSTOM_REAL) :: scale_kl,scale_kl_ani,scale_kl_rho
   real(kind=CUSTOM_REAL) :: rhol,mul,kappal,rho_kl,alpha_kl,beta_kl
   integer :: ispec,i,j,k,iglob
-  character(len=150) prname
 
   ! transverse isotropic parameters
   real(kind=CUSTOM_REAL), dimension(21) :: an_kl
@@ -352,7 +317,7 @@
     enddo
   enddo
 
-  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_PATH)
+  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH)
 
   ! For anisotropic kernels
   if (ANISOTROPIC_KL) then
@@ -473,39 +438,16 @@
 !-------------------------------------------------------------------------------------------------
 !
 
-  subroutine save_kernels_outer_core(myrank,scale_t,scale_displ, &
-                        rho_kl_outer_core,alpha_kl_outer_core, &
-                        rhostore_outer_core,kappavstore_outer_core, &
-                        deviatoric_outercore,nspec_beta_kl_outer_core,beta_kl_outer_core, &
-                        LOCAL_PATH)
+  subroutine save_kernels_outer_core()
 
+  use specfem_par
+  use specfem_par_outercore
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-  integer myrank
-
-  double precision :: scale_t,scale_displ
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE_ADJOINT) :: &
-    rho_kl_outer_core,alpha_kl_outer_core
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: &
-        rhostore_outer_core,kappavstore_outer_core
-
-  integer nspec_beta_kl_outer_core
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec_beta_kl_outer_core) :: &
-    beta_kl_outer_core
-  logical deviatoric_outercore
-
-  character(len=150) LOCAL_PATH
-
   ! local parameters
   real(kind=CUSTOM_REAL):: scale_kl
   real(kind=CUSTOM_REAL) :: rhol,kappal,rho_kl,alpha_kl,beta_kl
   integer :: ispec,i,j,k
-  character(len=150) prname
 
   scale_kl = scale_t/scale_displ * 1.d9
 
@@ -534,7 +476,8 @@
     enddo
   enddo
 
-  call create_name_database(prname,myrank,IREGION_OUTER_CORE,LOCAL_PATH)
+  call create_name_database(prname,myrank,IREGION_OUTER_CORE,LOCAL_TMP_PATH)
+
   open(unit=27,file=trim(prname)//'rho_kernel.bin',status='unknown',form='unformatted',action='write')
   write(27) rho_kl_outer_core
   close(27)
@@ -555,34 +498,17 @@
 !-------------------------------------------------------------------------------------------------
 !
 
-  subroutine save_kernels_inner_core(myrank,scale_t,scale_displ, &
-                          rho_kl_inner_core,beta_kl_inner_core,alpha_kl_inner_core, &
-                          rhostore_inner_core,muvstore_inner_core,kappavstore_inner_core, &
-                          LOCAL_PATH)
+  subroutine save_kernels_inner_core()
+
+  use specfem_par
+  use specfem_par_innercore
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-  integer myrank
-
-  double precision :: scale_t,scale_displ
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE_ADJOINT) :: &
-    rho_kl_inner_core, beta_kl_inner_core, alpha_kl_inner_core
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: &
-        rhostore_inner_core, kappavstore_inner_core,muvstore_inner_core
-
-  character(len=150) LOCAL_PATH
-
   ! local parameters
   real(kind=CUSTOM_REAL):: scale_kl
   real(kind=CUSTOM_REAL) :: rhol,mul,kappal,rho_kl,alpha_kl,beta_kl
   integer :: ispec,i,j,k
-  character(len=150) prname
 
-
   scale_kl = scale_t/scale_displ * 1.d9
 
   ! inner_core
@@ -606,7 +532,8 @@
     enddo
   enddo
 
-  call create_name_database(prname,myrank,IREGION_INNER_CORE,LOCAL_PATH)
+  call create_name_database(prname,myrank,IREGION_INNER_CORE,LOCAL_TMP_PATH)
+
   open(unit=27,file=trim(prname)//'rho_kernel.bin',status='unknown',form='unformatted',action='write')
   write(27) rho_kl_inner_core
   close(27)
@@ -623,34 +550,16 @@
 !-------------------------------------------------------------------------------------------------
 !
 
-  subroutine save_kernels_boundary_kl(myrank,scale_t,scale_displ, &
-                                  moho_kl,d400_kl,d670_kl,cmb_kl,icb_kl, &
-                                  LOCAL_PATH,HONOR_1D_SPHERICAL_MOHO)
+  subroutine save_kernels_boundary_kl()
 
+  use specfem_par
+  use specfem_par_crustmantle
+  use specfem_par_innercore
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-  integer myrank
-
-  double precision :: scale_t,scale_displ
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NSPEC2D_MOHO) :: moho_kl
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NSPEC2D_400) :: d400_kl
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NSPEC2D_670) :: d670_kl
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NSPEC2D_CMB) :: cmb_kl
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NSPEC2D_ICB) :: icb_kl
-
-  character(len=150) LOCAL_PATH
-
-  logical HONOR_1D_SPHERICAL_MOHO
-
   ! local parameters
   real(kind=CUSTOM_REAL):: scale_kl
-  character(len=150) prname
 
-
   scale_kl = scale_t/scale_displ * 1.d9
 
   ! scale the boundary kernels properly: *scale_kl gives s/km^3 and 1.d3 gives
@@ -661,7 +570,7 @@
   cmb_kl = cmb_kl * scale_kl * 1.d3
   icb_kl = icb_kl * scale_kl * 1.d3
 
-  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_PATH)
+  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH)
 
   if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then
     open(unit=27,file=trim(prname)//'moho_kernel.bin',status='unknown',form='unformatted',action='write')
@@ -695,23 +604,11 @@
 !-------------------------------------------------------------------------------------------------
 !
 
-  subroutine save_kernels_source_derivatives(nrec_local,NSOURCES,scale_displ,scale_t, &
-                                nu_source,moment_der,sloc_der,stshift_der,shdur_der,number_receiver_global)
+  subroutine save_kernels_source_derivatives()
 
+  use specfem_par
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-  integer nrec_local,NSOURCES
-  double precision :: scale_displ,scale_t
-
-  double precision :: nu_source(NDIM,NDIM,NSOURCES)
-  real(kind=CUSTOM_REAL) :: moment_der(NDIM,NDIM,nrec_local),sloc_der(NDIM,nrec_local), &
-                            stshift_der(nrec_local),shdur_der(nrec_local)
-
-  integer, dimension(nrec_local) :: number_receiver_global
-
   ! local parameters
   real(kind=CUSTOM_REAL),parameter :: scale_mass = RHOAV * (R_EARTH**3)
   integer :: irec_local
@@ -766,26 +663,14 @@
 !-------------------------------------------------------------------------------------------------
 !
 
-  subroutine save_kernels_hessian(myrank,scale_t,scale_displ, &
-                  hess_kl_crust_mantle,LOCAL_PATH)
+  subroutine save_kernels_hessian()
 
+  use specfem_par
+  use specfem_par_crustmantle
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-  integer myrank
-
-  double precision :: scale_t,scale_displ
-
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: &
-    hess_kl_crust_mantle
-
-  character(len=150) LOCAL_PATH
-
   ! local parameters
   real(kind=CUSTOM_REAL) :: scale_kl
-  character(len=150) prname
 
   ! scaling factors
   scale_kl = scale_t/scale_displ * 1.d9
@@ -794,7 +679,8 @@
   hess_kl_crust_mantle(:,:,:,:) = 2._CUSTOM_REAL * hess_kl_crust_mantle(:,:,:,:) * scale_kl
 
   ! stores into file
-  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_PATH)
+  call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH)
+
   open(unit=27,file=trim(prname)//'hess_kernel.bin',status='unknown',form='unformatted',action='write')
   write(27) hess_kl_crust_mantle
   close(27)

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_GLL_points.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_GLL_points.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_GLL_points.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -35,9 +35,9 @@
 
   ! set up GLL points, weights and derivation matrices
   call define_derivation_matrices(xigll,yigll,zigll,wxgll,wygll,wzgll, &
-         hprime_xx,hprime_yy,hprime_zz, &
-         hprimewgll_xx,hprimewgll_yy,hprimewgll_zz, &
-         wgllwgll_xy,wgllwgll_xz,wgllwgll_yz,wgll_cube)
+                                 hprime_xx,hprime_yy,hprime_zz, &
+                                 hprimewgll_xx,hprimewgll_yy,hprimewgll_zz, &
+                                 wgllwgll_xy,wgllwgll_xz,wgllwgll_yz,wgll_cube)
 
   if( USE_DEVILLE_PRODUCTS_VAL ) then
 

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_sources_receivers.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_sources_receivers.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/setup_sources_receivers.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -85,18 +85,26 @@
           Mzz(NSOURCES), &
           Mxy(NSOURCES), &
           Mxz(NSOURCES), &
-          Myz(NSOURCES), &
-          xi_source(NSOURCES), &
+          Myz(NSOURCES),stat=ier)
+  if( ier /= 0 ) call exit_MPI(myrank,'error allocating source arrays')
+
+  allocate(xi_source(NSOURCES), &
           eta_source(NSOURCES), &
-          gamma_source(NSOURCES), &
-          tshift_cmt(NSOURCES), &
+          gamma_source(NSOURCES),stat=ier)
+  if( ier /= 0 ) call exit_MPI(myrank,'error allocating source arrays')
+
+  allocate(tshift_cmt(NSOURCES), &
           hdur(NSOURCES), &
-          hdur_gaussian(NSOURCES), &
-          theta_source(NSOURCES), &
-          phi_source(NSOURCES), &
-          nu_source(NDIM,NDIM,NSOURCES),stat=ier)
+          hdur_gaussian(NSOURCES),stat=ier)
   if( ier /= 0 ) call exit_MPI(myrank,'error allocating source arrays')
 
+  allocate(theta_source(NSOURCES), &
+          phi_source(NSOURCES),stat=ier)
+  if( ier /= 0 ) call exit_MPI(myrank,'error allocating source arrays')
+
+  allocate(nu_source(NDIM,NDIM,NSOURCES),stat=ier)
+  if( ier /= 0 ) call exit_MPI(myrank,'error allocating source arrays')
+
   ! sources
   ! BS BS moved open statement and writing of first lines into sr.vtk before the
   ! call to locate_sources, where further write statements to that file follow
@@ -115,13 +123,14 @@
   ! locate sources in the mesh
   call locate_sources(NSOURCES,myrank,NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,ibool_crust_mantle, &
             xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
-            xigll,yigll,zigll,NPROCTOT_VAL,ELLIPTICITY_VAL,TOPOGRAPHY, &
+            xigll,yigll,zigll, &
+            ELLIPTICITY_VAL,TOPOGRAPHY, &
             sec,tshift_cmt,min_tshift_cmt_original,yr,jda,ho,mi,theta_source,phi_source, &
             NSTEP,DT,hdur,Mxx,Myy,Mzz,Mxy,Mxz,Myz, &
             islice_selected_source,ispec_selected_source, &
             xi_source,eta_source,gamma_source, nu_source, &
-            rspl,espl,espl2,nspl,ibathy_topo,NEX_XI,PRINT_SOURCE_TIME_FUNCTION, &
-            LOCAL_PATH,SIMULATION_TYPE)
+            rspl,espl,espl2,nspl,ibathy_topo,PRINT_SOURCE_TIME_FUNCTION, &
+            LOCAL_TMP_PATH,SIMULATION_TYPE)
 
   if(abs(minval(tshift_cmt)) > TINYVAL) call exit_MPI(myrank,'one tshift_cmt must be zero, others must be positive')
 
@@ -258,16 +267,20 @@
           ispec_selected_rec(nrec), &
           xi_receiver(nrec), &
           eta_receiver(nrec), &
-          gamma_receiver(nrec), &
-          station_name(nrec), &
+          gamma_receiver(nrec),stat=ier)
+  if( ier /= 0 ) call exit_MPI(myrank,'error allocating receiver arrays')
+
+  allocate(station_name(nrec), &
           network_name(nrec), &
           stlat(nrec), &
           stlon(nrec), &
           stele(nrec), &
-          stbur(nrec), &
-          nu(NDIM,NDIM,nrec),stat=ier)
+          stbur(nrec),stat=ier)
   if( ier /= 0 ) call exit_MPI(myrank,'error allocating receiver arrays')
 
+  allocate(nu(NDIM,NDIM,nrec),stat=ier)
+  if( ier /= 0 ) call exit_MPI(myrank,'error allocating receiver arrays')
+
   !  receivers
   if(myrank == 0) then
     write(IMAIN,*)
@@ -286,7 +299,8 @@
                       nrec,islice_selected_rec,ispec_selected_rec, &
                       xi_receiver,eta_receiver,gamma_receiver,station_name,network_name, &
                       stlat,stlon,stele,stbur,nu, &
-                      yr,jda,ho,mi,sec,NPROCTOT_VAL,ELLIPTICITY_VAL,TOPOGRAPHY, &
+                      yr,jda,ho,mi,sec, &
+                      ELLIPTICITY_VAL,TOPOGRAPHY, &
                       theta_source(1),phi_source(1),rspl,espl,espl2,nspl, &
                       ibathy_topo,RECEIVERS_CAN_BE_BURIED,NCHUNKS_VAL)
 
@@ -387,6 +401,9 @@
       call exit_MPI(myrank,'total number of receivers is incorrect')
   endif
 
+  ! frees arrays
+  deallocate(theta_source,phi_source)
+
   end subroutine setup_receivers
 
 
@@ -457,14 +474,6 @@
 
     ! stores source arrays
     call setup_sources_receivers_srcarr()
-!                      NSOURCES,myrank, &
-!                      ispec_selected_source,islice_selected_source, &
-!                      xi_source,eta_source,gamma_source, &
-!                      Mxx,Myy,Mzz,Mxy,Mxz,Myz, &
-!                      xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, &
-!                      etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
-!                      gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, &
-!                      xigll,yigll,zigll,sourcearrays)
 
   endif
 
@@ -504,41 +513,12 @@
 !
 
   subroutine setup_sources_receivers_srcarr()
-!                      NSOURCES,myrank, &
-!                      ispec_selected_source,islice_selected_source, &
-!                      xi_source,eta_source,gamma_source, &
-!                      Mxx,Myy,Mzz,Mxy,Mxz,Myz, &
-!                      xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, &
-!                      etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
-!                      gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, &
-!                      xigll,yigll,zigll,sourcearrays)
 
   use specfem_par
   use specfem_par_crustmantle
 
   implicit none
 
-!  include "constants.h"
-!  include "OUTPUT_FILES/values_from_mesher.h"
-!
-!  integer NSOURCES,myrank
-!
-!  integer, dimension(NSOURCES) :: islice_selected_source,ispec_selected_source
-!  double precision, dimension(NSOURCES) :: xi_source,eta_source,gamma_source
-!  double precision, dimension(NSOURCES) :: Mxx,Myy,Mzz,Mxy,Mxz,Myz
-!
-!  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: &
-!        xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle,&
-!        etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
-!        gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle
-!
-!  double precision, dimension(NGLLX) :: xigll
-!  double precision, dimension(NGLLY) :: yigll
-!  double precision, dimension(NGLLZ) :: zigll
-!
-!  real(kind=CUSTOM_REAL), dimension(NDIM,NGLLX,NGLLY,NGLLZ,NSOURCES) :: sourcearrays
-
-
   ! local parameters
   integer :: isource,iglob,ispec,i,j,k
   real(kind=CUSTOM_REAL), dimension(NDIM,NGLLX,NGLLY,NGLLZ) :: sourcearray

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -232,7 +232,7 @@
           ROTATE_SEISMOGRAMS_RT,HONOR_1D_SPHERICAL_MOHO,WRITE_SEISMOGRAMS_BY_MASTER,&
           SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE
 
-  character(len=150) OUTPUT_FILES,LOCAL_PATH,MODEL
+  character(len=150) :: OUTPUT_FILES,LOCAL_PATH,LOCAL_TMP_PATH,MODEL
   ! process/partition name
   character(len=150) :: prname
 
@@ -547,7 +547,7 @@
     normal_x_noise,normal_y_noise,normal_z_noise, mask_noise
   real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: noise_surface_movie
   integer :: irec_master_noise
-  integer :: nspec_top
+  integer :: NSPEC_TOP
 
   ! inner / outer elements crust/mantle region
   integer :: num_phase_ispec_crust_mantle

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_output.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_output.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_output.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -84,7 +84,7 @@
         endif
 
         call  write_movie_volume_strains(myrank,npoints_3dmovie, &
-                    LOCAL_PATH,MOVIE_VOLUME_TYPE,MOVIE_COARSE, &
+                    LOCAL_TMP_PATH,MOVIE_VOLUME_TYPE,MOVIE_COARSE, &
                     it,eps_trace_over_3_crust_mantle, &
                     epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, &
                     epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, &
@@ -94,7 +94,7 @@
       case( 2, 3 )
         ! output the Time Integral of Strain, or \mu*TIS
         call  write_movie_volume_strains(myrank,npoints_3dmovie, &
-                    LOCAL_PATH,MOVIE_VOLUME_TYPE,MOVIE_COARSE, &
+                    LOCAL_TMP_PATH,MOVIE_VOLUME_TYPE,MOVIE_COARSE, &
                     it,Ieps_trace_over_3_crust_mantle, &
                     Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, &
                     Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle, &
@@ -134,7 +134,7 @@
                         epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, &
                         epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, &
                         epsilondev_xz_inner_core,epsilondev_yz_inner_core, &
-                        LOCAL_PATH, &
+                        LOCAL_TMP_PATH, &
                         displ_crust_mantle,displ_inner_core,displ_outer_core, &
                         veloc_crust_mantle,veloc_inner_core,veloc_outer_core, &
                         accel_crust_mantle,accel_inner_core, &
@@ -148,7 +148,7 @@
 
         scalingval = scale_displ
         call write_movie_volume_vector(myrank,it,npoints_3dmovie, &
-                    LOCAL_PATH,MOVIE_VOLUME_TYPE, &
+                    LOCAL_TMP_PATH,MOVIE_VOLUME_TYPE, &
                     MOVIE_COARSE,ibool_crust_mantle,displ_crust_mantle, &
                     scalingval,mask_3dmovie,nu_3dmovie)
 
@@ -160,7 +160,7 @@
 
         scalingval = scale_veloc
         call write_movie_volume_vector(myrank,it,npoints_3dmovie, &
-                    LOCAL_PATH,MOVIE_VOLUME_TYPE, &
+                    LOCAL_TMP_PATH,MOVIE_VOLUME_TYPE, &
                     MOVIE_COARSE,ibool_crust_mantle,veloc_crust_mantle, &
                     scalingval,mask_3dmovie,nu_3dmovie)
 
@@ -176,7 +176,7 @@
 !      .and. it >= MOVIE_START .and. it <= MOVIE_STOP) then
 !    ! VTK file output
 !    ! displacement values
-!    !write(prname,'(a,i6.6,a)') trim(LOCAL_PATH)//'/'//'proc',myrank,'_'
+!    !write(prname,'(a,i6.6,a)') trim(LOCAL_TMP_PATH)//'/'//'proc',myrank,'_'
 !    !write(filename,'(a,a,i6.6)') prname(1:len_trim(prname)),'reg_3_displ_',it
 !    !call write_VTK_data_cr(idoubling_inner_core,NSPEC_INNER_CORE,NGLOB_INNER_CORE, &
 !    !                    xstore_inner_core,ystore_inner_core,zstore_inner_core,ibool_inner_core, &

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_volume.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_volume.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_movie_volume.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -32,44 +32,31 @@
 ! this subroutine counts the number of points and elements within the movie volume
 ! in this processor slice, and returns arrays that keep track of them, both in global and local indexing schemes
 
-  subroutine count_points_movie_volume(prname,ibool_crust_mantle, xstore_crust_mantle,ystore_crust_mantle, &
-                      zstore_crust_mantle,MOVIE_TOP,MOVIE_BOTTOM,MOVIE_WEST,MOVIE_EAST,MOVIE_NORTH,MOVIE_SOUTH, &
-                      MOVIE_COARSE,npoints_3dmovie,nspecel_3dmovie,num_ibool_3dmovie, &
-                      mask_ibool_3dmovie,mask_3dmovie)
+  subroutine count_points_movie_volume()
 
+  use specfem_par
+  use specfem_par_crustmantle
+  use specfem_par_movie
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-! input
-  integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle
-  real(kind=CUSTOM_REAL), dimension(NGLOB_CRUST_MANTLE) :: xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle
-  double precision :: MOVIE_TOP,MOVIE_BOTTOM,MOVIE_WEST,MOVIE_EAST,MOVIE_NORTH,MOVIE_SOUTH
-  logical :: MOVIE_COARSE
-  character(len=150) :: prname
-
-! output
-  integer :: npoints_3dmovie,nspecel_3dmovie
-  integer, dimension(NGLOB_CRUST_MANTLE) :: num_ibool_3dmovie
-  logical, dimension(NGLOB_CRUST_MANTLE) :: mask_ibool_3dmovie
-  logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_STRAIN_ONLY) :: mask_3dmovie
-
-! variables
-  integer :: ipoints_3dmovie,ispecel_3dmovie,ispec,iglob,i,j,k,NIT
+  ! local parameters
+  integer :: ipoints_3dmovie,ispecel_3dmovie,ispec,iglob,i,j,k,iNIT
   real(kind=custom_real) :: rval,thetaval,phival
 
   if(MOVIE_COARSE) then
-    NIT = NGLLX-1
+    iNIT = NGLLX-1
   else
-    NIT = 1
+    iNIT = 1
   endif
   ipoints_3dmovie=0
   num_ibool_3dmovie(:) = -99
   ispecel_3dmovie = 0
-  mask_ibool_3dmovie(:)=.false.
+  mask_ibool(:)=.false.
   mask_3dmovie(:,:,:,:)=.false.
+
   ! create name of database
+  write(prname,'(a,i6.6,a)') trim(LOCAL_TMP_PATH)//'/'//'proc',myrank,'_'
+
   open(unit=IOUT,file=trim(prname)//'movie3D_info.txt',status='unknown')
 
   !find and count points within given region for storing movie
@@ -85,13 +72,13 @@
             ( (phival < MOVIE_EAST .and. phival > MOVIE_WEST) .or. &
               ( (MOVIE_EAST < MOVIE_WEST) .and. (phival >MOVIE_EAST .or. phival < MOVIE_WEST) ) ) ) then
             ispecel_3dmovie=ispecel_3dmovie+1
-              do k=1,NGLLZ,NIT
-               do j=1,NGLLY,NIT
-                do i=1,NGLLX,NIT
+              do k=1,NGLLZ,iNIT
+               do j=1,NGLLY,iNIT
+                do i=1,NGLLX,iNIT
                  iglob    = ibool_crust_mantle(i,j,k,ispec)
-                 if(.not. mask_ibool_3dmovie(iglob)) then
+                 if(.not. mask_ibool(iglob)) then
                   ipoints_3dmovie = ipoints_3dmovie + 1
-                  mask_ibool_3dmovie(iglob)=.true.
+                  mask_ibool(iglob)=.true.
                   mask_3dmovie(i,j,k,ispec)=.true.
                   num_ibool_3dmovie(iglob)= ipoints_3dmovie
                  endif
@@ -112,49 +99,36 @@
 ! writes meshfiles to merge with solver snapshots for 3D volume movies.  Also computes and outputs
 ! the rotation matrix nu_3dmovie required to transfer to a geographic coordinate system
 
-  subroutine write_movie_volume_mesh(npoints_3dmovie,prname,ibool_crust_mantle,xstore_crust_mantle, &
-                         ystore_crust_mantle,zstore_crust_mantle, muvstore_crust_mantle_3dmovie, &
-                         mask_3dmovie,mask_ibool_3dmovie,num_ibool_3dmovie,nu_3dmovie,MOVIE_COARSE)
+  subroutine write_movie_volume_mesh()
 
+  use specfem_par
+  use specfem_par_crustmantle
+  use specfem_par_movie
   implicit none
 
-  include "constants.h"
-  include "OUTPUT_FILES/values_from_mesher.h"
-
-  !input
-  integer :: npoints_3dmovie
-  integer, dimension(NGLOB_CRUST_MANTLE) :: num_ibool_3dmovie
-  real(kind=CUSTOM_REAL), dimension(NGLOB_CRUST_MANTLE) :: xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle
-  real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_STR_OR_ATT) :: muvstore_crust_mantle_3dmovie
-  character(len=150) :: prname
-  logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_STRAIN_ONLY) :: mask_3dmovie
-  logical, dimension(NGLOB_CRUST_MANTLE) :: mask_ibool_3dmovie
-  logical :: MOVIE_COARSE
-  integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle
-
-  !output
-  real(kind=CUSTOM_REAL), dimension(3,3,npoints_3dmovie) :: nu_3dmovie
-  real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_mu
-
-  !variables
-  integer :: ipoints_3dmovie,ispecele,ispec,i,j,k,iglob,iglob1,iglob2,iglob3,iglob4,iglob5,iglob6,iglob7,iglob8
-  integer :: n1,n2,n3,n4,n5,n6,n7,n8,NIT
+  ! local parametes
+  integer :: ipoints_3dmovie,ispecele,ispec,i,j,k
+  integer :: iglob,iglob1,iglob2,iglob3,iglob4,iglob5,iglob6,iglob7,iglob8
+  integer :: n1,n2,n3,n4,n5,n6,n7,n8,iNIT
   real(kind=CUSTOM_REAL) :: rval,thetaval,phival,xval,yval,zval,st,ct,sp,cp
   real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_x,store_val3D_y, store_val3D_z
+  real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_mu
 
+!  character(len=150) :: prname
+
   if(NDIM /= 3) stop 'movie volume output requires NDIM = 3'
 
   if(MOVIE_COARSE) then
-    NIT = NGLLX-1
+    iNIT = NGLLX-1
   else
-    NIT = 1
+    iNIT = 1
   endif
 
   ipoints_3dmovie=0
   do ispec=1,NSPEC_CRUST_MANTLE
-    do k=1,NGLLZ,NIT
-      do j=1,NGLLY,NIT
-        do i=1,NGLLX,NIT
+    do k=1,NGLLZ,iNIT
+      do j=1,NGLLY,iNIT
+        do i=1,NGLLX,iNIT
           if(mask_3dmovie(i,j,k,ispec)) then
             ipoints_3dmovie=ipoints_3dmovie+1
             iglob= ibool_crust_mantle(i,j,k,ispec)
@@ -185,6 +159,9 @@
       enddo  !j
     enddo  !k
   enddo !ispec
+
+  write(prname,'(a,i6.6,a)') trim(LOCAL_TMP_PATH)//'/'//'proc',myrank,'_'
+
   open(unit=IOUT,file=trim(prname)//'movie3D_x.bin',status='unknown',form='unformatted')
   if(npoints_3dmovie>0) then
     write(IOUT) store_val3D_x(1:npoints_3dmovie)
@@ -218,20 +195,20 @@
     else
       iglob=ibool_crust_mantle(3,3,3,ispec)
     endif
-    if(mask_ibool_3dmovie(iglob)) then  !this element is in the region
+    if(mask_ibool(iglob)) then  !this element is in the region
       ispecele  = ispecele+1
-      do k=1,NGLLZ-1,NIT
-        do j=1,NGLLY-1,NIT
-          do i=1,NGLLX-1,NIT
+      do k=1,NGLLZ-1,iNIT
+        do j=1,NGLLY-1,iNIT
+          do i=1,NGLLX-1,iNIT
             ! if(mask_3dmovie(i,j,k,ispec)) then
             iglob1 = ibool_crust_mantle(i,j,k,ispec)
-            iglob2 = ibool_crust_mantle(i+NIT,j,k,ispec)
-            iglob3 = ibool_crust_mantle(i+NIT,j+NIT,k,ispec)
-            iglob4 = ibool_crust_mantle(i,j+NIT,k,ispec)
-            iglob5 = ibool_crust_mantle(i,j,k+NIT,ispec)
-            iglob6 = ibool_crust_mantle(i+NIT,j,k+NIT,ispec)
-            iglob7 = ibool_crust_mantle(i+NIT,j+NIT,k+NIT,ispec)
-            iglob8 = ibool_crust_mantle(i,j+NIT,k+NIT,ispec)
+            iglob2 = ibool_crust_mantle(i+iNIT,j,k,ispec)
+            iglob3 = ibool_crust_mantle(i+iNIT,j+iNIT,k,ispec)
+            iglob4 = ibool_crust_mantle(i,j+iNIT,k,ispec)
+            iglob5 = ibool_crust_mantle(i,j,k+iNIT,ispec)
+            iglob6 = ibool_crust_mantle(i+iNIT,j,k+iNIT,ispec)
+            iglob7 = ibool_crust_mantle(i+iNIT,j+iNIT,k+iNIT,ispec)
+            iglob8 = ibool_crust_mantle(i,j+iNIT,k+iNIT,ispec)
             n1 = num_ibool_3dmovie(iglob1)-1
             n2 = num_ibool_3dmovie(iglob2)-1
             n3 = num_ibool_3dmovie(iglob3)-1
@@ -256,7 +233,7 @@
 ! ---------------------------------------------
 
   subroutine write_movie_volume_strains(myrank,npoints_3dmovie, &
-                                        LOCAL_PATH,MOVIE_VOLUME_TYPE,MOVIE_COARSE, &
+                                        LOCAL_TMP_PATH,MOVIE_VOLUME_TYPE,MOVIE_COARSE, &
                                         it,eps_trace_over_3_crust_mantle, &
                                         epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, &
                                         epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, &
@@ -282,11 +259,11 @@
   logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_STRAIN_ONLY) :: mask_3dmovie
   logical :: MOVIE_COARSE
   real(kind=CUSTOM_REAL), dimension(3,3,npoints_3dmovie) :: nu_3dmovie
-  character(len=150) LOCAL_PATH,outputname
+  character(len=150) LOCAL_TMP_PATH,outputname
 
   ! variables
-  character(len=150) prname
-  integer :: ipoints_3dmovie,i,j,k,ispec,NIT
+  !character(len=150) prname
+  integer :: ipoints_3dmovie,i,j,k,ispec,iNIT
   real(kind=CUSTOM_REAL) :: muv_3dmovie
   real(kind=CUSTOM_REAL),dimension(3,3) :: eps_loc,eps_loc_new
   real(kind=CUSTOM_REAL),dimension(:),allocatable :: store_val3d_NN,store_val3d_EE,store_val3d_ZZ,&
@@ -311,17 +288,18 @@
       movie_prefix='P' ! potency, or integral of strain x \mu
   endif
   if(MOVIE_COARSE) then
-   NIT = NGLLX-1
+   iNIT = NGLLX-1
   else
-   NIT = 1
+   iNIT = 1
   endif
 
-  write(prname,"('proc',i6.6)") myrank
+  !write(prname,"('proc',i6.6)") myrank
+
   ipoints_3dmovie=0
   do ispec=1,NSPEC_CRUST_MANTLE
-   do k=1,NGLLZ,NIT
-    do j=1,NGLLY,NIT
-     do i=1,NGLLX,NIT
+   do k=1,NGLLZ,iNIT
+    do j=1,NGLLY,iNIT
+     do i=1,NGLLX,iNIT
       if(mask_3dmovie(i,j,k,ispec)) then
        ipoints_3dmovie=ipoints_3dmovie+1
        muv_3dmovie=muvstore_crust_mantle_3dmovie(i,j,k,ispec)
@@ -353,40 +331,40 @@
   if(ipoints_3dmovie /= npoints_3dmovie) stop 'did not find the right number of points for 3D movie'
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'NN',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_NN(1:npoints_3dmovie)
   close(27)
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'EE',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_EE(1:npoints_3dmovie)
   close(27)
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'ZZ',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_ZZ(1:npoints_3dmovie)
   close(27)
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'NE',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_NE(1:npoints_3dmovie)
   close(27)
 
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'NZ',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_NZ(1:npoints_3dmovie)
   close(27)
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'EZ',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_EZ(1:npoints_3dmovie)
   close(27)
 
   end subroutine write_movie_volume_strains
 
 ! ---------------------------------------------
-  subroutine write_movie_volume_vector(myrank,it,npoints_3dmovie,LOCAL_PATH,MOVIE_VOLUME_TYPE, &
+  subroutine write_movie_volume_vector(myrank,it,npoints_3dmovie,LOCAL_TMP_PATH,MOVIE_VOLUME_TYPE, &
                                       MOVIE_COARSE,ibool_crust_mantle,vector_crust_mantle, &
                                       scalingval,mask_3dmovie,nu_3dmovie)
   implicit none
@@ -403,10 +381,10 @@
   real(kind=CUSTOM_REAL), dimension(3) :: vector_local,vector_local_new
   logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_STRAIN_ONLY) :: mask_3dmovie
   logical :: MOVIE_COARSE
-  character(len=150) LOCAL_PATH
+  character(len=150) LOCAL_TMP_PATH
 
   ! variables
-  integer :: ipoints_3dmovie,i,j,k,ispec,NIT,iglob
+  integer :: ipoints_3dmovie,i,j,k,ispec,iNIT,iglob
   real(kind=CUSTOM_REAL),dimension(:),allocatable :: store_val3d_N,store_val3d_E,store_val3d_Z
   character(len=150) outputname
   character(len=2) movie_prefix
@@ -423,9 +401,9 @@
       movie_prefix='VE' ! velocity
   endif
   if(MOVIE_COARSE) then
-   NIT = NGLLX-1
+   iNIT = NGLLX-1
   else
-   NIT = 1
+   iNIT = 1
   endif
 
   if(CUSTOM_REAL == SIZE_REAL) then
@@ -436,9 +414,9 @@
 
   ipoints_3dmovie=0
   do ispec=1,NSPEC_CRUST_MANTLE
-   do k=1,NGLLZ,NIT
-    do j=1,NGLLY,NIT
-     do i=1,NGLLX,NIT
+   do k=1,NGLLZ,iNIT
+    do j=1,NGLLY,iNIT
+     do i=1,NGLLX,iNIT
       if(mask_3dmovie(i,j,k,ispec)) then
        ipoints_3dmovie=ipoints_3dmovie+1
        iglob = ibool_crust_mantle(i,j,k,ispec)
@@ -458,17 +436,17 @@
   if(ipoints_3dmovie /= npoints_3dmovie) stop 'did not find the right number of points for 3D movie'
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'N',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_N(1:npoints_3dmovie)
   close(27)
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'E',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_E(1:npoints_3dmovie)
   close(27)
 
   write(outputname,"('proc',i6.6,'_movie3D_',a,'Z',i6.6,'.bin')") myrank,movie_prefix,it
-  open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
+  open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted')
   write(27) store_val3d_Z(1:npoints_3dmovie)
   close(27)
 
@@ -485,7 +463,7 @@
                                       epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, &
                                       epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, &
                                       epsilondev_xz_inner_core,epsilondev_yz_inner_core, &
-                                      LOCAL_PATH, &
+                                      LOCAL_TMP_PATH, &
                                       displ_crust_mantle,displ_inner_core,displ_outer_core, &
                                       veloc_crust_mantle,veloc_inner_core,veloc_outer_core, &
                                       accel_crust_mantle,accel_inner_core, &
@@ -516,7 +494,7 @@
     epsilondev_xz_inner_core,epsilondev_yz_inner_core
 
 
-  character(len=150) LOCAL_PATH
+  character(len=150) LOCAL_TMP_PATH
 
   real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: displ_crust_mantle
   real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: veloc_crust_mantle
@@ -552,7 +530,7 @@
     ! these binary arrays can be converted into mesh format using the utilitiy ./bin/xcombine_vol_data
     ! old name format:     write(outputname,"('proc',i6.6,'_crust_mantle_div_displ_it',i6.6,'.bin')") myrank,it
     write(outputname,"('proc',i6.6,'_reg1_div_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) eps_trace_over_3_crust_mantle
     close(27)
@@ -560,7 +538,7 @@
     ! outer core
     if (NSPEC_OUTER_CORE_ADJOINT /= 1 ) then
       write(outputname,"('proc',i6.6,'_reg2_div_displ_it',i6.6,'.bin')") myrank,it
-      open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+      open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
       if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
       write(27)  ONE_THIRD * div_displ_outer_core
       close(27)
@@ -582,7 +560,7 @@
 
       ! old name format: write(outputname,"('proc',i6.6,'_outer_core_div_displ_it',i6.6,'.bin')") myrank,it
       write(outputname,"('proc',i6.6,'_reg2_div_displ_it',i6.6,'.bin')") myrank,it
-      open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+      open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
       if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
       write(27)  div_s_outer_core
       close(27)
@@ -593,7 +571,7 @@
     ! inner core
     ! old name format: write(outputname,"('proc',i6.6,'_inner_core_div_displ_proc_it',i6.6,'.bin')") myrank,it
     write(outputname,"('proc',i6.6,'_reg3_div_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) eps_trace_over_3_inner_core
     close(27)
@@ -606,7 +584,7 @@
     ! these binary files must be handled by the user, no further utilities available for this format
     ! crust mantle
     write(outputname,"('proc',i6.6,'_crust_mantle_epsdev_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//trim(outputname),status='unknown',form='unformatted')
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//trim(outputname),status='unknown',form='unformatted')
     write(27) epsilondev_xx_crust_mantle
     write(27) epsilondev_yy_crust_mantle
     write(27) epsilondev_xy_crust_mantle
@@ -615,7 +593,7 @@
     close(27)
     ! inner core
     write(outputname,"('proc',i6.6,'inner_core_epsdev_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//trim(outputname),status='unknown',form='unformatted')
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//trim(outputname),status='unknown',form='unformatted')
     write(27) epsilondev_xx_inner_core
     write(27) epsilondev_yy_inner_core
     write(27) epsilondev_xy_inner_core
@@ -629,7 +607,7 @@
     ! these binary arrays can be converted into mesh format using the utilitiy ./bin/xcombine_vol_data
     ! crust_mantle region
     write(outputname,"('proc',i6.6,'_reg1_epsdev_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     ! frobenius norm
     allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE))
@@ -656,7 +634,7 @@
 
     ! inner core
     write(outputname,"('proc',i6.6,'_reg3_epsdev_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     ! frobenius norm
     allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE))
@@ -706,7 +684,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg1_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -728,7 +706,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg2_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -750,7 +728,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg3_displ_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -777,7 +755,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg1_veloc_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -799,7 +777,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg2_veloc_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -821,7 +799,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg3_veloc_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -847,7 +825,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg1_accel_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -869,7 +847,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg2_accel_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)
@@ -891,7 +869,7 @@
       enddo
     enddo
     write(outputname,"('proc',i6.6,'_reg3_accel_it',i6.6,'.bin')") myrank,it
-    open(unit=27,file=trim(LOCAL_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
+    open(unit=27,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname),status='unknown',form='unformatted',iostat=ier)
     if( ier /= 0 ) call exit_MPI(myrank,'error opening file '//trim(outputname))
     write(27) tmp_data
     close(27)

Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90	2012-03-15 14:32:31 UTC (rev 19783)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90	2012-03-15 15:23:17 UTC (rev 19784)
@@ -107,7 +107,7 @@
       if( nrec_local > 0 ) &
         call write_adj_seismograms(seismograms,number_receiver_global, &
                                   nrec_local,it,nit_written,DT, &
-                                  NSTEP,NTSTEP_BETWEEN_OUTPUT_SEISMOS,t0,LOCAL_PATH)
+                                  NSTEP,NTSTEP_BETWEEN_OUTPUT_SEISMOS,t0,LOCAL_TMP_PATH)
         nit_written = it
     endif
     seismo_offset = seismo_offset + seismo_current
@@ -498,7 +498,7 @@
 
   subroutine write_adj_seismograms(seismograms,number_receiver_global, &
               nrec_local,it,nit_written,DT,NSTEP, &
-              NTSTEP_BETWEEN_OUTPUT_SEISMOS,hdur,LOCAL_PATH)
+              NTSTEP_BETWEEN_OUTPUT_SEISMOS,hdur,LOCAL_TMP_PATH)
 
   implicit none
 
@@ -508,7 +508,7 @@
   integer, dimension(nrec_local) :: number_receiver_global
   real(kind=CUSTOM_REAL), dimension(9,nrec_local,NSTEP) :: seismograms
   double precision :: hdur,DT
-  character(len=150) :: LOCAL_PATH
+  character(len=150) :: LOCAL_TMP_PATH
 
   integer :: irec,irec_local
   integer :: iorientation,isample
@@ -553,7 +553,7 @@
       write(sisname,"(a,i6.6,'.',a,'.',a3,'.sem')") 'S',irec,'NT',chn
 
       ! suppress white spaces if any
-      clean_LOCAL_PATH = adjustl(LOCAL_PATH)
+      clean_LOCAL_PATH = adjustl(LOCAL_TMP_PATH)
 
       ! create full final local path
       final_LOCAL_PATH = clean_LOCAL_PATH(1:len_trim(clean_LOCAL_PATH)) // '/'



More information about the CIG-COMMITS mailing list