[cig-commits] [commit] devel, master: Added parallel netcdf kernel write for CEM. (f90eb91)

[cig_noreply at geodynamics.org]

Repository : https://github.com/geodynamics/specfem3d_globe

On branches: devel,master
Link       : https://github.com/geodynamics/specfem3d_globe/compare/bc58e579b3b0838a0968725a076f5904845437ca...be63f20cbb6f462104e949894dbe205d2398cd7f

>---------------------------------------------------------------

commit f90eb9158cf23ae04831c668f8c114fcdb427fee
Author: Michael Afanasiev <michael.afanasiev at erdw.ethz.ch>
Date:   Tue Sep 23 15:19:21 2014 +0200

    Added parallel netcdf kernel write for CEM.


>---------------------------------------------------------------

f90eb9158cf23ae04831c668f8c114fcdb427fee
 src/specfem3D/read_write_netcdf.f90 | 58 ++++++++++++++++++++++++++++++++-----
 src/specfem3D/save_kernels.F90      |  8 ++++-
 2 files changed, 58 insertions(+), 8 deletions(-)

diff --git a/src/specfem3D/read_write_netcdf.f90 b/src/specfem3D/read_write_netcdf.f90
index 639e705..7f66816 100644
--- a/src/specfem3D/read_write_netcdf.f90
+++ b/src/specfem3D/read_write_netcdf.f90
@@ -1,5 +1,7 @@
 subroutine write_kernel_netcdf (fName, param)
   
+  ! This guy converts the kernels to global numbering, and writes to a NetCDF file in parallel.
+  
   use :: mpi
   use :: netcdf
   use :: specfem_par
@@ -14,14 +16,25 @@ subroutine write_kernel_netcdf (fName, param)
   real (kind=CUSTOM_REAL), dimension (:), allocatable :: paramUnpacked
   real (kind=CUSTOM_REAL), dimension (NGLLX, NGLLY, NGLLZ, NSPEC_CRUST_MANTLE) :: param
   
-  integer, parameter :: shuOn=1
-  integer, parameter :: comLvl=9
-  integer, parameter :: comOn=1
-  integer            :: ncid, stat
-  integer            :: i, j, k, ispec, iglob
+  integer, parameter         :: NDIMS=2  
+  integer, parameter         :: shuOn=1
+  integer, parameter         :: comLvl=9
+  integer, parameter         :: comOn=1
+                            
+  integer                    :: ncid, stat
+  integer                    :: i, j, k, ispec, iglob
+  integer                    :: commWorldSize
+  integer                    :: procDimId, kernDimId, varId
+  
+  integer, dimension (NDIMS) :: start, count, ids
+  
+  ! Get the total number of processors.
+  call world_size (commWorldSize)
   
+  ! Allocate the kernel dump array.
   allocate (paramUnpacked (NGLOB_CRUST_MANTLE))
   
+  ! Convert kernels to global numbering.
   do ispec = 1, NSPEC_CRUST_MANTLE
     do k = 1, NGLLZ
       do j = 1, NGLLY
@@ -35,13 +48,44 @@ subroutine write_kernel_netcdf (fName, param)
     end do
   end do
   
-  call checkNC ( nf90_create (fName, IOR(NF90_NETCDF4, NF90_MPIIO), ncid, &
+  ! Create the parallel NetCDF file.
+  call checkNC (nf90_create (fName, IOR(NF90_NETCDF4, NF90_MPIIO), ncid, &
     comm = MPI_COMM_WORLD, info = MPI_INFO_NULL))
     
+  ! Define the processor array.
+  call checkNC (nf90_def_dim (ncid, 'glob', NGLOB_CRUST_MANTLE, kernDimId))
+  call checkNC (nf90_def_dim (ncid, 'proc', commWorldSize, procDimID))
+  
+  ! Sort ids into array.
+  ids = (/ kernDimID, procDimID /)
+  
+  ! Define the kernel variable.
+  call checkNC (nf90_def_var (ncid, 'rawKernel', NF90_float, ids, varId))
+  
+  ! Deflate (compress) the variable. Doesn't work in parallel.
+  ! call checkNC (nf90_def_var_deflate (ncid, varId, shuOn, comOn, comLvl))
+  
+  ! End definition.
+  call checkNC (nf90_enddef (ncid))
+  
+  ! Each processor writes one row.
+  start = (/ 1, myRank + 1 /)
+  count = (/ NGLOB_CRUST_MANTLE, 1 /)
+  call checkNC (nf90_put_var (ncid, varId, paramUnpacked, start = start, count = count))
+  
+  ! Close the netcdf file.
+  call checkNC (nf90_close (ncid))
+  
+  ! Deallocate the kernel dump array.
+  deallocate (paramUnpacked)
+  
 end subroutine write_kernel_netcdf
 
 subroutine checkNC (status)
   
+  ! This little guy just checks for an error from the NetCDF libraries and throws a tantrum if
+  ! one's found.
+  
   use :: netcdf
   
   implicit none
@@ -50,7 +94,7 @@ subroutine checkNC (status)
   
   if (status /= nf90_noerr) then
     print *, trim (nf90_strerror (status))
-    stop 2
+    stop 'Netcdf error.'
   end if
   
 end subroutine checkNC
\ No newline at end of file
diff --git a/src/specfem3D/save_kernels.F90 b/src/specfem3D/save_kernels.F90
index a492754..0ee9e1f 100644
--- a/src/specfem3D/save_kernels.F90
+++ b/src/specfem3D/save_kernels.F90
@@ -435,9 +435,15 @@
         write(IOUT) rho_kl_crust_mantle
         close(IOUT)
 
+        ! Output these kernels as netcdf files -- one per processor.
 #if defined (CEM)
 
-        call write_kernel_netcdf ('alphav_kernel.nc', alphav_kl_crust_mantle)
+        call write_kernel_netcdf ('alphavKernelCrustMantle.nc', alphav_kl_crust_mantle)
+        call write_kernel_netcdf ('alphahKernelCrustMantle.nc', alphah_kl_crust_mantle)
+        call write_kernel_netcdf ('betavKernelCrustMantle.nc',  betav_kl_crust_mantle)
+        call write_kernel_netcdf ('betahKernelCrustMantle.nc',  betah_kl_crust_mantle)
+        call write_kernel_netcdf ('etaKernelCrustMantle.nc',    eta_kl_crust_mantle)
+        call write_kernel_netcdf ('rhoKernelCrustMantle.nc',    rho_kl_crust_mantle)
                 
 #endif