[cig-commits] [commit] master: MESHER/submit.csh: support for SuperMUC (b2efcd3)

[cig_noreply at geodynamics.org]

Repository : https://github.com/geodynamics/axisem

On branch  : master
Link       : https://github.com/geodynamics/axisem/compare/201b2a8c3949bb5acd44c5b1f0eeaba0012815d3...d9174f4e9c01eb069c0079f9a67d155de1997361

>---------------------------------------------------------------

commit b2efcd305a4fe97c2c69b76c875d807b3b4cb91d
Author: Simon Stähler <staehler at geophysik.uni-muenchen.de>
Date:   Wed Nov 12 13:32:57 2014 +0100

    MESHER/submit.csh: support for SuperMUC
    
     - if first argument is "SuperMUC", a rundir has to be defined as second argument, which allows multiple runs of xmesh in parallel.
     - movemesh.csh may take two arguments, where the first is then the rundir of the queued xmesh run


>---------------------------------------------------------------

b2efcd305a4fe97c2c69b76c875d807b3b4cb91d
 MESHER/movemesh.csh | 25 ++++++++++++++++---------
 MESHER/submit.csh   | 19 +++++++++++++++----
 2 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/MESHER/movemesh.csh b/MESHER/movemesh.csh
index 53da0f4..8e5f329 100755
--- a/MESHER/movemesh.csh
+++ b/MESHER/movemesh.csh
@@ -7,8 +7,18 @@ if ( ${#argv} < 1 || "$1" == "-h" ) then
     exit
 endif
 
+set homepath = `echo $PWD`
+set meshpath = "../SOLVER/MESHES/$1"
+
+if ( ${#argv} == 2 ) then
+  set xmeshrundir = $2
+else
+  set xmeshrundir = "."
+endif
+echo "Moving mesh from $xmeshrundir to $meshpath"
+
 # test if mesher finished without problems:
-if (`grep 'DONE WITH MESHER' OUTPUT | wc -l` != '1') then
+if (`grep 'DONE WITH MESHER' $xmeshrundir/OUTPUT | wc -l` != '1') then
   echo "ERROR: MESHER did not finish yet or encountered a problem."
   echo "       Check 'OUTPUT' file for more details."
   exit
@@ -16,9 +26,6 @@ else
   echo 'MESHER finished smoothly...'
 endif
 
-set homepath = `echo $PWD`
-set meshpath = "../SOLVER/MESHES/$1"
-
 if ( ! -d ../SOLVER/MESHES ) then
   mkdir ../SOLVER/MESHES
 endif
@@ -37,11 +44,11 @@ cd $homepath
 
 echo "Moving mesh to" $meshpath
 
-mv meshdb.dat* $meshpath
-mv mesh_params.h $meshpath
-mv OUTPUT $meshpath
+mv $xmeshrundir/meshdb.dat* $meshpath
+mv $xmeshrundir/mesh_params.h $meshpath
+mv $xmeshrundir/OUTPUT $meshpath
 #mv Diags $meshpath
-cp -p inparam_mesh $meshpath
+cp -p $xmeshrundir/inparam_mesh $meshpath
 
 set bgmodel = `grep "^BACKGROUND_MODEL" inparam_mesh | awk '{print $2}'`
 echo $bgmodel
@@ -65,7 +72,7 @@ cp -p xmesh $meshpath/Code
 cp -p submit.csh $meshpath/Code
 cp -p inparam_mesh $meshpath/Code
 
-mv Diags/* $meshpath
+mv $xmeshrundir/Diags/* $meshpath
 
 echo "Contents in" $meshpath ":"
 ls $meshpath
diff --git a/MESHER/submit.csh b/MESHER/submit.csh
index 3054a01..e5a93e6 100755
--- a/MESHER/submit.csh
+++ b/MESHER/submit.csh
@@ -65,6 +65,17 @@ else if ( $1 == 'slurmlocal' ) then
     aprun -n 1 ./xmesh > OUTPUT &
 
 else if ( $1 == 'SuperMUC') then
+
+    mkdir $2
+    cd $2 
+    mkdir Diags
+
+    cp ../xmesh ../inparam_mesh ../movemesh.csh .
+
+    if ( $bgmodel == 'external') then
+      cp ../$fnam_extmodel .
+    endif
+
     # Submit Mesher job on a FAT node on SuperMUC
     # Allows to use up to 40 threads with OpenMP
     set current_dir=$PWD
@@ -72,11 +83,11 @@ else if ( $1 == 'SuperMUC') then
     echo '#@ output = job_$(jobid).out '                         >> job.cmd
     echo '#@ error = job_$(jobid).err '                          >> job.cmd
     echo "#@ job_type = parallel "                               >> job.cmd
-    echo "#@ class = fat"                                        >> job.cmd
+    echo "#@ class = micro"                                      >> job.cmd
     echo "#@ total_tasks=1 "                                     >> job.cmd
     echo "#@ node = 1 "                                          >> job.cmd
     echo "#@ network.MPI = sn_all,not_shared,us "                >> job.cmd
-    echo "#@ wall_clock_limit = 1:00:00"                         >> job.cmd
+    echo "#@ wall_clock_limit = 0:10:00"                         >> job.cmd
     echo "#@ job_name = AXISEM_MESHER"                           >> job.cmd
     echo "#@ initialdir = $current_dir"                          >> job.cmd
     echo "#@ notification=always"                                >> job.cmd
@@ -85,9 +96,9 @@ else if ( $1 == 'SuperMUC') then
     echo "#@ queue "                                             >> job.cmd
     echo ". /etc/profile"                                        >> job.cmd
     echo ". /etc/profile.d/modules.sh"                           >> job.cmd
-    echo "export OMP_NUM_THREADS=40"                             >> job.cmd
+    echo "export OMP_NUM_THREADS=16"                             >> job.cmd
     echo "export KMP_AFFINITY=""granularity=core,compact,1"" "   >> job.cmd
-    echo "module load intel"                                     >> job.cmd
+    echo "module load netcdf/mpi"                                >> job.cmd
     echo "./xmesh > OUTPUT"                                      >> job.cmd
 
     llsubmit job.cmd