[cig-commits] [commit] master: Submit.csh: Options for SuperMUC (40eb3e9)

[cig_noreply at geodynamics.org]

Repository : https://github.com/geodynamics/axisem

On branch  : master
Link       : https://github.com/geodynamics/axisem/compare/b22757325075cce6c3044f97c0b9591687d53b79...40eb3e9d6991802f4d62ea03e4d20b5b4cf0ed34

>---------------------------------------------------------------

commit 40eb3e9d6991802f4d62ea03e4d20b5b4cf0ed34
Author: Simon Stähler <staehler at geophysik.uni-muenchen.de>
Date:   Wed Oct 29 13:31:14 2014 +0100

    Submit.csh: Options for SuperMUC
    
     - in both submit scripts. MESHER/submit.csh submits to a fat node to use OpenMP


>---------------------------------------------------------------

40eb3e9d6991802f4d62ea03e4d20b5b4cf0ed34
 MESHER/submit.csh | 29 +++++++++++++++++++++++++++++
 SOLVER/submit.csh | 26 ++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/MESHER/submit.csh b/MESHER/submit.csh
index c065cf7..3054a01 100755
--- a/MESHER/submit.csh
+++ b/MESHER/submit.csh
@@ -63,6 +63,35 @@ else if ( $1 == 'slurm' ) then
 else if ( $1 == 'slurmlocal' ) then 
     ######## slurm #######
     aprun -n 1 ./xmesh > OUTPUT &
+
+else if ( $1 == 'SuperMUC') then
+    # Submit Mesher job on a FAT node on SuperMUC
+    # Allows to use up to 40 threads with OpenMP
+    set current_dir=$PWD
+    echo "# This job command file calls the AxiSEM MESHER"        > job.cmd
+    echo '#@ output = job_$(jobid).out '                         >> job.cmd
+    echo '#@ error = job_$(jobid).err '                          >> job.cmd
+    echo "#@ job_type = parallel "                               >> job.cmd
+    echo "#@ class = fat"                                        >> job.cmd
+    echo "#@ total_tasks=1 "                                     >> job.cmd
+    echo "#@ node = 1 "                                          >> job.cmd
+    echo "#@ network.MPI = sn_all,not_shared,us "                >> job.cmd
+    echo "#@ wall_clock_limit = 1:00:00"                         >> job.cmd
+    echo "#@ job_name = AXISEM_MESHER"                           >> job.cmd
+    echo "#@ initialdir = $current_dir"                          >> job.cmd
+    echo "#@ notification=always"                                >> job.cmd
+    echo "#@ energy_policy_tag = Axisem_Mesher  "                >> job.cmd
+    echo "#@ minimize_time_to_solution = yes    "                >> job.cmd
+    echo "#@ queue "                                             >> job.cmd
+    echo ". /etc/profile"                                        >> job.cmd
+    echo ". /etc/profile.d/modules.sh"                           >> job.cmd
+    echo "export OMP_NUM_THREADS=40"                             >> job.cmd
+    echo "export KMP_AFFINITY=""granularity=core,compact,1"" "   >> job.cmd
+    echo "module load intel"                                     >> job.cmd
+    echo "./xmesh > OUTPUT"                                      >> job.cmd
+
+    llsubmit job.cmd
+
 else
     ######## SUBMIT LOCALLY #######
     #setenv OMP_NUM_THREADS 4
diff --git a/SOLVER/submit.csh b/SOLVER/submit.csh
index 7cab16e..aa567a4 100755
--- a/SOLVER/submit.csh
+++ b/SOLVER/submit.csh
@@ -377,6 +377,32 @@ foreach isim (${srcapp})
             echo "$mpiruncmd -n ${nodnum} $PWD/axisem  > $outputname " >> run_solver.pbs
             qsub run_solver.pbs
 
+        ############### SuperMUC ###################
+        else if ( $queue == 'SuperMUC') then
+            set current_dir=$PWD
+            @ nnodes = ($nodnum / 16)
+            echo "# This job command file is called job.cmd "             > job.cmd
+            echo '#@ output = job_$(jobid).out '                         >> job.cmd
+            echo '#@ error = job_$(jobid).err '                          >> job.cmd
+            echo "#@ job_type = parallel "                               >> job.cmd
+            echo "#@ class = general "                                   >> job.cmd
+            echo "#@ total_tasks=$nodnum "                               >> job.cmd
+            echo "#@ node = $nnodes "                                    >> job.cmd
+            echo "#@ network.MPI = sn_all,not_shared,us "                >> job.cmd
+            echo "#@ wall_clock_limit = 1:00:00"                         >> job.cmd
+            echo "#@ job_name = $outputname"                             >> job.cmd
+            echo "#@ initialdir = $current_dir"                          >> job.cmd
+            echo "#@ notification=always"                                >> job.cmd
+            echo "#@ energy_policy_tag = Axisem_Solver  "                >> job.cmd
+            echo "#@ minimize_time_to_solution = yes    "                >> job.cmd
+            echo "#@ queue "                                             >> job.cmd
+            echo ". /etc/profile"                                        >> job.cmd
+            echo ". /etc/profile.d/modules.sh"                           >> job.cmd
+            echo "module load netcdf/mpi"                                >> job.cmd
+            echo "module load intel"                                     >> job.cmd
+            echo "poe ./axisem > $outputname "                           >> job.cmd
+
+            llsubmit job.cmd
         endif
 
     ######## SUBMIT LOCALLY #######