[cig-commits] r6700 - in cs/pythia/trunk: mpi pyre/applications
pyre/schedulers
leif at geodynamics.org
leif at geodynamics.org
Thu Apr 26 15:06:25 PDT 2007
Author: leif
Date: 2007-04-26 15:06:24 -0700 (Thu, 26 Apr 2007)
New Revision: 6700
Added:
cs/pythia/trunk/pyre/schedulers/BatchScript.py
Modified:
cs/pythia/trunk/mpi/Application.py
cs/pythia/trunk/mpi/__init__.py
cs/pythia/trunk/pyre/applications/SuperScript.py
cs/pythia/trunk/pyre/schedulers/Job.py
cs/pythia/trunk/pyre/schedulers/Scheduler.py
cs/pythia/trunk/pyre/schedulers/SchedulerLSF.py
cs/pythia/trunk/pyre/schedulers/SchedulerNone.py
cs/pythia/trunk/pyre/schedulers/SchedulerPBS.py
cs/pythia/trunk/pyre/schedulers/__init__.py
Log:
[pythia v0.8.1.3b4]
Added ${job.id} macro. The job ID is also available from Python as
"app.job.id" (where 'app' is the Application instance).
Summary of built-in macros:
~~~ for all apps (serial or parallel) ~~~
${home} : user's home directory; on Unix, same as ${HOME}
${user} : login name
${hostname} : return value of gethostname()
${wd} : full path of working directory at startup
${pid} : process ID
~ in .cfg files only:
${td} : pathname of directory containing .cfg file
${basename} : filename of .cfg file, minus .cfg extension
~~~ within a parallel job ~~~
${job.name} : LSF/PBS job name specified using "job.name"
${job.id} : LSF/PBS job ID (a process ID if there is no scheduler)
${nodes} : number of machine nodes
~~~ under 'mpirun' ~~~
${rank} : result of MPI_Comm_rank(MPI_COMM_WORLD)
Modified: cs/pythia/trunk/mpi/Application.py
===================================================================
--- cs/pythia/trunk/mpi/Application.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/mpi/Application.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -36,6 +36,10 @@
def onLoginNode(self, *args, **kwds):
+ self.schedule(*args, **kwds)
+
+
+ def schedule(self, *args, **kwds):
import sys
path = self.pathString()
@@ -43,15 +47,18 @@
entry = self.entryName()
argv = self.getArgv(*args, **kwds)
state = self.getStateArgs('launch')
+ batchScriptArgs = self.getBatchScriptArgs()
# initialize the job
job = self.job
job.nodes = self.nodes
job.executable = self.jobExecutable
- job.arguments = ["--pyre-start", path, requires, "pyre.schedulers:jobstart", entry] + argv + state
+ job.arguments = (["--pyre-start", path, requires,
+ "pyre.schedulers:jobstart"] + batchScriptArgs +
+ [entry] + argv + state)
# for debugging purposes, add 'mpirun' command as a comment
- launcher = self.prepareLauncher()
+ launcher = self.prepareLauncher(argv + state)
job.comments.extend(["[%s] %s" % (launcher.name, comment) for comment in launcher.comments()])
# schedule
@@ -60,13 +67,12 @@
return
- def prepareLauncher(self, *args, **kwds):
+ def prepareLauncher(self, argv):
import sys
path = self.pathString()
requires = self.requires()
entry = self.entryName()
- argv = self.getArgv(*args, **kwds)
state = self.getStateArgs('compute')
# initialize the launcher
@@ -78,16 +84,42 @@
return launcher
+ def _onLauncherNode(self, *args, **kwds):
+ # This method should not be overriden in any subclass.
+ self.job.id = self.scheduler.jobId()
+ self.onLauncherNode(*args, **kwds)
+
+
def onLauncherNode(self, *args, **kwds):
+ self.launch(*args, **kwds)
- launcher = self.prepareLauncher()
+ def launch(self, *args, **kwds):
+
+ argv = self.getArgv(*args, **kwds)
+ launcher = self.prepareLauncher(argv)
+
# launch
launcher.launch()
return
+ def _onComputeNodes(self, *args, **kwds):
+ # This method should not be overriden in any subclass.
+
+ # Don't try this at home.
+ argv = kwds['argv']
+ for arg in argv:
+ if arg.startswith("--macros.job.id="):
+ self.job.id = arg.split('=')[1]
+ break
+
+ self.onComputeNodes(*args, **kwds)
+
+ return
+
+
def onComputeNodes(self, *args, **kwds):
self.main(*args, **kwds)
@@ -101,6 +133,12 @@
return state
+ def getBatchScriptArgs(self):
+ SchedulerClass = self.scheduler.__class__
+ schedulerClass = SchedulerClass.__module__ + ":" + SchedulerClass.__name__
+ return ["--scheduler-class=%s" % schedulerClass]
+
+
def __init__(self, name=None):
super(Application, self).__init__(name)
Modified: cs/pythia/trunk/mpi/__init__.py
===================================================================
--- cs/pythia/trunk/mpi/__init__.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/mpi/__init__.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -33,7 +33,7 @@
}
kwds = kwds.get('kwds', dict())
- kwds['message'] = 'onComputeNodes'
+ kwds['message'] = '_onComputeNodes'
kwds['macros'] = macros
try:
Modified: cs/pythia/trunk/pyre/applications/SuperScript.py
===================================================================
--- cs/pythia/trunk/pyre/applications/SuperScript.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/applications/SuperScript.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -52,13 +52,13 @@
def main(self, *args, **kwds):
+ args = kwds.get('args', [])
+ kwds = kwds.get('kwds', dict())
+ kwds['argv'] = [self.argv[-1]] + self.unprocessedArguments
self.runSubscript(*args, **kwds)
def runSubscript(self, *args, **kwds):
- args = kwds.get('args', [])
- kwds = kwds.get('kwds', dict())
- kwds['argv'] = [self.argv[-1]] + self.unprocessedArguments
self.subscript.run(*args, **kwds)
Added: cs/pythia/trunk/pyre/schedulers/BatchScript.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/BatchScript.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/schedulers/BatchScript.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# California Institute of Technology
+# (C) 2007 All Rights Reserved
+#
+# {LicenseText}
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+
+
+from pyre.applications import AppRunner
+from merlin import loadObject
+
+
+class BatchScript(AppRunner):
+
+
+ import pyre.inventory as pyre
+ schedulerClass = pyre.str("scheduler-class", default="pyre.schedulers:SchedulerNone")
+
+
+ def _init(self):
+ super(BatchScript, self)._init()
+ self.SchedulerClass = loadObject(self.schedulerClass)
+ return
+
+
+ def defineMacros(self, macros):
+ macros['job.id'] = self.SchedulerClass.jobId()
+ return
+
+
+ def runSubscript(self, *args, **kwds):
+ macros = kwds.setdefault('macros', {})
+ self.defineMacros(macros)
+ super(BatchScript, self).runSubscript(*args, **kwds)
+ return
+
+
+# end of file
Modified: cs/pythia/trunk/pyre/schedulers/Job.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/Job.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/schedulers/Job.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -45,12 +45,15 @@
def __init__(self):
super(Job, self).__init__()
self.nodes = 1
+ self.id = None
def getStateArgs(self, stage):
state = []
if stage == 'launch':
state.append("--macros.job.name=%s" % self.task)
+ elif stage == 'compute':
+ state.append("--macros.job.id=%s" % self.id)
return state
Modified: cs/pythia/trunk/pyre/schedulers/Scheduler.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/Scheduler.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/schedulers/Scheduler.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -33,4 +33,9 @@
raise NotImplementedError("class '%s' must override 'schedule'" % self.__class__.__name__)
+ def jobId(cls):
+ raise NotImplementedError("class '%s' must override 'jobId'" % cls.__name__)
+ jobId = classmethod(jobId)
+
+
# end of file
Modified: cs/pythia/trunk/pyre/schedulers/SchedulerLSF.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/SchedulerLSF.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/schedulers/SchedulerLSF.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -12,6 +12,7 @@
from BatchScheduler import BatchScheduler
+import os, sys
class SchedulerLSF(BatchScheduler):
@@ -27,7 +28,6 @@
def schedule(self, job):
- import os, sys
import pyre.util as util
# Fix-up the job.
@@ -101,4 +101,9 @@
return
+ def jobId(cls):
+ return os.environ['LSB_JOBID']
+ jobId = classmethod(jobId)
+
+
# end of file
Modified: cs/pythia/trunk/pyre/schedulers/SchedulerNone.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/SchedulerNone.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/schedulers/SchedulerNone.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -12,6 +12,7 @@
from Scheduler import Scheduler
+import os, sys
class SchedulerNone(Scheduler):
@@ -21,8 +22,6 @@
def schedule(self, job):
- import os, sys
-
job.executable = os.path.abspath(job.executable)
argv = [job.executable] + job.arguments
@@ -46,4 +45,9 @@
return
+ def jobId(cls):
+ return str(os.getpid())
+ jobId = classmethod(jobId)
+
+
# end of file
Modified: cs/pythia/trunk/pyre/schedulers/SchedulerPBS.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/SchedulerPBS.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/schedulers/SchedulerPBS.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -12,6 +12,7 @@
from BatchScheduler import BatchScheduler
+import os, sys
class SchedulerPBS(BatchScheduler):
@@ -28,7 +29,6 @@
def schedule(self, job):
- import os, sys
import pyre.util as util
# Fix-up the job.
@@ -105,4 +105,9 @@
return resourceList
+ def jobId(cls):
+ return os.environ['PBS_JOBID']
+ jobId = classmethod(jobId)
+
+
# end of file
Modified: cs/pythia/trunk/pyre/schedulers/__init__.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/__init__.py 2007-04-26 22:03:45 UTC (rev 6699)
+++ cs/pythia/trunk/pyre/schedulers/__init__.py 2007-04-26 22:06:24 UTC (rev 6700)
@@ -13,6 +13,9 @@
from BatchScriptTemplate import BatchScriptTemplate
from Scheduler import Scheduler
+from SchedulerNone import SchedulerNone
+from SchedulerLSF import SchedulerLSF
+from SchedulerPBS import SchedulerPBS
from Job import Job
@@ -39,13 +42,14 @@
"""entry point for batch jobs"""
import sys
- from pyre.applications import start, AppRunner
+ from pyre.applications import start
+ from BatchScript import BatchScript
kwds = kwds.get('kwds', dict())
- kwds['message'] = 'onLauncherNode'
+ kwds['message'] = '_onLauncherNode'
return start(argv,
- applicationClass = AppRunner,
+ applicationClass = BatchScript,
kwds = kwds)
More information about the cig-commits
mailing list