[cig-commits] r5327 - in cs/pythia/trunk: . mpi pyre/applications
pyre/inventory pyre/inventory/cfg pyre/inventory/odb
pyre/inventory/properties pyre/launchers pyre/schedulers
pyre/schedulers/scripts/lsf pyre/util
leif at geodynamics.org
leif at geodynamics.org
Fri Nov 17 17:25:42 PST 2006
Author: leif
Date: 2006-11-17 17:25:41 -0800 (Fri, 17 Nov 2006)
New Revision: 5327
Modified:
cs/pythia/trunk/
cs/pythia/trunk/mpi/Application.py
cs/pythia/trunk/mpi/Launcher.py
cs/pythia/trunk/pyre/applications/AppRunner.py
cs/pythia/trunk/pyre/applications/Application.py
cs/pythia/trunk/pyre/applications/CommandlineParser.py
cs/pythia/trunk/pyre/inventory/Facility.py
cs/pythia/trunk/pyre/inventory/cfg/Parser.py
cs/pythia/trunk/pyre/inventory/odb/Curator.py
cs/pythia/trunk/pyre/inventory/properties/List.py
cs/pythia/trunk/pyre/launchers/Launcher.py
cs/pythia/trunk/pyre/schedulers/BatchScheduler.py
cs/pythia/trunk/pyre/schedulers/Job.py
cs/pythia/trunk/pyre/schedulers/scripts/lsf/batch.sh.tmpl
cs/pythia/trunk/pyre/util/__init__.py
cs/pythia/trunk/setup.cfg
cs/pythia/trunk/setup.py
Log:
Pythia v0.8.1.0b5:
* Use merlin instead of setuptools. Moved a few methods from Pythia
into Merlin, because that's where they belong.
* Allow a trailing comma in list properties, per Eh's request, and for
consistency with Python.
* Fixed an obscure bug where list properties that defaulted to the
empty list would interfere with each other when modified with a
side-effect producing method such as extend() or append().
* Fixed an undeclared variable bug in CommandlineParser. It was on an
error path that isn't often executed. !@#$%* Python.
* Print the 'mpirun' command in the batch script as a comment, mainly
so that users can see and debug the 'mpirun' command when they use
--scheduler.dry. [Nowadays, the 'mpirun' command is otherwise
hidden: is no longer run directly from the batch script itself,
because now Pyre apps can do stuff at batch-script-run-time by
overriding onLauncherNode(). This functionality was introduced for
SPECFEM, which does pre- and post-job processing.]
* Flipped the 'scheduler.wait' default for BatchSchedulers (such as
LSF) from 'True' to 'False'. So, the default behavior is different
depending on where you are working: on a workstation or a Beowulf
with no scheduler, 'citcoms' will wait for the 'mpirun' command to
finish; but on a cluster, it will submit the job and exit. I think
this is what users expect.
* Fixed a bug where the launcher arguments were reversed. I have no
idea how this bug persisted as long as it did.
* Made .cfg files more flexible: allow dots on the left-hand side of
property assignment. I.e., instead of the following:
[CitcomS.solver]
datafile = cookbook1
[CitcomS.solver.ic]
num_perturbations = 1
one can now write instead:
[CitcomS.solver]
datafile = cookbook1
ic.num_perturbations = 1
Which one to choose is purely a matter of taste. I find it nice
because it cuts down on the number of sections in a .cfg file, and
eliminates the need for introducing a whole new section just for a
single property.
Property changes on: cs/pythia/trunk
___________________________________________________________________
Name: svn:externals
- ez_setup http://geodynamics.org/svn/cig/cs/ez_setup
+ archimedes http://geodynamics.org/svn/cig/cs/merlin/branches/v1/merlin/archimedes
Modified: cs/pythia/trunk/mpi/Application.py
===================================================================
--- cs/pythia/trunk/mpi/Application.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/mpi/Application.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -49,13 +49,17 @@
job.executable = self.jobExecutable
job.arguments = ["--pyre-start", path, requires, "pyre.schedulers:jobstart", entry] + argv
+ # for debugging purposes, add 'mpirun' command as a comment
+ launcher = self.prepareLauncher()
+ job.comments.extend(["[%s] %s" % (launcher.name, comment) for comment in launcher.comments()])
+
# schedule
self.scheduler.schedule(job)
return
- def onLauncherNode(self, *args, **kwds):
+ def prepareLauncher(self, *args, **kwds):
import sys
path = self.pathString()
@@ -67,14 +71,20 @@
launcher = self.launcher
launcher.nodes = self.nodes
launcher.executable = self.mpiExecutable
- launcher.arguments = ["--pyre-start", requires, path, "mpi:mpistart", entry] + argv
-
+ launcher.arguments = ["--pyre-start", path, requires, "mpi:mpistart", entry] + argv
+
+ return launcher
+
+
+ def onLauncherNode(self, *args, **kwds):
+
+ launcher = self.prepareLauncher()
+
# launch
launcher.launch()
return
-
def onComputeNodes(self, *args, **kwds):
self.main(*args, **kwds)
Modified: cs/pythia/trunk/mpi/Launcher.py
===================================================================
--- cs/pythia/trunk/mpi/Launcher.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/mpi/Launcher.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -34,17 +34,12 @@
def launch(self):
import os, sys
- self.executable = os.path.abspath(self.executable)
-
- argv = self._buildArgumentList()
- if not argv:
- return self.dry
-
+ argv = self.argv()
command = ' '.join(argv)
if self.dry:
print command
- return True
+ return
self._info.log("spawning: %s" % command)
status = os.spawnvp(os.P_WAIT, argv[0], argv)
@@ -53,10 +48,15 @@
sys.exit(statusMsg)
self._info.log(statusMsg)
- return True
+ return
+ def argv(self): return self._buildArgumentList()
+
+
def _buildArgumentList(self):
+ import os
+
if not self.nodes:
self.nodes = len(self.nodelist)
@@ -67,7 +67,7 @@
args = self.command.split(' ')
self._appendMpiRunArgs(args)
- args.append(self.executable)
+ args.append(os.path.abspath(self.executable))
args += self.arguments
return args
Modified: cs/pythia/trunk/pyre/applications/AppRunner.py
===================================================================
--- cs/pythia/trunk/pyre/applications/AppRunner.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/applications/AppRunner.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -21,7 +21,7 @@
def createSubscript(self, name):
- from pyre.util import loadObject
+ from merlin import loadObject
cls = loadObject(name)
return cls()
Modified: cs/pythia/trunk/pyre/applications/Application.py
===================================================================
--- cs/pythia/trunk/pyre/applications/Application.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/applications/Application.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -157,19 +157,10 @@
def workingSet(self):
"""Return the minimal working set for this application."""
-
- from pkg_resources import WorkingSet, Environment, parse_requirements
-
- requires = self.requires()
- workingSet = WorkingSet([])
- requirements = parse_requirements(requires)
-
- for dist in workingSet.resolve(requirements, Environment()):
- workingSet.add(dist)
+ from merlin import WorkingSet
+ return WorkingSet.minimal(self.requires())
- return workingSet
-
def path(self):
"""Return the minimal Python search path for this application."""
workingSet = self.workingSet()
Modified: cs/pythia/trunk/pyre/applications/CommandlineParser.py
===================================================================
--- cs/pythia/trunk/pyre/applications/CommandlineParser.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/applications/CommandlineParser.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -100,8 +100,8 @@
# dangling =
if len(tokens) > 1 and not tokens[1]:
- self._debug.log("tokens: bad expression: %s" % arg)
- raise CommandlineParser.CommandlineException("bad expression: '%s': no rhs" % arg)
+ self._debug.log("tokens: bad expression: %s" % candidate)
+ raise CommandlineParser.CommandlineException("bad expression: '%s': no rhs" % candidate)
# lhs, rhs
lhs = tokens[0]
Modified: cs/pythia/trunk/pyre/inventory/Facility.py
===================================================================
--- cs/pythia/trunk/pyre/inventory/Facility.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/inventory/Facility.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -170,7 +170,7 @@
objName = module + ':' + factoryName
try:
- from pyre.util import loadObject
+ from merlin import loadObject
factory = loadObject(objName)
except (ImportError, ValueError):
raise Facility.ComponentNotFound(
Modified: cs/pythia/trunk/pyre/inventory/cfg/Parser.py
===================================================================
--- cs/pythia/trunk/pyre/inventory/cfg/Parser.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/inventory/cfg/Parser.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -50,9 +50,13 @@
self.node = node
self.fp = fp
- def __setitem__(self, key, value):
+ def __setitem__(self, trait, value):
locator = locators.file(self.fp.name, self.fp.lineno)
- self.node.setProperty(key, value, locator)
+ path = trait.split('.')
+ key = path[-1]
+ path = path[:-1]
+ node = _getNode(self.node, path)
+ node.setProperty(key, value, locator)
dict.__setitem__(self, key, value)
class SectionDict(dict):
@@ -66,7 +70,7 @@
# Prevent 'ConfigParser' from creating section
# dictionaries; instead, create our own.
if not dict.__contains__(self, sectname):
- node = self._getNode(self.root, sectname.split('.'))
+ node = _getNode(self.root, sectname.split('.'))
cursect = Parser.Section(sectname, node, self.fp)
self[sectname] = cursect
return True
@@ -74,12 +78,6 @@
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
- def _getNode(self, node, path):
- if len(path) == 0:
- return node
- key = path[0].strip()
- return self._getNode(node.getNode(key), path[1:])
-
def __init__(self, root, defaults=None):
SafeConfigParser.__init__(self, defaults)
self._sections = Parser.SectionDict(root)
@@ -96,4 +94,11 @@
return optionstr
+def _getNode(node, path):
+ if len(path) == 0:
+ return node
+ key = path[0].strip()
+ return _getNode(node.getNode(key), path[1:])
+
+
# end of file
Modified: cs/pythia/trunk/pyre/inventory/odb/Curator.py
===================================================================
--- cs/pythia/trunk/pyre/inventory/odb/Curator.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/inventory/odb/Curator.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -191,7 +191,7 @@
systemDepository = self.setSystemDepository(system)
# create the built-in depositories
- from pkg_resources import resource_listdir, resource_isdir, resource_exists, resource_filename, Requirement
+ from merlin import resource_listdir, resource_isdir, resource_exists, resource_filename, Requirement
pythia = Requirement.parse("pythia")
entries = resource_listdir(pythia, "")
for entry in entries:
Modified: cs/pythia/trunk/pyre/inventory/properties/List.py
===================================================================
--- cs/pythia/trunk/pyre/inventory/properties/List.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/inventory/properties/List.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -18,7 +18,9 @@
class List(Property):
- def __init__(self, name, default=[], meta=None, validator=None):
+ def __init__(self, name, default=None, meta=None, validator=None):
+ if default is None:
+ default = list()
Property.__init__(self, name, "list", default, meta, validator)
return
@@ -31,6 +33,10 @@
text = text[:-1]
value = text.split(",")
+
+ # allow trailing comma
+ if len(value) and not value[-1]:
+ value.pop()
else:
value = text
Modified: cs/pythia/trunk/pyre/launchers/Launcher.py
===================================================================
--- cs/pythia/trunk/pyre/launchers/Launcher.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/launchers/Launcher.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -33,4 +33,12 @@
raise NotImplementedError("class '%s' must override 'launch'" % self.__class__.__name__)
+ def argv(self):
+ raise NotImplementedError("class '%s' must override 'argv'" % self.__class__.__name__)
+
+
+ def comments(self):
+ return ["command: " + ' '.join(self.argv())]
+
+
# end of file
Modified: cs/pythia/trunk/pyre/schedulers/BatchScheduler.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/BatchScheduler.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/schedulers/BatchScheduler.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -15,7 +15,12 @@
class BatchScheduler(Scheduler):
- pass
+ import pyre.inventory as pyre
+ # override the default for 'wait'
+ wait = pyre.bool("wait", default=False)
+ wait.meta['tip'] = """wait for the job to finish"""
+
+
# end of file
Modified: cs/pythia/trunk/pyre/schedulers/Job.py
===================================================================
--- cs/pythia/trunk/pyre/schedulers/Job.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/schedulers/Job.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -39,6 +39,8 @@
executable = pyre.str("executable")
arguments = pyre.list("arguments")
+ comments = pyre.list("comments")
+
def __init__(self):
super(Job, self).__init__()
Modified: cs/pythia/trunk/pyre/schedulers/scripts/lsf/batch.sh.tmpl
===================================================================
--- cs/pythia/trunk/pyre/schedulers/scripts/lsf/batch.sh.tmpl 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/schedulers/scripts/lsf/batch.sh.tmpl 2006-11-18 01:25:41 UTC (rev 5327)
@@ -41,5 +41,12 @@
${job.executable} @echo ' '.join($job.arguments)
-# submit with:
+ at if $job.comments
+# ~~~~ comments ~~~~
+ at for line in $job.comments
+# ${line}
+ at end for
+ at end if
+
+# ~~~~ submit command ~~~~
# ${scheduler.batchCommand} < [script]
Modified: cs/pythia/trunk/pyre/util/__init__.py
===================================================================
--- cs/pythia/trunk/pyre/util/__init__.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/pyre/util/__init__.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -44,22 +44,6 @@
return expandMacros(raw, substitutions)
-def loadObject(name):
- """Load and return the object referenced by <name> ==
- some.module[:some.attr].
-
- Derived from pkg_resources:EntryPoint.
-
- """
-
- module, attrs = name.split(':')
- attrs = attrs.split('.')
- obj = __import__(module, globals(), globals(), ['__name__'])
- for attr in attrs:
- obj = getattr(obj, attr)
- return obj
-
-
# version
__id__ = "$Id: __init__.py,v 1.1.1.1 2005/03/08 16:13:41 aivazis Exp $"
Modified: cs/pythia/trunk/setup.cfg
===================================================================
--- cs/pythia/trunk/setup.cfg 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/setup.cfg 2006-11-18 01:25:41 UTC (rev 5327)
@@ -1,4 +1,4 @@
[egg_info]
-tag_build = b4
+tag_build = b5
#tag_svn_revision = 1
Modified: cs/pythia/trunk/setup.py
===================================================================
--- cs/pythia/trunk/setup.py 2006-11-18 00:48:49 UTC (rev 5326)
+++ cs/pythia/trunk/setup.py 2006-11-18 01:25:41 UTC (rev 5327)
@@ -1,8 +1,8 @@
-from ez_setup import use_setuptools
-use_setuptools()
+from archimedes import use_merlin
+use_merlin()
-from setuptools import setup, find_packages
+from merlin import setup, find_packages
setup(
More information about the cig-commits
mailing list