[cig-commits] r7369 - in cs/buildbot/trunk: . bin buildbot
buildbot/scripts
leif at geodynamics.org
leif at geodynamics.org
Thu Jun 21 21:16:44 PDT 2007
Author: leif
Date: 2007-06-21 21:16:44 -0700 (Thu, 21 Jun 2007)
New Revision: 7369
Added:
cs/buildbot/trunk/bin/mkpkg
cs/buildbot/trunk/bin/ship
cs/buildbot/trunk/buildbot/meta.py
cs/buildbot/trunk/buildbot/scripts/packager.py
cs/buildbot/trunk/buildbot/scripts/shipper.py
Modified:
cs/buildbot/trunk/buildbot/bs.py
cs/buildbot/trunk/buildbot/config.py
cs/buildbot/trunk/buildbot/lines.py
cs/buildbot/trunk/setup.py
Log:
Added the ability to automatically package binaries, and upload them
to the main server. It only works on Linux at present.
Also, build PETSc like this:
make PETSC_MAKE_STOP_ON_ERROR=
This forces the build to stop with an error code when there is a
failure. Things were turning green, giving the appearance of success,
when in fact, the PETSc build was failing. (!!!)
Added: cs/buildbot/trunk/bin/mkpkg
===================================================================
--- cs/buildbot/trunk/bin/mkpkg 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/bin/mkpkg 2007-06-22 04:16:44 UTC (rev 7369)
@@ -0,0 +1,4 @@
+#!/usr/bin/python
+
+from buildbot.scripts import packager
+packager.mkpkg()
Property changes on: cs/buildbot/trunk/bin/mkpkg
___________________________________________________________________
Name: svn:executable
+ *
Added: cs/buildbot/trunk/bin/ship
===================================================================
--- cs/buildbot/trunk/bin/ship 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/bin/ship 2007-06-22 04:16:44 UTC (rev 7369)
@@ -0,0 +1,4 @@
+#!/usr/bin/python
+
+from buildbot.scripts import shipper
+shipper.ship()
Property changes on: cs/buildbot/trunk/bin/ship
___________________________________________________________________
Name: svn:executable
+ *
Modified: cs/buildbot/trunk/buildbot/bs.py
===================================================================
--- cs/buildbot/trunk/buildbot/bs.py 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/buildbot/bs.py 2007-06-22 04:16:44 UTC (rev 7369)
@@ -222,7 +222,7 @@
s(step.Compile,
description=["compiling"] + desc,
descriptionDone=desc + ["compile"],
- command=["make"],
+ command=["make", "PETSC_MAKE_STOP_ON_ERROR="],
workdir=workdir,
env=env,
timeout=3600, # Building Sieve takes a long time...
Modified: cs/buildbot/trunk/buildbot/config.py
===================================================================
--- cs/buildbot/trunk/buildbot/config.py 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/buildbot/config.py 2007-06-22 04:16:44 UTC (rev 7369)
@@ -1,10 +1,12 @@
class BuildConfig(object):
- def __init__(self, name, env, slaves=None, configureArgs=None, basedir=None,
+
+ def __init__(self, name, env, tag=None, slaves=None, configureArgs=None, basedir=None,
CPPFLAGS=None, LDFLAGS=None, LIBS=None):
self.name = name
self.env = env
+ self.tag = tag
# I designed BuildConfig & BuildSlave when I thought BuildBot
# did load balancing. But, in fact, BuildBot does load
@@ -25,6 +27,53 @@
return
+
+ def postSteps(self, line, buildEnv, buildConfig, env,
+ desc, workdir, configureArgs):
+ return []
+
+
+
+class BinaryBuildConfig(BuildConfig):
+
+ def __init__(self, name, env, **kwds):
+ kwds.setdefault('tag', 'binaries')
+ BuildConfig.__init__(self, name, env, **kwds)
+ return
+
+
+ def postSteps(self, line, buildEnv, buildConfig, env,
+ desc, workdir, configureArgs):
+
+ from buildbot.process import step
+ from buildbot.process.factory import s
+ from buildbot.steps.shell import WithProperties
+
+ buildnumber = WithProperties("%d", "buildnumber")
+ revision = WithProperties("%s", "revision")
+
+ steps = [
+ s(step.ShellCommand,
+ description=["packaging"] + desc,
+ descriptionDone=desc + ["packaging"],
+ command=["mkpkg", buildnumber, revision],
+ workdir=workdir,
+ env=env,
+ haltOnFailure=True,
+ ),
+ s(step.ShellCommand,
+ description=["shipping"] + desc,
+ descriptionDone=desc + ["shipping"],
+ command=["ship", buildnumber, revision],
+ workdir=workdir,
+ env=env,
+ haltOnFailure=True,
+ ),
+ ]
+
+ return steps
+
+
defaultConfigs = [BuildConfig("default", {"PYTHON": "python2.4"})]
@@ -148,6 +197,8 @@
slaves = buildConfig.slaves or buildEnv.defaultSlaves
builder = line.newBuilder(buildEnv, buildConfig)
+ if builder is None:
+ continue
builder['slavenames'] = [slave.name for slave in slaves]
import os
Modified: cs/buildbot/trunk/buildbot/lines.py
===================================================================
--- cs/buildbot/trunk/buildbot/lines.py 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/buildbot/lines.py 2007-06-22 04:16:44 UTC (rev 7369)
@@ -87,8 +87,10 @@
def newBuilder(self, buildEnv, buildConfig):
+ buildFactory = self.newBuildFactory(buildEnv, buildConfig)
+ if buildFactory is None:
+ return None
builddir = 'build/' + buildEnv.configFilename(buildConfig) + "/" + self.project.name + "/" + self.name
- buildFactory = self.newBuildFactory(buildEnv, buildConfig)
builder = {
'name' : self.builderName(buildEnv, buildConfig),
'builddir' : builddir,
@@ -108,12 +110,11 @@
env.update(self.env)
steps = []
+
+ configs = self.buildConfigs(buildEnv, buildConfig)
+ if configs is None:
+ return None
- if self.configs:
- configs = self.configs
- else:
- configs = {"__default__": []}
-
for config, configureArgs in configs.iteritems():
if config != "__default__":
desc = [config]
@@ -132,10 +133,41 @@
desc, workdir, configureArgs)
steps += self.buildSteps(buildEnv, buildConfig, env,
desc, workdir, configureArgs)
+
+ steps += buildConfig.postSteps(self, buildEnv, buildConfig, env,
+ desc, workdir, configureArgs)
return BuildFactory(steps)
+ def buildConfigs(self, buildEnv, buildConfig):
+ if buildConfig.tag:
+ # If the build config is tagged, and we have more than one
+ # config, only create a builder if we have a config
+ # corresponding to the tag. This is so that the
+ # 'binaries' build knows what to build.
+ if self.configs is None:
+ return {buildConfig.tag: []}
+ if len(self.configs) == 1:
+ return {buildConfig.tag: self.configs.values()[0]}
+ c = self.configs.get(buildConfig.tag, None)
+ if c is None:
+ return None
+ configs = {}
+ configs[buildConfig.tag] = c
+ return configs
+
+ # NYI: reject build configurations that don't support the
+ # languages we require.
+
+ if self.configs:
+ configs = self.configs
+ else:
+ configs = {"__default__": []}
+
+ return configs
+
+
def sourceSteps(self, **kwds):
"""Return a list of BuildSteps which checks-out my source from
the repository, or downloads my source tarball."""
Added: cs/buildbot/trunk/buildbot/meta.py
===================================================================
--- cs/buildbot/trunk/buildbot/meta.py 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/buildbot/meta.py 2007-06-22 04:16:44 UTC (rev 7369)
@@ -0,0 +1,47 @@
+
+
+from ConfigParser import ConfigParser
+from distutils.sysconfig import parse_makefile
+import os, sys
+
+
+class Packaging(object):
+ _name = 'packaging'
+ _attrs = ['directories', 'programs', 'scripts', 'libraries', 'python']
+
+
+class Config(object):
+
+ sections = [Packaging]
+
+ #@classmethod
+ def readConfigFromSubversionProp(cls):
+
+ parser = ConfigParser()
+
+ cfg = "buildbot:config"
+ argv = ["svn", "propget", cfg, "."]
+ i, o = os.popen4(argv)
+ i.close()
+ parser.readfp(o, cfg)
+
+ config = Config()
+
+ for sectionCls in cls.sections:
+ sectionDct = parser._sections.get(sectionCls._name, None)
+ if sectionDct is not None:
+ section = sectionCls()
+ for attr in section._attrs:
+ value = sectionDct.get(attr, "")
+ value = value.split()
+ setattr(section, attr, value)
+ setattr(config, sectionCls._name, section)
+
+ return config
+ readConfigFromSubversionProp = classmethod(readConfigFromSubversionProp)
+
+
+def getPackageAndVersion():
+ # NYI: This is Autotools-specific.
+ makefile = parse_makefile("Makefile")
+ return makefile['PACKAGE'], makefile['VERSION']
Added: cs/buildbot/trunk/buildbot/scripts/packager.py
===================================================================
--- cs/buildbot/trunk/buildbot/scripts/packager.py 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/buildbot/scripts/packager.py 2007-06-22 04:16:44 UTC (rev 7369)
@@ -0,0 +1,86 @@
+
+import os, sys, platform
+from buildbot.meta import Config, getPackageAndVersion
+
+
+def mkpkg():
+ buildnumber = int(sys.argv[1])
+ revision = sys.argv[2]
+ if not revision:
+ revision = "forced"
+ opSys = platform.system().lower()
+ arch = opSys + "-" + platform.machine()
+ tag = "b%04d-%s" % (buildnumber, revision)
+ python = os.environ['PYTHON']
+ prefix = os.environ['PREFIX']
+ package, version = getPackageAndVersion()
+ distdir = package + "-" + version
+ print "tag:", tag
+ print "python:", python
+ print "prefix:", prefix
+ print "distdir:", distdir
+
+ config = Config.readConfigFromSubversionProp()
+ if config.packaging is None:
+ sys.exit('no packaging configuation; try "svn propedit buildbot:config ."')
+
+ packingList = []
+ stripList = []
+
+ for d in config.packaging.directories:
+ packingList.append(d)
+
+ for p in config.packaging.programs:
+ p = "bin/" + p
+ packingList.append(p)
+ stripList.append(p)
+
+ for s in config.packaging.scripts:
+ packingList.append("bin/" + s)
+
+ for l in config.packaging.libraries:
+ l = "lib/" + l
+ packingList.append(l)
+ stripList.append(l)
+
+ for p in config.packaging.python:
+ p = "lib/" + python + "/site-packages/" + p
+ packingList.append(p)
+
+ if opSys == "linux":
+ # We only need one of these; do it on Linux only.
+ archive = distdir + ".tar.gz"
+ taggedArchive = tag + "-" + archive
+ os.system("make dist")
+ os.system("mv " + archive + " " + taggedArchive)
+
+ print "strip", stripList
+ print "pack", packingList
+
+ workdir = os.getcwd()
+ os.chdir(prefix)
+
+ strip = "strip"
+ if opSys == "darwin":
+ # Just plain "strip" renders our Python interpreter unusable
+ # by extension modules.
+ strip = "strip -S"
+ status = os.system(strip + " " + " ".join(stripList))
+ if status != 0:
+ sys.exit("strip: exit %d" % status)
+
+ os.chdir(workdir)
+
+ distdir_arch = distdir + "-" + arch
+ os.system("rm " + distdir_arch)
+ os.system("ln -s " + prefix + " " + distdir_arch)
+ packingList = [distdir_arch + "/" + m for m in packingList]
+ archive = distdir_arch + ".tar.gz"
+ taggedArchive = tag + "-" + archive
+ os.system("tar cvzf " + archive + " " + " ".join(packingList))
+ if status != 0:
+ sys.exit("tar: exit %d" % status)
+
+ os.system("mv " + archive + " " + taggedArchive)
+
+ return
Added: cs/buildbot/trunk/buildbot/scripts/shipper.py
===================================================================
--- cs/buildbot/trunk/buildbot/scripts/shipper.py 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/buildbot/scripts/shipper.py 2007-06-22 04:16:44 UTC (rev 7369)
@@ -0,0 +1,12 @@
+
+import os, sys
+
+
+def ship():
+ buildnumber = int(sys.argv[1])
+ revision = sys.argv[2]
+ if not revision:
+ revision = "forced"
+ tag = "b%04d-%s" % (buildnumber, revision)
+ command = "scp -o BatchMode=yes " + tag + "* www.geodynamics.org:/home/buildbot/public_html/shipping"
+ os.system(command)
Modified: cs/buildbot/trunk/setup.py
===================================================================
--- cs/buildbot/trunk/setup.py 2007-06-22 03:48:19 UTC (rev 7368)
+++ cs/buildbot/trunk/setup.py 2007-06-22 04:16:44 UTC (rev 7369)
@@ -32,7 +32,7 @@
improve.
"""
-scripts = ["bin/buildbot"]
+scripts = ["bin/buildbot", "bin/mkpkg", "bin/ship"]
if sys.platform == "win32":
scripts.append("contrib/windows/buildbot.bat")
scripts.append("contrib/windows/buildbot_service.py")
More information about the cig-commits
mailing list