[cig-commits] r6999 - cs/buildbot/trunk/buildbot
leif at geodynamics.org
leif at geodynamics.org
Wed May 30 03:33:03 PDT 2007
Author: leif
Date: 2007-05-30 03:33:02 -0700 (Wed, 30 May 2007)
New Revision: 6999
Modified:
cs/buildbot/trunk/buildbot/bs.py
cs/buildbot/trunk/buildbot/config.py
cs/buildbot/trunk/buildbot/lines.py
cs/buildbot/trunk/buildbot/projects.py
cs/buildbot/trunk/buildbot/repositories.py
Log:
Added code to download source packages. This allows dependencies to
be built & installed automatically by BuiltBot -- just as if they were
any other project -- so I don't have to run around and install them
manually on each builder.
Also: add a config description to the build steps in the waterfall
("pyrized", "traditional") so people understand why their projects are
being built multiple times. Cleaned-up slave assignment -- the goofy
code that was there was left-over from when I mistakenly thought
BuildBot did rudamentary load balancing.
Modified: cs/buildbot/trunk/buildbot/bs.py
===================================================================
--- cs/buildbot/trunk/buildbot/bs.py 2007-05-30 06:24:24 UTC (rev 6998)
+++ cs/buildbot/trunk/buildbot/bs.py 2007-05-30 10:33:02 UTC (rev 6999)
@@ -2,18 +2,18 @@
from buildbot.process import step
from buildbot.process.factory import s
-
+
class BuildSystem(object): # BuildFactory?
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs, line):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs, line):
raise NotImplementedError()
def configureArgs(self, buildEnv, buildConfig, env, configureArgs, line):
args = (
buildEnv.getConfigureArgs(self, env, buildConfig) +
configureArgs +
- line.project.configureArgs[self]
+ line.project.configureArgs.get(self, [])
)
return args
@@ -21,9 +21,11 @@
class Make(BuildSystem):
"""simply runs 'make'"""
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs, line):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs, line):
steps = [
s(step.Compile,
+ description=["compiling"] + desc,
+ descriptionDone=desc + ["compile"],
command="make all",
haltOnFailure=True),
]
@@ -35,20 +37,22 @@
class GNUBuildSystem(BuildSystem): # a.k.a. Autotools
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs, line):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs, line):
configureArgs = self.configureArgs(buildEnv, buildConfig, env, configureArgs, line)
steps = [
s(step.ShellCommand,
- description=["autoreconf"],
+ description=["autoreconf"] + desc,
+ descriptionDone=desc + ["autoreconf"],
command=["autoreconf", "-i"],
workdir=workdir,
env=env,
haltOnFailure=True,
),
s(step.ShellCommand,
- description=["configure"],
+ description=["configure"] + desc,
+ descriptionDone=desc + ["configuration"],
command=["./configure"] + configureArgs,
workdir=workdir,
env=env,
@@ -61,22 +65,22 @@
# it writes to 'srcdir'.
s(step.Compile,
- description=["compiling"],
- descriptionDone=["compile"],
+ description=["compiling"] + desc,
+ descriptionDone=desc + ["compile"],
command=["make"],
workdir=workdir,
env=env,
),
s(step.Compile,
- description=["installing"],
- descriptionDone=["installation"],
+ description=["installing"] + desc,
+ descriptionDone=desc + ["installation"],
command=["make", "install"],
workdir=workdir,
env=env,
),
s(step.Compile,
- description=["testing"],
- descriptionDone=["tests"],
+ description=["testing"] + desc,
+ descriptionDone=desc + ["tests"],
command=["make", "check"],
workdir=workdir,
env=env,
@@ -92,13 +96,14 @@
class SCons(BuildSystem):
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs, line):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs, line):
configureArgs = self.configureArgs(buildEnv, buildConfig, env, configureArgs, line)
steps = [
s(step.ShellCommand,
- description=["configure"],
+ description=["configure"] + desc,
+ descriptionDone=desc + ["configuration"],
command=["./configure.py", configureArgs],
haltOnFailure=True,
),
@@ -116,10 +121,15 @@
class Distutils(BuildSystem):
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs, line):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs, line):
+
+ configureArgs = self.configureArgs(buildEnv, buildConfig, env, configureArgs, line)
+
steps = [
s(step.Compile,
- command=["./setup.py", "build"],
+ command=["%(PYTHON)s" % env, "setup.py", "install"] + configureArgs,
+ description=["installing"] + desc,
+ descriptionDone=desc + ["installation"],
workdir=workdir,
),
]
@@ -131,13 +141,14 @@
class ASEBuildSystem(BuildSystem):
"""ANL SIDL Environment (ASE) BuildSystem (used by PETSc)"""
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs, line):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs, line):
configureArgs = self.configureArgs(buildEnv, buildConfig, env, configureArgs, line)
steps = [
s(step.ShellCommand,
- description=["configure"],
+ description=["configure"] + desc,
+ descriptionDone=desc + ["configuration"],
command=["./config/configure.py"] + configureArgs,
workdir=workdir,
env=env,
@@ -145,15 +156,16 @@
logfiles={"configure.log": "configure.log"},
),
s(step.Compile,
- description=["compiling"],
- descriptionDone=["compile"],
+ description=["compiling"] + desc,
+ descriptionDone=desc + ["compile"],
command=["make"],
workdir=workdir,
env=env,
+ timeout=3600, # Building Sieve takes a long time...
),
s(step.Compile,
- description=["installing"],
- descriptionDone=["installation"],
+ description=["installing"] + desc,
+ descriptionDone=desc + ["installation"],
command=["make", "install"],
workdir=workdir,
env=env,
@@ -168,10 +180,15 @@
class DebugBuildSystem(BuildSystem):
"""for debugging BuildBot"""
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs, line):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs, line):
+
+ configureArgs = self.configureArgs(buildEnv, buildConfig, env, configureArgs, line)
+
steps = [
s(step.Compile,
- command=["./setup.py", "build"],
+ command=["%(PYTHON)s" % env, "setup.py", "install"] + configureArgs,
+ description=["installing"] + desc,
+ descriptionDone=desc + ["installation"],
workdir=workdir,
),
]
Modified: cs/buildbot/trunk/buildbot/config.py
===================================================================
--- cs/buildbot/trunk/buildbot/config.py 2007-05-30 06:24:24 UTC (rev 6998)
+++ cs/buildbot/trunk/buildbot/config.py 2007-05-30 10:33:02 UTC (rev 6999)
@@ -1,12 +1,13 @@
class BuildConfig(object):
- def __init__(self, name, env):
+ def __init__(self, name, env, slave=None):
self.name = name
self.env = env
+ self.slave = slave
self.configureArgs = {}
-defaultConfigs = [BuildConfig("default", {"PYTHON": "python2.3"})]
+defaultConfigs = [BuildConfig("default", {"PYTHON": "python2.4"})]
class BuildSlave(object):
@@ -21,9 +22,9 @@
# (identical) BuildSlaves.
class BuildEnvironment(object):
- def __init__(self, name, buildSlaves, configs=defaultConfigs):
+ def __init__(self, name, defaultSlave=None, configs=defaultConfigs):
self.name = name
- self.buildSlaves = buildSlaves
+ self.defaultSlave = defaultSlave
self.configs = configs
return
@@ -56,12 +57,15 @@
return self.configName(config).replace(' ', '_')
def getConfigureArgs(self, bs, env, config):
- from buildbot.bs import gnu, scons, ase
+ from buildbot.bs import gnu, distutils, scons, ase
args = {
gnu: [
"--prefix=%(PREFIX)s" % env,
"LDFLAGS=-L%(PREFIX)s/lib -L%(DEPS_PREFIX)s/lib" % env,
"CPPFLAGS=-I%(PREFIX)s/include -I%(DEPS_PREFIX)s/include" % env,
+ ],
+ distutils: [
+ "--prefix=%(PREFIX)s" % env,
]
}
return args.get(bs, []) + config.configureArgs.get(bs, [])
@@ -90,9 +94,16 @@
return
def collectBots(self):
- for buildEnvironment in self.buildEnvironments:
- for buildSlave in buildEnvironment.buildSlaves:
- self.bots.append((buildSlave.name, buildSlave.password))
+ slaves = []
+ for env in self.buildEnvironments:
+ slave = env.defaultSlave
+ if slave and not slave in slaves:
+ slaves.append(slave)
+ for config in env.configs:
+ slave = config.slave
+ if slave and not slave in slaves:
+ slaves.append(slave)
+ self.bots = [(slave.name, slave.password) for slave in slaves]
return
@@ -109,13 +120,13 @@
for buildEnv in self.buildEnvironments:
- slavenames = [buildSlave.name for buildSlave in buildEnv.buildSlaves]
-
for config in buildEnv.configs:
env = buildEnv.getEnv(config, branch)
+ slave = config.slave or buildEnv.defaultSlave
+
builder = branch.newBuilder(buildEnv, config, env)
- builder['slavenames'] = [slavenames.pop(0)]
+ builder['slavenames'] = [slave.name]
import os
from os.path import isdir
Modified: cs/buildbot/trunk/buildbot/lines.py
===================================================================
--- cs/buildbot/trunk/buildbot/lines.py 2007-05-30 06:24:24 UTC (rev 6998)
+++ cs/buildbot/trunk/buildbot/lines.py 2007-05-30 10:33:02 UTC (rev 6999)
@@ -2,7 +2,7 @@
from buildbot.categories import Category
from buildbot.process.factory import BuildFactory
-from buildbot.scheduler import Scheduler
+from buildbot.scheduler import Scheduler, Nightly
import weakref
@@ -24,6 +24,7 @@
self.configs = {}
self.dependencies = []
+ self.preinstalledDependencies = []
self.scheduler = None
self._buildSystem = None
@@ -53,25 +54,31 @@
if self.configs:
configs = self.configs
else:
- configs = {"default": []}
+ configs = {"__default__": []}
for config, configureArgs in configs.iteritems():
- workdir = config
- steps.append(
- self.sourceStep(
+ if config != "__default__":
+ desc = [config]
+ workdir = config
+ else:
+ desc = []
+ workdir = "default"
+ steps.extend(
+ self.sourceSteps(
workdir = workdir,
mode = 'copy', # 'clobber', 'copy', 'update'
)
)
steps += self.buildSteps(buildEnv, buildConfig, env,
- workdir, configureArgs)
+ desc, workdir, configureArgs)
return BuildFactory(steps)
- def sourceStep(self, **kwds):
- """Return a BuildStep which checks-out my source from the repository."""
- return self.location.sourceStep(**kwds)
+ def sourceSteps(self, **kwds):
+ """Return a list of BuildSteps which checks-out my source from
+ the repository, or downloads my source tarball."""
+ return self.location.sourceSteps(**kwds)
def buildSystem(self):
@@ -84,11 +91,11 @@
self._buildSystem = bs
- def buildSteps(self, buildEnv, buildConfig, env, workdir, configureArgs):
+ def buildSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs):
buildSystem = self.buildSystem()
buildSteps = buildSystem.buildSteps(
buildEnv, buildConfig, env,
- workdir, configureArgs, self
+ desc, workdir, configureArgs, self
)
return buildSteps
@@ -125,4 +132,21 @@
**kwds)
+class Release(Line): # actually, the end of the line
+
+ def __init__(self, name, location, **kwds):
+ super(Release, self).__init__(
+ name = name,
+ location = location,
+ **kwds)
+
+ def createScheduler(self, builderNames, **kwds):
+ # Do release builds every night at 3am. Releases don't
+ # change, but the world does change around them -- good to
+ # test anyway. Plus, dependencies are currently handled
+ # here... this ensures they are installed on all slaves.
+ self.scheduler = Nightly(self.fullName(), builderNames, hour=3, minute=0)
+ return self.scheduler
+
+
# end of file
Modified: cs/buildbot/trunk/buildbot/projects.py
===================================================================
--- cs/buildbot/trunk/buildbot/projects.py 2007-05-30 06:24:24 UTC (rev 6998)
+++ cs/buildbot/trunk/buildbot/projects.py 2007-05-30 10:33:02 UTC (rev 6999)
@@ -2,7 +2,11 @@
from buildbot import util
from buildbot.bs import gnu
-from buildbot.lines import Trunk, Branch
+from buildbot.lines import Trunk, Branch, Release
+from buildbot.process import step
+from buildbot.process.factory import s
+from buildbot.steps import transfer
+from os.path import basename, splitext
class Project(util.ComparableMixin):
@@ -20,6 +24,10 @@
self.configureArgs = {}
+ self.trunk = None
+ self.release = None
+ if self.branches is None:
+ self.branches = {}
if self.owners is None:
self.owners = []
@@ -46,11 +54,65 @@
return
+ def addTarball(self, pathname):
+ version = "v" + splitext(splitext(pathname)[0])[0].split('-')[1]
+ location = TarballLocation(pathname)
+ self.release = Release(name = version,
+ location = location,
+ project = self)
+ return
+
+
def lines(self):
- yield self.trunk
+ if isinstance(self.branches, list):
+ raise RuntimeError("project '%s' was not included in 'initLayout'" % self.name)
+ if self.trunk:
+ yield self.trunk
+ if self.release:
+ yield self.release
for branch in self.branches.itervalues():
yield branch
return
+class TarballLocation(object):
+
+ def __init__(self, pathname):
+ self.pathname = pathname
+
+ def sourceSteps(self, **kwds):
+ filename = basename(self.pathname)
+ dirname = splitext(splitext(filename)[0])[0]
+ workdir = kwds['workdir']
+ steps = [
+ s(transfer.FileDownload,
+ mastersrc = self.pathname,
+ slavedest = filename,
+ workdir = ".",
+ ),
+ s(step.ShellCommand,
+ description=["extracting"],
+ descriptionDone=["extraction"],
+ command=["tar", "xzf", filename],
+ workdir = ".",
+ haltOnFailure=True,
+ ),
+ s(step.ShellCommand,
+ description=["cleaning"],
+ descriptionDone=["clean"],
+ command=["rm", "-rf", workdir],
+ workdir = ".",
+ haltOnFailure=True,
+ ),
+ s(step.ShellCommand,
+ description=["renaming"],
+ descriptionDone=["rename"],
+ command=["mv", dirname, workdir],
+ workdir = ".",
+ haltOnFailure=True,
+ )
+ ]
+ return steps
+
+
# end of file
Modified: cs/buildbot/trunk/buildbot/repositories.py
===================================================================
--- cs/buildbot/trunk/buildbot/repositories.py 2007-05-30 06:24:24 UTC (rev 6998)
+++ cs/buildbot/trunk/buildbot/repositories.py 2007-05-30 10:33:02 UTC (rev 6999)
@@ -12,6 +12,8 @@
"""a reference to a specific place (project & branch) in a repository"""
def __init__(self, repository):
self.repository = repository
+ def sourceSteps(self, **kwds):
+ raise NotImplementedError()
def trunkLocation(self):
raise NotImplementedError()
def branchLocation(self, name):
@@ -67,10 +69,10 @@
line = ['branches', name],
repository = self.repository)
return
- def sourceStep(self, **kwds):
- return self.repository.sourceStep(root = self.root(),
- defaultBranch = self.branch(),
- **kwds)
+ def sourceSteps(self, **kwds):
+ return [self.repository.sourceStep(root = self.root(),
+ defaultBranch = self.branch(),
+ **kwds)]
def root(self): return '/'.join(self.path) + '/'
def branch(self): return '/'.join(self.line)
More information about the cig-commits
mailing list