[cig-commits] r7026 - in cs/buildbot/trunk/buildbot: . changes
leif at geodynamics.org
leif at geodynamics.org
Thu May 31 20:24:34 PDT 2007
Author: leif
Date: 2007-05-31 20:24:33 -0700 (Thu, 31 May 2007)
New Revision: 7026
Modified:
cs/buildbot/trunk/buildbot/bs.py
cs/buildbot/trunk/buildbot/changes/changes.py
cs/buildbot/trunk/buildbot/config.py
cs/buildbot/trunk/buildbot/lines.py
cs/buildbot/trunk/buildbot/projects.py
Log:
Added stuff to build PETSc. Do a nightly build of all dependencies,
not just tarballs. Fixed change-filtering bug.
It turns out that FileDownload isn't robust enough to handle the PETSc
tarball (because of its size?), so now I just use 'wget'.
Modified: cs/buildbot/trunk/buildbot/bs.py
===================================================================
--- cs/buildbot/trunk/buildbot/bs.py 2007-06-01 00:34:41 UTC (rev 7025)
+++ cs/buildbot/trunk/buildbot/bs.py 2007-06-01 03:24:33 UTC (rev 7026)
@@ -18,6 +18,8 @@
configureArgs +
line.project.configureArgs.get(self, [])
)
+ # expand server-side environment variables
+ args = [arg % env for arg in args]
return args
Modified: cs/buildbot/trunk/buildbot/changes/changes.py
===================================================================
--- cs/buildbot/trunk/buildbot/changes/changes.py 2007-06-01 00:34:41 UTC (rev 7025)
+++ cs/buildbot/trunk/buildbot/changes/changes.py 2007-06-01 03:24:33 UTC (rev 7026)
@@ -219,9 +219,13 @@
self.changes = self.changes[-100:] # or something
def eventGenerator(self, categories):
+ from buildbot.categories import Category
+ from buildbot.projects import Project
+ other = Category(project=Project("other"))
for i in range(len(self.changes)-1, -1, -1):
c = self.changes[i]
- if c.category in categories:
+ category = c.category or other
+ if category in categories:
yield c
def getChangeNumbered(self, num):
Modified: cs/buildbot/trunk/buildbot/config.py
===================================================================
--- cs/buildbot/trunk/buildbot/config.py 2007-06-01 00:34:41 UTC (rev 7025)
+++ cs/buildbot/trunk/buildbot/config.py 2007-06-01 03:24:33 UTC (rev 7026)
@@ -1,12 +1,24 @@
class BuildConfig(object):
- def __init__(self, name, env, slave=None):
+ def __init__(self, name, env, slave=None, configureArgs=None):
self.name = name
self.env = env
+
+ # I designed BuildConfig & BuildSlave when I thought BuildBot
+ # did load balancing. But, in fact, BuildBot does load
+ # balancing only when the user repeatedly clicks the "Force
+ # Build" button for a single Builder (i.e., in a single column
+ # of the waterfall display). Therefore, for now, each
+ # BuildConfig is assigned to a unique slave. If 'slave' is
+ # 'None', it uses 'defaultSlave' from the parent
+ # BuildEnvironment.
self.slave = slave
- self.configureArgs = {}
+
+ self.configureArgs = configureArgs or {}
+ return
+
defaultConfigs = [BuildConfig("default", {"PYTHON": "python2.4"})]
@@ -32,11 +44,10 @@
basedir = "%(HOME)s"
filename = self.configFilename(config)
prefix = basedir + "/install/" + filename
- petsc_dir = prefix + "/opt/petsc-dev"
builddir = 'build/' + filename + "/" + branch.project.name + "/" + branch.name
env = {
+ "HOME": "%(HOME)s", # expanded by the slave
"PREFIX": prefix,
- "PETSC_DIR": petsc_dir,
"BUILDDIR": builddir,
}
env.update(config.env)
@@ -106,9 +117,13 @@
self.bots = [(slave.name, slave.password) for slave in slaves]
return
+ def collectSchedulers(self):
+ for branch in self.lines:
+ self.schedulers.extend(branch.schedulers)
+ return
+
def generate(self):
- from buildbot.scheduler import Scheduler, AnyBranchScheduler, Dependent
from buildbot.status import mail
self.collectLines()
@@ -141,26 +156,11 @@
# Add this builder to the global list of all builders.
self.builders.append(builder)
- # BuildBot does load balancing only when the user
- # repeatedly clicks the "Force Build" button for a single
- # Builder (i.e., in a single column of the waterfall
- # display). Therefore, we need to permute the list of
- # available slaves -- otherwise all the Linux/x86 builds
- # will be sent to the same machine. It is still somewhat
- # worthwhile to assign multiple BuildSlaves to each
- # Builder: it provides redundancy (in case a BuildSlave
- # goes offline).
-
- ## Doesn't work!!!
- #sn = slavenames.pop()
- #slavenames.insert(0, sn)
-
### for config in buildEnv.configs
### for buildEnv in buildEnvironments
- self.schedulers.append(branch.createScheduler(builderNames = builderNames,
- **self.schedulerKwds))
+ branch.generate(builderNames, self.schedulerKwds)
owners = branch.project.owners
if owners:
@@ -176,52 +176,37 @@
### for branch in lines
-
- # If A depends upon B, trigger a build of A when a build of B
- # completes successfully.
-
for branch in self.lines:
- for dep in branch.dependencies:
- name = branch.fullName() + " -> " + dep.fullName()
- upstream = dep.scheduler
- d = Dependent(name, upstream, branch.scheduler.builderNames,
- restamp=True)
- self.schedulers.append(d)
+ branch.postGenerate()
+ self.collectSchedulers()
- # Set-up schedulers.
+ return
- if False:
- # AnyBranchScheduler seems to be useless, because is tied to
- # a list of builders. Below, I give it the list of all
- # builders, which doesn't work. A check-in on
- # CitcomS/trunk triggers a build everywhere -- across all
- # projects -- but with each build checking-out the source
- # from the project/branch which triggered the build!!!
+ def notesTo(self):
+ # AnyBranchScheduler seems to be useless, because is tied to a
+ # list of builders. I gave it the list of all builders, which
+ # didn't work. A check-in on CitcomS/trunk triggered a build
+ # everywhere -- across all projects -- but with each build
+ # checking-out the source from the project/branch which
+ # triggered the build!!!
- # The build steps are tied to the builder. This
- # is why we need a separate set of builders (and columns) for each
- # project/branch combination. This means that, in the PyLith column,
- # it will try to build CitcomS source using PyLith's build steps.
+ # The build steps are tied to the builder. This is why we
+ # need a separate set of builders (and columns) for each
+ # project/branch combination. This means that, in the PyLith
+ # column, it will try to build CitcomS source using PyLith's
+ # build steps.
- # If every branch of a project had the same build steps, one
- # could use one AnyBranchScheduler for the entire project,
- # with a single set of builders for the entire project. But then --
- # it seems to me -- the different branches would step on
- # each other, since they would share the same build dir.
+ # If every branch of a project had the same build steps, one
+ # could use one AnyBranchScheduler for the entire project,
+ # with a single set of builders for the entire project. But then --
+ # it seems to me -- the different branches would step on
+ # each other, since they would share the same build dir.
- scheduler = AnyBranchScheduler(
- name = "AnyBranchScheduler",
- branches = [(branch.project.root, branch.path) for branch in self.lines],
- treeStableTimer = 5, #1*60,
- builderNames = [b['name'] for b in self.builders])
+ raise RuntimeError("comment only")
- self.schedulers.append(scheduler)
- return
-
-
def getBuildmasterConfig(self, c):
c['bots'] = self.bots
c['builders'] = self.builders
Modified: cs/buildbot/trunk/buildbot/lines.py
===================================================================
--- cs/buildbot/trunk/buildbot/lines.py 2007-06-01 00:34:41 UTC (rev 7025)
+++ cs/buildbot/trunk/buildbot/lines.py 2007-06-01 03:24:33 UTC (rev 7026)
@@ -2,7 +2,7 @@
from buildbot.categories import Category
from buildbot.process.factory import BuildFactory
-from buildbot.scheduler import Scheduler, Nightly
+from buildbot.scheduler import Scheduler, Nightly, Dependent
import weakref
@@ -24,14 +24,67 @@
self.configs = {}
self.dependencies = []
- self.preinstalledDependencies = []
- self.scheduler = None
+ # Builds which depend upon me pull configure args from here.
+ self.dependentConfigureArgs = {}
+ self.isDependency = False
+ self.builderNames = []
+ self.schedulers = []
+ self.changeScheduler = None
+
self._buildSystem = None
return
+ def dependencyConfigureArgs(self):
+ # If I depend upon PETSc (for example), I need "PETSC_DIR=xxx"
+ # on my 'configure' command line. This method handles that.
+ buildSystem = self.buildSystem()
+ args = []
+ for dep in self.dependencies:
+ args += dep.dependentConfigureArgs.get(buildSystem, [])
+ return args
+
+
+ def generate(self, builderNames, schedulerKwds):
+
+ # Save the list of builders that build me.
+ self.builderNames = builderNames
+
+ # Mark my dependencies.
+ for dep in self.dependencies:
+ dep.isDependency = True
+
+ # Trigger a build whenever my sources change.
+ self.changeScheduler = self.newChangeScheduler(schedulerKwds)
+ if self.changeScheduler:
+ self.schedulers.append(self.changeScheduler)
+
+ return
+
+
+ def postGenerate(self):
+
+ # If I depend upon another project, trigger a build of me
+ # every time a build of the other project completes
+ # successfully.
+ for dep in self.dependencies:
+ upstream = dep.changeScheduler
+ if upstream:
+ name = self.fullName() + " -> " + dep.fullName()
+ d = Dependent(name, upstream, self.builderNames,
+ restamp=True)
+ self.schedulers.append(d)
+
+ # If other projects depend upon me, make sure I am installed
+ # on all slaves eventually (within a 24-hour period).
+ if self.isDependency:
+ self.schedulers.append(self.newNightlyScheduler())
+
+ return
+
+
def newBuilder(self, buildEnv, buildConfig, env):
buildFactory = self.newBuildFactory(buildEnv, buildConfig, env)
builder = {
@@ -69,6 +122,7 @@
mode = 'copy', # 'clobber', 'copy', 'update'
)
)
+ configureArgs += self.dependencyConfigureArgs()
steps += self.generativeSteps(buildEnv, buildConfig, env,
desc, workdir, configureArgs)
steps += self.buildSteps(buildEnv, buildConfig, env,
@@ -115,17 +169,20 @@
return self.project.name + " " + self.name
- def createScheduler(self, builderNames, **kwds):
- self.scheduler = Scheduler(name = self.fullName(),
- root = self.location.root(),
- branch = self.location.branch(),
- category = self.category,
- builderNames = builderNames,
- **kwds)
- return self.scheduler
+ def newChangeScheduler(self, schedulerKwds):
+ s = Scheduler(name = "change " + self.fullName(),
+ root = self.location.root(),
+ branch = self.location.branch(),
+ category = self.category,
+ builderNames = self.builderNames,
+ **schedulerKwds)
+ return s
+ def newNightlyScheduler(self):
+ return Nightly("nightly " + self.fullName(), self.builderNames, hour=3, minute=0)
+
class Trunk(Line):
def __init__(self, location, **kwds):
super(Trunk, self).__init__(
@@ -151,14 +208,10 @@
location = location,
**kwds)
- def createScheduler(self, builderNames, **kwds):
- # Do release builds every night at 3am. Releases don't
- # change, but the world does change around them -- good to
- # test anyway. Plus, dependencies are currently handled
- # here... this ensures they are installed on all slaves.
- self.scheduler = Nightly(self.fullName(), builderNames, hour=3, minute=0)
- return self.scheduler
-
+ def newChangeScheduler(self, schedulerKwds):
+ # Releases don't change.
+ return None
+
def generativeSteps(self, buildEnv, buildConfig, env, desc, workdir, configureArgs):
return []
Modified: cs/buildbot/trunk/buildbot/projects.py
===================================================================
--- cs/buildbot/trunk/buildbot/projects.py 2007-06-01 00:34:41 UTC (rev 7025)
+++ cs/buildbot/trunk/buildbot/projects.py 2007-06-01 03:24:33 UTC (rev 7026)
@@ -5,7 +5,6 @@
from buildbot.lines import Trunk, Branch, Release
from buildbot.process import step
from buildbot.process.factory import s
-from buildbot.steps import transfer
from os.path import basename, splitext
@@ -54,9 +53,10 @@
return
- def addTarball(self, pathname):
- version = "v" + splitext(splitext(pathname)[0])[0].split('-')[1]
- location = TarballLocation(pathname)
+ def addTarball(self, url, version=None):
+ if version is None:
+ version = "v" + splitext(splitext(url)[0])[0].split('-')[1]
+ location = TarballLocation(url)
self.release = Release(name = version,
location = location,
project = self)
@@ -77,20 +77,29 @@
class TarballLocation(object):
- def __init__(self, pathname):
- self.pathname = pathname
+ def __init__(self, url):
+ self.url = url
def sourceSteps(self, **kwds):
- filename = basename(self.pathname)
+ filename = basename(self.url)
dirname = splitext(splitext(filename)[0])[0]
workdir = kwds['workdir']
steps = [
- s(transfer.FileDownload,
- mastersrc = self.pathname,
- slavedest = filename,
+ s(step.ShellCommand,
+ description=["cleaning"],
+ descriptionDone=["clean"],
+ command=["rm", "-f", filename],
workdir = ".",
+ haltOnFailure=True,
),
s(step.ShellCommand,
+ description=["downloading"],
+ descriptionDone=["download"],
+ command=["wget", self.url],
+ workdir = ".",
+ haltOnFailure=True,
+ ),
+ s(step.ShellCommand,
description=["extracting"],
descriptionDone=["extraction"],
command=["tar", "xzf", filename],
More information about the cig-commits
mailing list