[cig-commits] r6960 - in cs/buildbot/trunk/buildbot: . status steps
leif at geodynamics.org
leif at geodynamics.org
Thu May 24 17:59:20 PDT 2007
Author: leif
Date: 2007-05-24 17:59:19 -0700 (Thu, 24 May 2007)
New Revision: 6960
Added:
cs/buildbot/trunk/buildbot/config.py
cs/buildbot/trunk/buildbot/lines.py
cs/buildbot/trunk/buildbot/repositories.py
cs/buildbot/trunk/buildbot/steps/misc.py
cs/buildbot/trunk/buildbot/users.py
Modified:
cs/buildbot/trunk/buildbot/categories.py
cs/buildbot/trunk/buildbot/master.py
cs/buildbot/trunk/buildbot/projects.py
cs/buildbot/trunk/buildbot/status/html.py
Log:
Cleaned-up all the ad-hoc goo from my monstrous 'master.cfg' file and
merged it into BuildBot itself. My 'master.cfg' is now only 300
lines, and contains only data.
Modified: cs/buildbot/trunk/buildbot/categories.py
===================================================================
--- cs/buildbot/trunk/buildbot/categories.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/categories.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -1,7 +1,5 @@
-from buildbot.projects import Project
-
class Category(object):
def __init__(self, **kwds):
Added: cs/buildbot/trunk/buildbot/config.py
===================================================================
--- cs/buildbot/trunk/buildbot/config.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/config.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -0,0 +1,221 @@
+
+
+class BuildConfig(object):
+ def __init__(self, name, configureArgs, env):
+ self.name = name
+ self.configureArgs = configureArgs
+ self.env = env
+
+defaultConfigs = [BuildConfig("default", [], {"PYTHON": "python2.3"})]
+
+
+class BuildSlave(object):
+ def __init__(self, name, password):
+ self.name = name
+ self.password = password
+
+
+# These define the core build environments that are available to
+# projects. A project may create multiple Builds in the same
+# environment. A single BuildEnvironment may be served by several
+# (identical) BuildSlaves.
+
+class BuildEnvironment(object):
+ def __init__(self, name, buildSlaves, configs=defaultConfigs):
+ self.name = name
+ self.buildSlaves = buildSlaves
+ self.configs = configs
+ return
+
+ def getEnv(self, config, branch):
+ basedir = "%(HOME)s"
+ filename = self.configFilename(config)
+ prefix = basedir + "/install/" + filename
+ deps_prefix = basedir + "/deps/" + filename
+ petsc_dir = deps_prefix + "/petsc-dev"
+ builddir = 'build/' + filename + "/" + branch.project.name + "/" + branch.name
+ env = {
+ "PREFIX": prefix,
+ "DEPS_PREFIX": deps_prefix,
+ "PETSC_DIR": petsc_dir,
+ "BUILDDIR": builddir,
+ }
+ env.update(config.env)
+ bin = lambda p: p + "/bin"
+ sp = lambda p: p + "/lib/" + env['PYTHON'] + "/site-packages"
+ env.update({
+ "PATH": bin(prefix) + ":" + bin(deps_prefix),
+ "PYTHONPATH": sp(prefix) + ":" + sp(deps_prefix),
+ })
+ return env
+
+ def configName(self, config):
+ return self.name + " " + config.name
+
+ def configFilename(self, config):
+ return self.configName(config).replace(' ', '_')
+
+ def getConfigureArgs(self, env, config):
+ args = [
+ "--prefix=%(PREFIX)s" % env,
+ "LDFLAGS=-L%(PREFIX)s/lib -L%(DEPS_PREFIX)s/lib" % env,
+ "CPPFLAGS=-I%(PREFIX)s/include -I%(DEPS_PREFIX)s/include" % env,
+ ]
+ return args + config.configureArgs
+
+
+class Config(object):
+
+
+ def __init__(self):
+ self.addressBook = None
+ self.bots = []
+ self.buildEnvironments = []
+ self.builders = []
+ self.lines = []
+ self.projects = []
+ self.repositories = []
+ self.schedulerKwds = {}
+ self.schedulers = []
+ self.status = []
+ return
+
+
+ def collectLines(self):
+ for project in self.projects:
+ self.lines.extend(project.lines())
+ return
+
+ def collectBots(self):
+ for buildEnvironment in self.buildEnvironments:
+ for buildSlave in buildEnvironment.buildSlaves:
+ self.bots.append((buildSlave.name, buildSlave.password))
+ return
+
+
+ def generate(self):
+ from buildbot.scheduler import Scheduler, AnyBranchScheduler, Dependent
+ from buildbot.status import mail
+
+ self.collectLines()
+ self.collectBots()
+
+ for branch in self.lines:
+
+ builderNames = []
+
+ for buildEnv in self.buildEnvironments:
+
+ slavenames = [buildSlave.name for buildSlave in buildEnv.buildSlaves]
+
+ for config in buildEnv.configs:
+
+ env = buildEnv.getEnv(config, branch)
+ builder = branch.newBuilder(buildEnv, config, env)
+ builder['slavenames'] = [slavenames.pop(0)]
+
+ import os
+ from os.path import isdir
+ basedir = "/home/leif/sandbox/buildbot/pylith-master"
+ builddir = basedir + "/" + env['BUILDDIR']
+ if not isdir(builddir):
+ os.makedirs(builddir)
+
+ # Add this builder to the list of builders for this
+ # project/branch.
+ builderNames.append(builder['name'])
+
+ # Add this builder to the global list of all builders.
+ self.builders.append(builder)
+
+ # BuildBot does load balancing only when the user
+ # repeatedly clicks the "Force Build" button for a single
+ # Builder (i.e., in a single column of the waterfall
+ # display). Therefore, we need to permute the list of
+ # available slaves -- otherwise all the Linux/x86 builds
+ # will be sent to the same machine. It is still somewhat
+ # worthwhile to assign multiple BuildSlaves to each
+ # Builder: it provides redundancy (in case a BuildSlave
+ # goes offline).
+
+ ## Doesn't work!!!
+ #sn = slavenames.pop()
+ #slavenames.insert(0, sn)
+
+ ### for config in buildEnv.configs
+
+ ### for buildEnv in buildEnvironments
+
+ self.schedulers.append(branch.createScheduler(builderNames = builderNames,
+ **self.schedulerKwds))
+
+ owners = branch.project.owners
+ if owners:
+ # set-up a mail notifier for this branch
+ mn = mail.MailNotifier(fromaddr="buildbot at geodynamics.org",
+ mode='failing',
+ extraRecipients=[user.email for user in owners],
+ sendToInterestedUsers=False,
+ builders=builderNames,
+ lookup=self.addressBook,
+ )
+ self.status.append(mn)
+
+ ### for branch in lines
+
+
+ # If A depends upon B, trigger a build of A when a build of B
+ # completes successfully.
+
+ for branch in self.lines:
+ for dep in branch.dependencies:
+ name = branch.fullName() + " -> " + dep.fullName()
+ upstream = dep.scheduler
+ d = Dependent(name, upstream, branch.scheduler.builderNames,
+ restamp=True)
+ self.schedulers.append(d)
+
+
+ # Set-up schedulers.
+
+ if False:
+
+ # AnyBranchScheduler seems to be useless, because is tied to
+ # a list of builders. Below, I give it the list of all
+ # builders, which doesn't work. A check-in on
+ # CitcomS/trunk triggers a build everywhere -- across all
+ # projects -- but with each build checking-out the source
+ # from the project/branch which triggered the build!!!
+
+ # The build steps are tied to the builder. This
+ # is why we need a separate set of builders (and columns) for each
+ # project/branch combination. This means that, in the PyLith column,
+ # it will try to build CitcomS source using PyLith's build steps.
+
+ # If every branch of a project had the same build steps, one
+ # could use one AnyBranchScheduler for the entire project,
+ # with a single set of builders for the entire project. But then --
+ # it seems to me -- the different branches would step on
+ # each other, since they would share the same build dir.
+
+ scheduler = AnyBranchScheduler(
+ name = "AnyBranchScheduler",
+ branches = [(branch.project.root, branch.path) for branch in self.lines],
+ treeStableTimer = 5, #1*60,
+ builderNames = [b['name'] for b in self.builders])
+
+ self.schedulers.append(scheduler)
+
+ return
+
+
+ def getBuildmasterConfig(self, c):
+ c['bots'] = self.bots
+ c['builders'] = self.builders
+ c['projects'] = self.projects
+ c['schedulers'] = self.schedulers
+ c['status'] = self.status
+ return
+
+
+# end of file
Added: cs/buildbot/trunk/buildbot/lines.py
===================================================================
--- cs/buildbot/trunk/buildbot/lines.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/lines.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -0,0 +1,169 @@
+
+
+from buildbot.categories import Category
+from buildbot.process.factory import BuildFactory
+from buildbot.scheduler import Scheduler
+import weakref
+
+
+class Line(object):
+
+ def __init__(self, name, location, project):
+ self.name = name
+ self.location = location
+ ## cPickle.UnpickleableError: Cannot pickle <type 'weakproxy'> objects
+ #self.project = weakref.proxy(project)
+ self.project = project
+
+ self.configureArgs = []
+ self.dependencies = []
+ self.scheduler = None
+
+ return
+
+
+ def newBuilder(self, buildEnv, config, env):
+ buildFactory = self.newBuildFactory(buildEnv, config, env)
+ builder = {
+ 'name' : self.builderName(buildEnv, config),
+ 'builddir' : env['BUILDDIR'],
+ 'factory' : buildFactory,
+ 'category' : Category(project = self.project,
+ branch = self.name,
+ type = 'test'),
+ }
+ return builder
+
+
+ def builderName(self, buildEnv, config):
+ return self.project.name + " " + self.name + " " + buildEnv.configName(config)
+
+
+ def newBuildFactory(self, buildEnv, config, env):
+ if True:
+ steps = self.debugSteps()
+ else:
+ ## proj-specific
+ #steps.extend(depsSteps(buildEnv, config))
+
+ steps = self.buildSteps(buildEnv, config, env)
+
+ return BuildFactory(steps)
+
+
+ def sourceStep(self, **kwds):
+ """Return a BuildStep which checks-out my source from the repository."""
+ return self.location.sourceStep(**kwds)
+
+
+ def buildSteps(self, buildEnv, config, env):
+ from buildbot.process import step
+ from buildbot.process.factory import s
+
+ workdir = "build"
+
+ configureArgs = buildEnv.getConfigureArgs(env, config) + self.configureArgs
+
+ sourceStep = self.sourceStep(
+ workdir = workdir,
+ mode = 'copy', # 'clobber', 'copy', 'update'
+ )
+
+ steps = [
+ sourceStep,
+
+ s(step.ShellCommand,
+ description=["autoreconf"],
+ command=["autoreconf", "-i"],
+ workdir=workdir,
+ env=env,
+ haltOnFailure=True,
+ ),
+ s(step.ShellCommand,
+ description=["configure"],
+ command=["./configure"] + configureArgs,
+ workdir=workdir,
+ env=env,
+ haltOnFailure=True,
+ logfiles={"config.log": "config.log"},
+ ),
+ s(step.Compile,
+ description=["compiling"],
+ descriptionDone=["compile"],
+ command=["make"],
+ workdir=workdir,
+ env=env,
+ ),
+ s(step.Compile,
+ description=["installing"],
+ descriptionDone=["installation"],
+ command=["make", "install"],
+ workdir=workdir,
+ env=env,
+ ),
+ s(step.Compile,
+ description=["testing"],
+ descriptionDone=["tests"],
+ command=["make", "check"],
+ workdir=workdir,
+ env=env,
+ haltOnFailure=False,
+ ),
+ ]
+ return steps
+
+
+ def debugSteps(self):
+ from buildbot.process import step
+ from buildbot.process.factory import s
+
+ workdir = "build"
+
+ sourceStep = self.sourceStep(
+ workdir = workdir,
+ mode = 'clobber', # 'clobber', 'copy', 'update'
+ )
+
+ steps = [
+ sourceStep,
+
+ s(step.Compile,
+ command=["./setup.py", "build"],
+ workdir=workdir,
+ ),
+ ]
+ return steps
+
+
+ def fullName(self):
+ return self.project.name + " " + self.name
+
+
+ def createScheduler(self, builderNames, **kwds):
+ self.scheduler = Scheduler(name = self.fullName(),
+ root = self.location.root(),
+ branch = self.location.branch(),
+ builderNames = builderNames,
+ **kwds)
+ return self.scheduler
+
+
+
+class Trunk(Line):
+ def __init__(self, location, **kwds):
+ super(Trunk, self).__init__(
+ name = "trunk",
+ location = location.trunkLocation(),
+ **kwds)
+
+
+
+class Branch(Line):
+ def __init__(self, name, location, **kwds):
+ super(Branch, self).__init__(
+ name = name,
+ location = location.branchLocation(name),
+ **kwds)
+
+
+# end of file
Modified: cs/buildbot/trunk/buildbot/master.py
===================================================================
--- cs/buildbot/trunk/buildbot/master.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/master.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -359,7 +359,7 @@
def stopService(self):
for b in self.builders.values():
b.builder_status.addPointEvent(["master", "shutdown"])
- b.builder_status.saveYourself()
+ #b.builder_status.saveYourself()
return service.Service.stopService(self)
def getLockByID(self, lockid):
@@ -570,7 +570,7 @@
signal.signal(signal.SIGHUP, self._handleSIGHUP)
for b in self.botmaster.builders.values():
b.builder_status.addPointEvent(["master", "started"])
- b.builder_status.saveYourself()
+ #b.builder_status.saveYourself()
def useChanges(self, changes):
if self.change_svc:
@@ -930,7 +930,7 @@
log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
statusbag = old.builder_status
- statusbag.saveYourself() # seems like a good idea
+ #statusbag.saveYourself() # seems like a good idea
# TODO: if the basedir was changed, we probably need to make
# a new statusbag
new_builder = Builder(data, statusbag)
Modified: cs/buildbot/trunk/buildbot/projects.py
===================================================================
--- cs/buildbot/trunk/buildbot/projects.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/projects.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -1,13 +1,49 @@
+
from buildbot import util
+from buildbot.lines import Trunk, Branch
class Project(util.ComparableMixin):
- compare_attrs = ['name', 'url', 'owners', 'root']
- def __init__(self, name, url, owners, root):
+ compare_attrs = ['name', 'url']
+
+
+ def __init__(self, name, url=None, branches=None, owners=None):
self.name = name
self.url = url
+ self.branches = branches
self.owners = owners
- self.root = root
+
+ if self.owners is None:
+ self.owners = []
+
+ return
+
+
+ def initLayout(self, location):
+
+ self.trunk = Trunk(location = location,
+ project = self)
+
+ branches = {}
+ for name in self.branches:
+ branch = Branch(name = name,
+ location = location,
+ project = self)
+ branches[name] = branch
+
+ self.branches = branches
+
+ return
+
+
+ def lines(self):
+ yield self.trunk
+ for branch in self.branches.itervalues():
+ yield branch
+ return
+
+
+# end of file
Added: cs/buildbot/trunk/buildbot/repositories.py
===================================================================
--- cs/buildbot/trunk/buildbot/repositories.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/repositories.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -0,0 +1,104 @@
+
+
+from copy import copy
+from buildbot.process import step
+
+
+class Repository(object):
+
+ Step = None
+
+ class Location(object):
+ """a reference to a specific place (project & branch) in a repository"""
+ def __init__(self, repository):
+ self.repository = repository
+ def trunkLocation(self):
+ raise NotImplementedError()
+ def branchLocation(self, name):
+ raise NotImplementedError()
+
+ def sourceStep(self, **kwds):
+ """Return a BuildStep which checks-out source from me."""
+ from buildbot.process.factory import s
+ kwds.update(self.sourceStepKwds())
+ return s(self.Step, **kwds)
+
+ def sourceStepKwds(self):
+ raise NotImplementedError()
+
+
+class TreeRepository(Repository):
+
+ class Location(Repository.Location):
+ def __init__(self, path, **kwds):
+ super(TreeRepository.Location, self).__init__(**kwds)
+ self.path = path
+ def subdirectory(self, name):
+ s = copy(self)
+ s.path = s.path + [name]
+ return s
+
+ def rootLocation(self):
+ return self.Location(path=[], repository=self)
+
+ def initLayout(self, entry, location=None):
+ if location is None:
+ location = self.rootLocation()
+ if isinstance(entry, dict):
+ for name, subentry in entry.iteritems():
+ sublocation = location.subdirectory(name)
+ self.initLayout(subentry, sublocation)
+ else:
+ entry.initLayout(location)
+ return
+
+
+# This must be at global scope in order to be pickled.
+class SVNLocation(TreeRepository.Location):
+ def __init__(self, line=None, **kwds):
+ super(SVNRepository.Location, self).__init__(**kwds)
+ self.line = line
+ def trunkLocation(self):
+ return self.repository.Location(path = self.path,
+ line = ['trunk'],
+ repository = self.repository)
+ def branchLocation(self, name):
+ return self.repository.Location(path = self.path,
+ line = ['branches', name],
+ repository = self.repository)
+ return
+ def sourceStep(self, **kwds):
+ return self.repository.sourceStep(root = self.root(),
+ defaultBranch = self.branch(),
+ **kwds)
+ def root(self): return '/'.join(self.path) + '/'
+ def branch(self): return '/'.join(self.line)
+
+
+
+class SVNRepository(TreeRepository):
+
+ Step = step.SVN
+ Location = SVNLocation
+
+ def __init__(self, baseURL):
+ Repository.__init__(self)
+ self.baseURL = baseURL
+
+ def sourceStepKwds(self):
+ return {'baseURL': self.baseURL}
+
+
+class HGRepository(Repository):
+
+ Step = step.Mercurial
+
+ def __init__(self, baseURL):
+ Repository.__init__(self)
+ self.baseURL = baseURL
+
+ def sourceStepKwds(self):
+ return {'baseURL': self.baseURL}
+
+
+# end of file
Modified: cs/buildbot/trunk/buildbot/status/html.py
===================================================================
--- cs/buildbot/trunk/buildbot/status/html.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/status/html.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -1148,8 +1148,11 @@
project = self.getProject(categories)
if project:
# TODO: this is going to look really ugly
- topleft = "<a href=\"%s\">%s</a><br />last build" % \
- (project.url, project.name)
+ if project.url:
+ topleft = "<a href=\"%s\">%s</a><br />last build" % \
+ (project.url, project.name)
+ else:
+ topleft = "%s<br />last build" % project.name
else:
topleft = "last build"
data += ' <tr class="LastBuild">\n'
Added: cs/buildbot/trunk/buildbot/steps/misc.py
===================================================================
--- cs/buildbot/trunk/buildbot/steps/misc.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/steps/misc.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -0,0 +1,54 @@
+# special classes to handle PyLith's complexities
+
+
+from buildbot.process.step import ShellCommand
+
+
+class Deps(ShellCommand):
+
+ def createSummary(self, log):
+ from StringIO import StringIO
+
+ warnings = 0
+ errors = 0
+ skipped_misc = False
+ skipped_petsc = False
+ hostname = "unknown"
+
+ for line in StringIO(log.getText()):
+ if line.startswith("deps: warning: "):
+ warnings += 1
+ elif line.startswith("deps: error: "):
+ errors += 1
+ elif line.startswith("deps: skipped misc: "):
+ skipped_misc = True
+ elif line.startswith("deps: skipped petsc: "):
+ skipped_petsc = True
+ elif line.startswith("deps: hostname: "):
+ hostname = line[16:].strip()
+
+ self.descriptionDone = self.describe(True)[:]
+ self.descriptionDone.append("host: " + hostname)
+ if skipped_misc:
+ self.descriptionDone.append("skipped misc")
+ if skipped_petsc:
+ self.descriptionDone.append("skipped petsc")
+ if warnings:
+ self.descriptionDone.append("warn=%d" % warnings)
+ if errors:
+ self.descriptionDone.append("err=%d" % errors)
+
+ self.warnings = warnings
+ self.errors = errors
+
+ def evaluateCommand(self, cmd):
+ from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE #, SKIPPED, EXCEPTION
+ if cmd.rc != 0:
+ return FAILURE
+ #if cmd.log.getStderr(): return WARNINGS
+ if self.warnings or self.errors:
+ return WARNINGS
+ return SUCCESS
+
+
+# end of file
Added: cs/buildbot/trunk/buildbot/users.py
===================================================================
--- cs/buildbot/trunk/buildbot/users.py 2007-05-25 00:03:40 UTC (rev 6959)
+++ cs/buildbot/trunk/buildbot/users.py 2007-05-25 00:59:19 UTC (rev 6960)
@@ -0,0 +1,28 @@
+
+
+from buildbot import interfaces
+from buildbot.twcompat import implements
+
+
+class User:
+ def __init__(self, email):
+ self.email = email
+
+
+class AddressBook:
+ if implements:
+ implements(interfaces.IEmailLookup)
+ else:
+ __implements__ = interfaces.IEmailLookup,
+
+ def __init__(self, users):
+ self.users = users
+
+ def getAddress(self, user):
+ email = getattr(self.users, user).email
+ if email is None:
+ return self.users.admin.email
+ return email
+
+
+# end of file
More information about the cig-commits
mailing list