bug 377929 - mozilla2 buildbot setup. mozilla2 master.cfg + hgpoller.py r=rhelmer

git-svn-id: svn://10.0.0.236/trunk@231417 18797224-902f-48f8-a5cc-f745e15eee43

mozilla/tools/buildbot-configs/mozilla2/master.cfg

@@ -0,0 +1,143 @@
+# -*- python -*-
+# ex: set syntax=python:
+
+# This is a sample buildmaster config file. It must be installed as
+# 'master.cfg' in your buildmaster's base directory (although the filename
+# can be changed with the --basedir option to 'mktap buildbot master').
+
+# It has one job: define a dictionary named BuildmasterConfig. This
+# dictionary has a variety of keys to control different aspects of the
+# buildmaster. They are documented in docs/config.xhtml .
+
+import os.path
+from buildbot.changes import hgpoller
+from buildbot.scheduler import Scheduler, Periodic
+from buildbot.process import step, factory
+from buildbot.status import html, words, tinderbox
+from buildbot.steps.source import Mercurial
+from buildbot.steps.transfer import FileDownload
+
+s = factory.s
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+c['projectName'] = "Mozilla2"
+c['projectURL'] = "http://wiki.mozilla.org/Mozilla_2"
+c['buildbotURL'] = "" #XXX: fill me in
+
+####### BUILDSLAVES
+
+# the 'bots' list defines the set of allowable buildslaves. Each element is a
+# tuple of bot-name and bot-password. These correspond to values given to the
+# buildslave's mktap invocation.
+c['bots'] = [] #XXX: fill me in
+
+
+# 'slavePortnum' defines the TCP port to listen on. This must match the value
+# configured into the buildslaves (with their --master option)
+
+c['slavePortnum'] = 9989
+
+c['status'] = []
+c['status'].append(html.Waterfall(http_port=2004,
+                                  css="mozilla.css"))
+
+
+
+c['status'].append(tinderbox.TinderboxMailNotifier(
+                       fromaddr="", #XXX: fill me in
+                       tree="MozillaTest",
+                       extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org"],
+                       relayhost="localhost",
+                       logCompression="bzip2"))
+####### CHANGESOURCES
+
+# the 'sources' list tells the buildmaster how it should find out about
+# source code changes. Any class which implements IChangeSource can be added
+# to this list: there are several in buildbot/changes/*.py to choose from.
+
+
+c['sources'] = []
+c['sources'].append(hgpoller.HgPoller(
+                 hgURL = "http://hg.mozilla.org/",
+                 branch = "mozilla-central",
+                 pollInterval = 1 * 60))
+
+####### SCHEDULERS
+
+## configure the Schedulers
+
+c['schedulers'] = []
+c['schedulers'].append(Scheduler(name="mozilla-central",
+                                 branch="mozilla-central",
+                                 treeStableTimer=5*60,
+                                 builderNames=["mozilla2 firefox linux"]))
+
+
+####### BUILDERS
+
+# the 'builders' list defines the Builders. Each one is configured with a
+# dictionary, using the following keys:
+#  name (required): the name used to describe this bilder
+#  slavename (required): which slave to use, must appear in c['bots']
+#  builddir (required): which subdirectory to run the builder in
+#  factory (required): a BuildFactory to define how the build is run
+#  periodicBuildTime (optional): if set, force a build every N seconds
+
+
+# the first BuildStep is typically responsible for obtaining a copy of the
+# sources. There are source-obtaining Steps in buildbot/process/step.py for
+# CVS, SVN, and others.
+
+builders = []
+
+mozilla2_firefox_unix_steps = [
+        s(Mercurial, mode='update',
+          baseURL='http://hg.mozilla.org/',
+          defaultBranch='mozilla-central'),
+        s(step.ShellCommand, command=['python', 'client.py',
+                                      '--mozilla-repo=http://hg.mozilla.org/mozilla-central/',
+                                      'checkout'],
+          description='running client.py checkout',
+          descriptionDone='client.py checkout'),
+        s(FileDownload, mastersrc='mozconfig-firefox-linux',
+          slavedest='.mozconfig'),
+        s(step.ShellCommand, name='mozconfig contents',
+          command=['cat', '.mozconfig']),
+        s(step.Compile, command=['make', '-f', 'client.mk', 'build']),
+#XXX: for unit test boxes
+#        s(step.Test, command=['make', 'check'], workdir="build/obj-firefox")
+]
+
+mozilla2_firefox_unix_builder = {
+    'name': 'mozilla2 firefox linux',
+    'slavenames': [], #XXX: fill me in
+    'builddir': 'mozilla2',
+    'factory': factory.BuildFactory(mozilla2_firefox_unix_steps),
+    'category': 'HEAD',
+    }
+builders.append(mozilla2_firefox_unix_builder)
+
+c['builders'] = builders
+
+####### DEBUGGING OPTIONS
+
+# if you set 'debugPassword', then you can connect to the buildmaster with
+# the diagnostic tool in contrib/debugclient.py . From this tool, you can
+# manually force builds and inject changes, which may be useful for testing
+# your buildmaster without actually commiting changes to your repository (or
+# before you have a functioning 'sources' set up). The debug tool uses the
+# same port number as the slaves do: 'slavePortnum'.
+
+c['debugPassword'] = "" #XXX: fill me in
+
+# if you set 'manhole', you can ssh into the buildmaster and get an
+# interactive python shell, which may be useful for debugging buildbot
+# internals. It is probably only useful for buildbot developers. You can also
+# use an authorized_keys file, or plain telnet.
+#from buildbot import manhole
+#c['manhole'] = manhole.PasswordManhole("tcp:9999:interface=127.0.0.1",
+#                                       "admin", "password")
+

mozilla/tools/buildbot-configs/mozilla2/mozconfig-firefox-linux

@@ -0,0 +1,4 @@
+ac_add_options --enable-application=browser
+mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/obj-firefox
+ac_add_options --enable-optimize="-Os -gstabs+"
+ac_add_options --disable-debug

mozilla/tools/buildbot-configs/mozilla2/mozilla.css

@@ -0,0 +1,64 @@
+body {
+    background: url("http://www.mozilla.org/images/mozilla-banner.gif") 
+                no-repeat top left;
+    font-family: Verdana, Courier, sans-sarif;
+    font-size: 80%;
+    padding-top: 80px;
+}
+
+a:link, a:visited {
+    color: #000;
+}
+
+td.Event, td.BuildStep, td.Activity, td.Change, td.Time, td.Builder, td.LastBuild {
+    border: 1px solid #777;
+}
+
+td.Event, td.Change, td.Time, td.Builder {
+    background-color: #F3F3F3;
+}
+
+td.green {
+    background-color: #72ff75;
+}
+
+.table {
+    /* border-spacing: 2px; */
+    border-spacing: 0px;
+    width: 1%;
+    border-collapse: seperate;
+}
+
+td {
+    padding: 0.5ex 1ex;
+    text-align: center;
+    border: 1px solid #DDD;
+}
+
+/* Activity states */
+.idle {
+    background-color: #F3F3F3;
+}
+
+/* LastBuild, BuildStep states */
+.success {
+    background-color: #72ff75;
+}
+.failure {
+    background-color: red;
+}
+.warnings {
+    background-color: orange;
+}
+.exception {
+    background-color: #c000c0;
+}
+.start,.running {
+    background-color: yellow;
+}
+
+/* color trunk/branch builder links differently */
+
+.Builder a[href*="trunk"] {
+    color: #05e;
+}

mozilla/tools/buildbot/buildbot/changes/hgpoller.py

@@ -0,0 +1,113 @@
+import time
+import rfc822
+from urllib import urlopen
+from xml.dom import minidom, Node
+
+from twisted.python import log, failure
+from twisted.internet import defer, reactor
+from twisted.internet.task import LoopingCall
+
+from buildbot.changes import base, changes
+
+class HgPoller(base.ChangeSource):
+    """This source will poll a Mercurial server over HTTP using
+    the built-in RSS feed for changes and submit them to the
+    change master."""
+
+    compare_attrs = ['hgURL', 'branch', 'pollInterval']
+    parent = None
+    loop = None
+    volatile = ['loop']
+    working = False
+    
+    def __init__(self, hgURL, branch, pollInterval=30):
+        """
+        @type   hgURL:          string
+        @param  hgURL:          The base URL of the Hg repo
+                                (e.g. http://hg.mozilla.org/)
+        @type   branch:         string
+        @param  branch:         The branch to check (e.g. mozilla-central)
+        @type   pollInterval:   int
+        @param  pollInterval:   The time (in seconds) between queries for
+                                changes
+        """
+        
+        self.hgURL = hgURL
+        self.branch = branch
+        self.pollInterval = pollInterval
+        self.lastChange = time.time()
+        self.lastPoll = time.time()
+
+    def startService(self):
+        self.loop = LoopingCall(self.poll)
+        base.ChangeSource.startService(self)
+        reactor.callLater(0, self.loop.start, self.pollInterval)
+
+    def stopService(self):
+        self.loop.stop()
+        return base.ChangeSource.stopService(self)
+    
+    def describe(self):
+        return "Getting changes from the Mercurial repo at %s%s" % \
+               (self.hgURL, self.branch)
+    
+    def poll(self):
+        if self.working:
+            log.msg("Not polling because last poll is still working")
+        else:
+            self.working = True
+            d = self._get_changes()
+            d.addCallback(self._process_changes)
+            d.addCallbacks(self._finished_ok, self._finished_failure)
+
+    def _finished_ok(self, res):
+        assert self.working
+        self.working = False
+
+        return res
+
+    def _finished_failure(self, res):
+        log.msg("Hg poll failed: %s" % res)
+        assert self.working
+        self.working = False
+        return None
+
+    def _make_url(self):
+        return "%s%s/?rss-log" % (self.hgURL, self.branch)
+    
+    def _get_changes(self):
+        url = self._make_url()
+        log.msg("Polling Hg server at %s" % url)
+        
+        self.lastPoll = time.time()
+        return defer.maybeDeferred(urlopen, url)
+
+    def _parse_changes(self, query):
+        dom = minidom.parseString(query.read())
+        items = dom.getElementsByTagName("item")
+        changes = []
+        for i in items:
+            d = dict()
+            for k in ["description", "link", "author", "pubDate"]:
+                d[k] = i.getElementsByTagName(k)[0].firstChild.wholeText
+            # strip out HTML newlines
+            d["description"] = d["description"].replace("<br/>","")
+            # need to parse date with timezone, and turn into a UTC timestamp
+            d["pubDate"] = rfc822.mktime_tz(rfc822.parsedate_tz(d["pubDate"]) )
+            changes.append(d)
+        changes = [c for c in changes if c["pubDate"] > self.lastChange]
+        changes.reverse() # want t hem in reverse chronological order
+        return changes
+    
+    def _process_changes(self, query):
+        change_list = self._parse_changes(query)
+        for change in change_list:
+            c = changes.Change(who = change["author"],
+                               files = [], # sucks
+                               comments = change["description"],
+                               when = change["pubDate"],
+                               branch = self.branch)
+            self.parent.addChange(c)
+        self.lastChange = max(self.lastPoll, *[c["pubDate"] for c in
+                                                   change_list])
+