Bommels05/Mozilla
Bommels05/Mozilla
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 474343 - Upgrading staging talos to buildbot 0.7.9 r=anodelman

Browse Source 
git-svn-id: svn://10.0.0.236/trunk@255903 18797224-902f-48f8-a5cc-f745e15eee43
This commit is contained in:
catlee%mozilla.com 2009-01-23 13:53:48 +00:00
parent 8e5214f45c
commit b6d40c64b2
8 changed files with 1168 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
mozilla/tools/buildbot-configs/testing/talos/perf-staging/ftppoller.py Normal file
View File

@ -0,0 +1,167 @@
import time
import re
from urllib2 import urlopen, unquote
from twisted.python import log, failure
from twisted.internet import defer, reactor
from twisted.internet.task import LoopingCall
from buildbot.changes import base, changes
class InvalidResultError(Exception):
def __init__(self, value="InvalidResultError"):
self.value = value
def __str__(self):
return repr(self.value)
class EmptyResult(Exception):
pass
class FtpParser:
"""I parse the web page for possible builds to test"""
findBuildDirs = re.compile('^.*"(\d{10})\/".*$')
def __init__(self, query, searchString):
url = query.geturl()
pageContents = query.read()
query.close()
self.dirs = []
self.dates = []
lines = pageContents.split('\n')
#for parsing lists of directories
for line in lines:
if line == "": continue
match = re.match(self.findBuildDirs, line)
if match:
self.dirs.append(match.group(1))
#for parsing lists of files
findLastDate = re.compile('^.*"([^"]*' + searchString + ')".*(\d\d-[a-zA-Z]{3}-\d\d\d\d \d\d:\d\d).*$')
for line in lines:
match = re.match(findLastDate, line)
if match:
self.dates.append([match.group(1), url + match.group(1), time.mktime(time.strptime(match.group(2), "%d-%b-%Y %H:%M"))])
def getDirs(self):
return self.dirs
def getDates(self):
return self.dates
class FtpPoller(base.ChangeSource):
"""This source will poll an ftp directory for changes and submit
them to the change master."""
compare_attrs = ["ftpURLs", "pollInterval", "tree", "branch"]
parent = None # filled in when we're added
loop = None
volatile = ['loop']
working = 0
def __init__(self, branch="", tree="Firefox", pollInterval=30, ftpURLs=[], searchString=""):
"""
@type ftpURLs: list of strings
@param ftpURLs: The ftp directories to monitor
@type tree: string
@param tree: The tree to look for changes in.
For example, Firefox trunk is 'Firefox'
@type branch: string
@param branch: The branch to look for changes in. This must
match the 'branch' option for the Scheduler.
@type pollInterval: int
@param pollInterval: The time (in seconds) between queries for
changes
@type searchString: string
@param searchString: file type of the build we are looking for
"""
self.ftpURLs = ftpURLs
self.tree = tree
self.branch = branch
self.pollInterval = pollInterval
self.lastChanges = {}
for url in self.ftpURLs:
self.lastChanges[url] = time.time() - 3600
self.searchString = searchString
def startService(self):
self.loop = LoopingCall(self.poll)
base.ChangeSource.startService(self)
reactor.callLater(0, self.loop.start, self.pollInterval)
def stopService(self):
self.loop.stop()
return base.ChangeSource.stopService(self)
def describe(self):
str = ""
str += "Getting changes from ftp directory %s " \
% str(self.ftpURLs)
str += "<br>Using tree: %s, branch %s" % (self.tree, self.branch)
return str
def poll(self):
if self.working > 0:
log.msg("Not polling Tinderbox because last poll is still working (%s)" % (str(self.working)))
else:
for url in self.ftpURLs:
self.working = self.working + 1
d = self._get_changes(url)
d.addCallback(self._process_changes, 0)
d.addBoth(self._finished)
return
def _finished(self, res):
self.working = self.working - 1
def _get_changes(self, url):
log.msg("Polling ftp dir %s" % url)
return defer.maybeDeferred(urlopen, url)
def _process_changes(self, query, forceDate):
try:
url = query.geturl()
parser = FtpParser(query, self.searchString)
dirList = parser.getDirs()
dateList = parser.getDates()
except InvalidResultError, e:
log.msg("Could not process Tinderbox query: " + e.value)
return
except EmptyResult:
return
#figure out if there is a new directory that needs searching
for dir in dirList:
buildDate = int(dir)
if self.lastChanges[url] >= buildDate:
# change too old
continue
self.lastChanges[url] = buildDate
self.working = self.working + 1
d = self._get_changes(url + dir + '/')
d.addBoth(self._process_changes, buildDate)
d.addBoth(self._finished)
#if we have a new browser to test, test it
for buildname, fullpath, buildDate in dateList:
if (url in self.lastChanges):
if (self.lastChanges[url] >= buildDate):
# change too old
continue
if forceDate > 0:
buildDate = forceDate
else:
self.lastChanges[url] = buildDate
c = changes.Change(who = url,
comments = "success",
files = [buildname,],
branch = self.branch,
when = buildDate,
links = fullpath)
self.parent.addChange(c)
log.msg("found a browser to test (%s)" % (fullpath))

142
mozilla/tools/buildbot-configs/testing/talos/perf-staging/master.cfg
View File

@ -8,6 +8,7 @@
import os.path
# from buildbot.changes.freshcvs import FreshCVSSource
from buildbot.buildslave import BuildSlave
from buildbot.process import factory
from buildbot.scheduler import Scheduler, Periodic
from buildbot.status import html
@ -49,22 +50,24 @@ c['slavePortnum'] = 9987
## Slaves
##
c['bots'] = [("qm-pubuntu-stage01", "l1nux"),
("qm-ptiger-stage01", "mac1nt0sh"),
("qm-pxp-stage01", "w1nd3rs"),
("qm-pvista-stage01", "w1nd3rs"),
("qm-pleopard-stage01", "mac1nt0sh")]
c['slaves'] = [BuildSlave(name, password, max_builds=1) for (name, password) in [
("qm-pubuntu-stage01", "l1nux"),
("qm-ptiger-stage01", "mac1nt0sh"),
("qm-pxp-stage01", "w1nd3rs"),
("qm-pvista-stage01", "w1nd3rs"),
("qm-pleopard-stage01", "mac1nt0sh")
]]
##
## Status
##
c['status'] = []
c['status'].append(html.Waterfall(http_port=2008,
css="/build/perf-staging/waterfall.css"))
c['status'].append(html.WebStatus(http_port=2008, allowForce=True))
from buildbot.status import tinderbox
from buildbot.status.mail import MailNotifier
from tinderboxmailnotifier import TinderboxMailNotifier
c['status'].append(MailNotifier(
fromaddr="talos.buildbot@build.mozilla.org",
@ -80,16 +83,14 @@ c['status'].append(tinderbox.TinderboxMailNotifier(
tree="MozillaTest",
extraRecipients=["tinderbox-daemon@tinderbox.mozilla.org",],
relayhost="smtp.mozilla.org",
builders=["Linux talos trunk stage",
"MacOSX Darwin 8.8.1 talos trunk stage",
"MacOSX Darwin 9.0.0 talos trunk stage",
"WINNT 5.1 talos trunk stage",
"WINNT 6.0 talos trunk stage"],
builders=["Linux talos mozilla-central stage",
"MacOSX Darwin 8.8.1 talos mozilla-central stage",
"MacOSX Darwin 9.0.0 talos mozilla-central stage",
"WINNT 5.1 talos mozilla-central stage",
"WINNT 6.0 talos mozilla-central stage"],
# useChangeTime=True,
logCompression="bzip2"))
##
## Sources
##
@ -98,9 +99,9 @@ import ftppoller
reload(ftppoller)
from ftppoller import FtpPoller
c['sources'] = []
c['change_source'] = []
c['sources'].append(FtpPoller(
c['change_source'].append(FtpPoller(
tree = "Firefox",
branch = "MOZ2-WIN32-CENTRAL",
pollInterval = 2 * 60,
@ -108,15 +109,15 @@ c['sources'].append(FtpPoller(
searchString = WIN32_SEARCHSTRING)
)
c['sources'].append(FtpPoller(
c['change_source'].append(FtpPoller(
tree = "Firefox",
branch = "MOZ2-LINUX-CENTRAL",
pollInterval = 2 * 60,
ftpURLs = LINUX_MOZ2_CENTRAL_BUILDDIRS,
searchString = LINUX_SEARCHSTRING)
)
c['sources'].append(FtpPoller(
c['change_source'].append(FtpPoller(
tree = "Firefox",
branch = "MOZ2-MAC-CENTRAL",
pollInterval = 2 * 60,
@ -129,25 +130,23 @@ c['sources'].append(FtpPoller(
c['schedulers'] = []
## WINDOWS
c['schedulers'].append(Scheduler(name="head perfrun scheduler",
branch="HEAD",
#mozilla-central
c['schedulers'].append(Scheduler(name="moz2 central win32 scheduler",
branch="MOZ2-WIN32-CENTRAL",
treeStableTimer=5*60,
builderNames=["WINNT 5.1 talos trunk stage",
"WINNT 6.0 talos trunk stage"]
builderNames=["WINNT 5.1 talos mozilla-central stage",
"WINNT 6.0 talos mozilla-central stage"]
))
## LINUX
c['schedulers'].append(Scheduler(name="head perfrun linux scheduler",
branch="HEAD_LINUX",
c['schedulers'].append(Scheduler(name="moz2 central perfrun linux scheduler",
branch="MOZ2-LINUX-CENTRAL",
treeStableTimer=5*60,
builderNames=["Linux talos trunk stage"]
builderNames=["Linux talos mozilla-central stage"]
))
## MAC
c['schedulers'].append(Scheduler(name="head perfrun scheduler mac",
branch="HEAD-MAC",
c['schedulers'].append(Scheduler(name="moz2 central perfrun mac scheduler",
branch="MOZ2-MAC-CENTRAL",
treeStableTimer=5*60,
builderNames=["MacOSX Darwin 8.8.1 talos trunk stage",
"MacOSX Darwin 9.0.0 talos trunk stage"]
builderNames=["MacOSX Darwin 8.8.1 talos mozilla-central stage",
"MacOSX Darwin 9.0.0 talos mozilla-central stage"]
))
# the 'builders' list defines the Builders. Each one is configured with a
@ -161,87 +160,96 @@ c['schedulers'].append(Scheduler(name="head perfrun scheduler mac",
#STAGING MACHINES
graphConfig = ['--resultsServer', 'graphs-stage.mozilla.org', '--resultsLink', '/server/bulk.cgi']
stageConfig = graphConfig + ['--activeTests', 'ts:tsspider:twinopen']
stageConfig = graphConfig + ['--activeTests', 'tdhtml:ts:tsspider:tp']
nochromeConfig = graphConfig + ['--activeTests', 'tdhtml:ts:tsspider', '--noChrome']
basicConfig = 'configs/production.sample.config'
branchConfig = 'configs/production.sample.config.nogfx'
#nochromeConfig = 'configs/production.sample.config.nochrome'
fastConfig = 'configs/fast.production.sample.config'
fastManifest = 'configs/historic_manifest.txt'
jssConfig = 'configs/jss.production.sample.config'
basicTalosCmd = ['python', 'run_tests.py', '--noisy']
win32_trunk_stage_steps = TalosFactory(OS = 'win',
win32_moz2_stage_steps = TalosFactory(OS = 'win',
envName='vc8perf',
buildBranch="1.9",
buildBranch="moz-central-stage",
configOptions=stageConfig,
buildPath="..\\firefox\\firefox.exe",
talosCmd=basicTalosCmd)
vista_trunk_stage_steps = TalosFactory(OS = 'win',
vista_moz2_stage_steps = TalosFactory(OS = 'win',
envName='vc8perf',
buildBranch="1.9",
buildBranch="moz-central-stage",
configOptions=stageConfig,
buildPath="..\\firefox\\firefox.exe",
talosCmd=basicTalosCmd)
linux_trunk_stage_steps = TalosFactory(OS = 'linux',
linux_moz2_stage_steps = TalosFactory(OS = 'linux',
envName='linux',
buildBranch="1.9",
buildBranch="moz-central-stage",
configOptions=stageConfig,
buildPath="../firefox/firefox",
talosCmd=basicTalosCmd)
mac_trunk_stage_steps = TalosFactory(OS = 'tiger',
mac_moz2_stage_steps = TalosFactory(OS = 'tiger',
envName='mac',
buildBranch="1.9",
buildBranch="moz-central-stage",
configOptions=stageConfig,
buildPath="../GranParadiso.app/Contents/MacOS/firefox",
buildPath="../Minefield.app/Contents/MacOS/firefox",
talosCmd=basicTalosCmd)
leopard_trunk_stage_steps = TalosFactory(OS = 'leopard',
leopard_moz2_stage_steps = TalosFactory(OS = 'leopard',
envName='mac',
buildBranch="1.9",
buildBranch="moz-central-stage",
configOptions=stageConfig,
buildPath="../GranParadiso.app/Contents/MacOS/firefox",
buildPath="../Minefield.app/Contents/MacOS/firefox",
talosCmd=basicTalosCmd)
#STAGING
win32_trunk_stage_builder = {
'name': "WINNT 5.1 talos trunk stage",
win32_moz2_stage_builder = {
'name': "WINNT 5.1 talos mozilla-central stage",
'slavenames': ['qm-pxp-stage01'],
'builddir': "win32-trunk",
'factory': win32_trunk_stage_steps,
'factory': win32_moz2_stage_steps,
'category': "Firefox Trunk"
}
vista_trunk_stage_builder = {
'name': "WINNT 6.0 talos trunk stage",
vista_moz2_stage_builder = {
'name': "WINNT 6.0 talos mozilla-central stage",
'slavenames': ['qm-pvista-stage01'],
'builddir': "vista-trunk",
'factory': vista_trunk_stage_steps,
'factory': vista_moz2_stage_steps,
'category': "Firefox Trunk"
}
linux_trunk_stage_builder = {
'name': "Linux talos trunk stage",
linux_moz2_stage_builder = {
'name': "Linux talos mozilla-central stage",
'slavenames': ['qm-pubuntu-stage01'],
'builddir': "linux-trunk",
'factory': linux_trunk_stage_steps,
'factory': linux_moz2_stage_steps,
'category': "Firefox Trunk"
}
mac_trunk_stage_builder = {
'name': "MacOSX Darwin 8.8.1 talos trunk stage",
mac_moz2_stage_builder = {
'name': "MacOSX Darwin 8.8.1 talos mozilla-central stage",
'slavenames': ['qm-ptiger-stage01'],
'builddir': "mac-trunk",
'factory': mac_trunk_stage_steps,
'factory': mac_moz2_stage_steps,
'category': "Firefox Trunk"
}
leopard_trunk_stage_builder = {
'name': "MacOSX Darwin 9.0.0 talos trunk stage",
leopard_moz2_stage_builder = {
'name': "MacOSX Darwin 9.0.0 talos mozilla-central stage",
'slavenames': ['qm-pleopard-stage01'],
'builddir': "leopard-trunk",
'factory': leopard_trunk_stage_steps,
'factory': leopard_moz2_stage_steps,
'category': "Firefox Trunk"
}
c['builders'] = []
#STAGING
c['builders'].append(win32_trunk_stage_builder)
c['builders'].append(vista_trunk_stage_builder)
c['builders'].append(linux_trunk_stage_builder)
c['builders'].append(mac_trunk_stage_builder)
c['builders'].append(leopard_trunk_stage_builder)
c['builders'].append(win32_moz2_stage_builder)
c['builders'].append(vista_moz2_stage_builder)
c['builders'].append(linux_moz2_stage_builder)
c['builders'].append(mac_moz2_stage_builder)
c['builders'].append(leopard_moz2_stage_builder)

715
mozilla/tools/buildbot-configs/testing/talos/perf-staging/perfrunner.py Normal file
View File

@ -0,0 +1,715 @@
# -*- Python -*-
from twisted.python import log
from buildbot import buildset
from buildbot.scheduler import Scheduler
from buildbot.process.buildstep import BuildStep
from buildbot.process.factory import BuildFactory
from buildbot.buildset import BuildSet
from buildbot.sourcestamp import SourceStamp
from buildbot.steps.shell import ShellCommand
from buildbot.steps.transfer import FileDownload
from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION
import re, urllib, sys, os
from time import strptime, strftime, localtime
from datetime import datetime
from os import path
import copy
MozillaEnvironments = { }
# platform SDK location. we can build both from one generic template.
# modified from vc8 environment
MozillaEnvironments['vc8perf'] = {
"MOZ_CRASHREPORTER_NO_REPORT": '1',
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"CYGWINBASE": 'C:\\cygwin',
"PATH": 'C:\\Python24;' + \
'C:\\Python24\\Scripts;' + \
'C:\\cygwin\\bin;' + \
'C:\\WINDOWS\\System32;' + \
'C:\\program files\\gnuwin32\\bin;' + \
'C:\\WINDOWS;'
}
MozillaEnvironments['linux'] = {
"MOZ_CRASHREPORTER_NO_REPORT": '1',
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"DISPLAY": ":0",
}
MozillaEnvironments['mac'] = {
"MOZ_NO_REMOTE": '1',
"NO_EM_RESTART": '1',
"XPCOM_DEBUG_BREAK": 'warn',
"MOZ_CRASHREPORTER_NO_REPORT": '1',
# for extracting dmg's
"PAGER": '/bin/cat',
}
class MultiBuildScheduler(Scheduler):
"""Trigger N (default three) build requests based upon the same change request"""
def __init__(self, numberOfBuildsToTrigger=3, **kwargs):
self.numberOfBuildsToTrigger = numberOfBuildsToTrigger
Scheduler.__init__(self, **kwargs)
def fireTimer(self):
self.timer = None
self.nextBuildTime = None
changes = self.importantChanges + self.unimportantChanges
self.importantChanges = []
self.unimportantChanges = []
# submit
for i in range(0, self.numberOfBuildsToTrigger):
ss = UnmergableSourceStamp(changes=changes)
bs = buildset.BuildSet(self.builderNames, ss)
self.submit(bs)
class LatestFileURL:
sortByDateString = "?C=M;O=A"
def _retrievePageAtURL(self):
content = []
try:
opener = urllib.URLopener()
page = opener.open(self.url + self.sortByDateString)
content = page.readlines()
opener.close()
except:
print "unable to retrieve page at: " + self.url + " dated: " + self.sortByDateString
return content
def _populateDict(self):
'''Extract the latest filename from the given URL
* retrieve page at URL
* for each line
* grab filename URL
* if filename matches filenameSearchString
* grab datetime
* store date and filename URL in dictionary
* return latest filenameURL matching date'''
reDatestamp = re.compile('\d\d-[a-zA-Z]{3,3}-\d\d\d\d \d\d:\d\d')
reHREF = re.compile('href\="((?:\w|[.-])*)"')
content = self._retrievePageAtURL()
for line in content:
matchHREF = re.search(reHREF, line)
if matchHREF:
if self.filenameSearchString in matchHREF.group(1):
datetimeMatch = re.search(reDatestamp, line)
if datetimeMatch:
dts = line[datetimeMatch.start():datetimeMatch.end()]
timestamp = datetime(*strptime(dts, '%d-%b-%Y %H:%M')[0:6])
self.dateFileDict[timestamp] = matchHREF.group(1)
def _getDateKeys(self):
dbKeys = self.dateFileDict.keys()
if not dbKeys:
return []
return dbKeys
def getLatestFilename(self):
dateKeys = self._getDateKeys()
if not dateKeys: return ""
return self.dateFileDict[max(dateKeys)]
def _printDictionary(self):
keys = self._getDateKeys()
for timestamp in keys:
print "%s : %s" % (timestamp, self.dateFileDict[timestamp])
def testrun(self):
self._populateDict()
self._printDictionary()
name = self.getLatestFilename()
print self.url + name
def __init__(self, url, filenameSearchString):
self.url = url
self.filenameSearchString = filenameSearchString
self.dateFileDict = {}
self._populateDict()
class MozillaWgetLatestDated(ShellCommand):
"""Download built Firefox client from dated staging directory."""
haltOnFailure = True
def __init__(self, **kwargs):
self.branch = "HEAD"
if 'branch' in kwargs:
self.branch = kwargs['branch']
if not 'command' in kwargs:
kwargs['command'] = ["wget"]
ShellCommand.__init__(self, **kwargs)
def setBuild(self, build):
ShellCommand.setBuild(self, build)
self.changes = build.source.changes
#a full path is always provided by the poller
self.fileURL = self.changes[-1].links
self.filename = self.changes[-1].files[0]
def getFilename(self):
return self.filename
def describe(self, done=False):
return ["Wget Download"]
def start(self):
#urlGetter = LatestFileURL(self.url, self.filenameSearchString)
#self.filename = urlGetter.getLatestFilename()
#self.fileURL = self.url + self.filename
if self.branch:
self.setProperty("fileURL", self.fileURL)
self.setProperty("filename", self.filename)
self.setCommand(["wget", "-nv", "-N", self.fileURL])
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
if None != re.search('ERROR', cmd.logs['stdio'].getText()):
return FAILURE
return SUCCESS
class MozillaWgetLatest(ShellCommand):
"""Download built Firefox client from nightly staging directory."""
haltOnFailure = True
def __init__(self, **kwargs):
assert kwargs['url'] != ""
assert kwargs['filenameSearchString'] != ""
#if this change includes a link use it
if self.changes[-1].links:
self.url = self.changes[-1].links
else:
self.url = kwargs['url']
self.filenameSearchString = kwargs['filenameSearchString']
self.branch = "HEAD"
self.fileURL = ""
if 'branch' in kwargs:
self.branch = kwargs['branch']
if not 'command' in kwargs:
kwargs['command'] = ["wget"]
ShellCommand.__init__(self, **kwargs)
def getFilename(self):
return self.filename
def describe(self, done=False):
return ["Wget Download"]
def start(self):
urlGetter = LatestFileURL(self.url, self.filenameSearchString)
self.filename = urlGetter.getLatestFilename()
self.fileURL = self.url + self.filename
if self.branch:
self.setProperty("fileURL", self.fileURL)
self.setProperty("filename", self.filename)
self.setCommand(["wget", "-nv", "-N", self.fileURL])
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
if None != re.search('ERROR', cmd.logs['stdio'].getText()):
return FAILURE
return SUCCESS
class MozillaInstallZip(ShellCommand):
"""Install given file, unzipping to executablePath"""
def __init__(self, **kwargs):
self.filename = ""
self.branch = ""
if 'branch' in kwargs:
self.branch = kwargs['branch']
if 'filename' in kwargs:
self.filename = kwargs['filename']
if not 'command' in kwargs:
kwargs['command'] = ["unzip", "-o"]
ShellCommand.__init__(self, **kwargs)
def describe(self, done=False):
return ["Install zip"]
def start(self):
# removed the mkdir because this happens on the master, not the slave
if not self.filename:
if self.branch:
self.filename = self.getProperty("filename")
else:
return FAILURE
if self.filename:
self.command = self.command[:] + [self.filename]
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
if None != re.search('ERROR', cmd.logs['stdio'].getText()):
return FAILURE
if None != re.search('Usage:', cmd.logs['stdio'].getText()):
return FAILURE
return SUCCESS
class MozillaUpdateConfig(ShellCommand):
"""Configure YAML file for run_tests.py"""
def __init__(self, **kwargs):
self.addOptions = []
assert 'executablePath' in kwargs
assert 'branch' in kwargs
self.branch = kwargs['branch']
self.exePath = kwargs['executablePath']
if 'addOptions' in kwargs:
self.addOptions = kwargs['addOptions']
ShellCommand.__init__(self, **kwargs)
def setBuild(self, build):
ShellCommand.setBuild(self, build)
self.title = build.slavename
self.changes = build.source.changes
self.buildid = strftime("%Y%m%d%H%M", localtime(self.changes[-1].when))
if not self.command:
self.setCommand(["python", "PerfConfigurator.py", "-v", "-e", self.exePath, "-t", self.title, "-b", self.branch, "-d", self.buildid] + self.addOptions)
def describe(self, done=False):
return ["Update config"]
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
stdioText = cmd.logs['stdio'].getText()
if None != re.search('ERROR', stdioText):
return FAILURE
if None != re.search('USAGE:', stdioText):
return FAILURE
configFileMatch = re.search('outputName\s*=\s*(\w*?.yml)', stdioText)
if not configFileMatch:
return FAILURE
else:
self.setProperty("configFile", configFileMatch.group(1))
return SUCCESS
class MozillaRunPerfTests(ShellCommand):
"""Run the performance tests"""
def __init__(self, **kwargs):
if 'branch' in kwargs:
self.branch = kwargs['branch']
if not 'command' in kwargs:
kwargs['command'] = ["python", "run_tests.py"]
ShellCommand.__init__(self, **kwargs)
def describe(self, done=False):
return ["Run performance tests"]
def createSummary(self, log):
summary = []
for line in log.readlines():
if "RETURN:" in line:
summary.append(line.replace("RETURN:", "TinderboxPrint:"))
if "FAIL:" in line:
summary.append(line.replace("FAIL:", "TinderboxPrint:FAIL:"))
self.addCompleteLog('summary', "\n".join(summary))
def start(self):
"""docstring for start"""
self.command = copy.copy(self.command)
self.command.append(self.getProperty("configFile"))
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
stdioText = cmd.logs['stdio'].getText()
if SUCCESS != superResult:
return FAILURE
if None != re.search('ERROR', stdioText):
return FAILURE
if None != re.search('USAGE:', stdioText):
return FAILURE
if None != re.search('FAIL:', stdioText):
return WARNINGS
return SUCCESS
class MozillaInstallTarBz2(ShellCommand):
"""Install given file, unzipping to executablePath"""
def __init__(self, **kwargs):
self.filename = ""
self.branch = ""
if 'branch' in kwargs:
self.branch = kwargs['branch']
if 'filename' in kwargs:
self.filename = kwargs['filename']
if not 'command' in kwargs:
kwargs['command'] = ["tar", "-jvxf"]
ShellCommand.__init__(self, **kwargs)
def describe(self, done=False):
return ["Install tar.bz2"]
def start(self):
if not self.filename:
if self.branch:
self.filename = self.getProperty("filename")
else:
return FAILURE
if self.filename:
self.command = self.command[:] + [self.filename]
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
return SUCCESS
class MozillaInstallTarGz(ShellCommand):
"""Install given file, unzipping to executablePath"""
def __init__(self, **kwargs):
self.filename = ""
self.branch = ""
if 'branch' in kwargs:
self.branch = kwargs['branch']
if 'filename' in kwargs:
self.filename = kwargs['filename']
if not 'command' in kwargs:
kwargs['command'] = ["tar", "-zvxf"]
ShellCommand.__init__(self, **kwargs)
def describe(self, done=False):
return ["Install tar.gz"]
def start(self):
if not self.filename:
if self.branch:
self.filename = self.getProperty("filename")
else:
return FAILURE
if self.filename:
self.command = self.command[:] + [self.filename]
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
return SUCCESS
class MozillaWgetFromChange(ShellCommand):
"""Download built Firefox client from current change's filenames."""
haltOnFailure = True
def __init__(self, **kwargs):
self.branch = "HEAD"
self.fileURL = ""
self.filename = ""
self.filenameSearchString = "en-US.win32.zip"
if 'filenameSearchString' in kwargs:
self.filenameSearchString = kwargs['filenameSearchString']
if 'url' in kwargs:
self.url = kwargs['url']
else:
self.url = kwargs['build'].source.changes[-1].files[0]
if 'branch' in kwargs:
self.branch = kwargs['branch']
if not 'command' in kwargs:
kwargs['command'] = ["wget"]
ShellCommand.__init__(self, **kwargs)
def getFilename(self):
return self.filename
def describe(self, done=False):
return ["Wget Download"]
def start(self):
urlGetter = LatestFileURL(self.url, self.filenameSearchString)
self.filename = urlGetter.getLatestFilename()
self.fileURL = self.url + self.filename
if self.branch:
self.setProperty("fileURL", self.fileURL)
self.setProperty("filename", self.filename)
self.setCommand(["wget", "-nv", "-N", self.fileURL])
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
if None != re.search('ERROR', cmd.logs['stdio'].getText()):
return FAILURE
return SUCCESS
class MozillaUpdateConfigFromChange(ShellCommand):
"""Configure YAML file for run_tests.py"""
def __init__(self, **kwargs):
self.title = "default"
self.branch = ""
self.currentDate = ""
if 'build' in kwargs:
self.title = kwargs['build'].slavename
self.changes = kwargs['build'].source.changes
self.buildid = self.changes[-1].comments.split(',')[0]
if 'branch' in kwargs:
self.branch = kwargs['branch']
assert 'configPath' in kwargs
assert 'executablePath' in kwargs
self.configPath = kwargs['configPath']
self.exePath = kwargs['executablePath']
if not 'command' in kwargs:
kwargs['command'] = ["python", "PerfConfigurator.py", "-v", "-e",
self.exePath, "-c", self.configPath,
"-t", self.title, "-b", self.branch,
"-d", self.buildid]
ShellCommand.__init__(self, **kwargs)
def describe(self, done=False):
return ["Update config"]
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
stdioText = cmd.logs['stdio'].getText()
if None != re.search('ERROR', stdioText):
return FAILURE
if None != re.search('USAGE:', stdioText):
return FAILURE
configFileMatch = re.search('outputName\s*=\s*(\w*?.yml)', stdioText)
if not configFileMatch:
return FAILURE
else:
self.setProperty("configFile", configFileMatch.group(1))
return SUCCESS
class MozillaInstallDmg(ShellCommand):
"""Install given file, copying to workdir"""
def __init__(self, **kwargs):
self.filename = ""
self.branch = ""
if 'branch' in kwargs:
self.branch = kwargs['branch']
if 'filename' in kwargs:
self.filename = kwargs['filename']
if not 'command' in kwargs:
kwargs['command'] = ["bash", "installdmg.sh", "$FILENAME"]
ShellCommand.__init__(self, **kwargs)
def describe(self, done=False):
return ["Install dmg"]
def start(self):
if not self.filename:
if self.branch:
self.filename = self.getProperty("filename")
else:
return FAILURE
for i in range(len(self.command)):
if self.command[i] == "$FILENAME":
self.command[i] = self.filename
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
return SUCCESS
class MozillaInstallDmgEx(ShellCommand):
"""Install given file, copying to workdir"""
#This is a temporary class to test the new InstallDmg script without affecting the production mac machines
# if this works everything should be switched over the using it
def __init__(self, **kwargs):
self.filename = ""
self.branch = ""
if 'branch' in kwargs:
self.branch = kwargs['branch']
if 'filename' in kwargs:
self.filename = kwargs['filename']
if not 'command' in kwargs:
kwargs['command'] = ["expect", "installdmg.ex", "$FILENAME"]
ShellCommand.__init__(self, **kwargs)
def describe(self, done=False):
return ["Install dmg"]
def start(self):
if not self.filename:
if self.branch:
self.filename = self.getProperty("filename")
else:
return FAILURE
for i in range(len(self.command)):
if self.command[i] == "$FILENAME":
self.command[i] = self.filename
ShellCommand.start(self)
def evaluateCommand(self, cmd):
superResult = ShellCommand.evaluateCommand(self, cmd)
if SUCCESS != superResult:
return FAILURE
return SUCCESS
class TalosFactory(BuildFactory):
winClean = ["touch temp.zip &", "rm", "-rf", "*.zip", "talos/", "firefox/"]
macClean = "rm -vrf *"
linuxClean = "rm -vrf *"
def __init__(self, OS, envName, buildBranch, configOptions, buildPath, talosCmd, customManifest='', cvsRoot=":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot"):
BuildFactory.__init__(self)
if OS in ('linux', 'linuxbranch',):
cleanCmd = self.linuxClean
elif OS in ('win',):
cleanCmd = self.winClean
else:
cleanCmd = self.macClean
self.addStep(ShellCommand(
workdir=".",
description="Cleanup",
command=cleanCmd,
env=MozillaEnvironments[envName]))
#self.addStep(ShellCommand,
# command=["cvs", "-d", cvsRoot, "co", "-d", "talos",
# "mozilla/testing/performance/talos"],
# workdir=".",
# description="checking out talos",
# haltOnFailure=True,
# flunkOnFailure=True,
# env=MozillaEnvironments[envName])
self.addStep(FileDownload(
mastersrc="talos.zip",
slavedest="talos.zip",
workdir="."))
self.addStep(FileDownload,
mastersrc="scripts/count_and_reboot.py",
slavedest="count_and_reboot.py",
workdir=".")
self.addStep(MozillaInstallZip(
workdir=".",
branch=buildBranch,
haltOnFailure=True,
env=MozillaEnvironments[envName],
filename="talos.zip"))
self.addStep(FileDownload(
mastersrc="scripts/generate-tpcomponent.py",
slavedest="generate-tpcomponent.py",
workdir="talos/page_load_test"))
if customManifest != '':
self.addStep(FileDownload(
mastersrc=customManifest,
slavedest="manifest.txt",
workdir="talos/page_load_test"))
self.addStep(ShellCommand(
command=["python", "generate-tpcomponent.py"],
workdir="talos/page_load_test",
description="setting up pageloader",
haltOnFailure=True,
flunkOnFailure=True,
env=MozillaEnvironments[envName]))
self.addStep(MozillaWgetLatestDated(
workdir=".",
branch=buildBranch,
env=MozillaEnvironments[envName]))
#install the browser, differs based upon platform
if OS == 'linux':
self.addStep(MozillaInstallTarBz2(
workdir=".",
branch=buildBranch,
haltOnFailure=True,
env=MozillaEnvironments[envName]))
elif OS == 'linuxbranch': #special case for old linux builds
self.addStep(MozillaInstallTarGz(
workdir=".",
branch=buildBranch,
haltOnFailure=True,
env=MozillaEnvironments[envName]))
elif OS == 'win':
self.addStep(MozillaInstallZip(
workdir=".",
branch="1.9",
haltOnFailure=True,
env=MozillaEnvironments[envName]))
self.addStep(ShellCommand(
workdir="firefox/",
flunkOnFailure=False,
warnOnFailure=False,
description="chmod files (see msys bug)",
command=["chmod", "-v", "-R", "a+x", "."],
env=MozillaEnvironments[envName]))
elif OS == 'tiger':
self.addStep(FileDownload(
mastersrc="scripts/installdmg.sh",
slavedest="installdmg.sh",
workdir="."))
self.addStep(MozillaInstallDmg(
workdir=".",
branch=buildBranch,
haltOnFailure=True,
env=MozillaEnvironments[envName]))
else: #leopard
self.addStep(FileDownload(
mastersrc="scripts/installdmg.ex",
slavedest="installdmg.ex",
workdir="."))
self.addStep(MozillaInstallDmgEx(
workdir=".",
branch=buildBranch,
haltOnFailure=True,
env=MozillaEnvironments[envName]))
self.addStep(MozillaUpdateConfig(
workdir="talos/",
branch=buildBranch,
haltOnFailure=True,
executablePath=buildPath,
addOptions=configOptions,
env=MozillaEnvironments[envName]))
self.addStep(MozillaRunPerfTests(
warnOnWarnings=True,
workdir="talos/",
branch=buildBranch,
timeout=21600,
haltOnFailure=True,
command=talosCmd,
env=MozillaEnvironments[envName]))
self.addStep(ShellCommand(
flunkOnFailure=False,
warnOnFailure=False,
workdir='.',
description="reboot after 5 test runs",
command=["python", "count_and_reboot.py", "-f", "../talos_count.txt", "-n", "5", "-z"],
env=MozillaEnvironments[envName]))
def main(argv=None):
if argv is None:
argv = sys.argv
tester = LatestFileURL('http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/mozilla-central-linux/1222921630/ ', "en-US.linux-i686.tar.bz2")
tester.testrun()
return 0
if __name__ == '__main__':
main()

60
mozilla/tools/buildbot-configs/testing/talos/perf-staging/scripts/count_and_reboot.py Normal file
View File

@ -0,0 +1,60 @@
#!/usr/bin/env python
# encoding: utf-8
# Created by Chris AtLee on 2008-11-04
"""count_and_reboot.py [-n maxcount] -f countfile
Increments the value in countfile, and reboots the machine once the count
reaches or exceeds maxcount."""
import os, sys, time
if sys.platform in ('darwin', 'linux2'):
def reboot():
# -S means to accept password from stdin, which we then redirect from
# /dev/null
# This results in sudo not waiting forever for a password. If sudoers
# isn't set up properly, this will fail immediately
os.system("sudo -S reboot < /dev/null")
# After starting the shutdown, we go to sleep since the system can
# take a few minutes to shut everything down and reboot
time.sleep(600)
elif sys.platform == "win32":
# Windows
def reboot():
os.system("shutdown -r")
# After starting the shutdown, we go to sleep since the system can
# take a few minutes to shut everything down and reboot
time.sleep(600)
def increment_count(fname):
try:
current_count = int(open(fname).read())
except:
current_count = 0
current_count += 1
open(fname, "w").write("%i\n" % current_count)
return current_count
if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser(__doc__)
parser.add_option("-n", "--max-count", dest="maxcount", default=10,
help="reboot after <maxcount> runs", type="int")
parser.add_option("-f", "--count-file", dest="countfile", default=None,
help="file to record count in")
parser.add_option("-z", "--zero-count", dest="zero", default=False,
action="store_true", help="zero out the counter before rebooting")
options, args = parser.parse_args()
if not options.countfile:
parser.error("countfile is required")
if increment_count(options.countfile) >= options.maxcount:
if options.zero:
open(options.countfile, "w").write("0\n")
print "Reached max number of runs before reboot required, restarting machine..."
sys.stdout.flush()
reboot()

83
mozilla/tools/buildbot-configs/testing/talos/perf-staging/scripts/generate-tpcomponent.py Normal file
View File

@ -0,0 +1,83 @@
# Generates the proper chrome/ and component/ directories that can be dropped
# into a profile to enable the pageloader
# Original Author: Alice Nodelman (anodelman@mozilla.com)
# Modified by: Ben Hearsum (bhearsum@mozilla.com)
import os
import sys
import zipfile
import shutil
import tempfile
# create the temp directory
tmp_dir = tempfile.mkdtemp()
pageloader_dir = os.path.join(tmp_dir, "pageloader")
# where the chrome/ and component/ directories will be put
working_dir = "."
chrome_dir = os.path.join(working_dir, 'chrome')
components_dir = os.path.join(working_dir, 'components')
# where the pageloader will be checked out from
cvsroot = ":pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot"
module = "mozilla/layout/tools/pageloader"
def removedir(rdir):
if os.path.isdir(rdir):
for root, dirs, files in os.walk(rdir, topdown=False):
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
os.rmdir(rdir)
def zipdir(zip, zdir):
if os.path.isdir(zdir):
for root, dirs, files in os.walk(zdir, topdown=False):
for name in files:
zip.write(os.path.join(root, name), os.path.join(os.path.basename(zdir), name))
else:
zip.write(zdir)
# bail if the directories already exist
if os.path.exists(chrome_dir):
print "chrome/ directory exists, bailing out"
sys.exit(1)
if os.path.exists(components_dir):
print "components/ directory exists, bailing out"
sys.exit(1)
oldcwd = os.getcwd()
os.chdir(tmp_dir)
# exit if cvs throws an error
if os.system("cvs -d%s co -d pageloader %s" % (cvsroot, module)):
print "could not retrieve pageloader, bailing out"
sys.exit(1)
os.chdir(oldcwd)
#create the directory structure in the working_dir
os.mkdir(chrome_dir)
os.mkdir(os.path.join(chrome_dir, 'content'))
os.mkdir(components_dir)
#create the pageloader.manifest file
f = open(os.path.join(chrome_dir, 'pageloader.manifest'), 'w')
f.write('content pageloader jar:pageloader.jar!/content/\n')
f.close()
shutil.copy(os.path.join(pageloader_dir, 'pageloader.xul'), os.path.join(chrome_dir, 'content', 'pageloader.xul'))
shutil.copy(os.path.join(pageloader_dir, 'quit.js'), os.path.join(chrome_dir, 'content', 'quit.js'))
shutil.copy(os.path.join(pageloader_dir, 'pageloader.js'), os.path.join(chrome_dir, 'content', 'pageloader.js'))
shutil.copy(os.path.join(pageloader_dir, 'report.js'), os.path.join(chrome_dir, 'content', 'report.js'))
# create pageloader.jar
jar = zipfile.ZipFile(os.path.join(chrome_dir, 'pageloader.jar'), 'w')
zipdir(jar, os.path.join(chrome_dir, 'content'))
jar.close()
removedir(os.path.join(chrome_dir, 'content'))
shutil.copy(os.path.join(pageloader_dir, 'tp-cmdline.js'), os.path.join(components_dir, 'tp-cmdline.js'))
#get rid of the temporary directory
removedir(tmp_dir)

49
mozilla/tools/buildbot-configs/testing/talos/perf-staging/scripts/installdmg.ex Executable file
View File

@ -0,0 +1,49 @@
#!/usr/bin/expect
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is Mozilla Corporation Code.
#
# The Initial Developer of the Original Code is
# Bob Clary.
# Portions created by the Initial Developer are Copyright (C) 2007
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
# Bob Clary <bclary@bclary.com>
# Clint Talbert <ctalbert@mozilla.com>
# Alice Nodelman <anodelman@mozilla.com>
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****
system mkdir -p mnt
spawn hdiutil attach -verbose -noautoopen -mountpoint ./mnt $argv
expect {
"byte" {send "G"; exp_continue}
"END" {send "\r"; exp_continue}
"Y/N?" {send "Y\r"; exp_continue}
}
system rsync -a ./mnt/* .
system hdiutil detach mnt
system rm -rdf ./mnt

6
mozilla/tools/buildbot-configs/testing/talos/perf-staging/scripts/installdmg.sh Normal file
View File

@ -0,0 +1,6 @@
#!/bin/sh
mkdir -p mnt
echo "y" | PAGER="/bin/cat" hdiutil attach -verbose -noautoopen -mountpoint ./mnt $1
rsync -a ./mnt/* .
hdiutil detach mnt

13
mozilla/tools/buildbot-configs/testing/talos/perf-staging/tinderboxmailnotifier.py Normal file
View File

@ -0,0 +1,13 @@
from buildbot.status import tinderbox
class TinderboxMailNotifier(tinderbox.TinderboxMailNotifier):
"""
Subclass TinderboxMailNotifier to add the slave name to the build.
This makes tinderbox happy if we have more than one slave working
on one builder.
"""
def buildMessage(self, name, build, results):
return tinderbox.TinderboxMailNotifier.buildMessage(self,
name + ' ' +
build.getSlavename(),
build, results)