diff --git a/mozilla/tools/buildbotcustom/l10n/l10n.py b/mozilla/tools/buildbotcustom/l10n/l10n.py index 95ac9f0c8f2..a1b1e40126f 100644 --- a/mozilla/tools/buildbotcustom/l10n/l10n.py +++ b/mozilla/tools/buildbotcustom/l10n/l10n.py @@ -41,4 +41,4 @@ class BuildL10n(process.base.Build): raise Exception("No build found for %s on %s, bad mojo" % \ (self.builder.name, self.slavename)) process.base.Build.setupBuild(self, expectations) - self.setProperty('locale', bd.locale) + self.setProperty('locale', bd.locale, "Build") diff --git a/mozilla/tools/buildbotcustom/l10n/scheduler.py b/mozilla/tools/buildbotcustom/l10n/scheduler.py index 3568a983589..af20baf539d 100644 --- a/mozilla/tools/buildbotcustom/l10n/scheduler.py +++ b/mozilla/tools/buildbotcustom/l10n/scheduler.py @@ -24,6 +24,7 @@ # ***** END LICENSE BLOCK ***** from twisted.python import log +from twisted.web.client import getPage from twisted.internet import defer, reactor from buildbot.steps.shell import ShellCommand from buildbot.scheduler import Nightly, Periodic, Dependent @@ -37,9 +38,17 @@ class L10nMixin(object): to generate build objects as specified per list of locales """ - def __init__(self, scheduler, locales, - cvsRoot=':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot'): + def __init__(self, scheduler, locales, + localesURL = None, + repoType = 'cvs', + cvsRoot = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot'): self.scheduler = scheduler + self.repoType = repoType + # set a default localesURL accordingly to the repoType if none has been set + if not localesURL and repoType.find('hg') >= 0: + self.localesURL = "http://hg.mozilla.org/mozilla-central/raw-file/tip/browser/locales/all-locales" + elif not localesURL and repoType.find('cvs') >= 0: + self.localesURL = "mozilla/browser/locales/all-locales" self.CVSROOT = cvsRoot # we are going to have a queue per builder attached to the scheduler self.queue = {} @@ -87,12 +96,12 @@ class L10nMixin(object): for key in self.scheduler.builderNames: self.queue[key].insert(0, obj) # let's submit the BuildSet for this locale - self.scheduler.submit( + self.scheduler.submitBuildSet( buildset.BuildSet(self.scheduler.builderNames, self.NoMergeStamp(branch=self.scheduler.branch), self.scheduler.reason)) - def getLocales(self, path = 'mozilla/browser/locales/all-locales'): + def getLocales(self): """ It returns a list of locales if the user has set a list of locales in the scheduler OR it returns a defered @@ -101,11 +110,19 @@ class L10nMixin(object): log.msg('L10nMixin.getLocales():: The user has set a list of locales') return self.locales else: - args = ['cvs', '-q', '-d', self.CVSROOT, 'co', '-p', path] - # communicate() returns a tuple - stdio, stderr - # the output of cvs has a '\n' element at the end - # a last '' string is generated that we get rid of - return subprocess.Popen( args, stdout=subprocess.PIPE).communicate()[0].split('\n')[0:-1] + if self.repoType.find('cvs') >= 0: + args = ['cvs', '-q', '-d', self.CVSROOT, 'co', '-p', self.localesURL] + # communicate() returns a tuple - stdio, stderr + # the output of cvs has a '\n' element at the end + # a last '' string is generated that we get rid of + return subprocess.Popen( args, stdout=subprocess.PIPE).communicate()[0].split('\n')[0:-1] + else: #the repoType is 'hg' + #getPage returns a defered that will return a string + d = getPage(self.localesURL, timeout = 5 * 60) + def _getLocalesList(data): + return filter(None, data.split()) + d.addCallback(_getLocalesList) + return d def createL10nBuilds(self): """ @@ -139,10 +156,16 @@ class NightlyL10n(Nightly): 'dayOfWeek', 'branch') def __init__(self, name, builderNames, minute=0, hour='*', dayOfMonth='*', month='*', dayOfWeek='*', + repoType=None, localesURL=None, branch=None, cvsRoot=None, locales=None): Nightly.__init__(self, name, builderNames, minute, hour, dayOfMonth, month, dayOfWeek, branch) - self.helper = L10nMixin(self, locales, cvsRoot) + # To avoid breakage in 1.9.0 that uses this scheduler + # TODO: change repoType=None to repoType when fixed + if not repoType: + repoType = 'cvs' + self.helper = L10nMixin(self, locales, cvsRoot, repoType, + localesURL) def doPeriodicBuild(self): #Schedule the next run (as in Nightly's doPeriodicBuild) @@ -162,9 +185,14 @@ class DependentL10n(Dependent): compare_attrs = ('name', 'upstream', 'builders') def __init__(self, name, upstream, builderNames, + repoType, localesURL=None, cvsRoot=None, locales=None): - Dependent.__init__(name, upstream, builderNames) - self.helper = L10nMixin(self, locales, cvsRoot) + Dependent.__init__(self, name, upstream, builderNames) + # The next two lines has been added because of: + # _cbLoadedLocales's BuildSet submit needs them + self.branch = None + self.reason = None + self.helper = L10nMixin(self, locales, cvsRoot, repoType, localesURL) # ss is the source stamp that we don't use currently def upstreamBuilt(self, ss): diff --git a/mozilla/tools/buildbotcustom/steps/test.py b/mozilla/tools/buildbotcustom/steps/test.py index 21239863819..0934dee7984 100644 --- a/mozilla/tools/buildbotcustom/steps/test.py +++ b/mozilla/tools/buildbotcustom/steps/test.py @@ -120,6 +120,10 @@ class CompareBloatLogs(ShellCommand): bloatDiffPath = 'tools/rb/bloatdiff.pl' if 'bloatDiffPath' in kwargs: bloatDiffPath = kwargs['bloatDiffPath'] + testnameprefix = "" + if 'testnameprefix' in kwargs: + testnameprefix = kwargs['testnameprefix'] + " " + self.testnameprefix=testnameprefix self.name = "compare " + testname + "bloat logs" self.description = "compare " + testname + "bloat logs" self.descriptionDone = "compare " + testname + "bloat logs complete" @@ -158,9 +162,9 @@ class CompareBloatLogs(ShellCommand): summary += "leaks = %d\n" % leaks summary += "bloat = %d\n" % bloat - leaksAbbr = "RLk"; - leaksTestname = "refcnt_leaks" - leaksTestnameLabel = "refcnt Leaks" + leaksAbbr = "%sRLk" % self.testnameprefix + leaksTestname = ("%srefcnt_leaks" %self.testnameprefix).replace(' ', '_') + leaksTestnameLabel = "%srefcnt Leaks" % self.testnameprefix tinderLink = tinderboxPrint(leaksTestname, leaksTestnameLabel, @@ -192,6 +196,8 @@ class CompareLeakLogs(ShellCommand): platform = kwargs['platform'] assert platform.startswith('win32') or platform.startswith('macosx') \ or platform.startswith('linux') + assert 'objdir' in kwargs + self.objdir = kwargs['objdir'] if 'leakFailureThreshold' in kwargs: self.leakFailureThreshold = kwargs['leakFailureThreshold'] if not 'mallocLog' in kwargs: @@ -202,14 +208,18 @@ class CompareLeakLogs(ShellCommand): else: testname = "" self.testname = testname + if 'testnameprefix' in kwargs: + self.testnameprefix = kwargs['testnameprefix'] + " " + else: + self.testnameprefix = "" self.name = "compare " + testname + "leak logs" self.description = "compare " + testname + "leak logs" self.descriptionDone = "compare " + testname + "leak logs complete" if platform.startswith("win32"): - kwargs['command'] = ['obj-firefox\\dist\\bin\\leakstats.exe', + kwargs['command'] = ['%s\\dist\\bin\\leakstats.exe' % self.objdir, kwargs['mallocLog']] else: - kwargs['command'] = ['obj-firefox/dist/bin/leakstats', + kwargs['command'] = ['%s/dist/bin/leakstats' % self.objdir, kwargs['mallocLog']] ShellCommand.__init__(self, **kwargs) @@ -226,6 +236,13 @@ class CompareLeakLogs(ShellCommand): leakStats['new'] = {} summary = self.testname + " trace-malloc bloat test: leakstats\n" + lkAbbr = "%sLk" % self.testnameprefix + lkTestname = ("%strace_malloc_leaks" % prefix).replace(' ','_') + mhAbbr = "%sMH" % self.testnameprefix + mhTestname = ("%strace_malloc_maxheap" % prefix).replace(' ','_') + aAbbr = "%sA" % self.testnameprefix + aTestName = ("%strace_malloc_allocs" % prefix).replace(' ','_') + resultSet = 'new' for line in log.readlines(): summary += line