bug 377929: mozilla2 buildbot master setup - BuildSteps for uploading builds to stage & getting the build id from a build. r=rhelmer,cf, patch=me
git-svn-id: svn://10.0.0.236/trunk@245728 18797224-902f-48f8-a5cc-f745e15eee43
This commit is contained in:
bhearsum%mozilla.com
parent
b8b833a7fb
commit
3459ac9e27
0
mozilla/tools/buildbotcustom/steps/__init__.py
Normal file
0
mozilla/tools/buildbotcustom/steps/__init__.py
Normal file
30
mozilla/tools/buildbotcustom/steps/misc.py
Normal file
30
mozilla/tools/buildbotcustom/steps/misc.py
Normal file
@ -0,0 +1,30 @@
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
from buildbot.status.builder import FAILURE, SUCCESS
|
||||
|
||||
class GetBuildID(ShellCommand):
|
||||
"""Retrieves the BuildID from a Mozilla tree (using platform.ini) and sets
|
||||
it as a build property ('buildid'). If defined, uses objdir as it's base.
|
||||
"""
|
||||
description=['getting buildid']
|
||||
descriptionDone=['get buildid']
|
||||
haltOnFailure=True
|
||||
|
||||
def __init__(self, objdir="", **kwargs):
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
self.addFactoryArguments(objdir=objdir)
|
||||
|
||||
self.objdir = objdir
|
||||
self.command = ['python', 'config/printconfigsetting.py',
|
||||
'%s/dist/bin/application.ini' % self.objdir,
|
||||
'App', 'BuildID']
|
||||
|
||||
def commandComplete(self, cmd):
|
||||
buildid = ""
|
||||
try:
|
||||
buildid = cmd.logs['stdio'].getText().strip().rstrip()
|
||||
self.setProperty('buildid', buildid)
|
||||
except:
|
||||
log.msg("Could not find BuildID or BuildID invalid")
|
||||
log.msg("Found: %s" % buildid)
|
||||
return FAILURE
|
||||
return SUCCESS
|
||||
240
mozilla/tools/buildbotcustom/steps/transfer.py
Normal file
240
mozilla/tools/buildbotcustom/steps/transfer.py
Normal file
@ -0,0 +1,240 @@
|
||||
from os import path
|
||||
from time import strftime, strptime
|
||||
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
|
||||
class MozillaStageUpload(ShellCommand):
|
||||
def __init__(self, objdir, username, milestone, platform, remoteHost,
|
||||
remoteBasePath, group=None, chmodMode=755, sshKey=None,
|
||||
releaseToDated=True, releaseToLatest=True,
|
||||
releaseToTinderboxBuilds=True, **kwargs):
|
||||
"""
|
||||
@type objdir: string
|
||||
@param objdir: The obj directory used for the build. This is needed to
|
||||
find the packages in the source tree.
|
||||
|
||||
@type username: string
|
||||
@param username: The username used to login with on the remote host.
|
||||
The buildslaves should have passwordless logins to
|
||||
this account.
|
||||
|
||||
@type milestone: string
|
||||
@param milestone: The milestone of the build (eg, trunk)
|
||||
|
||||
@type platform: string
|
||||
@param platform: The platform we are uploading for. One of 'win32',
|
||||
'linux', or 'macosx'.
|
||||
|
||||
@type remoteHost: string
|
||||
@param remoteHost: The server to upload the builds to.
|
||||
|
||||
@type remoteBasePath: string
|
||||
@param remoteBasePath: The directory on the server used as a base path
|
||||
for these builds. eg:
|
||||
/home/ftp/pub/firefox
|
||||
|
||||
@type group: string
|
||||
@param group: If group is set, any files uploaded will be chgrp'ed to
|
||||
it. Default: None
|
||||
|
||||
@type chmodMode: int
|
||||
@param chmodMode: The mode used when fixing permissions on remoteHost.
|
||||
Default: 755
|
||||
|
||||
@type sshKey: string
|
||||
@param sshKey: If defined, the filename of the ssh key to use. It
|
||||
should be relative to ${HOME}/.ssh/. Default: None
|
||||
|
||||
@type releaseToDated: bool
|
||||
@param releaseToDated: If True, builds will be pushed to
|
||||
'remoteBasePath'/nightly/yyyy/mm/yyyy-mm-dd-hh-milestone. This
|
||||
directory will also be symlinked in 'remoteBasePath'. Generally,
|
||||
this should be True for nightlies. Default: True
|
||||
|
||||
@type releaseToLatest: bool
|
||||
@param releaseToLatest: If True, builds will be pushed to
|
||||
'remoteBasePath'/nightly/latest-milestone. If
|
||||
releaseToDated=True, builds will be copied from
|
||||
'remoteBasePath'/yyyy/mm/yyyy-mm-dd-hh-milestone. Otherwise,
|
||||
builds will be uploaded from the slave. Generally, this should
|
||||
be True for nightlies. Default: True
|
||||
|
||||
@type releaseToTinderboxBuilds: bool
|
||||
@param releaseToTinderboxBuilds: If True, builds will be pushed to
|
||||
'remoteBasePath'/tinderbox-builds/$hostname. This should
|
||||
generally be set to True for all builds. Default: True
|
||||
"""
|
||||
|
||||
ShellCommand.__init__(self, **kwargs)
|
||||
self.addFactoryArguments(objdir=objdir,
|
||||
username=username,
|
||||
milestone=milestone,
|
||||
platform=platform,
|
||||
remoteHost=remoteHost,
|
||||
remoteBasePath=remoteBasePath,
|
||||
group=group,
|
||||
chmodMode=chmodMode,
|
||||
sshKey=sshKey,
|
||||
releaseToDated=releaseToDated,
|
||||
releaseToLatest=releaseToLatest,
|
||||
releaseToTinderboxBuilds=releaseToTinderboxBuilds)
|
||||
|
||||
assert platform in ('win32', 'linux', 'macosx')
|
||||
self.objdir = objdir
|
||||
self.username = username
|
||||
self.milestone = milestone
|
||||
self.platform = platform
|
||||
self.remoteHost = remoteHost
|
||||
self.remoteBasePath = remoteBasePath
|
||||
self.group = group
|
||||
self.chmodMode = chmodMode
|
||||
self.sshKey = sshKey
|
||||
self.releaseToDated = releaseToDated
|
||||
self.releaseToLatest = releaseToLatest
|
||||
self.releaseToTinderboxBuilds = releaseToTinderboxBuilds
|
||||
|
||||
self.description = ["uploading package(s) to", remoteHost]
|
||||
self.descriptionDone = ["upload package(s) to", remoteHost]
|
||||
|
||||
def _getBaseCommand(self, ssh=False, scp=False):
|
||||
assert not (ssh and scp)
|
||||
assert (ssh or scp)
|
||||
|
||||
command = ""
|
||||
# scp cannot use the '-l' format
|
||||
if ssh:
|
||||
command += 'ssh'
|
||||
command += ' -l ' + self.username
|
||||
else:
|
||||
command += 'scp'
|
||||
|
||||
if self.sshKey:
|
||||
# surprisingly, this works on Windows (probably because Buildbot)
|
||||
# gets started from MSYS
|
||||
command += ' -i ' + '~/.ssh/%s' % self.sshKey
|
||||
return command
|
||||
|
||||
def getBuildID(self):
|
||||
# the build id is extracted in a previous step and set as a build
|
||||
# property
|
||||
buildid = self.getProperty("buildid")
|
||||
return strftime("%Y-%m-%d-%H", strptime(buildid, "%Y%m%d%H"))
|
||||
|
||||
def getPackageDirectory(self):
|
||||
return '%s-%s' % (self.getBuildID(), self.milestone)
|
||||
|
||||
def getPackageGlob(self):
|
||||
# i can't find a better way to do this.
|
||||
if self.platform == "win32":
|
||||
return '%s/dist/*.zip %s/dist/install/sea/*.exe' % (self.objdir,
|
||||
self.objdir)
|
||||
if self.platform == "macosx":
|
||||
return '%s/dist/*.dmg' % self.objdir
|
||||
if self.platform == "linux":
|
||||
return '%s/dist/*.tar.bz2' % self.objdir
|
||||
|
||||
def getLongDatedPath(self):
|
||||
buildid = self.getBuildID()
|
||||
fullRemotePath = path.join(self.remoteBasePath, 'nightly',
|
||||
buildid.split('-')[0], # the year
|
||||
buildid.split('-')[1], # the month
|
||||
self.getPackageDirectory()
|
||||
)
|
||||
return fullRemotePath
|
||||
|
||||
def getLatestPath(self):
|
||||
return path.join(self.remoteBasePath, 'nightly',
|
||||
'latest-%s' % self.milestone)
|
||||
|
||||
def getTinderboxBuildsPath(self):
|
||||
return path.join(self.remoteBasePath, 'tinderbox-builds',
|
||||
self.step_status.build.builder.getName())
|
||||
|
||||
def createDirCommand(self, dir):
|
||||
return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
|
||||
' mkdir -p ' + dir
|
||||
|
||||
def uploadCommand(self, dir):
|
||||
return self._getBaseCommand(scp=True) + ' ' + self.getPackageGlob() + \
|
||||
' ' + self.username + '@' + self.remoteHost + ':' + \
|
||||
dir
|
||||
|
||||
def chmodCommand(self, dir):
|
||||
return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
|
||||
' chmod -R ' + str(self.chmodMode) + ' ' + dir
|
||||
|
||||
def chgrpCommand(self, dir):
|
||||
return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
|
||||
' chgrp -R ' + self.group + ' ' + dir
|
||||
|
||||
def syncCommand(self, src, dst):
|
||||
# rsync needs trailing slashes
|
||||
src += '/'
|
||||
dst += '/'
|
||||
return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
|
||||
' rsync -av ' + src + ' ' + dst
|
||||
|
||||
def symlinkDateDirCommand(self, datedDir):
|
||||
# Tinderbox client tests for existence rather than forcibly re-linking.
|
||||
# Because ShellCommand's get passed through cmd.exe, which has
|
||||
# weird and awful quoting rules we cannot do anything that requires
|
||||
# quotes, ie. no 'if [ ! -h ... ]'
|
||||
return self._getBaseCommand(ssh=True) + ' ' + self.remoteHost + \
|
||||
' ln -fs ' + datedDir + ' ' + path.join(self.remoteBasePath,
|
||||
'nightly')
|
||||
|
||||
def start(self):
|
||||
datedDir = self.getLongDatedPath()
|
||||
latestDir = self.getLatestPath()
|
||||
tinderboxBuildsDir = self.getTinderboxBuildsPath()
|
||||
|
||||
commands = []
|
||||
if self.releaseToDated:
|
||||
# 1) Create the directory on the staging server.
|
||||
# 2) Upload the package(s).
|
||||
# 3) Fix the permissions on the package(s).
|
||||
# 4) Maybe adjust the group on the package(s).
|
||||
# 5) Symlink the longer dated directory to the shorter one.
|
||||
cmd = ""
|
||||
cmd += self.createDirCommand(datedDir) + " && " + \
|
||||
self.uploadCommand(datedDir) + " && " + \
|
||||
self.chmodCommand(datedDir)
|
||||
if self.group:
|
||||
cmd += " && " + self.chgrpCommand(datedDir)
|
||||
cmd += " && " + self.symlinkDateDirCommand(datedDir)
|
||||
commands.append(cmd)
|
||||
|
||||
if self.releaseToLatest:
|
||||
# 1) Create the directory on the staging server.
|
||||
# 2) If there was a dated release, rsync those files to the
|
||||
# latest-(milestone) directory.
|
||||
# 3) If not, upload the package(s).
|
||||
# 4) Fix the permissions on the package(s).
|
||||
# 5) Maybe adjust the group on the package(s).
|
||||
cmd = ""
|
||||
cmd += self.createDirCommand(latestDir) + " && "
|
||||
if self.releaseToDated:
|
||||
cmd += self.syncCommand(datedDir, latestDir) + " && "
|
||||
else:
|
||||
cmd += self.uploadCommand(latestDir) + " && "
|
||||
cmd += self.chmodCommand(latestDir)
|
||||
if self.group:
|
||||
cmd += " && " + self.chgrpCommand(latestDir)
|
||||
commands.append(cmd)
|
||||
|
||||
if self.releaseToTinderboxBuilds:
|
||||
# 1) Create the directory on the staging server.
|
||||
# 2) Upload the package(s).
|
||||
# 3) Fix the permissions on the package(s).
|
||||
# 4) Maybe adjust the group on the package(s).
|
||||
cmd = ""
|
||||
cmd += self.createDirCommand(tinderboxBuildsDir) + " && " + \
|
||||
self.uploadCommand(tinderboxBuildsDir) + " && " + \
|
||||
self.chmodCommand(tinderboxBuildsDir)
|
||||
if self.group:
|
||||
cmd += " && " + self.chgrpCommand(tinderboxBuildsDir)
|
||||
commands.append(cmd)
|
||||
|
||||
finalCommand = ' && '.join(commands)
|
||||
self.setCommand(finalCommand)
|
||||
ShellCommand.start(self)
|
||||
Loading…
x
Reference in New Issue
Block a user