Bommels05/Mozilla
Bommels05/Mozilla
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 607389 - generate partial updates at build time for releases. p=rail,r=nthomas

Browse Source 
git-svn-id: svn://10.0.0.236/trunk@263279 18797224-902f-48f8-a5cc-f745e15eee43
This commit is contained in:
cltbld 2012-01-18 12:25:38 +00:00
parent d4b824f06a
commit f79bfed788
3 changed files with 142 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
mozilla/tools/patcher/MozAUSConfig.pm
View File

@ -62,7 +62,7 @@ use strict;
use vars qw( @RUN_MODES
$DEFAULT_APP $DEFAULT_MODE $DEFAULT_CONFIG_FILE
$DEFAULT_DOWNLOAD_DIR $DEFAULT_DELIVERABLE_DIR
$DEFAULT_MAR_NAME
$DEFAULT_MAR_NAME $DEFAULT_CHECKSUMS_NAME
);
@RUN_MODES = qw( build-tools
@ -75,6 +75,7 @@ $DEFAULT_APP = 'MyApp';
$DEFAULT_DOWNLOAD_DIR = 'downloads';
$DEFAULT_DELIVERABLE_DIR = 'temp';
$DEFAULT_MAR_NAME = '%app%-%version%.%locale%.%platform%.complete.mar';
$DEFAULT_CHECKSUMS_NAME = '%app%-%version%.%locale%.%platform%.checksums';
sub new
{
@ -114,7 +115,7 @@ sub ProcessCommandLineArgs
Getopt::Long::GetOptions(\%args,
'help|h|?', 'man', 'version', 'app=s', 'brand=s', 'config=s', 'verbose',
'dry-run', 'tools-dir=s', 'download-dir=s', 'deliverable-dir=s',
'tools-revision=s', 'partial-patchlist-file=s', @RUN_MODES)
'tools-revision=s', 'partial-patchlist-file=s', 'use-checksums', @RUN_MODES)
or return 0;
$this->{'mConfigFilename'} = defined($args{'config'}) ? $args{'config'} :
@ -133,6 +134,7 @@ sub ProcessCommandLineArgs
# Is this a dry run, and we'll just print what we *would* do?
$this->{'dryRun'} = defined($args{'dryRun'}) ? 1 : 0;
$this->{'useChecksums'} = defined($args{'use-checksums'}) ? 1 : 0;
## Expects to be the dir that $mToolsDir/mozilla/[all the tools] will be in.
$this->{'mToolsDir'} = defined($args{'mToolsDir'}) ? $args{'mToolsDir'} : getcwd();
@ -740,7 +742,8 @@ sub RemoveBrokenUpdates
my $partial_pathname = "$u/ftp/$gen_partial_path";
# Go to next iteration if this partial patch already exists.
next if -e $partial_pathname;
next if -e $partial_pathname ||
exists $MozAUSLib::SNIPPET_CHECKSUM_HASH_CACHE->{$partial_pathname};
$i++;
if ( ! -f $from_path or
@ -802,6 +805,12 @@ sub IsDryRun
return 1 == $this->{'dryRun'};
}
sub useChecksums
{
my $this = shift;
return 1 == $this->{'useChecksums'};
}
sub GetToolsRevision
{
my $this = shift;

89
mozilla/tools/patcher/MozAUSLib.pm
View File

@ -44,12 +44,15 @@ package MozAUSLib;
use Cwd;
use File::Path;
use File::Basename;
use File::Copy qw(move copy);
use English;
use File::Spec::Functions;
use Data::Dumper;
use MozBuild::Util qw(RunShellCommand MkdirWithPath HashFile);
use MozAUSConfig;
require Exporter;
@ -62,6 +65,7 @@ require Exporter;
GetSnippetDirFromChannel
CachedHashFile
$OBJDIR
PrepopulateHashCache
);
use strict;
@ -79,6 +83,7 @@ use vars qw($OBJDIR $MAR_BIN $MBSDIFF_BIN $MAKE_BIN
$DEFAULT_SNIPPET_BASE_DIR $DEFAULT_SNIPPET_TEST_DIR
$SNIPPET_CHECKSUM_HASH_CACHE);
$Data::Dumper::Indent = 1;
$OBJDIR = 'obj';
$MAR_BIN = "$OBJDIR/dist/host/bin/mar";
$MBSDIFF_BIN = "$OBJDIR/dist/host/bin/mbsdiff";
@ -126,6 +131,84 @@ $DEFAULT_SNIPPET_TEST_DIR = $DEFAULT_SNIPPET_BASE_DIR . '.test';
$SNIPPET_CHECKSUM_HASH_CACHE = {};
sub PrepopulateHashCache {
my %args = @_;
my $config = $args{'config'};
my $startdir = getcwd();
my $deliverableDir = EnsureDeliverablesDir(config => $config);
chdir($deliverableDir);
my $from = $config->GetCurrentUpdate()->{'from'};
my $to = $config->GetCurrentUpdate()->{'to'};
my $rl_config = $config->{'mAppConfig'}->{'release'}->{$to};
my $rlp_config = $rl_config->{'platforms'};
my @platforms = sort(keys(%{$rlp_config}));
for my $p (@platforms) {
my $platform_locales = $rlp_config->{$p}->{'locales'};
for my $l (@$platform_locales) {
chdir(catfile($deliverableDir, $to, 'ftp'));
my $checksums_file = SubstitutePath(
path => $MozAUSConfig::DEFAULT_CHECKSUMS_NAME,
platform => $p,
locale => $l,
version => $to,
app => lc($config->GetApp()));
my $partial_mar_path = SubstitutePath(
path => $config->GetCurrentUpdate()->{'partial'}->{'path'},
platform => $p,
locale => $l);
my $complete_mar_path = SubstitutePath(
path => $config->GetCurrentUpdate()->{'complete'}->{'path'},
platform => $p,
locale => $l);
my $complete_mar_local_path = SubstitutePath(
path => $MozAUSConfig::DEFAULT_MAR_NAME,
platform => $p,
locale => $l,
version => $to,
app => lc($config->GetApp()));
open F, "$checksums_file" || die("can't read $checksums_file\n");
while (my $line = <F>){
chomp($line);
my ($hash, $hash_type, $size, $fname) = split /\s/, $line;
$hash_type = uc($hash_type);
my $file = undef;
if ($fname =~ m/partial.mar$/ &&
basename($fname) eq basename($partial_mar_path)){
$file = $partial_mar_path;
} elsif ($fname =~ m/complete.mar$/ &&
basename($fname) eq basename($complete_mar_path)){
$file = $complete_mar_path;
}
if (defined($file)){
my @files = (catfile("$from-$to", "ftp", $file));
# work around snippet creation functions using
# different places for complete mars
if ($file =~ m/complete.mar$/){
push(@files, catfile($to, "ftp", $complete_mar_local_path));
}
for my $f (@files){
if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f})) {
$SNIPPET_CHECKSUM_HASH_CACHE->{$f} = {};
}
if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{$hash_type})) {
$SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{$hash_type} = $hash;
}
if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{'size'})) {
$SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{'size'} = $size;
}
}
}
}
close F;
}
}
chdir($startdir);
# print Data::Dumper::Dumper($SNIPPET_CHECKSUM_HASH_CACHE);
}
sub CachedHashFile {
my %args = @_;
@ -142,9 +225,13 @@ sub CachedHashFile {
}
if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType})) {
if ($checksumType eq 'size'){
$SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{'size'} = (stat($file))[7];
} else {
$SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType} =
HashFile(file => $file, type => $checksumType);
}
}
return $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType};
@ -250,7 +337,7 @@ sub CreatePartialMarFile
}
if ( not -r $fromCompleteMar) {
print STDERR "CreatePartialMarFile: $fromCompleteMardoesn't exist!";
print STDERR "CreatePartialMarFile: $fromCompleteMar doesn't exist!";
return -1;
}

59
mozilla/tools/patcher/patcher2.pl
View File

@ -61,7 +61,8 @@ use MozAUSLib qw(CreatePartialMarFile
ValidateToolsDirectory SubstitutePath
GetSnippetDirFromChannel
CachedHashFile
$OBJDIR);
$OBJDIR
PrepopulateHashCache);
use MozBuild::Util qw(MkdirWithPath RunShellCommand DownloadFile);
@ -80,10 +81,9 @@ use vars qw($PID_FILE
$DEFAULT_HGROOT
$DEFAULT_SCHEMA_VERSION
@SCHEMA_2_OPTIONAL_ATTR
@COMPUTED_URLS
$ST_SIZE );
@COMPUTED_URLS);
$PID_FILE = 'patcher2.pid';
$PID_FILE = catfile(getcwd(), 'patcher2.pid');
$DEFAULT_HASH_TYPE = 'SHA512';
$DEFAULT_CVSROOT = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot';
$DEFAULT_HGROOT = 'http://hg.mozilla.org/mozilla-central';
@ -105,7 +105,6 @@ $DEFAULT_SCHEMA_VERSION = 2;
showSurvey
actions);
$ST_SIZE = 7;
sub main {
Startup();
@ -116,7 +115,7 @@ sub main {
PrintUsage(exitCode => 1) if ($config eq undef);
if (not ($config->RequestedStep('build-tools') or
if (not $config->useChecksums() and not ($config->RequestedStep('build-tools') or
$config->RequestedStep('build-tools-hg')) and
not ValidateToolsDirectory(toolsDir => $config->GetToolsDir())) {
my $badDir = $config->GetToolsDir();
@ -130,19 +129,23 @@ __END_TOOLS_ERROR__
my $startdir = getcwd();
my $deliverableDir = EnsureDeliverablesDir(config => $config);
#printf("PRE-REMOVE-BROKEN-UPDATES:\n\n%s", Data::Dumper::Dumper($config));
$config->RemoveBrokenUpdates();
#printf("POST-REMOVE-BROKEN:\n\n%s", Data::Dumper::Dumper($config));
BuildTools(config => $config, fromHg => 0) if $config->RequestedStep('build-tools');
BuildTools(config => $config, fromHg => 1) if $config->RequestedStep('build-tools-hg');
run_download_complete_patches(config => $config) if $config->RequestedStep('download');
if ($config->useChecksums()) {
PrepopulateHashCache(config => $config);
}
#printf("PRE-REMOVE-BROKEN-UPDATES:\n\n%s", Data::Dumper::Dumper($config));
$config->RemoveBrokenUpdates();
#printf("POST-REMOVE-BROKEN:\n\n%s", Data::Dumper::Dumper($config));
if ($config->RequestedStep('create-patches')) {
if (!$config->useChecksums()) {
CreatePartialPatches(config => $config);
CreateCompletePatches(config => $config);
}
}
if ($config->RequestedStep('(create-patches|create-patchinfo)')) {
CreatePartialPatchinfo(config => $config);
@ -185,6 +188,7 @@ Options:
Only applicable when in "create-patches" mode
--tools-revision=TAG Specify tag to use for build-tools checkout
Only applicable when in "build-tools" mode
--use-checksums Use checksums files instead of MAR files.
__END_USAGE__
exit($exitCode) if (defined($exitCode));
@ -329,6 +333,9 @@ sub download_complete_patches {
my $r_config = $config->{'mAppConfig'}->{'release'};
my @releases = ($fromReleaseVersion, $toReleaseVersion);
if ($config->useChecksums()){
@releases = ($toReleaseVersion);
}
for my $r (@releases) {
my $rl_config = $r_config->{$r};
@ -345,13 +352,20 @@ sub download_complete_patches {
chdir($relPath);
my $download_url = $rl_config->{'completemarurl'};
if ($config->useChecksums()){
$download_url = $rl_config->{'checksumsurl'};
}
$download_url = SubstitutePath(path => $download_url,
platform => $p,
version => $r,
locale => $l);
my $output_filename = $MozAUSConfig::DEFAULT_MAR_NAME;
if ($config->useChecksums()){
$output_filename = $MozAUSConfig::DEFAULT_CHECKSUMS_NAME;
}
my $output_filename = SubstitutePath(
path => $MozAUSConfig::DEFAULT_MAR_NAME,
path => $output_filename,
platform => $p,
locale => $l,
version => $r,
@ -392,7 +406,6 @@ sub download_complete_patches {
DownloadFile(url => $download_url,
dest => $output_filename );
}
chdir(catfile($deliverableDir, $r, 'ftp'));
my $end_time = time();
@ -802,8 +815,8 @@ sub CreateCompletePatchinfo {
$complete_patch->{'info_path'} = catfile($aus_prefix,
'complete.txt');
# Go to next iteration if this partial patch already exists.
next if ( -e $complete_patch->{'info_path'} or ! -e $complete_pathname );
next if ( -e $complete_patch->{'info_path'} or
(! $config->useChecksums() && ! -e $complete_pathname ) );
$i++;
#printf("partial = %s", Data::Dumper::Dumper($partial_patch));
@ -833,7 +846,8 @@ sub CreateCompletePatchinfo {
$complete_patch->{'build_id'} = $to->{'build_id'};
$complete_patch->{'appv'} = $prettySnippetToAppVersion;
$complete_patch->{'extv'} = $to->{'extv'};
$complete_patch->{'size'} = (stat($to_path))[$ST_SIZE];
$complete_patch->{'size'} = CachedHashFile(
file => $to_path, type => 'size');
my $channelSpecificUrlKey = $c . '-url';
@ -1102,7 +1116,8 @@ sub CreatePastReleasePatchinfo {
$completePatch->{'build_id'} = $patchLocaleNode->{'build_id'};
$completePatch->{'appv'} = $prettySnippetToAppVersion;
$completePatch->{'extv'} = $patchLocaleNode->{'extv'};
$completePatch->{'size'} = (stat($to_path))[$ST_SIZE];
$completePatch->{'size'} = CachedHashFile(
file => $to_path, type => 'size');
my $channelSpecificUrlKey = $channel . '-url';
@ -1246,7 +1261,8 @@ sub CreatePartialPatchinfo {
my $partialPatchHash = CachedHashFile(file => $partial_pathname,
type => $DEFAULT_HASH_TYPE);
my $partialPatchSize = (stat($partial_pathname))[$ST_SIZE];
my $partialPatchSize = CachedHashFile(
file => $partial_pathname, type => 'size');
my $gen_complete_path = SubstitutePath(path => $complete_path,
platform => $p,
@ -1264,7 +1280,8 @@ sub CreatePartialPatchinfo {
file => $complete_pathname,
type => $DEFAULT_HASH_TYPE);
my $completePatchSize = (stat($complete_pathname))[$ST_SIZE];
my $completePatchSize = CachedHashFile(
file => $complete_pathname, type => 'size');
my $computed_urls = {};
foreach my $attr (@COMPUTED_URLS) {
@ -1322,8 +1339,8 @@ sub CreatePartialPatchinfo {
$partial_patch->{'info_path'} = catfile($aus_prefix,
'partial.txt');
# Go to next iteration if this partial patch already exists.
next if ( -e $partial_patch->{'info_path'} or ! -e $snippetPathname );
next if ( -e $partial_patch->{'info_path'} or
(! $config->useChecksums() && ! -e $snippetPathname) );
$i++;
# This is just prettyfication for the PrintProgress() call,