From f79bfed788e30eebced39ed9f29dca1bc866ffb9 Mon Sep 17 00:00:00 2001 From: cltbld Date: Wed, 18 Jan 2012 12:25:38 +0000 Subject: [PATCH] Bug 607389 - generate partial updates at build time for releases. p=rail,r=nthomas git-svn-id: svn://10.0.0.236/trunk@263279 18797224-902f-48f8-a5cc-f745e15eee43 --- mozilla/tools/patcher/MozAUSConfig.pm | 15 ++++- mozilla/tools/patcher/MozAUSLib.pm | 93 ++++++++++++++++++++++++++- mozilla/tools/patcher/patcher2.pl | 63 +++++++++++------- 3 files changed, 142 insertions(+), 29 deletions(-) diff --git a/mozilla/tools/patcher/MozAUSConfig.pm b/mozilla/tools/patcher/MozAUSConfig.pm index a5b96e33a60..e20a7da1587 100644 --- a/mozilla/tools/patcher/MozAUSConfig.pm +++ b/mozilla/tools/patcher/MozAUSConfig.pm @@ -62,7 +62,7 @@ use strict; use vars qw( @RUN_MODES $DEFAULT_APP $DEFAULT_MODE $DEFAULT_CONFIG_FILE $DEFAULT_DOWNLOAD_DIR $DEFAULT_DELIVERABLE_DIR - $DEFAULT_MAR_NAME + $DEFAULT_MAR_NAME $DEFAULT_CHECKSUMS_NAME ); @RUN_MODES = qw( build-tools @@ -75,6 +75,7 @@ $DEFAULT_APP = 'MyApp'; $DEFAULT_DOWNLOAD_DIR = 'downloads'; $DEFAULT_DELIVERABLE_DIR = 'temp'; $DEFAULT_MAR_NAME = '%app%-%version%.%locale%.%platform%.complete.mar'; +$DEFAULT_CHECKSUMS_NAME = '%app%-%version%.%locale%.%platform%.checksums'; sub new { @@ -114,7 +115,7 @@ sub ProcessCommandLineArgs Getopt::Long::GetOptions(\%args, 'help|h|?', 'man', 'version', 'app=s', 'brand=s', 'config=s', 'verbose', 'dry-run', 'tools-dir=s', 'download-dir=s', 'deliverable-dir=s', - 'tools-revision=s', 'partial-patchlist-file=s', @RUN_MODES) + 'tools-revision=s', 'partial-patchlist-file=s', 'use-checksums', @RUN_MODES) or return 0; $this->{'mConfigFilename'} = defined($args{'config'}) ? $args{'config'} : @@ -133,6 +134,7 @@ sub ProcessCommandLineArgs # Is this a dry run, and we'll just print what we *would* do? $this->{'dryRun'} = defined($args{'dryRun'}) ? 1 : 0; + $this->{'useChecksums'} = defined($args{'use-checksums'}) ? 1 : 0; ## Expects to be the dir that $mToolsDir/mozilla/[all the tools] will be in. $this->{'mToolsDir'} = defined($args{'mToolsDir'}) ? $args{'mToolsDir'} : getcwd(); @@ -740,7 +742,8 @@ sub RemoveBrokenUpdates my $partial_pathname = "$u/ftp/$gen_partial_path"; # Go to next iteration if this partial patch already exists. - next if -e $partial_pathname; + next if -e $partial_pathname || + exists $MozAUSLib::SNIPPET_CHECKSUM_HASH_CACHE->{$partial_pathname}; $i++; if ( ! -f $from_path or @@ -802,6 +805,12 @@ sub IsDryRun return 1 == $this->{'dryRun'}; } +sub useChecksums +{ + my $this = shift; + return 1 == $this->{'useChecksums'}; +} + sub GetToolsRevision { my $this = shift; diff --git a/mozilla/tools/patcher/MozAUSLib.pm b/mozilla/tools/patcher/MozAUSLib.pm index ec500787bd3..71c41aeca79 100644 --- a/mozilla/tools/patcher/MozAUSLib.pm +++ b/mozilla/tools/patcher/MozAUSLib.pm @@ -44,12 +44,15 @@ package MozAUSLib; use Cwd; use File::Path; +use File::Basename; use File::Copy qw(move copy); use English; use File::Spec::Functions; +use Data::Dumper; use MozBuild::Util qw(RunShellCommand MkdirWithPath HashFile); +use MozAUSConfig; require Exporter; @@ -62,6 +65,7 @@ require Exporter; GetSnippetDirFromChannel CachedHashFile $OBJDIR + PrepopulateHashCache ); use strict; @@ -79,6 +83,7 @@ use vars qw($OBJDIR $MAR_BIN $MBSDIFF_BIN $MAKE_BIN $DEFAULT_SNIPPET_BASE_DIR $DEFAULT_SNIPPET_TEST_DIR $SNIPPET_CHECKSUM_HASH_CACHE); +$Data::Dumper::Indent = 1; $OBJDIR = 'obj'; $MAR_BIN = "$OBJDIR/dist/host/bin/mar"; $MBSDIFF_BIN = "$OBJDIR/dist/host/bin/mbsdiff"; @@ -126,6 +131,84 @@ $DEFAULT_SNIPPET_TEST_DIR = $DEFAULT_SNIPPET_BASE_DIR . '.test'; $SNIPPET_CHECKSUM_HASH_CACHE = {}; +sub PrepopulateHashCache { + my %args = @_; + my $config = $args{'config'}; + my $startdir = getcwd(); + my $deliverableDir = EnsureDeliverablesDir(config => $config); + chdir($deliverableDir); + my $from = $config->GetCurrentUpdate()->{'from'}; + my $to = $config->GetCurrentUpdate()->{'to'}; + my $rl_config = $config->{'mAppConfig'}->{'release'}->{$to}; + my $rlp_config = $rl_config->{'platforms'}; + my @platforms = sort(keys(%{$rlp_config})); + for my $p (@platforms) { + my $platform_locales = $rlp_config->{$p}->{'locales'}; + for my $l (@$platform_locales) { + chdir(catfile($deliverableDir, $to, 'ftp')); + + my $checksums_file = SubstitutePath( + path => $MozAUSConfig::DEFAULT_CHECKSUMS_NAME, + platform => $p, + locale => $l, + version => $to, + app => lc($config->GetApp())); + my $partial_mar_path = SubstitutePath( + path => $config->GetCurrentUpdate()->{'partial'}->{'path'}, + platform => $p, + locale => $l); + my $complete_mar_path = SubstitutePath( + path => $config->GetCurrentUpdate()->{'complete'}->{'path'}, + platform => $p, + locale => $l); + my $complete_mar_local_path = SubstitutePath( + path => $MozAUSConfig::DEFAULT_MAR_NAME, + platform => $p, + locale => $l, + version => $to, + app => lc($config->GetApp())); + + open F, "$checksums_file" || die("can't read $checksums_file\n"); + while (my $line = ){ + chomp($line); + my ($hash, $hash_type, $size, $fname) = split /\s/, $line; + $hash_type = uc($hash_type); + my $file = undef; + if ($fname =~ m/partial.mar$/ && + basename($fname) eq basename($partial_mar_path)){ + $file = $partial_mar_path; + } elsif ($fname =~ m/complete.mar$/ && + basename($fname) eq basename($complete_mar_path)){ + $file = $complete_mar_path; + } + if (defined($file)){ + my @files = (catfile("$from-$to", "ftp", $file)); + # work around snippet creation functions using + # different places for complete mars + if ($file =~ m/complete.mar$/){ + push(@files, catfile($to, "ftp", $complete_mar_local_path)); + } + for my $f (@files){ + if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f})) { + $SNIPPET_CHECKSUM_HASH_CACHE->{$f} = {}; + } + if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{$hash_type})) { + $SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{$hash_type} = $hash; + } + if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{'size'})) { + $SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{'size'} = $size; + } + } + } + } + close F; + } + } + chdir($startdir); + # print Data::Dumper::Dumper($SNIPPET_CHECKSUM_HASH_CACHE); +} + + sub CachedHashFile { my %args = @_; @@ -142,8 +225,12 @@ sub CachedHashFile { } if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType})) { - $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType} = - HashFile(file => $file, type => $checksumType); + if ($checksumType eq 'size'){ + $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{'size'} = (stat($file))[7]; + } else { + $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType} = + HashFile(file => $file, type => $checksumType); + } } return $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType}; @@ -250,7 +337,7 @@ sub CreatePartialMarFile } if ( not -r $fromCompleteMar) { - print STDERR "CreatePartialMarFile: $fromCompleteMardoesn't exist!"; + print STDERR "CreatePartialMarFile: $fromCompleteMar doesn't exist!"; return -1; } diff --git a/mozilla/tools/patcher/patcher2.pl b/mozilla/tools/patcher/patcher2.pl index ed601af4307..ebdd8a9ae91 100755 --- a/mozilla/tools/patcher/patcher2.pl +++ b/mozilla/tools/patcher/patcher2.pl @@ -61,7 +61,8 @@ use MozAUSLib qw(CreatePartialMarFile ValidateToolsDirectory SubstitutePath GetSnippetDirFromChannel CachedHashFile - $OBJDIR); + $OBJDIR + PrepopulateHashCache); use MozBuild::Util qw(MkdirWithPath RunShellCommand DownloadFile); @@ -80,10 +81,9 @@ use vars qw($PID_FILE $DEFAULT_HGROOT $DEFAULT_SCHEMA_VERSION @SCHEMA_2_OPTIONAL_ATTR - @COMPUTED_URLS - $ST_SIZE ); + @COMPUTED_URLS); -$PID_FILE = 'patcher2.pid'; +$PID_FILE = catfile(getcwd(), 'patcher2.pid'); $DEFAULT_HASH_TYPE = 'SHA512'; $DEFAULT_CVSROOT = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot'; $DEFAULT_HGROOT = 'http://hg.mozilla.org/mozilla-central'; @@ -105,7 +105,6 @@ $DEFAULT_SCHEMA_VERSION = 2; showSurvey actions); -$ST_SIZE = 7; sub main { Startup(); @@ -116,7 +115,7 @@ sub main { PrintUsage(exitCode => 1) if ($config eq undef); - if (not ($config->RequestedStep('build-tools') or + if (not $config->useChecksums() and not ($config->RequestedStep('build-tools') or $config->RequestedStep('build-tools-hg')) and not ValidateToolsDirectory(toolsDir => $config->GetToolsDir())) { my $badDir = $config->GetToolsDir(); @@ -130,18 +129,22 @@ __END_TOOLS_ERROR__ my $startdir = getcwd(); my $deliverableDir = EnsureDeliverablesDir(config => $config); - #printf("PRE-REMOVE-BROKEN-UPDATES:\n\n%s", Data::Dumper::Dumper($config)); - $config->RemoveBrokenUpdates(); - #printf("POST-REMOVE-BROKEN:\n\n%s", Data::Dumper::Dumper($config)); - BuildTools(config => $config, fromHg => 0) if $config->RequestedStep('build-tools'); BuildTools(config => $config, fromHg => 1) if $config->RequestedStep('build-tools-hg'); run_download_complete_patches(config => $config) if $config->RequestedStep('download'); + if ($config->useChecksums()) { + PrepopulateHashCache(config => $config); + } + #printf("PRE-REMOVE-BROKEN-UPDATES:\n\n%s", Data::Dumper::Dumper($config)); + $config->RemoveBrokenUpdates(); + #printf("POST-REMOVE-BROKEN:\n\n%s", Data::Dumper::Dumper($config)); if ($config->RequestedStep('create-patches')) { - CreatePartialPatches(config => $config); - CreateCompletePatches(config => $config); + if (!$config->useChecksums()) { + CreatePartialPatches(config => $config); + CreateCompletePatches(config => $config); + } } if ($config->RequestedStep('(create-patches|create-patchinfo)')) { @@ -185,6 +188,7 @@ Options: Only applicable when in "create-patches" mode --tools-revision=TAG Specify tag to use for build-tools checkout Only applicable when in "build-tools" mode +--use-checksums Use checksums files instead of MAR files. __END_USAGE__ exit($exitCode) if (defined($exitCode)); @@ -329,6 +333,9 @@ sub download_complete_patches { my $r_config = $config->{'mAppConfig'}->{'release'}; my @releases = ($fromReleaseVersion, $toReleaseVersion); + if ($config->useChecksums()){ + @releases = ($toReleaseVersion); + } for my $r (@releases) { my $rl_config = $r_config->{$r}; @@ -345,13 +352,20 @@ sub download_complete_patches { chdir($relPath); my $download_url = $rl_config->{'completemarurl'}; + + if ($config->useChecksums()){ + $download_url = $rl_config->{'checksumsurl'}; + } $download_url = SubstitutePath(path => $download_url, platform => $p, version => $r, locale => $l); - + my $output_filename = $MozAUSConfig::DEFAULT_MAR_NAME; + if ($config->useChecksums()){ + $output_filename = $MozAUSConfig::DEFAULT_CHECKSUMS_NAME; + } my $output_filename = SubstitutePath( - path => $MozAUSConfig::DEFAULT_MAR_NAME, + path => $output_filename, platform => $p, locale => $l, version => $r, @@ -392,7 +406,6 @@ sub download_complete_patches { DownloadFile(url => $download_url, dest => $output_filename ); } - chdir(catfile($deliverableDir, $r, 'ftp')); my $end_time = time(); @@ -802,8 +815,8 @@ sub CreateCompletePatchinfo { $complete_patch->{'info_path'} = catfile($aus_prefix, 'complete.txt'); - # Go to next iteration if this partial patch already exists. - next if ( -e $complete_patch->{'info_path'} or ! -e $complete_pathname ); + next if ( -e $complete_patch->{'info_path'} or + (! $config->useChecksums() && ! -e $complete_pathname ) ); $i++; #printf("partial = %s", Data::Dumper::Dumper($partial_patch)); @@ -833,7 +846,8 @@ sub CreateCompletePatchinfo { $complete_patch->{'build_id'} = $to->{'build_id'}; $complete_patch->{'appv'} = $prettySnippetToAppVersion; $complete_patch->{'extv'} = $to->{'extv'}; - $complete_patch->{'size'} = (stat($to_path))[$ST_SIZE]; + $complete_patch->{'size'} = CachedHashFile( + file => $to_path, type => 'size'); my $channelSpecificUrlKey = $c . '-url'; @@ -1102,7 +1116,8 @@ sub CreatePastReleasePatchinfo { $completePatch->{'build_id'} = $patchLocaleNode->{'build_id'}; $completePatch->{'appv'} = $prettySnippetToAppVersion; $completePatch->{'extv'} = $patchLocaleNode->{'extv'}; - $completePatch->{'size'} = (stat($to_path))[$ST_SIZE]; + $completePatch->{'size'} = CachedHashFile( + file => $to_path, type => 'size'); my $channelSpecificUrlKey = $channel . '-url'; @@ -1246,7 +1261,8 @@ sub CreatePartialPatchinfo { my $partialPatchHash = CachedHashFile(file => $partial_pathname, type => $DEFAULT_HASH_TYPE); - my $partialPatchSize = (stat($partial_pathname))[$ST_SIZE]; + my $partialPatchSize = CachedHashFile( + file => $partial_pathname, type => 'size'); my $gen_complete_path = SubstitutePath(path => $complete_path, platform => $p, @@ -1264,7 +1280,8 @@ sub CreatePartialPatchinfo { file => $complete_pathname, type => $DEFAULT_HASH_TYPE); - my $completePatchSize = (stat($complete_pathname))[$ST_SIZE]; + my $completePatchSize = CachedHashFile( + file => $complete_pathname, type => 'size'); my $computed_urls = {}; foreach my $attr (@COMPUTED_URLS) { @@ -1322,8 +1339,8 @@ sub CreatePartialPatchinfo { $partial_patch->{'info_path'} = catfile($aus_prefix, 'partial.txt'); - # Go to next iteration if this partial patch already exists. - next if ( -e $partial_patch->{'info_path'} or ! -e $snippetPathname ); + next if ( -e $partial_patch->{'info_path'} or + (! $config->useChecksums() && ! -e $snippetPathname) ); $i++; # This is just prettyfication for the PrintProgress() call,