Bug 607389 - generate partial updates at build time for releases. p=rail,r=nthomas

git-svn-id: svn://10.0.0.236/trunk@263279 18797224-902f-48f8-a5cc-f745e15eee43

mozilla/tools/patcher/MozAUSConfig.pm

@@ -62,7 +62,7 @@ use strict;
 use vars qw( @RUN_MODES
              $DEFAULT_APP $DEFAULT_MODE $DEFAULT_CONFIG_FILE
              $DEFAULT_DOWNLOAD_DIR $DEFAULT_DELIVERABLE_DIR
-             $DEFAULT_MAR_NAME
+             $DEFAULT_MAR_NAME $DEFAULT_CHECKSUMS_NAME
            );
 
 @RUN_MODES = qw( build-tools
@@ -75,6 +75,7 @@ $DEFAULT_APP = 'MyApp';
 $DEFAULT_DOWNLOAD_DIR = 'downloads';
 $DEFAULT_DELIVERABLE_DIR = 'temp';
 $DEFAULT_MAR_NAME = '%app%-%version%.%locale%.%platform%.complete.mar';
+$DEFAULT_CHECKSUMS_NAME = '%app%-%version%.%locale%.%platform%.checksums';
 
 sub new
 {
@@ -114,7 +115,7 @@ sub ProcessCommandLineArgs
     Getopt::Long::GetOptions(\%args,
      'help|h|?', 'man', 'version', 'app=s', 'brand=s', 'config=s', 'verbose',
      'dry-run', 'tools-dir=s', 'download-dir=s', 'deliverable-dir=s',
-     'tools-revision=s', 'partial-patchlist-file=s', @RUN_MODES)
+     'tools-revision=s', 'partial-patchlist-file=s', 'use-checksums', @RUN_MODES)
      or return 0;
 
     $this->{'mConfigFilename'} = defined($args{'config'}) ? $args{'config'} :
@@ -133,6 +134,7 @@ sub ProcessCommandLineArgs
 
     # Is this a dry run, and we'll just print what we *would* do?
     $this->{'dryRun'} = defined($args{'dryRun'}) ? 1 : 0;
+    $this->{'useChecksums'} = defined($args{'use-checksums'}) ? 1 : 0;
 
     ## Expects to be the dir that $mToolsDir/mozilla/[all the tools] will be in.
     $this->{'mToolsDir'} = defined($args{'mToolsDir'}) ? $args{'mToolsDir'} : getcwd();
@@ -740,7 +742,8 @@ sub RemoveBrokenUpdates
                 my $partial_pathname = "$u/ftp/$gen_partial_path";
 
                 # Go to next iteration if this partial patch already exists.
-                next if -e $partial_pathname;
+                next if -e $partial_pathname ||
+                 exists $MozAUSLib::SNIPPET_CHECKSUM_HASH_CACHE->{$partial_pathname};
                 $i++;
 
                 if ( ! -f $from_path or
@@ -802,6 +805,12 @@ sub IsDryRun
     return 1 == $this->{'dryRun'};
 }
 
+sub useChecksums
+{
+    my $this = shift;
+    return 1 == $this->{'useChecksums'};
+}
+
 sub GetToolsRevision
 {
     my $this = shift;

mozilla/tools/patcher/MozAUSLib.pm

@@ -44,12 +44,15 @@ package MozAUSLib;
 
 use Cwd;
 use File::Path;
+use File::Basename;
 use File::Copy qw(move copy);
 use English;
 
 use File::Spec::Functions;
+use Data::Dumper;
 
 use MozBuild::Util qw(RunShellCommand MkdirWithPath HashFile);
+use MozAUSConfig;
 
 require Exporter;
 
@@ -62,6 +65,7 @@ require Exporter;
                 GetSnippetDirFromChannel
                 CachedHashFile
                 $OBJDIR
+                PrepopulateHashCache
                );
 
 use strict;
@@ -79,6 +83,7 @@ use vars qw($OBJDIR $MAR_BIN $MBSDIFF_BIN $MAKE_BIN
             $DEFAULT_SNIPPET_BASE_DIR $DEFAULT_SNIPPET_TEST_DIR
             $SNIPPET_CHECKSUM_HASH_CACHE);
 
+$Data::Dumper::Indent = 1;
 $OBJDIR = 'obj';
 $MAR_BIN = "$OBJDIR/dist/host/bin/mar";
 $MBSDIFF_BIN = "$OBJDIR/dist/host/bin/mbsdiff";
@@ -126,6 +131,84 @@ $DEFAULT_SNIPPET_TEST_DIR = $DEFAULT_SNIPPET_BASE_DIR . '.test';
 
 $SNIPPET_CHECKSUM_HASH_CACHE = {};
 
+sub PrepopulateHashCache {
+   my %args = @_;
+   my $config = $args{'config'};
+   my $startdir = getcwd();
+   my $deliverableDir = EnsureDeliverablesDir(config => $config);
+   chdir($deliverableDir);
+   my $from = $config->GetCurrentUpdate()->{'from'};
+   my $to = $config->GetCurrentUpdate()->{'to'};
+   my $rl_config = $config->{'mAppConfig'}->{'release'}->{$to};
+   my $rlp_config = $rl_config->{'platforms'};
+   my @platforms = sort(keys(%{$rlp_config}));
+   for my $p (@platforms) {
+      my $platform_locales = $rlp_config->{$p}->{'locales'};
+      for my $l (@$platform_locales) {
+         chdir(catfile($deliverableDir, $to, 'ftp'));
+
+         my $checksums_file = SubstitutePath(
+          path => $MozAUSConfig::DEFAULT_CHECKSUMS_NAME,
+          platform => $p,
+          locale => $l,
+          version => $to,
+          app => lc($config->GetApp()));
+         my $partial_mar_path = SubstitutePath(
+          path => $config->GetCurrentUpdate()->{'partial'}->{'path'},
+          platform => $p,
+          locale => $l);
+         my $complete_mar_path = SubstitutePath(
+          path => $config->GetCurrentUpdate()->{'complete'}->{'path'},
+          platform => $p,
+          locale => $l);
+         my $complete_mar_local_path = SubstitutePath(
+          path => $MozAUSConfig::DEFAULT_MAR_NAME,
+          platform => $p,
+          locale => $l,
+          version => $to,
+          app => lc($config->GetApp()));
+
+         open F, "$checksums_file" || die("can't read $checksums_file\n");
+         while (my $line = <F>){
+            chomp($line);
+            my ($hash, $hash_type, $size, $fname) = split /\s/, $line;
+            $hash_type = uc($hash_type);
+            my $file = undef;
+            if ($fname =~ m/partial.mar$/ && 
+              basename($fname) eq basename($partial_mar_path)){
+               $file = $partial_mar_path;
+            } elsif ($fname =~ m/complete.mar$/ &&
+              basename($fname) eq basename($complete_mar_path)){
+               $file = $complete_mar_path;
+            }
+            if (defined($file)){
+               my @files = (catfile("$from-$to", "ftp", $file));
+               # work around snippet creation functions using
+               # different places for complete mars
+               if ($file =~ m/complete.mar$/){
+                 push(@files, catfile($to, "ftp", $complete_mar_local_path));
+               }
+               for my $f (@files){
+                  if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f})) {
+                     $SNIPPET_CHECKSUM_HASH_CACHE->{$f} = {};
+                  }
+                  if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{$hash_type})) {
+                     $SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{$hash_type} = $hash;
+                  }
+                  if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{'size'})) {
+                     $SNIPPET_CHECKSUM_HASH_CACHE->{$f}->{'size'} = $size;
+                  }
+               }
+            }
+         }
+         close F;
+      }
+   }
+   chdir($startdir);
+   # print Data::Dumper::Dumper($SNIPPET_CHECKSUM_HASH_CACHE);
+}
+
+
 sub CachedHashFile {
    my %args = @_;
 
@@ -142,9 +225,13 @@ sub CachedHashFile {
    }
 
    if (! exists($SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType})) {
+      if ($checksumType eq 'size'){
+         $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{'size'} = (stat($file))[7];
+      } else {
         $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType} =
          HashFile(file => $file, type => $checksumType);
      }
+   }
 
    return $SNIPPET_CHECKSUM_HASH_CACHE->{$file}->{$checksumType};
 
@@ -250,7 +337,7 @@ sub CreatePartialMarFile
     }
 
     if ( not -r $fromCompleteMar) {
-        print STDERR "CreatePartialMarFile: $fromCompleteMardoesn't exist!";
+        print STDERR "CreatePartialMarFile: $fromCompleteMar doesn't exist!";
         return -1;
     }
 

mozilla/tools/patcher/patcher2.pl

@@ -61,7 +61,8 @@ use MozAUSLib qw(CreatePartialMarFile
                  ValidateToolsDirectory SubstitutePath
                  GetSnippetDirFromChannel
                  CachedHashFile
-                 $OBJDIR);
+                 $OBJDIR
+                 PrepopulateHashCache);
 
 use MozBuild::Util qw(MkdirWithPath RunShellCommand DownloadFile);
 
@@ -80,10 +81,9 @@ use vars qw($PID_FILE
             $DEFAULT_HGROOT
             $DEFAULT_SCHEMA_VERSION
             @SCHEMA_2_OPTIONAL_ATTR
-            @COMPUTED_URLS
+            @COMPUTED_URLS);
-            $ST_SIZE );
 
-$PID_FILE = 'patcher2.pid';
+$PID_FILE = catfile(getcwd(), 'patcher2.pid');
 $DEFAULT_HASH_TYPE = 'SHA512';
 $DEFAULT_CVSROOT = ':pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot';
 $DEFAULT_HGROOT  = 'http://hg.mozilla.org/mozilla-central';
@@ -105,7 +105,6 @@ $DEFAULT_SCHEMA_VERSION = 2;
                              showSurvey
                              actions);
 
-$ST_SIZE = 7;
 
 sub main {
     Startup();
@@ -116,7 +115,7 @@ sub main {
     PrintUsage(exitCode => 1) if ($config eq undef);
 
 
-    if (not ($config->RequestedStep('build-tools') or
+    if (not $config->useChecksums() and not ($config->RequestedStep('build-tools') or
              $config->RequestedStep('build-tools-hg')) and
         not ValidateToolsDirectory(toolsDir => $config->GetToolsDir())) {
         my $badDir = $config->GetToolsDir();
@@ -130,19 +129,23 @@ __END_TOOLS_ERROR__
     my $startdir = getcwd();
     my $deliverableDir = EnsureDeliverablesDir(config => $config);
 
-    #printf("PRE-REMOVE-BROKEN-UPDATES:\n\n%s", Data::Dumper::Dumper($config));
-    $config->RemoveBrokenUpdates();
-    #printf("POST-REMOVE-BROKEN:\n\n%s", Data::Dumper::Dumper($config));
-
     BuildTools(config => $config, fromHg => 0) if $config->RequestedStep('build-tools');
     BuildTools(config => $config, fromHg => 1) if $config->RequestedStep('build-tools-hg');
 
     run_download_complete_patches(config => $config) if $config->RequestedStep('download');
+    if ($config->useChecksums()) {
+        PrepopulateHashCache(config => $config);
+    }
+    #printf("PRE-REMOVE-BROKEN-UPDATES:\n\n%s", Data::Dumper::Dumper($config));
+    $config->RemoveBrokenUpdates();
+    #printf("POST-REMOVE-BROKEN:\n\n%s", Data::Dumper::Dumper($config));
 
     if ($config->RequestedStep('create-patches')) {
+        if (!$config->useChecksums()) {
             CreatePartialPatches(config => $config);
             CreateCompletePatches(config => $config);
         }
+    }
 
     if ($config->RequestedStep('(create-patches|create-patchinfo)')) {
         CreatePartialPatchinfo(config => $config);
@@ -185,6 +188,7 @@ Options:
                                 Only applicable when in "create-patches" mode
 --tools-revision=TAG            Specify tag to use for build-tools checkout
                                 Only applicable when in "build-tools" mode
+--use-checksums                 Use checksums files instead of MAR files.
 
 __END_USAGE__
    exit($exitCode) if (defined($exitCode));
@@ -329,6 +333,9 @@ sub download_complete_patches {
 
     my $r_config = $config->{'mAppConfig'}->{'release'};
     my @releases = ($fromReleaseVersion, $toReleaseVersion);
+    if ($config->useChecksums()){
+        @releases = ($toReleaseVersion);
+    }
 
     for my $r (@releases) {
         my $rl_config = $r_config->{$r};
@@ -345,13 +352,20 @@ sub download_complete_patches {
                 chdir($relPath);
 
                 my $download_url = $rl_config->{'completemarurl'};
+
+                if ($config->useChecksums()){
+                    $download_url = $rl_config->{'checksumsurl'};
+                }
                 $download_url = SubstitutePath(path => $download_url,
                                                platform => $p,
                                                version => $r,
                                                locale => $l);
-
+                my $output_filename = $MozAUSConfig::DEFAULT_MAR_NAME;
+                if ($config->useChecksums()){
+                  $output_filename = $MozAUSConfig::DEFAULT_CHECKSUMS_NAME;
+                }
                 my $output_filename = SubstitutePath(
-                 path => $MozAUSConfig::DEFAULT_MAR_NAME,
+                 path => $output_filename,
                  platform => $p,
                  locale => $l,
                  version => $r,
@@ -392,7 +406,6 @@ sub download_complete_patches {
                     DownloadFile(url => $download_url,
                      dest => $output_filename );
                 }
-
                 chdir(catfile($deliverableDir, $r, 'ftp'));
 
                 my $end_time = time();
@@ -802,8 +815,8 @@ sub CreateCompletePatchinfo {
                         $complete_patch->{'info_path'} = catfile($aus_prefix,
                          'complete.txt');
     
-                        # Go to next iteration if this partial patch already exists.
+                        next if ( -e $complete_patch->{'info_path'} or
-                        next if ( -e $complete_patch->{'info_path'} or ! -e $complete_pathname );
+                         (! $config->useChecksums() && ! -e $complete_pathname ) );
                         $i++;
     
                         #printf("partial = %s", Data::Dumper::Dumper($partial_patch));
@@ -833,7 +846,8 @@ sub CreateCompletePatchinfo {
                         $complete_patch->{'build_id'} = $to->{'build_id'};
                         $complete_patch->{'appv'} = $prettySnippetToAppVersion;
                         $complete_patch->{'extv'} = $to->{'extv'};
-                        $complete_patch->{'size'} = (stat($to_path))[$ST_SIZE];
+                        $complete_patch->{'size'} = CachedHashFile(
+                         file => $to_path, type => 'size');
     
                         my $channelSpecificUrlKey = $c . '-url';
     
@@ -1102,7 +1116,8 @@ sub CreatePastReleasePatchinfo {
                         $completePatch->{'build_id'} = $patchLocaleNode->{'build_id'};
                         $completePatch->{'appv'} = $prettySnippetToAppVersion;
                         $completePatch->{'extv'} = $patchLocaleNode->{'extv'};
-                        $completePatch->{'size'} = (stat($to_path))[$ST_SIZE];
+                        $completePatch->{'size'} = CachedHashFile(
+                         file => $to_path, type => 'size');
     
                         my $channelSpecificUrlKey = $channel . '-url';
     
@@ -1246,7 +1261,8 @@ sub CreatePartialPatchinfo {
 
                 my $partialPatchHash = CachedHashFile(file => $partial_pathname,
                                                 type => $DEFAULT_HASH_TYPE);
-                my $partialPatchSize = (stat($partial_pathname))[$ST_SIZE];
+                my $partialPatchSize = CachedHashFile(
+                 file => $partial_pathname, type => 'size');
 
                 my $gen_complete_path = SubstitutePath(path => $complete_path,
                                                        platform => $p,
@@ -1264,7 +1280,8 @@ sub CreatePartialPatchinfo {
                  file => $complete_pathname,
                  type => $DEFAULT_HASH_TYPE);
 
-                my $completePatchSize = (stat($complete_pathname))[$ST_SIZE];
+                my $completePatchSize = CachedHashFile(
+                 file => $complete_pathname, type => 'size');
 
                 my $computed_urls = {};
                 foreach my $attr (@COMPUTED_URLS) {
@@ -1322,8 +1339,8 @@ sub CreatePartialPatchinfo {
                         $partial_patch->{'info_path'} = catfile($aus_prefix,
                                                                 'partial.txt');
     
-                        # Go to next iteration if this partial patch already exists.
+                        next if ( -e $partial_patch->{'info_path'}  or
-                        next if ( -e $partial_patch->{'info_path'} or ! -e $snippetPathname );
+                         (! $config->useChecksums() && ! -e $snippetPathname) );
                         $i++;
     
                         # This is just prettyfication for the PrintProgress() call,