* Tagged 0.7.

files and hard-links them to each other to save disk space.

  Currently it doesn't actually do the hard-linking, it just reports
  the amount of space saved if it did.

Makefile.am

@@ -29,6 +29,7 @@ init-state:
 	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(prefix)/store
+	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 #	$(bindir)/nix-store --init
 else
 init-state:

NEWS

@@ -1,3 +1,25 @@
+Version 0.7
+
+* Binary patching.  When upgrading components using pre-built binaries
+  (through nix-pull / nix-channel), Nix can automatically download and
+  apply binary patches to already installed components instead of full
+  downloads.  Patching is "smart": if there is a *sequence* of patches
+  to an installed component, Nix will use it.  Patches are currently
+  generated automatically between Nixpkgs (pre-)releases.
+
+* Simplifications to the substitute mechanism.
+
+* Nix-pull now stores downloaded manifests in /nix/var/nix/manifests.
+
+* Metadata on files in the Nix store is canonicalised after builds:
+  the last-modified timestamp is set to 0 (00:00:00 1/1/1970), the
+  mode is set to 0444 or 0555 (readable and possibly executable by
+  all; setuid/setgid bits are dropped), and the group is set to the
+  default.  This ensures that the result of a build and an
+  installation through a substitute is the same; and that timestamp
+  dependencies are revealed.
+
+
 Version 0.6
 
 Major changes include the following:

configure.ac

@@ -1,11 +1,11 @@
-AC_INIT(nix, "0.6")
+AC_INIT(nix, "0.7")
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
 AM_INIT_AUTOMAKE
 
 # Change to `1' to produce a `stable' release (i.e., the `preREVISION'
 # suffix is not added).
-STABLE=0
+STABLE=1
 
 # Put the revision number in the version.
 if test "$STABLE" != "1"; then
@@ -189,13 +189,13 @@ AC_CONFIG_FILES([Makefile
    src/nix-instantiate/Makefile
    src/nix-env/Makefile
    src/log2xml/Makefile
+   src/bsdiff-4.2/Makefile
    scripts/Makefile
    corepkgs/Makefile
    corepkgs/fetchurl/Makefile
    corepkgs/nar/Makefile
    corepkgs/buildenv/Makefile
    corepkgs/channels/Makefile
-   corepkgs/nix-pull/Makefile
    doc/Makefile
    doc/manual/Makefile
    misc/Makefile

corepkgs/Makefile.am

@@ -1 +1 @@
-SUBDIRS = fetchurl nar buildenv channels nix-pull
+SUBDIRS = fetchurl nar buildenv channels

corepkgs/nar/nar.sh.in

@@ -6,10 +6,14 @@ export PATH=/bin:/usr/bin
 echo "packing $path into $out..."
 mkdir $out
 dst=$out/$(basename $path).nar.bz2
-@bindir@/nix-store --dump "$path" | @bzip2@ > $dst
+@bindir@/nix-store --dump "$path" > tmp
 
-if test "${PIPESTATUS[0]}" != "0"; then exit 1; fi
+@bzip2@ < tmp > $dst
 
-md5=$(md5sum -b $dst | cut -c1-32)
+narHash=$(md5sum -b tmp | cut -c1-32)
 if test $? != 0; then exit 1; fi
-echo $md5 > $out/md5
+echo $narHash > $out/nar-hash
+
+narbz2Hash=$(md5sum -b $dst | cut -c1-32)
+if test $? != 0; then exit 1; fi
+echo $narbz2Hash > $out/narbz2-hash

corepkgs/nix-pull/Makefile.am

@@ -1,11 +0,0 @@
-all-local: builder.sh
-
-install-exec-local:
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
-	$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
-	$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
-
-include ../../substitute.mk
-
-EXTRA_DIST = default.nix builder.sh.in

corepkgs/nix-pull/builder.sh.in

@@ -1,34 +0,0 @@
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-mkdir $out
-
-cat > $out/fetch <<EOF
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-echo "downloading \$2..."
-
-export PRINT_PATH=1
-result=(\$(@bindir@/nix-prefetch-url \$2))
-
-hash=\${result[0]}
-path=\${result[1]}
-
-if test "\$hash" != "\$3"; then
-    echo "hash is \$hash, expected \$3"
-    exit 1
-fi
-
-echo "unpacking into \$1..."
-
-if ! @bunzip2@ < "\$path" | @bindir@/nix-store --restore "\$1"; then
-    exit 1
-fi
-
-exit 0
-EOF
-
-chmod +x $out/fetch

corepkgs/nix-pull/default.nix

@@ -1,7 +0,0 @@
-{system}:
-
-derivation {
-  name = "nix-pull";
-  builder = ./builder.sh;
-  inherit system;
-}

doc/manual/introduction.xml

@@ -75,7 +75,7 @@ be adapted easily to achieve similar policies.</para></listitem>
 
 <listitem><para>Nix component builds aim to be <quote>pure</quote>;
 that is, unaffected by anything other than the declared dependencies.
-This means that if a component was built succesfully once, it can be
+This means that if a component was built successfully once, it can be
 rebuilt again on another machine and the result will be the same.  We
 cannot <emphasis>guarantee</emphasis> this (e.g., if the build depends
 on the time-of-day), but Nix (and the tools in the Nix Packages
@@ -113,6 +113,13 @@ software deployment also apply here: for instance, the ability
 trivially to have multiple configurations at the same time, or the
 ability to do rollbacks.</para></listitem>
 
+<listitem><para>Nix can efficiently upgrade between different versions
+of a component through <emphasis>binary patching</emphasis>.  If
+patches are available on a server, and you try to install a new
+version of some component, Nix will automatically apply a patch (or
+sequence of patches), if available, to transform the installed
+component into the new version.</para></listitem> 
+
 </itemizedlist>
 
 </para>

doc/manual/writing-nix-expressions.xml

@@ -575,7 +575,7 @@ builder becomes even shorter:
 
 <programlisting>
 . $stdenv/setup
-genericBuilder</programlisting>
+genericBuild</programlisting>
 
 In fact, <varname>mkDerivation</varname> provides a default builder
 that looks exactly like that, so it is actually possible to omit the
@@ -1253,6 +1253,17 @@ command-line argument.  See <xref linkend='sec-standard-environment'
   performed by looking for the hash parts of file names of the
   inputs.</para></listitem>
 
+  <listitem><para>After the build, Nix sets the last-modified
+  timestamp on all files in the build result to 0 (00:00:00 1/1/1970
+  UTC), sets the group to the default group, and sets the mode of the
+  file to 0444 or 0555 (i.e., read-only, with execute permission
+  enabled if the file was originally executable).  Note that possible
+  <literal>setuid</literal> and <literal>setgid</literal> bits are
+  cleared.  Setuid and setgid programs are not currently supported by
+  Nix.  This is because the Nix archives used in deployment have no
+  concept of ownership information, and because it makes the build
+  result dependent on the user performing the build.</para></listitem>
+
 </itemizedlist>
 
 </para>
@@ -1371,7 +1382,7 @@ following:
 </para>
 
 <para>The <filename>setup</filename> script also exports a function
-called <function>genericBuilder</function> that knows how to build
+called <function>genericBuild</function> that knows how to build
 typical Autoconf-style components.  It can be customised to perform
 builds for any type of component.  It is advisable to use
 <function>genericBuild</function> since it provides facilities that

externals/Makefile.am

@@ -34,12 +34,12 @@ endif
 
 # CWI ATerm
 
-ATERM = aterm-2.2
+ATERM = aterm-2.3.1
 
 $(ATERM).tar.gz:
 	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.2 from"
-	@echo "  http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.2.tar.gz"
+	@echo "Please download version 2.3.1 from"
+	@echo "  http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.3.1.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 

scripts/Makefile.am

@@ -4,16 +4,15 @@ bin_SCRIPTS = nix-collect-garbage \
 
 noinst_SCRIPTS = nix-profile.sh
 
-nix-pull nix-push: readmanifest.pm
+nix-pull nix-push: readmanifest.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm
+install-exec-local: readmanifest.pm download-using-manifests.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
-# !!! don't overwrite local modifications
-	$(INSTALL_DATA) prebuilts.conf $(DESTDIR)$(sysconfdir)/nix/prebuilts.conf
 
 include ../substitute.mk
 
@@ -21,5 +20,6 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-pull.in nix-push.in nix-profile.sh.in \
   nix-prefetch-url.in nix-install-package.in \
   nix-channel.in \
-  prebuilts.conf readmanifest.pm.in \
-  nix-build.in
+  readmanifest.pm.in \
+  nix-build.in \
+  download-using-manifests.pl.in

scripts/download-using-manifests.pl.in

@@ -0,0 +1,246 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use readmanifest;
+
+my $manifestDir = "@localstatedir@/nix/manifests";
+my $logFile = "@localstatedir@/log/nix/downloads";
+
+open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
+
+
+# Check the arguments.
+die unless scalar @ARGV == 1;
+my $targetPath = $ARGV[0];
+
+my $date = `date`;
+chomp $date;
+print LOGFILE "$$ get $targetPath $date\n";
+
+print "\n*** Trying to download/patch `$targetPath'\n";
+
+
+# Load all manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+for my $manifest (glob "$manifestDir/*.nixmanifest") {
+#    print STDERR "reading $manifest\n";
+    readManifest $manifest, \%narFiles, \%patches, \%successors;
+}
+
+
+# Build a graph of all store paths that might contribute to the
+# construction of $targetPath, and the special node "start".  The
+# edges are either patch operations, or downloads of full NAR files.
+# The latter edges only occur between "start" and a store path.
+
+my %graph;
+
+$graph{"start"} = {d => 0, pred => undef, edges => []};
+
+my @queue = ();
+my $queueFront = 0;
+my %done;
+
+sub addToQueue {
+    my $v = shift;
+    return if defined $done{$v};
+    $done{$v} = 1;
+    push @queue, $v;
+}
+
+sub addNode {
+    my $u = shift;
+    $graph{$u} = {d => 999999999999, pred => undef, edges => []}
+        unless defined $graph{$u};
+}
+
+sub addEdge {
+    my $u = shift;
+    my $v = shift;
+    my $w = shift;
+    my $type = shift;
+    my $info = shift;
+    addNode $u;
+    push @{$graph{$u}->{edges}},
+        {weight => $w, start => $u, end => $v, type => $type, info => $info};
+    my $n = scalar @{$graph{$u}->{edges}};
+}
+
+addToQueue $targetPath;
+
+sub isValidPath {
+    my $p = shift;
+    system "nix-store --isvalid '$p' 2> /dev/null";
+    return $? == 0;
+}
+
+while ($queueFront < scalar @queue) {
+    my $u = $queue[$queueFront++];
+#    print "$u\n";
+
+    addNode $u;
+
+    # If the path already exists, it has distance 0 from the "start"
+    # node.
+    if (isValidPath($u)) {
+        addEdge "start", $u, 0, "present", undef;
+    }
+
+    else {
+
+        # Add patch edges.
+        my $patchList = $patches{$u};
+        foreach my $patch (@{$patchList}) {
+            if (isValidPath($patch->{basePath})) {
+                # !!! this should be cached
+                my $hash = `nix-hash "$patch->{basePath}"`;
+                chomp $hash;
+#                print "  MY HASH is $hash\n";
+                if ($hash ne $patch->{baseHash}) {
+                    print LOGFILE "$$ rejecting $patch->{basePath}\n";
+                    next;
+                }
+            }
+#            print "  PATCH from $patch->{basePath}\n";
+            addToQueue $patch->{basePath};
+            addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
+        }
+
+        # Add NAR file edges to the start node.
+        my $narFileList = $narFiles{$u};
+        foreach my $narFile (@{$narFileList}) {
+#            print "  NAR from $narFile->{url}\n";
+            addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
+            if ($u eq $targetPath) {
+                print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
+            }
+        }
+
+    }
+}
+
+
+# Run Dijkstra's shortest path algorithm to determine the shortest
+# sequence of download and/or patch actions that will produce
+# $targetPath.
+
+sub byDistance { # sort by distance, reversed
+    return -($graph{$a}->{d} <=> $graph{$b}->{d});
+}
+
+my @todo = keys %graph;
+
+while (scalar @todo > 0) {
+
+    # Remove the closest element from the todo list.
+    @todo = sort byDistance @todo;
+    my $u = pop @todo;
+
+    my $u_ = $graph{$u};
+
+#    print "IN $u $u_->{d}\n";
+
+    foreach my $edge (@{$u_->{edges}}) {
+        my $v_ = $graph{$edge->{end}};
+        if ($v_->{d} > $u_->{d} + $edge->{weight}) {
+            $v_->{d} = $u_->{d} + $edge->{weight};
+            # Store the edge; to edge->start is actually the
+            # predecessor.
+            $v_->{pred} = $edge; 
+#            print "  RELAX $edge->{end} $v_->{d}\n";
+        }
+    }
+}
+
+
+# Retrieve the shortest path from "start" to $targetPath.
+my @path = ();
+my $cur = $targetPath;
+die "don't know how to produce $targetPath\n"
+    unless defined $graph{$targetPath}->{pred};
+while ($cur ne "start") {
+    push @path, $graph{$cur}->{pred};
+    $cur = $graph{$cur}->{pred}->{start};
+}
+
+
+# Traverse the shortest path, perform the actions described by the
+# edges.
+my $curStep = 1;
+my $maxStep = scalar @path;
+
+sub downloadFile {
+    my $url = shift;
+    my $hash = shift;
+    $ENV{"PRINT_PATH"} = 1;
+    $ENV{"QUIET"} = 1;
+    my ($hash2, $path) = `nix-prefetch-url '$url' '$hash'`;
+    chomp $hash2;
+    chomp $path;
+    die "hash mismatch" if $hash ne $hash2;
+    return $path;
+}
+
+while (scalar @path > 0) {
+    my $edge = pop @path;
+    my $u = $edge->{start};
+    my $v = $edge->{end};
+
+    print "\n*** Step $curStep/$maxStep: ";
+    $curStep++;
+
+    if ($edge->{type} eq "present") {
+        print "using already present path `$v'\n";
+        print LOGFILE "$$ present $v\n";
+    }
+
+    elsif ($edge->{type} eq "patch") {
+        my $patch = $edge->{info};
+        print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
+
+        print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n";
+
+        # Download the patch.
+        print "  downloading patch...\n";
+        my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
+
+        # Turn the base path into a NAR archive, to which we can
+        # actually apply the patch.
+        print "  packing base path...\n";
+        system "nix-store --dump $patch->{basePath} > /tmp/nar";
+        die "cannot dump `$patch->{basePath}'" if ($? != 0);
+
+        # Apply the patch.
+        print "  applying patch...\n";
+        system "@libexecdir@/bspatch /tmp/nar /tmp/nar2 $patchPath";
+        die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
+
+        # Unpack the resulting NAR archive into the target path.
+        print "  unpacking patched archive...\n";
+        system "nix-store --restore $v < /tmp/nar2";
+        die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
+    }
+
+    elsif ($edge->{type} eq "narfile") {
+        my $narFile = $edge->{info};
+        print "downloading `$narFile->{url}' into `$v'\n";
+
+        print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
+        
+        # Download the archive.
+        print "  downloading archive...\n";
+        my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
+
+        # Unpack the archive into the target path.
+        print "  unpacking archive...\n";
+        system "bunzip2 < '$narFilePath' | nix-store --restore '$v'";
+        die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
+    }
+}
+
+
+print LOGFILE "$$ success\n";
+close LOGFILE;

scripts/generate-patches.pl

@@ -0,0 +1,285 @@
+#! /usr/bin/perl -w -I/home/eelco/nix/scripts
+
+use strict;
+use POSIX qw(tmpnam);
+use readmanifest;
+
+die unless scalar @ARGV == 5;
+
+my $cacheDir = $ARGV[0];
+my $patchesDir = $ARGV[1];
+my $patchesURL = $ARGV[2];
+my $srcDir = $ARGV[3];
+my $dstDir = $ARGV[4];
+
+my $tmpdir;
+do { $tmpdir = tmpnam(); }
+until mkdir $tmpdir, 0777;
+
+print "TEMP = $tmpdir\n";
+
+#END { rmdir $tmpdir; }
+
+my %srcNarFiles;
+my %srcPatches;
+my %srcSuccessors;
+
+my %dstNarFiles;
+my %dstPatches;
+my %dstSuccessors;
+
+readManifest "$srcDir/MANIFEST",
+    \%srcNarFiles, \%srcPatches, \%srcSuccessors;
+
+readManifest "$dstDir/MANIFEST",
+    \%dstNarFiles, \%dstPatches, \%dstSuccessors;
+
+
+sub findOutputPaths {
+    my $narFiles = shift;
+    my $successors = shift;
+
+    my %outPaths;
+    
+    foreach my $p (keys %{$narFiles}) {
+
+        # Ignore store expressions.
+        next if ($p =~ /\.store$/);
+        
+        # Ignore builders (too much ambiguity -- they're all called
+        # `builder.sh').
+        next if ($p =~ /\.sh$/);
+        next if ($p =~ /\.patch$/);
+        
+        # Don't bother including tar files etc.
+        next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
+
+        $outPaths{$p} = 1;
+    }
+
+    return %outPaths;
+}
+
+print "finding src output paths...\n";
+my %srcOutPaths = findOutputPaths \%srcNarFiles, \%srcSuccessors;
+
+print "finding dst output paths...\n";
+my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
+
+
+sub getNameVersion {
+    my $p = shift;
+    $p =~ /\/[0-9a-f]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
+    my $name = $1;
+    my $version = $2;
+    $name =~ s/^-//;
+    $version =~ s/^-//;
+    return ($name, $version);
+}
+
+
+# A quick hack to get a measure of the `distance' between two
+# versions: it's just the position of the first character that differs
+# (or 999 if they are the same).
+sub versionDiff {
+    my $s = shift;
+    my $t = shift;
+    my $i;
+    return 999 if $s eq $t;
+    for ($i = 0; $i < length $s; $i++) {
+        return $i if $i >= length $t or
+            substr($s, $i, 1) ne substr($t, $i, 1);
+    }
+    return $i;
+}
+
+
+sub getNarBz2 {
+    my $narFiles = shift;
+    my $storePath = shift;
+    
+    my $narFileList = $$narFiles{$storePath};
+    die "missing store expression $storePath" unless defined $narFileList;
+
+    my $narFile = @{$narFileList}[0];
+    die unless defined $narFile;
+
+    $narFile->{url} =~ /\/([^\/]+)$/;
+    die unless defined $1;
+    return "$cacheDir/$1";
+}
+
+
+sub containsPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $basePath = shift;
+    my $patchList = $$patches{$storePath};
+    return 0 if !defined $patchList;
+    my $found = 0;
+    foreach my $patch (@{$patchList}) {
+        # !!! baseHash might differ
+        return 1 if $patch->{basePath} eq $basePath;
+    }
+    return 0;
+}
+
+
+# For each output path in the destination, see if we need to / can
+# create a patch.
+
+print "creating patches...\n";
+
+foreach my $p (keys %dstOutPaths) {
+
+    # If exactly the same path already exists in the source, skip it.
+    next if defined $srcOutPaths{$p};
+    
+#    print "  $p\n";
+
+    # If not, then we should find the path in the source that is
+    # `most' likely to be present on a system that wants to install
+    # this path.
+
+    (my $name, my $version) = getNameVersion $p;
+
+    my @closest = ();
+    my $closestVersion;
+    my $minDist = -1; # actually, larger means closer
+
+    # Find all source paths with the same name.
+
+    foreach my $q (keys %srcOutPaths) {
+        (my $name2, my $version2) = getNameVersion $q;
+        if ($name eq $name2) {
+            my $dist = versionDiff $version, $version2;
+            if ($dist > $minDist) {
+                $minDist = $dist;
+                @closest = ($q);
+                $closestVersion = $version2;
+            } elsif ($dist == $minDist) {
+                push @closest, $q;
+            }
+        }
+    }
+
+    if (scalar(@closest) == 0) {
+        print "  NO BASE: $p\n";
+        next;
+    }
+
+    foreach my $closest (@closest) {
+
+        # Generate a patch between $closest and $p.
+        print "  $p <- $closest\n";
+
+        # If the patch already exists, skip it.
+        if (containsPatch(\%srcPatches, $p, $closest) ||
+            containsPatch(\%dstPatches, $p, $closest))
+        {
+            print "    skipping, already exists\n";
+            next;
+        }
+
+#        next;
+        
+        my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
+        my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
+        
+        system("bunzip2 < $srcNarBz2 > $tmpdir/A") == 0
+            or die "cannot unpack $srcNarBz2";
+
+        system("bunzip2 < $dstNarBz2 > $tmpdir/B") == 0
+            or die "cannot unpack $dstNarBz2";
+
+        system("bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
+            or die "cannot compute binary diff";
+
+        my $baseHash = `nix-hash --flat $tmpdir/A` or die;
+        chomp $baseHash;
+
+        my $narHash = `nix-hash --flat $tmpdir/B` or die;
+        chomp $narHash;
+
+        my $narDiffHash = `nix-hash --flat $tmpdir/DIFF` or die;
+        chomp $narDiffHash;
+
+        my $narDiffSize = (stat "$tmpdir/DIFF")[7];
+        my $dstNarBz2Size = (stat $dstNarBz2)[7];
+
+        if ($narDiffSize >= $dstNarBz2Size) {
+            print "    rejecting; patch bigger than full archive\n";
+            next;
+        }
+    
+        my $finalName =
+            "$narDiffHash-$name-$closestVersion-to-$version.nar-bsdiff";
+
+        print "    size $narDiffSize; full size $dstNarBz2Size\n";
+        
+        if (-e "$patchesDir/$finalName") {
+            print "    not copying, already exists\n";
+        }
+
+        else {
+
+            system("cp '$tmpdir/DIFF' '$patchesDir/$finalName.tmp'") == 0
+                or die "cannot copy diff";
+            
+            rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
+                or die "cannot rename $patchesDir/$finalName.tmp";
+            
+        }
+        
+        # Add the patch to the manifest.
+        addPatch \%dstPatches, $p,
+            { url => "$patchesURL/$finalName", hash => $narDiffHash
+            , size => $narDiffSize
+            , basePath => $closest, baseHash => $baseHash
+            , narHash => $narHash, patchType => "nar-bsdiff"
+            };
+    }
+}
+
+
+# Add in any potentially useful patches in the source (namely, those
+# patches that produce either paths in the destination or paths that
+# can be used as the base for other useful patches).
+
+my $changed;
+do {
+    # !!! we repeat this to reach the transitive closure; inefficient
+    $changed = 0;
+
+    foreach my $p (keys %srcPatches) {
+        my $patchList = $srcPatches{$p};
+
+        my $include = 0;
+
+        # Is path $p included in the destination?  If so, include
+        # patches that produce it.
+        $include = 1 if (defined $dstNarFiles{$p});
+
+        # Is path $p a path that serves as a base for paths in the
+        # destination?  If so, include patches that produce it.
+        foreach my $q (keys %dstPatches) {
+            foreach my $patch (@{$dstPatches{$q}}) {
+                # !!! check baseHash
+                $include = 1 if ($p eq $patch->{basePath});
+            }
+        }
+
+        if ($include) {
+            foreach my $patch (@{$patchList}) {
+                $changed = 1 if addPatch \%dstPatches, $p, $patch;
+            }
+        }            
+        
+    }
+    
+} while $changed;
+
+
+# Rewrite the manifest of the destination (with the new patches).
+writeManifest "$dstDir/MANIFEST",
+    \%dstNarFiles, \%dstPatches, \%dstSuccessors;

scripts/nix-channel.in

@@ -53,6 +53,10 @@ sub addChannel {
 sub update {
     readChannels;
 
+    # Get rid of all the old substitutes.
+    system "@bindir@/nix-store --clear-substitutes";
+    die "cannot clear substitutes" if ($? != 0);
+
     # Pull cache manifests.
     foreach my $url (@channels) {
         print "pulling cache manifest from `$url'\n";

scripts/nix-prefetch-url.in

@@ -1,35 +1,41 @@
 #! @shell@ -e
 
 url=$1
+hash=$2
 
 if test -z "$url"; then
     echo "syntax: nix-prefetch-url URL" >&2
     exit 1
 fi
 
-# !!! race
-tmpPath1=@storedir@/nix-prefetch-url-$$
+# Determine the hash, unless it was given.
+if test -z "$hash"; then
 
-# Test whether we have write permission in the store.  If not, fetch
-# to /tmp and don't copy to the store.  This is a hack to make this
-# script at least work somewhat in setuid installations.
-if ! touch $tmpPath1 2> /dev/null; then
-    echo "(cannot write to the store, result won't be cached)" >&2
-    dummyMode=1
-    tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
-fi
+    # !!! race
+    tmpPath1=@storedir@/nix-prefetch-url-$$
 
-# Perform the checkout.
-@curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
+    # Test whether we have write permission in the store.  If not,
+    # fetch to /tmp and don't copy to the store.  This is a hack to
+    # make this script at least work somewhat in setuid installations.
+    if ! touch $tmpPath1 2> /dev/null; then
+        echo "(cannot write to the store, result won't be cached)" >&2
+        dummyMode=1
+        tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
+    fi
 
-# Compute the hash.
-hash=$(@bindir@/nix-hash --flat $tmpPath1)
-echo "hash is $hash" >&2
+    # Perform the checkout.
+    @curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
+
+    # Compute the hash.
+    hash=$(@bindir@/nix-hash --flat $tmpPath1)
+    if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
+
+    # Rename it so that the fetchurl builder can find it.
+    if test "$dummyMode" != 1; then
+        tmpPath2=@storedir@/nix-prefetch-url-$hash
+        test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
+    fi
 
-# Rename it so that the fetchurl builder can find it.
-if test "$dummyMode" != 1; then
-    tmpPath2=@storedir@/nix-prefetch-url-$hash
-    test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
 fi
 
 # Create a Nix expression that does a fetchurl.
@@ -41,9 +47,11 @@ storeExpr=$( \
 # Realise it.
 finalPath=$(@bindir@/nix-store -qnB --force-realise $storeExpr)
     
-echo "path is $finalPath" >&2
+if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
 
-rm -rf $tmpPath1 $tmpPath2 || true
+if test -n "$tmpPath1" -o -n "$tmpPath2"; then
+    rm -rf $tmpPath1 $tmpPath2 || true
+fi
 
 echo $hash
 

scripts/nix-pull.in

@@ -10,53 +10,52 @@ do { $tmpdir = tmpnam(); }
 until mkdir $tmpdir, 0777;
 
 my $manifest = "$tmpdir/manifest";
-my $confFile = "@sysconfdir@/nix/prebuilts.conf";
 
-#END { unlink $manifest; rmdir $tmpdir; }
+END { unlink $manifest; rmdir $tmpdir; }
 
 
 # Obtain URLs either from the command line or from a configuration file.
-my %storePaths2urls;
-my %urls2hashes;
+my %narFiles;
+my %patches;
 my %successors;
-sub doURL {
+
+sub processURL {
     my $url = shift;
-    processURL $manifest, $url, \%storePaths2urls, \%urls2hashes, \%successors;
+
+    $url =~ s/\/$//;
+    print "obtaining list of Nix archives at $url...\n";
+
+    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
+           "'$url' > '$manifest'") == 0
+           or die "curl failed: $?";
+
+    readManifest $manifest, \%narFiles, \%patches, \%successors;
+
+    my $baseName = "unnamed";
+    if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
+        $baseName = $1;
+    }
+
+    my $hash = `@bindir@/nix-hash --flat '$manifest'`
+        or die "cannot hash `$manifest'";
+    chomp $hash;
+    
+    my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest";
+    
+    system("mv '$manifest' '$finalPath'") == 0
+        or die "cannot move `$manifest' to `$finalPath";
 }
-if (scalar @ARGV > 0) {
-    while (@ARGV) {
-        my $url = shift @ARGV;
-	doURL $url;
-    }
-} else {
-    open CONFFILE, "<$confFile";
-    while (<CONFFILE>) {
-        chomp;
-        if (/^\s*(\S+)\s*(\#.*)?$/) {
-            my $url = $1;
-	    doURL $url;
-        }
-    }
-    close CONFFILE;
+
+while (@ARGV) {
+    my $url = shift @ARGV;
+    processURL $url;
 }
 
 
-my $size = scalar (keys %storePaths2urls);
+my $size = scalar (keys %narFiles);
 print "$size store paths in manifest\n";
 
 
-# Instantiate a store expression that builds the substitute program
-# (the program that fetches URLs and unpacks them into the store).
-my $nixExpr =
-    "(import @datadir@/nix/corepkgs/nix-pull) " .
-    "{system = \"@system@\";}";
-
-print STDERR "instantiating store expression...\n";
-my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
-    or die "cannot instantiate Nix expression";
-chomp $storeExpr;
-
-
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
 
@@ -65,13 +64,13 @@ my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --substitute")
 
 close READ;
 
-foreach my $storePath (keys %storePaths2urls) {
-    print WRITE "$storePath\n";
-    print WRITE "$storeExpr\n";
-    print WRITE "/fetch\n";
-    print WRITE "2\n";
-    print WRITE "$storePaths2urls{$storePath}\n";
-    print WRITE "$urls2hashes{$storePaths2urls{$storePath}}\n";
+foreach my $storePath (keys %narFiles) {
+    my $narFileList = $narFiles{$storePath};
+    foreach my $narFile (@{$narFileList}) {
+        print WRITE "$storePath\n";
+        print WRITE "@libexecdir@/nix/download-using-manifests.pl\n";
+        print WRITE "0\n";
+    }
 }
 
 close WRITE;

scripts/nix-push.in

@@ -2,6 +2,7 @@
 
 use strict;
 use POSIX qw(tmpnam);
+use readmanifest;
 
 my $tmpdir;
 do { $tmpdir = tmpnam(); }
@@ -25,7 +26,7 @@ my $manifest_put_url = shift @ARGV;
 
 # From the given store expressions, determine the requisite store
 # paths.
-my %storepaths;
+my %storePaths;
 
 foreach my $storeexpr (@ARGV) {
     die unless $storeexpr =~ /^\//;
@@ -39,12 +40,12 @@ foreach my $storeexpr (@ARGV) {
     while (<PATHS>) {
         chomp;
         die "bad: $_" unless /^\//;
-        $storepaths{$_} = "";
+        $storePaths{$_} = "";
     }
     close PATHS;
 }
 
-my @storepaths = keys %storepaths;
+my @storePaths = keys %storePaths;
 
 
 # For each path, create a Nix expression that turns the path into
@@ -52,14 +53,14 @@ my @storepaths = keys %storepaths;
 open NIX, ">$nixfile";
 print NIX "[";
 
-foreach my $storepath (@storepaths) {
-    die unless ($storepath =~ /\/[0-9a-z]{32}.*$/);
+foreach my $storePath (@storePaths) {
+    die unless ($storePath =~ /\/[0-9a-z]{32}.*$/);
 
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
         "((import @datadir@/nix/corepkgs/nar/nar.nix) " .
-        # !!! $storepath should be represented as a closure
-        "{path = \"$storepath\"; system = \"@system@\";}) ";
+        # !!! $storePath should be represented as a closure
+        "{path = \"$storePath\"; system = \"@system@\";}) ";
     
     print NIX $nixexpr;
 }
@@ -108,53 +109,65 @@ while (scalar @tmp > 0) {
 # Create the manifest.
 print STDERR "creating manifest...\n";
 
-open MANIFEST, ">$manifest";
+my %narFiles;
+my %patches;
+my %successors;
 
 my @nararchives;
-for (my $n = 0; $n < scalar @storepaths; $n++) {
-    my $storepath = $storepaths[$n];
+for (my $n = 0; $n < scalar @storePaths; $n++) {
+    my $storePath = $storePaths[$n];
     my $nardir = $narpaths[$n];
     
-    $storepath =~ /\/([^\/]*)$/;
+    $storePath =~ /\/([^\/]*)$/;
     my $basename = $1;
     defined $basename or die;
 
     my $narname = "$basename.nar.bz2";
 
     my $narfile = "$nardir/$narname";
-    (-f $narfile) or die "narfile for $storepath not found";
+    (-f $narfile) or die "narfile for $storePath not found";
     push @nararchives, $narfile;
 
-    open MD5, "$nardir/md5" or die "cannot open hash";
-    my $hash = <MD5>;
-    chomp $hash;
-    $hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
+    open MD5, "$nardir/narbz2-hash" or die "cannot open narbz2-hash";
+    my $narbz2Hash = <MD5>;
+    chomp $narbz2Hash;
+    $narbz2Hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
     close MD5;
 
-    print MANIFEST "{\n";
-    print MANIFEST "  StorePath: $storepath\n";
-    print MANIFEST "  NarURL: $archives_get_url/$narname\n";
-    print MANIFEST "  MD5: $hash\n";
+    open MD5, "$nardir/nar-hash" or die "cannot open nar-hash";
+    my $narHash = <MD5>;
+    chomp $narHash;
+    $narHash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
+    close MD5;
+    
+    my $narbz2Size = (stat $narfile)[7];
 
-    if ($storepath =~ /\.store$/) {
-        open PREDS, "@bindir@/nix-store --query --predecessors $storepath |" or die "cannot run nix";
+    $narFiles{$storePath} = [
+        { url => $archives_get_url/$narname
+        , hash => $narbz2Hash
+        , size => $narbz2Size
+        , narHash => $narHash
+        }
+    ];
+                            
+    if ($storePath =~ /\.store$/) {
+        open PREDS, "@bindir@/nix-store --query --predecessors $storePath |" or die "cannot run nix";
         while (<PREDS>) {
             chomp;
             die unless (/^\//);
             my $pred = $_;
             # Only include predecessors that are themselves being
             # pushed.
-            if (defined $storepaths{$pred}) {
-                print MANIFEST "  SuccOf: $pred\n";
+            if (defined $storePaths{$pred}) {
+                $successors{$pred} = $storePath;
             }
         }
         close PREDS;
     }
 
-    print MANIFEST "}\n";
 }
 
-close MANIFEST;
+writeManifest $manifest, \%narFiles, \%patches, \%successors;
 
 
 # Upload the archives.

scripts/optimise-store.pl

@@ -0,0 +1,69 @@
+#! /usr/bin/perl -w
+
+use strict;
+
+{ my $ofh = select STDOUT;
+  $| = 1;
+  select $ofh;
+}
+
+#my @paths = ("/nix/store/caef3a49150506d233f474322a824e50-glibc-2.3.3", "/nix/store/a8a9d585d1ad4b1bc911be7743b3b996-glibc-2.3.3");
+my @paths = ("/nix/store");
+
+my $tmpfile = "/tmp/nix-optimise-hash-list";
+#my $tmpfile = "/data/nix-optimise-hash-list";
+
+system("find @paths -type f -print0 | xargs -0 md5sum -- > $tmpfile") == 0
+    or die "cannot hash store files";
+
+system("sort $tmpfile > $tmpfile.sorted") == 0
+    or die "cannot sort list";
+
+open LIST, "<$tmpfile.sorted" or die;
+
+my $prevFile;
+my $prevHash;
+
+my $totalSpace = 0;
+my $savedSpace = 0;
+
+my $files = 0;
+
+while (<LIST>) {
+#    print "D";
+    /^([0-9a-f]*)\s+(.*)$/ or die;
+    my $curFile = $2;
+    my $curHash = $1;
+
+#    print "A";
+    my $fileSize = (stat $curFile)[7];
+#    print "B";
+#    my $fileSize = 1;
+    $totalSpace += $fileSize;
+
+    if (defined $prevHash && $curHash eq $prevHash) {
+        
+#        print "$curFile = $prevFile\n";
+
+        $savedSpace += $fileSize;
+        
+    } else {
+        $prevFile = $curFile;
+        $prevHash = $curHash;
+    }
+
+    print "." if ($files++ % 100 == 0);
+    #print ".";
+
+#    print "C";
+}
+
+print "\n";
+
+print "total space = $totalSpace\n";
+print "saved space = $savedSpace\n";
+my $savings = ($savedSpace / $totalSpace) * 100.0;
+print "savings = $savings %\n";
+
+
+close LIST;

scripts/readmanifest.pm.in

@@ -1,27 +1,54 @@
 use strict;
 
-sub processURL {
+
+sub addPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $patch = shift;
+
+    $$patches{$storePath} = []
+        unless defined $$patches{$storePath};
+
+    my $patchList = $$patches{$storePath};
+
+    my $found = 0;
+    foreach my $patch2 (@{$patchList}) {
+        if ($patch2->{url} eq $patch->{url}) {
+            if ($patch2->{hash} eq $patch->{hash}) {
+                $found = 1 if ($patch2->{basePath} eq $patch->{basePath});
+            } else {
+                die "conflicting hashes for URL $patch->{url}, " .
+                    "namely $patch2->{hash} and $patch->{hash}";
+            }
+        }
+    }
+    
+    push @{$patchList}, $patch if !$found;
+
+    return !$found;
+}
+
+
+sub readManifest {
     my $manifest = shift;
-    my $url = shift;
-    my $storePaths2urls = shift;
-    my $urls2hashes = shift;
+    my $narFiles = shift;
+    my $patches = shift;
     my $successors = shift;
 
-    $url =~ s/\/$//;
-    print "obtaining list of Nix archives at $url...\n";
-
-    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
-           "'$url' > '$manifest'") == 0
-           or die "curl failed: $?";
-        
     open MANIFEST, "<$manifest";
 
     my $inside = 0;
+    my $type;
 
     my $storePath;
-    my $narurl;
+    my $url;
     my $hash;
+    my $size;
     my @preds;
+    my $basePath;
+    my $baseHash;
+    my $patchType;
+    my $narHash;
 
     while (<MANIFEST>) {
         chomp;
@@ -29,39 +56,82 @@ sub processURL {
         next if (/^$/);
 
         if (!$inside) {
-            if (/^\{$/) { 
+
+            if (/^\s*(\w*)\s*\{$/) {
+                $type = $1;
+                $type = "narfile" if $type eq "";
                 $inside = 1;
                 undef $storePath;
-                undef $narurl;
+                undef $url;
                 undef $hash;
+                undef $size;
                 @preds = ();
+                undef $narHash;
+                undef $basePath;
+                undef $baseHash;
+                undef $patchType;
 	    }
-            else { die "bad line: $_"; }
+
         } else {
+            
             if (/^\}$/) {
                 $inside = 0;
 
-		$$storePaths2urls{$storePath} = $narurl;
-		$$urls2hashes{$narurl} = $hash;
+                if ($type eq "narfile") {
 
-                foreach my $p (@preds) {
-		    $$successors{$p} = $storePath;
+                    $$narFiles{$storePath} = []
+                        unless defined $$narFiles{$storePath};
+
+                    my $narFileList = $$narFiles{$storePath};
+
+                    my $found = 0;
+                    foreach my $narFile (@{$narFileList}) {
+                        if ($narFile->{url} eq $url) {
+                            if ($narFile->{hash} eq $hash) {
+                                $found = 1;
+                            } else {
+                                die "conflicting hashes for URL $url, " .
+                                    "namely $narFile->{hash} and $hash";
+                            }
+                        }
+                    }
+                    if (!$found) {
+                        push @{$narFileList},
+                            { url => $url, hash => $hash, size => $size
+                            , narHash => $narHash
+                            };
+                    }
+                
+                    foreach my $p (@preds) {
+                        $$successors{$p} = $storePath;
+                    }
+
+                }
+
+                elsif ($type eq "patch") {
+                    addPatch $patches, $storePath,
+                        { url => $url, hash => $hash, size => $size
+                        , basePath => $basePath, baseHash => $baseHash
+                        , narHash => $narHash, patchType => $patchType
+                        };
                 }
 
             }
-            elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
-                $storePath = $1;
-            }
-            elsif (/^\s*NarURL:\s*(\S+)\s*$/) {
-                $narurl = $1;
-	    }
-            elsif (/^\s*MD5:\s*(\S+)\s*$/) {
-                $hash = $1;
-            }
-            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
-                push @preds, $1;
-            }
-            else { die "bad line: $_"; }
+            
+            elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; }
+            elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; }
+            elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; }
+            elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; }
+            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { push @preds, $1; }
+            elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
+            elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
+            elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
+            elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+
+            # Compatibility;
+            elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
+            elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = $1; }
+
         }
     }
 
@@ -69,4 +139,55 @@ sub processURL {
 }
 
 
+sub writeManifest
+{
+    my $manifest = shift;
+    my $narFiles = shift;
+    my $patches = shift;
+    my $successors = shift;
+
+    open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
+
+    foreach my $storePath (keys %{$narFiles}) {
+        my $narFileList = $$narFiles{$storePath};
+        foreach my $narFile (@{$narFileList}) {
+            print MANIFEST "{\n";
+            print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  NarURL: $narFile->{url}\n";
+            print MANIFEST "  MD5: $narFile->{hash}\n";
+            print MANIFEST "  NarHash: $narFile->{narHash}\n";
+            print MANIFEST "  Size: $narFile->{size}\n";
+            foreach my $p (keys %{$successors}) { # !!! quadratic
+                if ($$successors{$p} eq $storePath) {
+                    print MANIFEST "  SuccOf: $p\n";
+                }
+            }
+            print MANIFEST "}\n";
+        }
+    }
+    
+    foreach my $storePath (keys %{$patches}) {
+        my $patchList = $$patches{$storePath};
+        foreach my $patch (@{$patchList}) {
+            print MANIFEST "patch {\n";
+            print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  NarURL: $patch->{url}\n";
+            print MANIFEST "  MD5: $patch->{hash}\n";
+            print MANIFEST "  NarHash: $patch->{narHash}\n";
+            print MANIFEST "  Size: $patch->{size}\n";
+            print MANIFEST "  BasePath: $patch->{basePath}\n";
+            print MANIFEST "  BaseHash: $patch->{baseHash}\n";
+            print MANIFEST "  Type: $patch->{patchType}\n";
+            print MANIFEST "}\n";
+        }
+    }
+    
+    
+    close MANIFEST;
+
+    rename("$manifest.tmp", $manifest)
+        or die "cannot rename $manifest.tmp: $!";
+}
+
+
 return 1;

scripts/remove-patches.pl

@@ -0,0 +1,19 @@
+#! /usr/bin/perl -w -I/home/eelco/nix/scripts
+
+use strict;
+use readmanifest;
+
+for my $p (@ARGV) {
+
+    my %narFiles;
+    my %patches;
+    my %successors;
+
+    readManifest $p,
+        \%narFiles, \%patches, \%successors;
+
+    %patches = ();
+    
+    writeManifest $p,
+        \%narFiles, \%patches, \%successors;
+}

scripts/update-manifest.pl

@@ -0,0 +1,53 @@
+#! /usr/bin/perl -w -I.
+
+use strict;
+use readmanifest;
+
+die unless scalar @ARGV == 2;
+
+my $cache = $ARGV[0];
+my $manifest = $ARGV[1];
+my %narFiles;
+my %patches;
+my %successors;
+
+readManifest $manifest, \%narFiles, \%patches, \%successors;
+
+foreach my $storePath (keys %narFiles) {
+    my $narFileList = $narFiles{$storePath};
+
+    foreach my $narFile (@{$narFileList}) {
+        if (!defined $narFile->{size} or
+            !defined $narFile->{narHash})
+        {
+            $narFile->{url} =~ /\/([^\/]+)$/;
+            die unless defined $1;
+            my $fn = "$cache/$1";
+            
+            my @info = stat $fn or die;
+            $narFile->{size} = $info[7];
+
+            my $narHash;
+            my $hashFile = "$fn.NARHASH";
+            if (-e $hashFile) {
+                open HASH, "<$hashFile" or die;
+                $narHash = <HASH>;
+                close HASH;
+            } else {
+                print "$fn\n";
+                $narHash = `bunzip2 < '$fn' | nix-hash --flat /dev/stdin` or die;
+                open HASH, ">$hashFile" or die;
+                print HASH $narHash;
+                close HASH;
+            }
+            chomp $narHash;
+            $narFile->{narHash} = $narHash;
+        }
+    }
+}
+
+if (! -e "$manifest.backup") {
+    system "mv --reply=no '$manifest' '$manifest.backup'";
+}
+
+writeManifest $manifest, \%narFiles, \%patches, \%successors;

src/Makefile.am

@@ -1,5 +1,5 @@
 SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
- libexpr nix-instantiate nix-env log2xml
+ libexpr nix-instantiate nix-env log2xml bsdiff-4.2
 
 EXTRA_DIST = aterm-helper.pl
 

src/bsdiff-4.2/LICENSE

@@ -0,0 +1,121 @@
+BSD Protection License
+February 2002
+
+Preamble
+--------
+
+The Berkeley Software Distribution ("BSD") license has proven very effective
+over the years at allowing for a wide spread of work throughout both
+commercial and non-commercial products.  For programmers whose primary
+intention is to improve the general quality of available software, it is
+arguable that there is no better license than the BSD license, as it
+permits improvements to be used wherever they will help, without idealogical
+or metallic constraint.
+
+This is of particular value to those who produce reference implementations
+of proposed standards: The case of TCP/IP clearly illustrates that freely
+and universally available implementations leads the rapid acceptance of
+standards -- often even being used instead of a de jure standard (eg, OSI
+network models).
+
+With the rapid proliferation of software licensed under the GNU General
+Public License, however, the continued success of this role is called into
+question.  Given that the inclusion of a few lines of "GPL-tainted" work
+into a larger body of work will result in restricted distribution -- and
+given that further work will likely build upon the "tainted" portions,
+making them difficult to remove at a future date -- there are inevitable
+circumstances where authors would, in order to protect their goal of
+providing for the widespread usage of their work, wish to guard against
+such "GPL-taint".
+
+In addition, one can imagine that companies which operate by producing and
+selling (possibly closed-source) code would wish to protect themselves
+against the rise of a GPL-licensed competitor.  While under existing
+licenses this would mean not releasing their code under any form of open
+license, if a license existed under which they could incorporate any
+improvements back into their own (commercial) products then they might be
+far more willing to provide for non-closed distribution.
+
+For the above reasons, we put forth this "BSD Protection License": A
+license designed to retain the freedom granted by the BSD license to use
+licensed works in a wide variety of settings, both non-commercial and
+commercial, while protecting the work from having future contributors
+restrict that freedom.
+
+The precise terms and conditions for copying, distribution, and
+modification follow.
+
+BSD PROTECTION LICENSE
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION, AND MODIFICATION
+----------------------------------------------------------------
+
+0. Definitions.
+   a) "Program", below, refers to any program or work distributed under
+      the terms of this license.
+   b) A "work based on the Program", below, refers to either the Program
+      or any derivative work under copyright law.
+   c) "Modification", below, refers to the act of creating derivative works.
+   d) "You", below, refers to each licensee.
+
+1. Scope.
+   This license governs the copying, distribution, and modification of the
+   Program.  Other activities are outside the scope of this license; The
+   act of running the Program is not restricted, and the output from the
+   Program is covered only if its contents constitute a work based on the
+   Program.
+
+2. Verbatim copies.
+   You may copy and distribute verbatim copies of the Program as you
+   receive it, in any medium, provided that you conspicuously and
+   appropriately publish on each copy an appropriate copyright notice; keep
+   intact all the notices that refer to this License and to the absence of
+   any warranty; and give any other recipients of the Program a copy of this
+   License along with the Program.
+
+3. Modification and redistribution under closed license.
+   You may modify your copy or copies of the Program, and distribute
+   the resulting derivative works, provided that you meet the
+   following conditions:
+   a) The copyright notice and disclaimer on the Program must be reproduced
+      and included in the source code, documentation, and/or other materials
+      provided in a manner in which such notices are normally distributed.
+   b) The derivative work must be clearly identified as such, in order that
+      it may not be confused with the original work.
+   c) The license under which the derivative work is distributed must
+      expressly prohibit the distribution of further derivative works.
+
+4. Modification and redistribution under open license.
+   You may modify your copy or copies of the Program, and distribute
+   the resulting derivative works, provided that you meet the
+   following conditions:
+   a) The copyright notice and disclaimer on the Program must be reproduced
+      and included in the source code, documentation, and/or other materials
+      provided in a manner in which such notices are normally distributed.
+   b) You must clearly indicate the nature and date of any changes made
+      to the Program.  The full details need not necessarily be included in
+      the individual modified files, provided that each modified file is
+      clearly marked as such and instructions are included on where the
+      full details of the modifications may be found.
+   c) You must cause any work that you distribute or publish, that in whole
+      or in part contains or is derived from the Program or any part
+      thereof, to be licensed as a whole at no charge to all third
+      parties under the terms of this License.
+
+5. Implied acceptance.
+   You may not copy or distribute the Program or any derivative works except
+   as expressly provided under this license.  Consequently, any such action
+   will be taken as implied acceptance of the terms of this license.
+
+6. NO WARRANTY.
+   THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+   INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+   AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL
+   THE COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+   REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE FOR ANY DIRECT,
+   INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+   ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING, BUT
+   NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+   TORT, EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+   POSSIBILITY OF SUCH DAMAGES.

src/bsdiff-4.2/Makefile.am

@@ -0,0 +1,7 @@
+libexec_PROGRAMS = bsdiff bspatch
+
+bsdiff_SOURCES = bsdiff.c
+
+bspatch_SOURCES = bspatch.c
+
+AM_CFLAGS = -O3 -DBZIP2=\"$(bzip2)\"

src/bsdiff-4.2/Makefile.old

@@ -0,0 +1,18 @@
+CFLAGS		+=	-O3
+.ifdef BZIP2
+CFLAGS		+=	-DBZIP2=\"${BZIP2}\"
+.endif
+
+PREFIX		?=	/usr/local
+INSTALL_PROGRAM	?=	${INSTALL} -c -s -m 555
+INSTALL_MAN	?=	${INSTALL} -c -m 444
+
+all:		bsdiff bspatch
+bsdiff:		bsdiff.c
+bspatch:	bspatch.c
+
+install:
+	${INSTALL_PROGRAM} bsdiff bspatch ${PREFIX}/bin
+.ifndef WITHOUT_MAN
+	${INSTALL_MAN} bsdiff.1 bspatch.1 ${PREFIX}/man/man1
+.endif

src/bsdiff-4.2/bsdiff.1

@@ -0,0 +1,36 @@
+.Dd May 18, 2003
+.Dt BSDIFF 1
+.Os FreeBSD
+.Sh NAME
+.Nm bsdiff
+.Nd generate a patch between two binary files
+.Sh SYNOPSIS
+.Nm
+.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
+.Sh DESCRIPTION
+.Nm
+compares
+.Ao Ar oldfile Ac
+to
+.Ao Ar newfile Ac
+and writes to
+.Ao Ar patchfile Ac
+a binary patch suitable for use by bspatch(1).
+When
+.Ao Ar oldfile Ac
+and
+.Ao Ar newfile Ac
+are two versions of an executable program, the
+patches produced are on average a factor of five smaller
+than those produced by any other binary patch tool known
+to the author.
+.Pp
+.Nm
+uses memory equal to 17 times the size of 
+.Ao Ar oldfile Ac ,
+and requires
+an absolute minimum working set size of 8 times the size of oldfile.
+.Sh SEE ALSO
+.Xr bspatch 1
+.Sh AUTHORS
+.An Colin Percival Aq cperciva@daemonology.net

src/bsdiff-4.2/bsdiff.c

@@ -0,0 +1,385 @@
+/*
+  bsdiff.c -- Binary patch generator.
+
+  Copyright 2003 Colin Percival
+
+  For the terms under which this work may be distributed, please see
+  the adjoining file "LICENSE".
+*/
+
+#ifndef BZIP2
+#define BZIP2 "/usr/bin/bzip2"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <err.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+#define MIN(x,y) (((x)<(y)) ? (x) : (y))
+
+void split(off_t *I,off_t *V,off_t start,off_t len,off_t h)
+{
+	off_t i,j,k,x,tmp,jj,kk;
+
+	if(len<16) {
+		for(k=start;k<start+len;k+=j) {
+			j=1;x=V[I[k]+h];
+			for(i=1;k+i<start+len;i++) {
+				if(V[I[k+i]+h]<x) {
+					x=V[I[k+i]+h];
+					j=0;
+				};
+				if(V[I[k+i]+h]==x) {
+					tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp;
+					j++;
+				};
+			};
+			for(i=0;i<j;i++) V[I[k+i]]=k+j-1;
+			if(j==1) I[k]=-1;
+		};
+		return;
+	};
+
+	x=V[I[start+len/2]+h];
+	jj=0;kk=0;
+	for(i=start;i<start+len;i++) {
+		if(V[I[i]+h]<x) jj++;
+		if(V[I[i]+h]==x) kk++;
+	};
+	jj+=start;kk+=jj;
+
+	i=start;j=0;k=0;
+	while(i<jj) {
+		if(V[I[i]+h]<x) {
+			i++;
+		} else if(V[I[i]+h]==x) {
+			tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp;
+			j++;
+		} else {
+			tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp;
+			k++;
+		};
+	};
+
+	while(jj+j<kk) {
+		if(V[I[jj+j]+h]==x) {
+			j++;
+		} else {
+			tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp;
+			k++;
+		};
+	};
+
+	if(jj>start) split(I,V,start,jj-start,h);
+
+	for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1;
+	if(jj==kk-1) I[jj]=-1;
+
+	if(start+len>kk) split(I,V,kk,start+len-kk,h);
+}
+
+void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize)
+{
+	off_t buckets[256];
+	off_t i,h,len;
+
+	for(i=0;i<256;i++) buckets[i]=0;
+	for(i=0;i<oldsize;i++) buckets[old[i]]++;
+	for(i=1;i<256;i++) buckets[i]+=buckets[i-1];
+	for(i=255;i>0;i--) buckets[i]=buckets[i-1];
+	buckets[0]=0;
+
+	for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i;
+	I[0]=oldsize;
+	for(i=0;i<oldsize;i++) V[i]=buckets[old[i]];
+	V[oldsize]=0;
+	for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1;
+	I[0]=-1;
+
+	for(h=1;I[0]!=-(oldsize+1);h+=h) {
+		len=0;
+		for(i=0;i<oldsize+1;) {
+			if(I[i]<0) {
+				len-=I[i];
+				i-=I[i];
+			} else {
+				if(len) I[i-len]=-len;
+				len=V[I[i]]+1-i;
+				split(I,V,i,len,h);
+				i+=len;
+				len=0;
+			};
+		};
+		if(len) I[i-len]=-len;
+	};
+
+	for(i=0;i<oldsize+1;i++) I[V[i]]=i;
+}
+
+off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize)
+{
+	off_t i;
+
+	for(i=0;(i<oldsize)&&(i<newsize);i++)
+		if(old[i]!=new[i]) break;
+
+	return i;
+}
+
+off_t search(off_t *I,u_char *old,off_t oldsize,
+		u_char *new,off_t newsize,off_t st,off_t en,off_t *pos)
+{
+	off_t x,y;
+
+	if(en-st<2) {
+		x=matchlen(old+I[st],oldsize-I[st],new,newsize);
+		y=matchlen(old+I[en],oldsize-I[en],new,newsize);
+
+		if(x>y) {
+			*pos=I[st];
+			return x;
+		} else {
+			*pos=I[en];
+			return y;
+		}
+	};
+
+	x=st+(en-st)/2;
+	if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) {
+		return search(I,old,oldsize,new,newsize,x,en,pos);
+	} else {
+		return search(I,old,oldsize,new,newsize,st,x,pos);
+	};
+}
+
+void offtout(off_t x,u_char *buf)
+{
+	off_t y;
+
+	if(x<0) y=-x; else y=x;
+
+		buf[0]=y%256;y-=buf[0];
+	y=y/256;buf[1]=y%256;y-=buf[1];
+	y=y/256;buf[2]=y%256;y-=buf[2];
+	y=y/256;buf[3]=y%256;y-=buf[3];
+	y=y/256;buf[4]=y%256;y-=buf[4];
+	y=y/256;buf[5]=y%256;y-=buf[5];
+	y=y/256;buf[6]=y%256;y-=buf[6];
+	y=y/256;buf[7]=y%256;
+
+	if(x<0) buf[7]|=0x80;
+}
+
+int main(int argc,char *argv[])
+{
+	int fd,p[2];
+	pid_t pid;
+	u_char *old,*new;
+	off_t oldsize,newsize;
+	off_t *I,*V;
+
+	off_t scan,pos,len;
+	off_t lastscan,lastpos,lastoffset;
+	off_t oldscore,scsc;
+
+	off_t s,Sf,lenf,Sb,lenb;
+	off_t overlap,Ss,lens;
+	off_t i;
+
+	off_t dblen,eblen;
+	u_char *db,*eb;
+
+	u_char buf[8];
+	u_char header[32];
+
+	if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
+
+	/* Allocate oldsize+1 bytes instead of oldsize bytes to ensure
+		that we never try to malloc(0) and get a NULL pointer */
+	if(((fd=open(argv[1],O_RDONLY,0))<0) ||
+		((oldsize=lseek(fd,0,SEEK_END))==-1) ||
+		((old=malloc(oldsize+1))==NULL) ||
+		(lseek(fd,0,SEEK_SET)!=0) ||
+		(read(fd,old,oldsize)!=oldsize) ||
+		(close(fd)==-1)) err(1,"%s",argv[1]);
+
+	if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) ||
+		((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL);
+
+	qsufsort(I,V,old,oldsize);
+
+	free(V);
+
+	/* Allocate newsize+1 bytes instead of newsize bytes to ensure
+		that we never try to malloc(0) and get a NULL pointer */
+	if(((fd=open(argv[2],O_RDONLY,0))<0) ||
+		((newsize=lseek(fd,0,SEEK_END))==-1) ||
+		((new=malloc(newsize+1))==NULL) ||
+		(lseek(fd,0,SEEK_SET)!=0) ||
+		(read(fd,new,newsize)!=newsize) ||
+		(close(fd)==-1)) err(1,"%s",argv[2]);
+
+	if(((db=malloc(newsize+1))==NULL) ||
+		((eb=malloc(newsize+1))==NULL)) err(1,NULL);
+	dblen=0;
+	eblen=0;
+
+	if((fd=open(argv[3],O_CREAT|O_TRUNC|O_WRONLY,0666))<0)
+		err(1,"%s",argv[3]);
+
+	/* Header is
+		0	8	 "BSDIFF40"
+		8	8	length of bzip2ed ctrl block
+		16	8	length of bzip2ed diff block
+		24	8	length of new file */
+	/* File is
+		0	32	Header
+		32	??	Bzip2ed ctrl block
+		??	??	Bzip2ed diff block
+		??	??	Bzip2ed extra block */
+	memcpy(header,"BSDIFF40",8);
+	memset(header+8,0,24);
+	if(write(fd,header,32)!=32) err(1,"%s",argv[3]);
+
+	if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
+	if(pid==0) {
+		if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
+			(dup2(p[0],0)==-1) || (close(fd)==-1) ||
+			(close(p[0])==-1) || (close(p[1])==-1))
+			err(1,NULL);
+		execl(BZIP2,BZIP2,"-zc",NULL);
+		err(1,"%s",BZIP2);
+	};
+	if(close(p[0])==-1) err(1,NULL);
+
+	scan=0;len=0;
+	lastscan=0;lastpos=0;lastoffset=0;
+	while(scan<newsize) {
+		oldscore=0;
+
+		for(scsc=scan+=len;scan<newsize;scan++) {
+			len=search(I,old,oldsize,new+scan,newsize-scan,
+					0,oldsize,&pos);
+
+			for(;scsc<scan+len;scsc++)
+			if((scsc+lastoffset<oldsize) &&
+				(old[scsc+lastoffset] == new[scsc]))
+				oldscore++;
+
+			if(((len==oldscore) && (len!=0)) || 
+				(len>oldscore+8)) break;
+
+			if((scan+lastoffset<oldsize) &&
+				(old[scan+lastoffset] == new[scan]))
+				oldscore--;
+		};
+
+		if((len!=oldscore) || (scan==newsize)) {
+			s=0;Sf=0;lenf=0;
+			for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
+				if(old[lastpos+i]==new[lastscan+i]) s++;
+				i++;
+				if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
+			};
+
+			lenb=0;
+			if(scan<newsize) {
+				s=0;Sb=0;
+				for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
+					if(old[pos-i]==new[scan-i]) s++;
+					if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
+				};
+			};
+
+			if(lastscan+lenf>scan-lenb) {
+				overlap=(lastscan+lenf)-(scan-lenb);
+				s=0;Ss=0;lens=0;
+				for(i=0;i<overlap;i++) {
+					if(new[lastscan+lenf-overlap+i]==
+					   old[lastpos+lenf-overlap+i]) s++;
+					if(new[scan-lenb+i]==
+					   old[pos-lenb+i]) s--;
+					if(s>Ss) { Ss=s; lens=i+1; };
+				};
+
+				lenf+=lens-overlap;
+				lenb-=lens;
+			};
+
+			for(i=0;i<lenf;i++)
+				db[dblen+i]=new[lastscan+i]-old[lastpos+i];
+			for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
+				eb[eblen+i]=new[lastscan+lenf+i];
+
+			dblen+=lenf;
+			eblen+=(scan-lenb)-(lastscan+lenf);
+
+			offtout(lenf,buf);
+			if(write(p[1],buf,8)!=8) err(1,NULL);
+			offtout((scan-lenb)-(lastscan+lenf),buf);
+			if(write(p[1],buf,8)!=8) err(1,NULL);
+			offtout((pos-lenb)-(lastpos+lenf),buf);
+			if(write(p[1],buf,8)!=8) err(1,NULL);
+
+			lastscan=scan-lenb;
+			lastpos=pos-lenb;
+			lastoffset=pos-scan;
+		};
+	};
+
+	if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
+
+	if((len=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
+	offtout(len-32,buf);
+	if((lseek(fd,8,SEEK_SET)!=8) || (write(fd,buf,8)!=8))
+		err(1,"%s",argv[3]);
+	offtout(newsize,buf);
+	if((lseek(fd,24,SEEK_SET)!=24) || (write(fd,buf,8)!=8))
+		err(1,"%s",argv[3]);
+
+	if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
+	if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
+	if(pid==0) {
+		if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
+			(dup2(p[0],0)==-1) || (close(fd)==-1) ||
+			(close(p[0])==-1) || (close(p[1])==-1))
+			err(1,NULL);
+		execl(BZIP2,BZIP2,"-zc",NULL);
+		err(1,"%s",BZIP2);
+	};
+	if(close(p[0])==-1) err(1,NULL);
+	if(write(p[1],db,dblen)!=dblen) err(1,NULL);
+	if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
+
+	if((newsize=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
+	offtout(newsize-len,buf);
+	if((lseek(fd,16,SEEK_SET)!=16) || (write(fd,buf,8)!=8))
+		err(1,"%s",argv[3]);
+
+	if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
+	if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
+	if(pid==0) {
+		if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
+			(dup2(p[0],0)==-1) || (close(fd)==-1) ||
+			(close(p[0])==-1) || (close(p[1])==-1))
+			err(1,NULL);
+		execl(BZIP2,BZIP2,"-zc",NULL);
+		err(1,"%s",BZIP2);
+	};
+	if(close(p[0])==-1) err(1,NULL);
+	if(write(p[1],eb,eblen)!=eblen) err(1,NULL);
+	if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
+	if(close(fd)==-1) err(1,"%s",argv[3]);
+
+	free(db);
+	free(eb);
+	free(I);
+	free(old);
+	free(new);
+
+	return 0;
+}

src/bsdiff-4.2/bspatch.1

@@ -0,0 +1,32 @@
+.Dd May 18, 2003
+.Dt BSPATCH 1
+.Os FreeBSD
+.Sh NAME
+.Nm bspatch
+.Nd apply a patch built with bsdiff(1)
+.Sh SYNOPSIS
+.Nm
+.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
+.Sh DESCRIPTION
+.Nm
+generates
+.Ao Ar newfile Ac
+from
+.Ao Ar oldfile Ac
+and
+.Ao Ar patchfile Ac
+where
+.Ao Ar patchfile Ac
+is a binary patch built by bsdiff(1).
+.Pp
+.Nm
+uses memory equal to the size of 
+.Ao Ar oldfile Ac
+plus the size of 
+.Ao Ar newfile Ac ,
+but can tolerate a very small working set without a dramatic loss
+of performance.
+.Sh SEE ALSO
+.Xr bsdiff 1
+.Sh AUTHORS
+.An Colin Percival Aq cperciva@daemonology.net

src/bsdiff-4.2/bspatch.c

@@ -0,0 +1,216 @@
+/*
+  .c -- Binary patcher
+
+  Copyright 2003,2004 Colin Percival
+
+  For the terms under which this work may be distributed, please see
+  the adjoining file "LICENSE".
+*/
+
+#ifndef BZIP2
+#define BZIP2 "/usr/bin/bzip2"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+ssize_t loopread(int d,void *buf,size_t nbytes)
+{
+	ssize_t ptr,lenread;
+
+	for(ptr=0;ptr<nbytes;ptr+=lenread) {
+		lenread=read(d,buf+ptr,nbytes-ptr);
+		if(lenread==0) return ptr;
+		if(lenread==-1) return -1;
+	};
+	return ptr;
+}
+
+int bz2read(int fd,off_t offset,off_t len,char * fname,pid_t * pids)
+{
+	int p0[2],p1[2];
+	u_char * data;
+
+	if((pipe(p0)==-1) || (pipe(p1)==-1)) err(1,NULL);
+
+	if((pids[0]=fork())==-1) err(1,NULL);
+	if(pids[0]==0) {
+		if(close(0) || close(1) || close(p0[0]) ||
+			close(p1[0]) || close(p1[1])) err(1,NULL);
+		if((data=malloc(len+1))==NULL) err(1,NULL);
+		if((pread(fd,data,len,offset)!=len) || close(fd))
+			err(1,"%s",fname);
+		if((write(p0[1],data,len)!=len) || close(p0[1]))
+			err(1,NULL);
+		free(data);
+		_exit(0);
+	};
+
+	if((pids[1]=fork())==-1) err(1,NULL);
+	if(pids[1]==0) {
+		if(close(0) || close(1) || close(p0[1]) ||
+			close(p1[0])) err(1,NULL);
+		if((dup2(p0[0],0)==-1) || close(p0[0])) err(1,NULL);
+		if((dup2(p1[1],1)==-1) || close(p1[1])) err(1,NULL);
+		if(close(fd)==-1) err(1,"%s",fname);
+
+		execl(BZIP2,BZIP2,"-dc",NULL);
+		err(1,"%s",BZIP2);
+	};
+
+	if(close(p0[0]) || close(p0[1]) || close(p1[1])) err(1,NULL);
+
+	return p1[0];
+}
+
+off_t offtin(u_char *buf)
+{
+	off_t y;
+
+	y=buf[7]&0x7F;
+	y=y*256;y+=buf[6];
+	y=y*256;y+=buf[5];
+	y=y*256;y+=buf[4];
+	y=y*256;y+=buf[3];
+	y=y*256;y+=buf[2];
+	y=y*256;y+=buf[1];
+	y=y*256;y+=buf[0];
+
+	if(buf[7]&0x80) y=-y;
+
+	return y;
+}
+
+int main(int argc,char * argv[])
+{
+	int fd,ctrlpipe,diffpipe,extrapipe;
+	pid_t pids[6];
+	ssize_t patchsize,oldsize,newsize;
+	ssize_t bzctrllen,bzdatalen;
+	u_char header[32],buf[8];
+	int version=0;
+	u_char *old, *new;
+	off_t oldpos,newpos;
+	off_t ctrl[3];
+	off_t lenread;
+	off_t i;
+
+	if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
+
+	if(((fd=open(argv[3],O_RDONLY,0))<0) ||
+		((patchsize=lseek(fd,0,SEEK_END))==-1) ||
+		(lseek(fd,0,SEEK_SET)!=0)) err(1,"%s",argv[3]);
+	if(patchsize<32) errx(1,"Corrupt patch\n");
+
+	/*
+	  Ok, this is going to be messy.  There are two different patch
+	formats which we need to support.
+
+	  The old format (pre-4.0) is:
+		0	8	"QSUFDIFF" or "BSDIFF30"
+		8	8	X
+		16	8	Y
+		24	8	sizeof(newfile)
+		32	X	bzip2(control block)
+		32+X	Y	bzip2(data block)
+	with control block a set of pairs (x,y) meaning "seek forward
+	in oldfile by y bytes, and add the next x bytes to x bytes from
+	the data block".
+
+	  The new format (4.0) is:
+		0	8	"BSDIFF40"
+		8	8	X
+		16	8	Y
+		24	8	sizeof(newfile)
+		32	X	bzip2(control block)
+		32+X	Y	bzip2(diff block)
+		32+X+Y	???	bzip2(extra block)
+	with control block a set of triples (x,y,z) meaning "add x bytes
+	from oldfile to x bytes from the diff block; copy y bytes from the
+	extra block; seek forwards in oldfile by z bytes".
+	*/
+
+	if(read(fd,header,32)!=32) err(1,"%s",argv[3]);
+	if(memcmp(header,"QSUFDIFF",8)==0) version=1;
+	if(memcmp(header,"BSDIFF30",8)==0) version=1;
+	if(memcmp(header,"BSDIFF40",8)==0) version=2;
+
+	if(!version) errx(1,"Corrupt patch\n");
+
+	bzctrllen=offtin(header+8);
+	bzdatalen=offtin(header+16);
+	newsize=offtin(header+24);
+	if((bzctrllen<0) || (bzdatalen<0) || (newsize<0) ||
+		((version==1) && (32+bzctrllen+bzdatalen!=patchsize)))
+		errx(1,"Corrupt patch\n");
+
+	ctrlpipe=bz2read(fd,32,bzctrllen,argv[3],pids);
+	diffpipe=bz2read(fd,32+bzctrllen,bzdatalen,argv[3],pids+2);
+	if(version==2) {
+		extrapipe=bz2read(fd,32+bzctrllen+bzdatalen,
+			patchsize-(32+bzctrllen+bzdatalen),argv[3],pids+4);
+	};
+
+	if(close(fd)==-1) err(1,"%s",argv[3]);
+	if(((fd=open(argv[1],O_RDONLY,0))<0) ||
+		((oldsize=lseek(fd,0,SEEK_END))==-1) ||
+		((old=malloc(oldsize+1))==NULL) ||
+		(lseek(fd,0,SEEK_SET)!=0) ||
+		(read(fd,old,oldsize)!=oldsize) ||
+		(close(fd)==-1)) err(1,"%s",argv[1]);
+	if((new=malloc(newsize+1))==NULL) err(1,NULL);
+
+	oldpos=0;newpos=0;
+	while(newpos<newsize) {
+		for(i=0;i<=version;i++) {
+			if((lenread=loopread(ctrlpipe,buf,8))<0) err(1,NULL);
+			if(lenread<8) errx(1,"Corrupt patch\n");
+			ctrl[i]=offtin(buf);
+		};
+
+		if(version==1) oldpos+=ctrl[1];
+
+		if(newpos+ctrl[0]>newsize) errx(1,"Corrupt patch\n");
+		if((lenread=loopread(diffpipe,new+newpos,ctrl[0]))<0)
+			err(1,NULL);
+		if(lenread!=ctrl[0]) errx(1,"Corrupt patch\n");
+		for(i=0;i<ctrl[0];i++)
+			if((oldpos+i>=0) && (oldpos+i<oldsize))
+				new[newpos+i]+=old[oldpos+i];
+		newpos+=ctrl[0];
+		oldpos+=ctrl[0];
+
+		if(version==2) {
+			if(newpos+ctrl[1]>newsize) errx(1,"Corrupt patch\n");
+			if((lenread=loopread(extrapipe,new+newpos,ctrl[1]))<0)
+				err(1,NULL);
+			if(lenread!=ctrl[1]) errx(1,"Corrupt patch\n");
+
+			newpos+=ctrl[1];
+			oldpos+=ctrl[2];
+		};
+	};
+
+	if(loopread(ctrlpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
+	if(loopread(diffpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
+	if(version==2)
+		if(loopread(extrapipe,buf,1)!=0) errx(1,"Corrupt patch\n");
+
+	if(close(ctrlpipe) || close(diffpipe) || 
+		((version==2) && close(extrapipe)))
+		err(1,NULL);
+	for(i=0;i<(version+1)*2;i++) waitpid(pids[i],NULL,0);
+
+	if(((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) ||
+		(write(fd,new,newsize)!=newsize) || (close(fd)==-1))
+		err(1,"%s",argv[2]);
+
+	free(new);
+	free(old);
+
+	return 0;
+}

src/libstore/normalise.cc

@@ -5,9 +5,11 @@
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/time.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
+#include <utime.h>
 
 #include "normalise.hh"
 #include "references.hh"
@@ -298,6 +300,60 @@ Path queryNormalForm(const Path & nePath)
 }
 
 
+/* "Fix", or canonicalise, the meta-data of the files in a store path
+   after it has been built.  In particular:
+   - the last modification date on each file is set to 0 (i.e.,
+     00:00:00 1/1/1970 UTC)
+   - the permissions are set of 444 or 555 (i.e., read-only with or
+     without execute permission; setuid bits etc. are cleared)
+   - the owner and group are set to the Nix user and group, if we're
+     in a setuid Nix installation
+*/
+void canonicalisePathMetaData(const Path & path)
+{
+    checkInterrupt();
+
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+	throw SysError(format("getting attributes of path `%1%'") % path);
+
+    if (!S_ISLNK(st.st_mode)) {
+
+        /* Mask out all type related bits. */
+        mode_t mode = st.st_mode & ~S_IFMT;
+        
+        if (mode != 0444 && mode != 0555) {
+            mode = (st.st_mode & S_IFMT)
+                 | 0444
+                 | (st.st_mode & S_IXUSR ? 0111 : 0);
+            if (chmod(path.c_str(), mode) == -1)
+                throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
+        }
+
+        if (st.st_uid != getuid() || st.st_gid != getgid()) {
+            if (chown(path.c_str(), getuid(), getgid()) == -1)
+                throw SysError(format("changing owner/group of `%1%' to %2%/%3%")
+                    % path % getuid() % getgid());
+        }
+
+        if (st.st_mtime != 0) {
+            struct utimbuf utimbuf;
+            utimbuf.actime = st.st_atime;
+            utimbuf.modtime = 0;
+            if (utime(path.c_str(), &utimbuf) == -1) 
+                throw SysError(format("changing modification time of `%1%'") % path);
+        }
+
+    }
+
+    if (S_ISDIR(st.st_mode)) {
+        Strings names = readDirectory(path);
+	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+	    canonicalisePathMetaData(path + "/" + *i);
+    }
+}
+
+
 
 //////////////////////////////////////////////////////////////////////
 
@@ -1009,7 +1065,7 @@ void NormalisationGoal::createClosure()
         }
         nf.closure.roots.insert(path);
 
-	makePathReadOnly(path);
+	canonicalisePathMetaData(path);
 
 	/* For this output path, find the references to other paths contained
 	   in it. */
@@ -1361,9 +1417,6 @@ private:
     /* The current substitute. */
     Substitute sub;
 
-    /* The normal form of the substitute store expression. */
-    Path nfSub;
-
     /* Pipe for the substitute's standard output/error. */
     Pipe logPipe;
 
@@ -1385,8 +1438,6 @@ public:
     /* The states. */
     void init();
     void tryNext();
-    void exprNormalised();
-    void exprRealised();
     void tryToRun();
     void finished();
 
@@ -1451,26 +1502,7 @@ void SubstitutionGoal::tryNext()
     sub = subs.front();
     subs.pop_front();
 
-    /* Realise the substitute store expression. */
-    nrFailed = 0;
-    addWaitee(worker.makeRealisationGoal(sub.storeExpr));
-
-    state = &SubstitutionGoal::exprRealised;
-}
-
-
-void SubstitutionGoal::exprRealised()
-{
-    trace("substitute store expression realised");
-
-    if (nrFailed != 0) {
-        tryNext();
-        return;
-    }
-
-    /* !!! the storeExpr doesn't have to be a derivation, right? */
-    nfSub = queryNormalForm(sub.storeExpr);
-    
+    /* Wait until we can run the substitute program. */
     state = &SubstitutionGoal::tryToRun;
     worker.waitForBuildSlot(shared_from_this());
 }
@@ -1502,15 +1534,8 @@ void SubstitutionGoal::tryToRun()
 
     printMsg(lvlInfo,
         format("substituting path `%1%' using substituter `%2%'")
-        % storePath % sub.storeExpr);
+        % storePath % sub.program);
     
-    /* What's the substitute program? */
-    StoreExpr expr = storeExprFromPath(nfSub);
-    assert(expr.type == StoreExpr::neClosure);
-    assert(!expr.closure.roots.empty());
-    Path program =
-        canonPath(*expr.closure.roots.begin() + "/" + sub.program);
-
     logPipe.create();
 
     /* Remove the (stale) output path if it exists. */
@@ -1536,12 +1561,12 @@ void SubstitutionGoal::tryToRun()
             /* Fill in the arguments. */
             Strings args(sub.args);
             args.push_front(storePath);
-            args.push_front(baseNameOf(program));
+            args.push_front(baseNameOf(sub.program));
             const char * * argArr = strings2CharPtrs(args);
 
-            execv(program.c_str(), (char * *) argArr);
+            execv(sub.program.c_str(), (char * *) argArr);
             
-            throw SysError(format("executing `%1%'") % program);
+            throw SysError(format("executing `%1%'") % sub.program);
             
         } catch (exception & e) {
             cerr << format("substitute error: %1%\n") % e.what();
@@ -1593,7 +1618,7 @@ void SubstitutionGoal::finished()
 
         printMsg(lvlInfo,
             format("substitution of path `%1%' using substituter `%2%' failed: %3%")
-            % storePath % sub.storeExpr % e.msg());
+            % storePath % sub.program % e.msg());
         
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
@@ -1601,7 +1626,7 @@ void SubstitutionGoal::finished()
         return;
     }
 
-    makePathReadOnly(storePath);
+    canonicalisePathMetaData(storePath);
 
     Transaction txn;
     createStoreTransaction(txn);

src/libstore/store.cc

@@ -41,12 +41,13 @@ static TableId dbSuccessors = 0;
 */
 static TableId dbSuccessorsRev = 0;
 
-/* dbSubstitutes :: Path -> [(Path, Path, [string])]
+/* dbSubstitutes :: Path -> [[Path]]
 
    Each pair $(p, subs)$ tells Nix that it can use any of the
-   substitutes in $subs$ to build path $p$.  Each substitute is a
-   tuple $(storeExpr, program, args)$ (see the type `Substitute' in
-   `store.hh').
+   substitutes in $subs$ to build path $p$.  Each substitute defines a
+   command-line invocation of a program (i.e., the first list element
+   is the full path to the program, the remaining elements are
+   arguments).
 
    The main purpose of this is for distributed caching of derivates.
    One system can compute a derivate and put it on a website (as a Nix
@@ -56,18 +57,10 @@ static TableId dbSuccessorsRev = 0;
 */
 static TableId dbSubstitutes = 0;
 
-/* dbSubstitutesRev :: Path -> [Path]
-
-   The reverse mapping of dbSubstitutes; it maps store expressions
-   back to the paths for which they are substitutes.
-*/
-static TableId dbSubstitutesRev = 0;
-
 
 bool Substitute::operator == (const Substitute & sub)
 {
-    return storeExpr == sub.storeExpr
-        && program == sub.program
+    return program == sub.program
         && args == sub.args;
 }
 
@@ -86,7 +79,6 @@ void openDB()
     dbSuccessors = nixDB.openTable("successors");
     dbSuccessorsRev = nixDB.openTable("successors-rev");
     dbSubstitutes = nixDB.openTable("substitutes");
-    dbSubstitutesRev = nixDB.openTable("substitutes-rev");
 }
 
 
@@ -300,10 +292,13 @@ static Substitutes readSubstitutes(const Transaction & txn,
             break;
         }
         Strings ss2 = unpackStrings(*i);
-        if (ss2.size() != 3) throw Error("malformed substitute");
+        if (ss2.size() == 3) {
+            /* Another old-style substitute. */
+            continue;
+        }
+        if (ss2.size() != 2) throw Error("malformed substitute");
         Strings::iterator j = ss2.begin();
         Substitute sub;
-        sub.storeExpr = *j++;
         sub.program = *j++;
         sub.args = unpackStrings(*j++);
         subs.push_back(sub);
@@ -322,7 +317,6 @@ static void writeSubstitutes(const Transaction & txn,
          i != subs.end(); ++i)
     {
         Strings ss2;
-        ss2.push_back(i->storeExpr);
         ss2.push_back(i->program);
         ss2.push_back(packStrings(i->args));
         ss.push_back(packStrings(ss2));
@@ -332,14 +326,9 @@ static void writeSubstitutes(const Transaction & txn,
 }
 
 
-typedef map<Path, Paths> SubstitutesRev;
-
-
 void registerSubstitutes(const Transaction & txn,
     const SubstitutePairs & subPairs)
 {
-    SubstitutesRev revMap;
-    
     for (SubstitutePairs::const_iterator i = subPairs.begin();
          i != subPairs.end(); ++i)
     {
@@ -347,7 +336,6 @@ void registerSubstitutes(const Transaction & txn,
         const Substitute & sub(i->second);
 
         assertStorePath(srcPath);
-        assertStorePath(sub.storeExpr);
     
         Substitutes subs = readSubstitutes(txn, srcPath);
 
@@ -357,19 +345,7 @@ void registerSubstitutes(const Transaction & txn,
         subs.push_front(sub);
         
         writeSubstitutes(txn, srcPath, subs);
-
-        Paths & revs = revMap[sub.storeExpr];
-        if (revs.empty())
-            nixDB.queryStrings(txn, dbSubstitutesRev, sub.storeExpr, revs);
-        if (find(revs.begin(), revs.end(), srcPath) == revs.end())
-            revs.push_back(srcPath);
     }
-
-    /* Re-write the reverse mapping in one go to prevent Theta(n^2)
-       performance.  (This would occur because the data fields of the
-       `substitutes-rev' table are lists). */
-    for (SubstitutesRev::iterator i = revMap.begin(); i != revMap.end(); ++i)
-        nixDB.setStrings(txn, dbSubstitutesRev, i->first, i->second);
 }
 
 
@@ -379,6 +355,44 @@ Substitutes querySubstitutes(const Path & srcPath)
 }
 
 
+static void unregisterPredecessors(const Path & path, Transaction & txn)
+{
+    /* Remove any successor mappings to this path (but not *from*
+       it). */
+    Paths revs;
+    nixDB.queryStrings(txn, dbSuccessorsRev, path, revs);
+    for (Paths::iterator i = revs.begin(); i != revs.end(); ++i)
+        nixDB.delPair(txn, dbSuccessors, *i);
+    nixDB.delPair(txn, dbSuccessorsRev, path);
+}
+ 
+
+void clearSubstitutes()
+{
+    Transaction txn(nixDB);
+    
+    /* Iterate over all paths for which there are substitutes. */
+    Paths subKeys;
+    nixDB.enumTable(txn, dbSubstitutes, subKeys);
+    for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
+
+        /* If this path has not become valid in the mean-while, delete
+           any successor mappings *to* it.  This is to preserve the
+           invariant the all successors are `usable' as opposed to
+           `valid' (i.e., the successor must be valid *or* have at
+           least one substitute). */
+        if (!isValidPath(*i)) {
+            unregisterPredecessors(*i, txn);
+        }
+        
+        /* Delete all substitutes for path *i. */
+        nixDB.delPair(txn, dbSubstitutes, *i);
+    }
+
+    txn.commit();
+}
+
+
 void registerValidPath(const Transaction & txn, const Path & _path)
 {
     Path path(canonPath(_path));
@@ -393,36 +407,7 @@ static void invalidatePath(const Path & path, Transaction & txn)
     debug(format("unregistering path `%1%'") % path);
 
     nixDB.delPair(txn, dbValidPaths, path);
-
-    /* Remove any successor mappings to this path (but not *from*
-       it). */
-    Paths revs;
-    nixDB.queryStrings(txn, dbSuccessorsRev, path, revs);
-    for (Paths::iterator i = revs.begin(); i != revs.end(); ++i)
-        nixDB.delPair(txn, dbSuccessors, *i);
-    nixDB.delPair(txn, dbSuccessorsRev, path);
-
-    /* Remove any substitute mappings to this path. */
-    revs.clear();
-    nixDB.queryStrings(txn, dbSubstitutesRev, path, revs);
-    for (Paths::iterator i = revs.begin(); i != revs.end(); ++i) {
-        Substitutes subs = readSubstitutes(txn, *i), subs2;
-        bool found = false;
-        for (Substitutes::iterator j = subs.begin(); j != subs.end(); ++j)
-            if (j->storeExpr != path)
-                subs2.push_back(*j);
-            else
-                found = true;
-        // !!!        if (!found) throw Error("integrity error in substitutes mapping");
-        writeSubstitutes(txn, *i, subs);
-
-	/* If path *i now has no substitutes left, and is not valid,
-	   then it too should be invalidated.  This is because it may
-	   be a substitute or successor. */
-	if (subs.size() == 0 && !isValidPathTxn(*i, txn))
-	    invalidatePath(*i, txn);
-    }
-    nixDB.delPair(txn, dbSubstitutesRev, path);
+    unregisterPredecessors(path, txn);
 }
 
 
@@ -548,30 +533,11 @@ void verifyStore()
     Paths subKeys;
     nixDB.enumTable(txn, dbSubstitutes, subKeys);
     for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
-        Substitutes subs = readSubstitutes(txn, *i), subs2;
-        for (Substitutes::iterator j = subs.begin(); j != subs.end(); ++j)
-            if (validPaths.find(j->storeExpr) == validPaths.end())
-                printMsg(lvlError,
-                    format("found substitute mapping to non-existent path `%1%'")
-                    % j->storeExpr);
-            else
-                subs2.push_back(*j);
-        if (subs.size() != subs2.size())
-            writeSubstitutes(txn, *i, subs2);
-	if (subs2.size() > 0)
+        Substitutes subs = readSubstitutes(txn, *i);
+	if (subs.size() > 0)
 	    usablePaths.insert(*i);
-    }
-
-    /* Check that the keys of the reverse substitute mappings are
-       valid paths. */ 
-    Paths rsubKeys;
-    nixDB.enumTable(txn, dbSubstitutesRev, rsubKeys);
-    for (Paths::iterator i = rsubKeys.begin(); i != rsubKeys.end(); ++i) {
-        if (validPaths.find(*i) == validPaths.end()) {
-            printMsg(lvlError,
-                format("found reverse substitute mapping for non-existent path `%1%'") % *i);
-            nixDB.delPair(txn, dbSubstitutesRev, *i);
-        }
+        else
+            nixDB.delPair(txn, dbSubstitutes, *i);
     }
 
     /* Check that the values of the successor mappings are usable

src/libstore/store.hh

@@ -14,10 +14,6 @@ using namespace std;
    network). */
 struct Substitute
 {
-    /* Store expression to be normalised and realised in order to
-       obtain `program'. */
-    Path storeExpr;
-
     /* Program to be executed to create the store path.  Must be in
        the output path of `storeExpr'. */
     Path program;
@@ -73,6 +69,9 @@ void registerSubstitutes(const Transaction & txn,
 /* Return the substitutes expression for the given path. */
 Substitutes querySubstitutes(const Path & srcPath);
 
+/* Deregister all substitutes. */
+void clearSubstitutes();
+
 /* Register the validity of a path. */
 void registerValidPath(const Transaction & txn, const Path & path);
 

src/nix-store/help.txt

@@ -11,6 +11,7 @@ Operations:
 
   --successor: register a successor expression (dangerous!)
   --substitute: register a substitute expression (dangerous!)
+  --clear-substitute: clear all substitutes
   --validpath: register path validity (dangerous!)
   --isvalid: check path validity
 

src/nix-store/main.cc

@@ -166,7 +166,6 @@ static void opSubstitute(Strings opFlags, Strings opArgs)
         Substitute sub;
         getline(cin, srcPath);
         if (cin.eof()) break;
-        getline(cin, sub.storeExpr);
         getline(cin, sub.program);
         string s;
         getline(cin, s);
@@ -186,6 +185,17 @@ static void opSubstitute(Strings opFlags, Strings opArgs)
 }
 
 
+static void opClearSubstitutes(Strings opFlags, Strings opArgs)
+{
+    if (!opFlags.empty()) throw UsageError("unknown flag");
+    if (!opArgs.empty())
+        throw UsageError("no arguments expected");
+
+    clearSubstitutes();
+}
+
+
+
 static void opValidPath(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -354,6 +364,8 @@ void run(Strings args)
             op = opSuccessor;
         else if (arg == "--substitute")
             op = opSubstitute;
+        else if (arg == "--clear-substitutes")
+            op = opClearSubstitutes;
         else if (arg == "--validpath")
             op = opValidPath;
         else if (arg == "--isvalid")

tests/fallback.sh

@@ -4,7 +4,7 @@ echo "store expr is $storeExpr"
 
 # Register a non-existant successor (and a nox-existant substitute).
 suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
-(echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute
+(echo $suc && echo $TOP/no-such-program && echo 0) | $TOP/src/nix-store/nix-store --substitute
 $TOP/src/nix-store/nix-store --successor $storeExpr $suc
 
 outPath=$($TOP/src/nix-store/nix-store -qnf --fallback "$storeExpr")

tests/parallel.builder.sh

@@ -10,7 +10,7 @@ while ! ln -s x $shared.lock; do
 done
 test -f $shared.cur || echo 0 > $shared.cur
 test -f $shared.max || echo 0 > $shared.max
-new=$(($(cat $shared.cur) + 1))
+new=$(expr $(cat $shared.cur) + 1)
 if test $new -gt $(cat $shared.max); then
     echo $new > $shared.max
 fi
@@ -28,5 +28,5 @@ while ! ln -s x $shared.lock; do
     sleep 1
 done
 test -f $shared.cur || echo 0 > $shared.cur
-echo $(($(cat $shared.cur) - 1)) > $shared.cur
+echo $(expr $(cat $shared.cur) - 1) > $shared.cur
 rm $shared.lock

tests/substitutes.sh

@@ -6,21 +6,22 @@ echo "store expr is $storeExpr"
 outPath=$($TOP/src/nix-store/nix-store -qvvvvv "$storeExpr")
 echo "output path is $outPath"
 
-# Instantiate the substitute program.
-subExpr=$($TOP/src/nix-instantiate/nix-instantiate substituter.nix)
-echo "store expr is $subExpr"
+# Build the substitute program.
+subProgram=$($TOP/src/nix-store/nix-store -qnf \
+    $($TOP/src/nix-instantiate/nix-instantiate substituter.nix))/substituter
+echo "substitute program is $subProgram"
 
 regSub() {
-    (echo $1 && echo $2 && echo "/substituter" && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
+    (echo $1 && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
 }
 
 # Register a fake successor, and a substitute for it.
 suc=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-s.store
-regSub $suc $subExpr
+regSub $suc $subProgram
 $TOP/src/nix-store/nix-store --successor $storeExpr $suc
 
 # Register a substitute for the output path.
-regSub $outPath $subExpr
+regSub $outPath $subProgram
 
 
 $TOP/src/nix-store/nix-store -rvvvvv "$storeExpr"

tests/substitutes2.sh

@@ -6,32 +6,34 @@ echo "store expr is $storeExpr"
 outPath=$($TOP/src/nix-store/nix-store -qvvvvv "$storeExpr")
 echo "output path is $outPath"
 
-# Instantiate the substitute program.
-subExpr=$($TOP/src/nix-instantiate/nix-instantiate substituter.nix)
-echo "store expr is $subExpr"
+# Build the substitute program.
+subProgram=$($TOP/src/nix-store/nix-store -qnf \
+    $($TOP/src/nix-instantiate/nix-instantiate substituter.nix))/substituter
+echo "substitute program is $subProgram"
 
-# Instantiate the failing substitute program.
-subExpr2=$($TOP/src/nix-instantiate/nix-instantiate substituter2.nix)
-echo "store expr is $subExpr2"
+# Build the failing substitute program.
+subProgram2=$($TOP/src/nix-store/nix-store -qnf \
+    $($TOP/src/nix-instantiate/nix-instantiate substituter2.nix))/substituter
+echo "failing substitute program is $subProgram2"
 
 regSub() {
-    (echo $1 && echo $2 && echo "/substituter" && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
+    (echo $1 && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
 }
 
 # Register a fake successor, and a substitute for it.
 suc=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab-s.store
-regSub $suc $subExpr
+regSub $suc $subProgram
 $TOP/src/nix-store/nix-store --successor $storeExpr $suc
 
 # Register a failing substitute for it (it takes precedence).
-regSub $suc $subExpr2
+regSub $suc $subProgram2
 
 # Register a substitute for the output path.
-regSub $outPath $subExpr
+regSub $outPath $subProgram
 
 # Register another substitute for the output path.  This one will
 # produce other output. 
-regSub $outPath $subExpr2
+regSub $outPath $subProgram2
 
 
 $TOP/src/nix-store/nix-store -rvvvvv "$storeExpr"