* Sync with thesis: do not include store location in the hash

computation (in the intensional model).

Makefile.am

@@ -1,5 +1,6 @@
 SUBDIRS = externals src scripts corepkgs doc misc tests
-EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh svn-revision
+EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
+  svn-revision nix.conf.example
 
 include ./substitute.mk
 
@@ -12,6 +13,11 @@ relname:
 	echo -n $(distdir) > relname
 
 install-data-local: init-state
+	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
+	$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix
+	if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
+		$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
+	fi
 
 if INIT_STATE
 if SETUID_HACK
@@ -22,13 +28,16 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/db
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix/drvs
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(prefix)/store
+	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 #	$(bindir)/nix-store --init
 else
 init-state:

NEWS

@@ -1,4 +1,196 @@
-Version 0.6
+Version 0.9
+
+* Unpacking of patch sequences is much faster now by not doing
+  redundant unpacking and repacking of intermediate paths.
+
+
+Version 0.8 (April 11, 2005)
+
+NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).  As a
+result, `nix-pull' manifests and channels built for Nix 0.7 and below
+will now work anymore.  However, the Nix expression language has not
+changed, so you can still build from source.  Also, existing user
+environments continue to work.  Nix 0.8 will automatically upgrade the
+database schema of previous installations when it is first run.
+
+If you get the error message
+
+  you have an old-style manifest `/nix/var/nix/manifests/[...]';
+  please delete it
+
+you should delete previously downloaded manifests:
+
+  $ rm /nix/var/nix/manifests/*
+
+If `nix-channel' gives the error message
+
+  manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST'
+  is too old (i.e., for Nix <= 0.7)
+
+then you should unsubscribe from the offending channel (`nix-channel
+--remove URL'; leave out `/MANIFEST'), and subscribe to the same URL,
+with `channels' replaced by `channels-v3' (e.g.,
+http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable).
+
+Nix 0.8 has the following improvements:
+
+* The cryptographic hashes used in store paths are now 160 bits long,
+  but encoded in base-32 so that they are still only 32 characters
+  long (e.g., /nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0).
+  (This is actually a 160 bit truncation of a SHA-256 hash.)
+
+* Big cleanups and simplifications of the basic store semantics.  The
+  notion of "closure store expressions" is gone (and so is the notion
+  of "successors"); the file system references of a store path are now
+  just stored in the database.
+
+  For instance, given any store path, you can query its closure:
+
+    $ nix-store -qR $(which firefox)
+    ... lots of paths ...
+
+  Also, Nix now remembers for each store path the derivation that
+  built it (the "deriver"):
+
+    $ nix-store -qR $(which firefox)
+    /nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv
+
+  So to see the build-time dependencies, you can do
+  
+    $ nix-store -qR $(nix-store -qd $(which firefox))
+
+  or, in a nicer format:
+
+    $ nix-store -q --tree $(nix-store -qd $(which firefox))
+
+  File system references are also stored in reverse.  For instance,
+  you can query all paths that directly or indirectly use a certain
+  Glibc:
+
+    $ nix-store -q --referers-closure \
+        /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+
+* The concept of fixed-output derivations has been formalised.
+  Previously, functions such as `fetchurl' in Nixpkgs used a hack
+  (namely, explicitly specifying a store path hash) to prevent changes
+  to, say, the URL of the file from propagating upwards through the
+  dependency graph, causing rebuilds of everything.  This can now be
+  done cleanly by specifying the `outputHash' and `outputHashAlgo'
+  attributes.  Nix itself checks that the content of the output has
+  the specified hash.  (This is important for maintaining certain
+  invariants necessary for future work on secure shared stores.)
+
+* One-click installation :-)  It is now possible to install any
+  top-level component in Nixpkgs directly, through the web - see,
+  e.g., http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/.  All you
+  have to do is associate `/nix/bin/nix-install-package' with the MIME
+  type `application/nix-package' (or the extension `.nixpkg'), and
+  clicking on a package link will cause it to be installed, with all
+  appropriate dependencies.  If you just want to install some specific
+  application, this is easier than subscribing to a channel.
+
+* `nix-store -r PATHS' now builds all the derivations PATHS in
+  parallel.  Previously it did them sequentially (though exploiting
+  possible parallelism between subderivations).  This is nice for
+  build farms.
+
+* `nix-channel' has new operations `--list' and `--remove'.
+
+* New ways of installing components into user environments:
+
+  - Copy from another user environment:
+
+      $ nix-env -i --from-profile .../other-profile firefox
+
+  - Install a store derivation directly (bypassing the Nix expression
+    language entirely):
+
+      $ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv
+
+    (This is used to implement `nix-install-package', which is
+    therefore immune to evolution in the Nix expression language.)
+
+  - Install an already built store path directly:
+
+      $ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1
+
+  - Install the result of a Nix expression specified as a command-line
+    argument:
+
+      $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'
+
+    The difference with the normal installation mode is that `-E' does
+    not use the `name' attributes of derivations.  Therefore, this can
+    be used to disambiguate multiple derivations with the same name.
+
+* A hash of the contents of a store path is now stored in the database
+  after a succesful build.  This allows you to check whether store
+  paths have been tampered with: `nix-store --verify --check-contents'.
+    
+* Implemented a concurrent garbage collector.  It is now always safe
+  to run the garbage collector, even if other Nix operations are
+  happening simultaneously.
+
+  However, there can still be GC races if you use `nix-instantiate'
+  and `nix-store -r' directly to build things.  To prevent races, use
+  the `--add-root' flag of those commands.
+
+* The garbage collector now finally deletes paths in the right order
+  (i.e., topologically sorted under the `references' relation), thus
+  making it safe to interrupt the collector without risking a store
+  that violates the closure invariant.
+
+* Likewise, the substitute mechanism now downloads files in the right
+  order, thus preserving the closure invariant at all times.
+
+* The result of `nix-build' is now registered as a root of the garbage
+  collector.  If the `./result' link is deleted, the GC root
+  disappears automatically.
+
+* The behaviour of the garbage collector can be changed globally by
+  setting options in `/nix/etc/nix/nix.conf'.
+
+  - `gc-keep-derivations' specifies whether deriver links should be
+    followed when searching for live paths.
+
+  - `gc-keep-outputs' specifies whether outputs of derivations should
+    be followed when searching for live paths.
+  
+  - `env-keep-derivations' specifies whether user environments should
+    store the paths of derivations when they are added (thus keeping
+    the derivations alive).
+
+* New `nix-env' query flags `--drv-path' and `--out-path'.
+
+* `fetchurl' allows SHA-1 and SHA-256 in addition to MD5.  Just
+  specify the attribute `sha1' or `sha256' instead of `md5'.
+
+* Manual updates.
+
+  
+Version 0.7 (January 12, 2005)
+
+* Binary patching.  When upgrading components using pre-built binaries
+  (through nix-pull / nix-channel), Nix can automatically download and
+  apply binary patches to already installed components instead of full
+  downloads.  Patching is "smart": if there is a *sequence* of patches
+  to an installed component, Nix will use it.  Patches are currently
+  generated automatically between Nixpkgs (pre-)releases.
+
+* Simplifications to the substitute mechanism.
+
+* Nix-pull now stores downloaded manifests in /nix/var/nix/manifests.
+
+* Metadata on files in the Nix store is canonicalised after builds:
+  the last-modified timestamp is set to 0 (00:00:00 1/1/1970), the
+  mode is set to 0444 or 0555 (readable and possibly executable by
+  all; setuid/setgid bits are dropped), and the group is set to the
+  default.  This ensures that the result of a build and an
+  installation through a substitute is the same; and that timestamp
+  dependencies are revealed.
+
+
+Version 0.6 (November 14, 2004)
 
 Major changes include the following:
 

README

@@ -1,5 +1,9 @@
-                             *** Nix ***
-
 For installation and usage instructions, please read the manual, which
 can be found in `docs/manual/manual.html', and additionally at the Nix
 website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
+
+
+Acknowledgments
+
+This product includes software developed by the OpenSSL Project for
+use in the OpenSSL Toolkit (http://www.OpenSSL.org/)

blacklisting/check-env.pl

@@ -0,0 +1,252 @@
+#! /usr/bin/perl -w -I /home/eelco/.nix-profile/lib/site_perl
+
+use strict;
+use XML::LibXML;
+#use XML::Simple;
+
+my $blacklistFN = shift @ARGV;
+die unless defined $blacklistFN;
+my $userEnv = shift @ARGV;
+die unless defined $userEnv;
+
+
+# Read the blacklist.
+my $parser = XML::LibXML->new();
+my $blacklist = $parser->parse_file($blacklistFN)->getDocumentElement;
+
+#print $blacklist->toString() , "\n";
+
+
+# Get all the elements of the user environment.
+my $userEnvElems = `nix-store --query --references '$userEnv'`;
+die "cannot query user environment elements" if $? != 0;
+my @userEnvElems = split ' ', $userEnvElems;
+
+
+my %storePathHashes;
+
+
+sub getElemNodes {
+    my $node = shift;
+    my @elems = ();
+    foreach my $node ($node->getChildNodes) {
+        push @elems, $node if $node->nodeType == XML_ELEMENT_NODE;
+    }
+    return @elems;
+}
+
+
+my %referencesCache;
+sub getReferences {
+    my $path = shift;
+    return $referencesCache{$path} if defined $referencesCache{$path};
+    
+    my $references = `nix-store --query --references '$path'`;
+    die "cannot query references" if $? != 0;
+    $referencesCache{$path} = [split ' ', $references];
+    
+    return $referencesCache{$path};
+}
+
+
+my %attrsCache;
+sub getAttr {
+    my $path = shift;
+    my $name = shift;
+    my $key = "$path/$name";
+    return $referencesCache{$key} if defined $referencesCache{$key};
+
+    my $value = `nix-store --query --binding '$name' '$path' 2> /dev/null`;
+    $value = "" if $? != 0; # !!!
+    chomp $value;
+    $referencesCache{$key} = $value;
+
+    return $value;
+}
+
+
+sub evalCondition;
+
+
+sub traverse {
+    my $done = shift;
+    my $set = shift;
+    my $path = shift;
+    my $stopCondition = shift;
+
+    return if defined $done->{$path};
+    $done->{$path} = 1;
+    $set->{$path} = 1;
+
+#    print "  in $path\n";
+
+    if (!evalCondition({$path => 1}, $stopCondition)) {
+#        print "  STOPPING in $path\n";
+        return;
+    }
+
+    # Get the requisites of the deriver.
+
+    foreach my $reference (@{getReferences $path}) {
+        traverse($done, $set, $reference, $stopCondition);
+    }
+}
+
+
+sub evalSet {
+    my $inSet = shift;
+    my $expr = shift;
+    my $name = $expr->getName;
+    
+    if ($name eq "traverse") {
+        my $stopCondition = (getElemNodes $expr)[0];
+        my $done = { };
+        my $set = { };
+        foreach my $path (keys %{$inSet}) {
+            traverse($done, $set, $path, $stopCondition);
+        }
+        return $set;
+    }
+
+    else {
+        die "unknown element `$name'";
+    }
+}
+
+
+# Function for evaluating conditions.
+sub evalCondition {
+    my $storePaths = shift;
+    my $condition = shift;
+    my $elemName = $condition->getName;
+    
+    if ($elemName eq "containsSource") {
+        my $hash = $condition->attributes->getNamedItem("hash")->getValue;
+        foreach my $path (keys %{$storePathHashes{$hash}}) {
+            return 1 if defined $storePaths->{$path};
+        }
+        return 0;
+    }
+
+    elsif ($elemName eq "hasName") { 
+        my $nameRE = $condition->attributes->getNamedItem("name")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            return 1 if $path =~ /$nameRE/;
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "hasAttr") { 
+        my $name = $condition->attributes->getNamedItem("name")->getValue;
+        my $valueRE = $condition->attributes->getNamedItem("value")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            if ($path =~ /\.drv$/) {
+                my $value = getAttr($path, $name);
+#                print "    $path $name $value\n";
+                return 1 if $value =~ /$valueRE/;
+            }
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "and") {
+        my $result = 1;
+        foreach my $node (getElemNodes $condition) {
+            $result &= evalCondition($storePaths, $node);
+        }
+        return $result;
+    }
+
+    elsif ($elemName eq "not") {
+        return !evalCondition($storePaths, (getElemNodes $condition)[0]);
+    }
+    
+    elsif ($elemName eq "within") {
+        my @elems = getElemNodes $condition;
+        my $set = evalSet($storePaths, $elems[0]);
+        return evalCondition($set, $elems[1]);
+    }
+
+    elsif ($elemName eq "true") {
+        return 1;
+    }
+
+    elsif ($elemName eq "false") {
+        return 0;
+    }
+
+    else {
+        die "unknown element `$elemName'";
+    }
+}
+
+
+sub evalOr {
+    my $storePaths = shift;
+    my $nodes = shift;
+
+    my $result = 0;
+    foreach my $node (@{$nodes}) {
+        $result |= evalCondition($storePaths, $node);
+    }
+    
+    return $result;
+}
+
+
+# Iterate over all elements, check them.
+foreach my $userEnvElem (@userEnvElems) {
+
+    # Get the deriver of this path.
+    my $deriver = `nix-store --query --deriver '$userEnvElem'`;
+    die "cannot query deriver" if $? != 0;
+    chomp $deriver;
+
+    if ($deriver eq "unknown-deriver") {
+#        print "  deriver unknown, cannot check sources\n";
+        next;
+    }
+
+    print "CHECKING $userEnvElem\n";
+
+
+    # Get the requisites of the deriver.
+#    my $requisites = `nix-store --query --requisites --include-outputs '$deriver'`;
+#    die "cannot query requisites" if $? != 0;
+#    my @requisites = split ' ', $requisites;
+
+
+    # Get the hashes of the requisites.
+#    my $hashes = `nix-store --query --hash @requisites`;
+#    die "cannot query hashes" if $? != 0;
+#    my @hashes = split ' ', $hashes;
+#    for (my $i = 0; $i < scalar @requisites; $i++) {
+#        die unless $i < scalar @hashes;
+#        my $hash = $hashes[$i];
+#        $storePathHashes{$hash} = {} unless defined $storePathHashes{$hash};
+#        my $r = $storePathHashes{$hash}; # !!! fix
+#        $$r{$requisites[$i]} = 1;
+#    }
+
+
+    # Evaluate each blacklist item.
+    foreach my $item ($blacklist->getChildrenByTagName("item")) {
+        my $itemId = $item->getAttributeNode("id")->getValue;
+#        print "  CHECKING FOR $itemId\n";
+
+        my $condition = ($item->getChildrenByTagName("condition"))[0];
+        die unless $condition;
+
+        # Evaluate the condition.
+        my @elems = getElemNodes $condition;
+        if (evalOr({$deriver => 1}, \@elems)) {
+            # Oops, condition triggered.
+            my $reason = ($item->getChildrenByTagName("reason"))[0]->getChildNodes->to_literal;
+            $reason =~ s/\s+/ /g;
+            $reason =~ s/^\s+//g;
+
+            print "    VULNERABLE TO `$itemId': $reason\n";
+        }
+    }
+}
+

configure.ac

@@ -1,7 +1,7 @@
-AC_INIT(nix, "0.6")
+AC_INIT(nix, "0.9")
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
-AM_INIT_AUTOMAKE
+AM_INIT_AUTOMAKE([dist-bzip2])
 
 # Change to `1' to produce a `stable' release (i.e., the `preREVISION'
 # suffix is not added).
@@ -81,17 +81,19 @@ AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 NEED_PROG(perl, perl)
+NEED_PROG(tar, tar)
+                
+NEED_PROG(cat, cat)
+AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
+  [path of cat, mkdir, etc.]),
+  coreutils=$withval, coreutils=$(dirname $cat))
+AC_SUBST(coreutils)
 
 AC_ARG_WITH(docbook-catalog, AC_HELP_STRING([--with-docbook-catalog=PATH],
   [path of the DocBook XML DTD]),
   docbookcatalog=$withval, docbookcatalog=/docbook-dtd-missing)
 AC_SUBST(docbookcatalog)
 
-AC_ARG_WITH(docbook-ebnf-catalog, AC_HELP_STRING([--with-docbook-ebnf-catalog=PATH],
-  [path of the DocBook XML EBNF module DTD]),
-  docbookebnfcatalog=$withval, docbookcatalog=/docbook-ebnf-dtd-missing)
-AC_SUBST(docbookebnfcatalog)
-
 AC_ARG_WITH(docbook-xsl, AC_HELP_STRING([--with-docbook-xsl=PATH],
   [path of the DocBook XSL stylesheets]),
   docbookxsl=$withval, docbookxsl=/docbook-xsl-missing)
@@ -189,13 +191,12 @@ AC_CONFIG_FILES([Makefile
    src/nix-instantiate/Makefile
    src/nix-env/Makefile
    src/log2xml/Makefile
+   src/bsdiff-4.2/Makefile
    scripts/Makefile
    corepkgs/Makefile
-   corepkgs/fetchurl/Makefile
    corepkgs/nar/Makefile
    corepkgs/buildenv/Makefile
    corepkgs/channels/Makefile
-   corepkgs/nix-pull/Makefile
    doc/Makefile
    doc/manual/Makefile
    misc/Makefile

corepkgs/Makefile.am

@@ -1 +1 @@
-SUBDIRS = fetchurl nar buildenv channels nix-pull
+SUBDIRS = nar buildenv channels

corepkgs/buildenv/builder.pl.in

@@ -25,6 +25,7 @@ sub createLinks {
         
 	if ($srcFile =~ /\/propagated-build-inputs$/ ||
             $srcFile =~ /\/nix-support$/ ||
+            $srcFile =~ /\/perllocal.pod$/ ||
             $srcFile =~ /\/log$/)
         {
             # Do nothing.
@@ -72,13 +73,27 @@ sub createLinks {
 
 my %done;
 
+sub addPkg;
 sub addPkg {
     my $pkgDir = shift;
 
     return if (defined $done{$pkgDir});
     $done{$pkgDir} = 1;
 
+    print "adding $pkgDir\n";
     createLinks("$pkgDir", "$out");
+
+    my $propagatedFN = "$pkgDir/nix-support/propagated-build-inputs";
+    if (-e $propagatedFN) {
+        open PROP, "<$propagatedFN" or die;
+        my $propagated = <PROP>;
+        close PROP;
+        my @propagated = split ' ', $propagated;
+        foreach my $p (@propagated) {
+            addPkg $p;
+        }
+    }
+        
 }
 
 
@@ -86,7 +101,6 @@ my @args = split ' ', $ENV{"derivations"};
 
 while (scalar @args > 0) {
     my $drvPath = shift @args;
-    print "adding $drvPath\n";
     addPkg($drvPath);
 }
 

corepkgs/channels/unpack.sh.in

@@ -1,9 +1,7 @@
 #! @shell@ -e
 
-export PATH=/bin:/usr/bin # !!! impure
-
-mkdir $out
-mkdir $out/tmp
+@coreutils@/mkdir $out
+@coreutils@/mkdir $out/tmp
 cd $out/tmp
 
 expr=$out/default.nix
@@ -12,8 +10,8 @@ echo '[' > $expr
 nr=0
 for i in $inputs; do
     echo "unpacking $i"
-    @bunzip2@ < $i | tar xvf -
-    mv * ../$nr # !!! hacky
+    @bunzip2@ < $i | @tar@ xf -
+    @coreutils@/mv * ../$nr # !!! hacky
     echo "(import ./$nr)" >> $expr
     nr=$(($nr + 1))
 done
@@ -21,4 +19,4 @@ done
 echo ']' >> $expr
 
 cd ..
-rmdir tmp
+@coreutils@/rmdir tmp

corepkgs/fetchurl/Makefile.am

@@ -1,11 +0,0 @@
-all-local: builder.sh
-
-install-exec-local:
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-
-include ../../substitute.mk
-
-EXTRA_DIST = default.nix builder.sh.in

corepkgs/fetchurl/builder.sh.in

@@ -1,19 +0,0 @@
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-echo "downloading $url into $out"
-
-prefetch=@storedir@/nix-prefetch-url-$md5
-if test -f "$prefetch"; then
-    echo "using prefetched $prefetch";
-    mv $prefetch $out
-else
-    @curl@ --fail --location --max-redirs 20 "$url" > "$out"
-fi
-
-actual=$(@bindir@/nix-hash --flat $out)
-if test "$actual" != "$md5"; then
-    echo "hash is $actual, expected $md5"
-    exit 1
-fi

corepkgs/fetchurl/default.nix

@@ -1,8 +0,0 @@
-{system, url, md5}:
-
-derivation {
-  name = baseNameOf (toString url);
-  builder = ./builder.sh;
-  id = md5;
-  inherit system url md5;
-}

corepkgs/nar/Makefile.am

@@ -1,13 +1,11 @@
-all-local: nar.sh unnar.sh
+all-local: nar.sh
 
 install-exec-local:
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_DATA) nar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_PROGRAM) nar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_DATA) unnar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_PROGRAM) unnar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
 
 include ../../substitute.mk
 
-EXTRA_DIST = nar.nix nar.sh.in unnar.nix unnar.sh.in
+EXTRA_DIST = nar.nix nar.sh.in

corepkgs/nar/nar.nix

@@ -1,6 +1,5 @@
-{system, path}: derivation {
+{system, path, hashAlgo}: derivation {
   name = "nar";
   builder = ./nar.sh;
-  system = system;
-  path = path;
+  inherit system path hashAlgo;
 }

corepkgs/nar/nar.sh.in

@@ -1,15 +1,14 @@
 #! @shell@ -e
 
-# !!! impure; fix this
-export PATH=/bin:/usr/bin
-
 echo "packing $path into $out..."
-mkdir $out
-dst=$out/$(basename $path).nar.bz2
-@bindir@/nix-store --dump "$path" | @bzip2@ > $dst
+@coreutils@/mkdir $out
+dst=$out/tmp.nar.bz2
+@bindir@/nix-store --dump "$path" > tmp
 
-if test "${PIPESTATUS[0]}" != "0"; then exit 1; fi
+@bzip2@ < tmp > $dst
 
-md5=$(md5sum -b $dst | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $md5 > $out/md5
+@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
+
+@bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
+
+@coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2

corepkgs/nar/unnar.nix

@@ -1,7 +0,0 @@
-{system, narFile, outPath}: derivation {
-  name = "unnar";
-  builder = ./unnar.sh;
-  system = system;
-  narFile = narFile;
-  outPath = outPath;
-}

corepkgs/nar/unnar.sh.in

@@ -1,4 +0,0 @@
-#! @shell@ -e
-
-echo "unpacking $narFile to $out..."
-@bunzip2@ < $narFile | @bindir@/nix-store --restore "$out"

corepkgs/nix-pull/Makefile.am

@@ -1,11 +0,0 @@
-all-local: builder.sh
-
-install-exec-local:
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
-	$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
-	$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
-
-include ../../substitute.mk
-
-EXTRA_DIST = default.nix builder.sh.in

corepkgs/nix-pull/builder.sh.in

@@ -1,34 +0,0 @@
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-mkdir $out
-
-cat > $out/fetch <<EOF
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-echo "downloading \$2..."
-
-export PRINT_PATH=1
-result=(\$(@bindir@/nix-prefetch-url \$2))
-
-hash=\${result[0]}
-path=\${result[1]}
-
-if test "\$hash" != "\$3"; then
-    echo "hash is \$hash, expected \$3"
-    exit 1
-fi
-
-echo "unpacking into \$1..."
-
-if ! @bunzip2@ < "\$path" | @bindir@/nix-store --restore "\$1"; then
-    exit 1
-fi
-
-exit 0
-EOF
-
-chmod +x $out/fetch

corepkgs/nix-pull/default.nix

@@ -1,7 +0,0 @@
-{system}:
-
-derivation {
-  name = "nix-pull";
-  builder = ./builder.sh;
-  inherit system;
-}

doc/manual/Makefile.am

@@ -1,37 +1,41 @@
-ENV = SGML_CATALOG_FILES=$(docbookcatalog):$(docbookebnfcatalog)
+ENV = SGML_CATALOG_FILES=$(docbookcatalog)
 
 XMLLINT = $(ENV) $(xmllint) $(xmlflags) --catalogs
 XSLTPROC = $(ENV) $(xsltproc) $(xmlflags) --catalogs \
  --param section.autolabel 1 \
  --param section.label.includes.component.label 1 \
  --param html.stylesheet \'style.css\' \
- --param xref.with.number.and.title 0
+ --param xref.with.number.and.title 1 \
+ --param toc.section.depth 3
 
-man1_MANS = nix-env.1 nix-store.1 nix-instantiate.1 \
+man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
- nix-prefetch-url.1
+ nix-prefetch-url.1 nix-channel.1
 
 FIGURES = figures/user-environments.png
 
-SOURCES = manual.xml introduction.xml installation.xml \
+MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  package-management.xml writing-nix-expressions.xml \
  build-farm.xml \
  $(man1_MANS:.1=.xml) \
  troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
- quick-start.xml nix-lang-ref.xml style.css images
+ env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
+ conf-file.xml \
+ style.css images
 
-manual.is-valid: $(SOURCES) version.xml
-	$(XMLLINT) --noout --valid manual.xml
+manual.is-valid: $(MANUAL_SRCS) version.txt
+	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --valid -
 	touch $@
 
-version.xml:
-	echo -n $(VERSION) > version.xml
+version.txt:
+	echo -n $(VERSION) > version.txt
 
-man $(MANS): $(SOURCES) manual.is-valid
-	$(XSLTPROC) $(docbookxsl)/manpages/docbook.xsl manual.xml
+man $(MANS): $(MANUAL_SRCS) manual.is-valid
+	$(XSLTPROC) --nonet --xinclude $(docbookxsl)/manpages/docbook.xsl manual.xml
 
-manual.html: $(SOURCES) manual.is-valid images
-	$(XSLTPROC) --output manual.html $(docbookxsl)/html/docbook.xsl manual.xml
+manual.html: $(MANUAL_SRCS) manual.is-valid images
+	$(XSLTPROC) --nonet --xinclude --output manual.html \
+	  $(docbookxsl)/html/docbook.xsl manual.xml
 
 all-local: manual.html
 
@@ -50,8 +54,8 @@ images:
 	cp $(docbookxsl)/images/callouts/*.png images/callouts
 	chmod +w -R images
 
-KEEP = manual.html manual.is-valid version.xml $(MANS)
+KEEP = manual.html manual.is-valid version.txt $(MANS)
 
-EXTRA_DIST = $(SOURCES) $(FIGURES) $(KEEP)
+EXTRA_DIST = $(MANUAL_SRCS) $(FIGURES) $(KEEP)
 
 DISTCLEANFILES = $(KEEP)

doc/manual/bugs.xml

@@ -2,90 +2,25 @@
 
 <itemizedlist>
 
-    <listitem>
-      <para>
-        The man-pages generated from the DocBook documentation are ugly.
-      </para>
-    </listitem>
+<listitem><para>The man-pages generated from the DocBook documentation
+are ugly.</para></listitem>
 
-    <listitem>
-      <para>
-        Generations properly form a tree.  E.g., if after switching to
-        generation 39, we perform an installation action, a generation
-        43 is created which is a descendant of 39, not 42.  So a
-        rollback from 43 ought to go back to 39.  This is not
-        currently implemented; generations form a linear sequence.
-      </para>
-    </listitem>
+<listitem><para>Generations properly form a tree.  E.g., if after
+switching to generation 39, we perform an installation action, a
+generation 43 is created which is a descendant of 39, not 42.  So a
+rollback from 43 ought to go back to 39.  This is not currently
+implemented; generations form a linear sequence.</para></listitem>
 
-    <listitem>
-      <para>
-        Unify the concepts of successors and substitutes into a
-        general notion of <emphasis>equivalent expressions</emphasis>.
-        Expressions are equivalent if they have the same target paths
-        with the same identifiers.  However, even though they are
-        functionally equivalent, they may differ stronly with respect
-        to their <emphasis>performance characteristics</emphasis>.
-        For example, realising a closure expression is more efficient
-        that realising the derivation expression from which it was
-        produced.  On the other hand, distributing sources may be more
-        efficient (storage- or bandwidth-wise) than distributing
-        binaries.  So we need to be able to attach weigths or
-        priorities or performance annotations to expressions; Nix can
-        then choose the most efficient expression dependent on the
-        context.
-      </para>
-    </listitem>
-
-    <listitem>
-      <para>
-        <emphasis>Build management.</emphasis>  In principle it is already
-        possible to do build management using Nix (by writing builders that
-        perform appropriate build steps), but the Nix expression language is
-        not yet powerful enough to make this pleasant (?).  The language should
-        be extended with features from the <ulink
-          url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
-        Another interesting idea is to write a <command>make</command>
-        implementation that uses Nix as a back-end to support <ulink
-          url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink> 
-        build files.
-      </para>
-    </listitem>
-
-    <listitem>
-      <para>
-        There are race conditions between the garbage collector and
-        other Nix tools.  For instance, when we run
-        <command>nix-env</command> to build and install a derivation
-        and run the garbage collector at the same time, the garbage
-        collector may kick in exactly between the build and
-        installation steps, i.e., before the newly built derivation
-        has become reachable from a root of the garbage collector.
-      </para>
-
-      <para>
-        One solution would be for these programs to properly register
-        temporary roots for the collector.  Another would be to use
-        stop-the-world garbage collection: if any tool is running, the
-        garbage collector blocks, and vice versa.  These solutions do
-        not solve the situation where multiple tools are involved,
-        e.g.,
-
-        <screen>
-$ nix-store -r $(nix-instantiate foo.nix)</screen>
-
-        since even if <command>nix-instantiate</command> where to
-        register a temporary root, it would be released by the time
-        <command>nix-store</command> is started.  A solution would be
-        to write the intermediate value to a file that is used as a
-        root to the collector, e.g.,
-        
-        <screen>
-$ nix-instantiate foo.nix > /nix/var/nix/roots/bla
-$ nix-store -r $(cat /nix/var/nix/roots/bla)</screen>
-
-      </para>
-    </listitem>
+<listitem><para><emphasis>Build management.</emphasis> In principle it
+is already possible to do build management using Nix (by writing
+builders that perform appropriate build steps), but the Nix expression
+language is not yet powerful enough to make this pleasant (?).  The
+language should be extended with features from the <ulink
+url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
+Another interesting idea is to write a <command>make</command>
+implementation that uses Nix as a back-end to support <ulink
+url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink>
+build files.</para></listitem>
 
 <listitem><para>For security, <command>nix-push</command> manifests
 should be digitally signed, and <command>nix-pull</command> should
@@ -94,15 +29,18 @@ need to be signed, since the manifest contains cryptographic hashes of
 these files (and <filename>fetchurl.nix</filename> checks
 them).</para></listitem>
 
-<listitem><para>We should switch away from MD5, since it has been
-more-or-less cracked.  We don't currently depend very much on the
-collision-resistance of MD5, but we will once we start sharing build
-results between users.</para></listitem>
-
 <listitem><para>It would be useful to have an option in
 <command>nix-env --delete-generations</command> to remove non-current
 generations older than a certain age.</para></listitem>
 
+<listitem><para>There should be a flexible way to change the user
+environment builder.  Currently, you have to replace
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>,
+which is hard-coded into <command>nix-env</command>.  Also, the
+default builder should be more powerful.  For instance, there should
+be some way to specify priorities to resolve
+collisions.</para></listitem>
+
 </itemizedlist>
 
 </appendix>

doc/manual/build-farm.xml

@@ -65,7 +65,10 @@ will call whenever it wants to build a derivation.  The build hook
 will perform it in the usual way if possible, or it can accept it, in
 which case it is responsible for somehow getting the inputs of the
 build to another machine, doing the build there, and getting the
-results back.</para>
+results back.  The details of the build hook protocol are described in
+the documentation of the <link
+linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar>
+variable</link>.</para>
 
 <example id='ex-remote-systems'><title>Remote machine configuration:
 <filename>remote-systems.conf</filename></title>

doc/manual/conf-file.xml

@@ -0,0 +1,82 @@
+<sect1 id="sec-conf-file"><title>Nix configuration file</title>
+
+<para>A number of persistent settings of Nix are stored in the file
+<filename><replaceable>prefix</replaceable>/etc/nix/nix.conf</filename>.
+This file is a list of <literal><replaceable>name</replaceable> =
+<replaceable>value</replaceable></literal> pairs, one per line.
+Comments start with a <literal>#</literal> character.  An example
+configuration file is shown in <xref linkend="ex-nix-conf" />.</para>
+
+<example id='ex-nix-conf'><title>Nix configuration file</title>
+
+<programlisting>
+gc-keep-outputs = true       # Nice for developers
+gc-keep-derivations = true   # Idem
+env-keep-derivations = false
+</programlisting>
+</example>
+
+<para>The following variables are currently available: 
+
+<variablelist>
+
+  <varlistentry id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
+
+    <listitem><para>If <literal>true</literal>, the garbage collector
+    will keep the outputs of non-garbage derivations.  If
+    <literal>false</literal> (default), outputs will be deleted unless
+    they are GC roots themselves (or reachable from other roots).</para>
+ 
+    <para>In general, outputs must be registered as roots separately.
+    However, even if the output of a derivation is registered as a
+    root, the collector will still delete store paths that are used
+    only at build time (e.g., the C compiler, or source tarballs
+    downloaded from the network).  To prevent it from doing so, set
+    this option to <literal>true</literal>.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>true</literal> (default), the garbage
+    collector will keep the derivations from which non-garbage store
+    paths were built.  If <literal>false</literal>, they will be
+    deleted unless explicitly registered as a root (or reachable from
+    other roots).</para>
+
+    <para>Keeping derivation around is useful for querying and
+    traceability (e.g., it allows you to ask with what dependencies or
+    options a store path was built), so by default this option is on.
+    Turn it off to safe a bit of disk space (or a lot if
+    <literal>gc-keep-outputs</literal> is also turned on).</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><literal>env-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>false</literal> (default), derivations
+    are not stored in Nix user environments.  That is, the derivation
+    any build-time-only dependencies may be garbage-collected.</para>
+
+    <para>If <literal>true</literal>, when you add a Nix derivation to
+    a user environment, the path of the derivation is stored in the
+    user environment.  Thus, the derivation will not be
+    garbage-collected until the user environment generation is deleted
+    (<command>nix-env --delete-generations</command>).  To prevent
+    build-time-only dependencies from being collected, you should also
+    turn on <literal>gc-keep-outputs</literal>.</para>
+
+    <para>The difference between this option and
+    <literal>gc-keep-derivations</literal> is that this one is
+    “sticky”: it applies to any user environment created while this
+    option was enabled, while <literal>gc-keep-derivations</literal>
+    only applies at the moment the garbage collector is
+    run.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+</sect1>

doc/manual/env-common.xml

@@ -0,0 +1,274 @@
+<sect1 id="sec-common-env"><title>Common environment variables</title>
+
+<para>Most Nix commands interpret the following environment variables:</para>
+
+<variablelist>
+
+  
+<varlistentry><term><envar>NIX_ROOT</envar></term>
+
+  <listitem><para>If <envar>NIX_ROOT</envar> is set, the Nix command
+  will on startup perform a <function>chroot()</function> to the
+  specified directory.  This is useful in certain bootstrapping
+  situations (e.g., when installing a Nix installation onto a hard
+  disk from CD-ROM).</para></listitem>
+
+</varlistentry>
+
+  
+<varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term>
+
+  <listitem>
+
+  <para>Normally, the Nix store directory (typically
+  <filename>/nix/store</filename>) is not allowed to contain any
+  symlink components.  This is to prevent “impure” builds.  Builders
+  sometimes “canonicalise” paths by resolving all symlink components.
+  Thus, builds on different machines (with
+  <filename>/nix/store</filename> resolving to different locations)
+  could yield different results.  This is generally not a problem,
+  except when builds are deployed to machines where
+  <filename>/nix/store</filename> resolves differently.  If you are
+  sure that you’re not going to do that, you can set
+  <envar>NIX_IGNORE_SYMLINK_STORE</envar> to <envar>1</envar>.</para>
+
+  <para>Note that if you’re symlinking the Nix store so that you can
+  put it on another file system than the root file system, on Linux
+  you’re better off using <literal>bind</literal> mount points, e.g.,
+
+  <screen>
+$ mkdir /nix   
+$ mount -o bind /mnt/otherdisk/nix /nix</screen>
+
+  Consult the <citerefentry><refentrytitle>mount</refentrytitle>
+  <manvolnum>8</manvolnum></citerefentry> manual page for details.</para>
+
+  </listitem>
+
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STORE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix store (default
+  <filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DATA_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix static data
+  directory (default
+  <filename><replaceable>prefix</replaceable>/share</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix log directory
+  (default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STATE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix state directory
+  (default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DB_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix database (default
+  <filename><replaceable>$NIX_STATE_DIR</replaceable>/db</filename>, i.e.,
+  <filename><replaceable>prefix</replaceable>/var/nix/db</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_CONF_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix configuration
+  directory (default
+  <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_TYPE</envar></term>
+
+  <listitem><para>Equivalent to the <link
+  linkend="opt-log-type"><option>--log-type</option>
+  option</link>.</para></listitem>
+
+</varlistentry>
+  
+
+<varlistentry><term><envar>TMPDIR</envar></term>
+
+  <listitem><para>Use the specified directory to store temporary
+  files.  In particular, this includes temporary build directories;
+  these can take up substantial amounts of disk space.  The default is
+  <filename>/tmp</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term>
+
+  <listitem>
+
+  <para>Specifies the location of the <emphasis>build hook</emphasis>,
+  which is a program (typically some script) that Nix will call
+  whenever it wants to build a derivation.  This is used to implement
+  distributed builds (see <xref linkend="sec-distributed-builds"
+  />).  The protocol by which the calling Nix process and the build
+  hook communicate is as follows.</para>
+
+  <para>The build hook is called with the following command-line
+  arguments:
+
+  <orderedlist>
+
+    <listitem><para>A boolean value <literal>0</literal> or
+    <literal>1</literal> specifying whether Nix can locally execute
+    more builds, as per the <link
+    linkend="opt-max-jobs"><option>--max-jobs</option> option</link>.
+    The purpose of this argument is to allow the hook to not have to
+    maintain bookkeeping for the local machine.</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the local machine
+    (e.g., <literal>i686-linux</literal>).</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the derivation,
+    i.e., its <link linkend="attr-system"><varname>system</varname>
+    attribute</link>.</para></listitem>
+
+    <listitem><para>The store path of the derivation.</para></listitem>
+
+  </orderedlist>
+
+  </para>
+
+  <para>On the basis of this information, and whatever persistent
+  state the build hook keeps about other machines and their current
+  load, it has to decide what to do with the build.  It should print
+  out on file descriptor 3 one of the following responses (terminated
+  by a newline, <literal>"\n"</literal>):
+
+  <variablelist>
+
+    <varlistentry><term><literal>decline</literal></term>
+
+      <listitem><para>The build hook is not willing or able to perform
+      the build; the calling Nix process should do the build itself,
+      if possible.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>postpone</literal></term>
+
+      <listitem><para>The build hook cannot perform the build now, but
+      can do so in the future (e.g., because all available build slots
+      on remote machines are in use).  The calling Nix process should
+      postpone this build until at least one currently running build
+      has terminated.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>accept</literal></term>
+
+      <listitem><para>The build hook has accepted the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the build hook accepts the build, it is possible that it is
+  no longer necessary to do the build because some other process has
+  performed the build in the meantime.  To prevent races, the hook
+  must read from file descriptor 4 a single line that tells it whether
+  to continue:
+
+  <variablelist>
+
+    <varlistentry><term><literal>cancel</literal></term>
+
+      <listitem><para>The build has already been done, so the hook
+      should exit.</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><literal>okay</literal></term>
+
+      <listitem><para>The hook should proceed with the build.  At this
+      point, the calling Nix process has acquired locks on the output
+      path, so no other Nix process will perform the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the hook has been told to proceed, Nix will store in the
+  hook’s current directory a number of text files that contain
+  information about the derivation:
+
+  <variablelist>
+
+    <varlistentry><term><filename>inputs</filename></term>
+
+      <listitem><para>The set of store paths that are inputs to the
+      build process (one per line).  These have to be copied
+      <emphasis>to</emphasis> the remote machine (in addition to the
+      store derivation itself).</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><filename>outputs</filename></term>
+
+      <listitem><para>The set of store paths that are outputs of the
+      derivation (one per line).  These have to be copied
+      <emphasis>from</emphasis> the remote machine if the build
+      succeeds.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><filename>references</filename></term>
+
+      <listitem><para>The reference graph of the inputs, in the format
+      accepted by the command <command>nix-store
+      --register-validity</command>.  It is necessary to run this
+      command on the remote machine after copying the inputs to inform
+      Nix on the remote machine that the inputs are valid
+      paths.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>The hook should copy the inputs to the remote machine,
+  register the validity of the inputs, perform the remote build, and
+  copy the outputs back to the local machine.  An exit code other than
+  <literal>0</literal> indicates that the hook has failed.</para>
+
+  </listitem>
+
+
+</varlistentry>
+
+
+</variablelist>
+
+</sect1>

doc/manual/glossary.xml

@@ -0,0 +1,163 @@
+<appendix><title>Glossary</title>
+
+<glosslist>
+
+
+<glossentry id="gloss-derivation"><glossterm>derivation</glossterm>
+
+  <glossdef><para>A description of a build action.  The result of a
+  derivation is a store object.  Derivations are typically specified
+  in Nix expressions using the <link
+  linkend="ssec-derivation"><function>derivation</function>
+  primitive</link>.  These are translated into low-level
+  <emphasis>store derivations</emphasis> (implicitly by
+  <command>nix-env</command> and <command>nix-build</command>, or
+  explicitly by <command>nix-instantiate</command>).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store</glossterm>
+
+  <glossdef><para>The location in the file system where store objects
+  live.  Typically <filename>/nix/store</filename>.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store path</glossterm>
+
+  <glossdef><para>The location in the file system of a store object,
+  i.e., an immediate child of the Nix store
+  directory.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store object</glossterm>
+
+  <glossdef><para>A file that is an immediate child of the Nix store
+  directory.  These can be regular files, but also entire directory
+  trees.  Store objects can be sources (objects copied from outside of
+  the store), derivation outputs (objects produced by running a build
+  action), or derivations (files describing a build
+  action).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-substitute"><glossterm>substitute</glossterm>
+
+  <glossdef><para>A substitute is a command invocation stored in the
+  Nix database that describes how to build a store object, bypassing
+  normal the build mechanism (i.e., derivations).  Typically, the
+  substitute builds the store object by downloading a pre-built
+  version of the store object from some server.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>purity</glossterm>
+
+  <glossdef><para>The assumption that equal Nix derivations when run
+  always produce the same output.  This cannot be guaranteed in
+  general (e.g., a builder can rely on external inputs such as the
+  network or the system time) but the Nix model assumes
+  it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>Nix expression</glossterm>
+
+  <glossdef><para>A high-level description of software components and
+  compositions thereof.  Deploying software using Nix entails writing
+  Nix expressions for your components.  Nix expressions are translated
+  to derivations that are stored in the Nix store.  These derivations
+  can then be built.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-reference"><glossterm>reference</glossterm>
+
+  <glossdef><para>A store path <varname>P</varname> is said to have a
+  reference to a store path <varname>Q</varname> if the store object
+  at <varname>P</varname> contains the path <varname>Q</varname>
+  somewhere.  This implies than an execution involving
+  <varname>P</varname> potentially needs <varname>Q</varname> to be
+  present.  The <emphasis>references</emphasis> of a store path are
+  the set of store paths to which it has a reference.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-closure"><glossterm>closure</glossterm>
+
+  <glossdef><para>The closure of a store path is the set of store
+  paths that are directly or indirectly “reachable” from that store
+  path; that is, it’s the closure of the path under the <link
+  linkend="gloss-reference">references</link> relation.  For instance,
+  if the store object at path <varname>P</varname> contains a
+  reference to path <varname>Q</varname>, then <varname>Q</varname> is
+  in the closure of <varname>P</varname>.  For correct deployment it
+  is necessary to deploy whole closures, since otherwise at runtime
+  files could be missing.  The command <command>nix-store
+  -qR</command> prints out closures of store paths.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-output-path"><glossterm>output path</glossterm>
+
+  <glossdef><para>A store path produced by a derivation.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-deriver"><glossterm>deriver</glossterm>
+
+  <glossdef><para>The deriver of an <link
+  linkend="gloss-output-path">output path</link> is the store
+  derivation that built it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-validity"><glossterm>validity</glossterm>
+
+  <glossdef><para>A store path is considered
+  <emphasis>valid</emphasis> if it exists in the file system, is
+  listed in the Nix database as being valid, and if all paths in its
+  closure are also valid.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-user-env"><glossterm>user environment</glossterm>
+
+  <glossdef><para>An automatically generated store object that
+  consists of a set of symlinks to “active” applications, i.e., other
+  store paths.  These are generated automatically by <link
+  linkend="sec-nix-env"><command>nix-env</command></link>.  See <xref
+  linkend="sec-profiles" />.</para>
+
+  </glossdef>
+  
+</glossentry>
+
+
+<glossentry id="gloss-profile"><glossterm>profile</glossterm>
+
+  <glossdef><para>A symlink to the current <link
+  linkend="gloss-user-env">user environment</link> of a user, e.g.,
+  <filename>/nix/var/nix/profiles/default</filename>.</para></glossdef>
+
+</glossentry>
+
+
+
+</glosslist>
+
+
+</appendix>

doc/manual/installation.xml

@@ -5,8 +5,8 @@
 
 <para>The easiest way to obtain Nix is to download a <ulink
 url='http://www.cs.uu.nl/groups/ST/Trace/Nix'>source
-distribution</ulink>.  RPMs for Red Hat 9 are also available.  These
-distributions are generated automatically.</para>
+distribution</ulink>.  RPMs for Red Hat, SuSE, and Fedore Core are
+also available.</para>
 
 <para>Alternatively, the most recent sources of Nix can be obtained
 from its <ulink

doc/manual/introduction.xml

@@ -75,7 +75,7 @@ be adapted easily to achieve similar policies.</para></listitem>
 
 <listitem><para>Nix component builds aim to be <quote>pure</quote>;
 that is, unaffected by anything other than the declared dependencies.
-This means that if a component was built succesfully once, it can be
+This means that if a component was built successfully once, it can be
 rebuilt again on another machine and the result will be the same.  We
 cannot <emphasis>guarantee</emphasis> this (e.g., if the build depends
 on the time-of-day), but Nix (and the tools in the Nix Packages
@@ -113,6 +113,13 @@ software deployment also apply here: for instance, the ability
 trivially to have multiple configurations at the same time, or the
 ability to do rollbacks.</para></listitem>
 
+<listitem><para>Nix can efficiently upgrade between different versions
+of a component through <emphasis>binary patching</emphasis>.  If
+patches are available on a server, and you try to install a new
+version of some component, Nix will automatically apply a patch (or
+sequence of patches), if available, to transform the installed
+component into the new version.</para></listitem> 
+
 </itemizedlist>
 
 </para>
@@ -123,11 +130,7 @@ collection.  It also discusses some advanced topics, such as setting
 up a Nix-based build farm, and doing service deployment using
 Nix.</para>
 
-<warning><para>This manual is a work in progress.  It's quite likely
-to be incomplete, inconsistent with the current implementation, or
-simply wrong.</para></warning>
-
-<note><para>Some background information on Nix can be found in two
+<note><para>Some background information on Nix can be found in three
 papers.  The ICSE 2004 paper <ulink
 url='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'><citetitle>Imposing
 a Memory Management Discipline on Software
@@ -138,6 +141,10 @@ different versions and variants of packages.  The LISA 2004 paper
 url='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'><citetitle>Nix:
 A Safe and Policy-Free System for Software
 Deployment</citetitle></ulink> gives a more general discussion of Nix
-from a system-administration perspective.</para></note>
+from a system-administration perspective.  The CBSE 2005 paper <ulink
+url='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'><citetitle>Efficient
+Upgrading in a Purely Functional Component Deployment Model
+</citetitle></ulink> is about transparent patch deployment in
+Nix.</para></note>
 
 </chapter>

doc/manual/manual.xml

@@ -3,31 +3,14 @@
     PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
     "http://www.docbook.org/xml/4.3/docbook-xml-4.3.zip"
 [
-<!ENTITY introduction SYSTEM "introduction.xml">
-<!ENTITY quick-start SYSTEM "quick-start.xml">
-<!ENTITY installation SYSTEM "installation.xml">
-<!ENTITY package-management SYSTEM "package-management.xml">
-<!ENTITY writing-nix-expressions SYSTEM "writing-nix-expressions.xml">
-<!ENTITY build-farm SYSTEM "build-farm.xml">
-<!ENTITY opt-common SYSTEM "opt-common.xml">
-<!ENTITY opt-common-syn SYSTEM "opt-common-syn.xml">
-<!ENTITY nix-env SYSTEM "nix-env.xml">
-<!ENTITY nix-store SYSTEM "nix-store.xml">
-<!ENTITY nix-instantiate SYSTEM "nix-instantiate.xml">
-<!ENTITY nix-collect-garbage SYSTEM "nix-collect-garbage.xml">
-<!ENTITY nix-push SYSTEM "nix-push.xml">
-<!ENTITY nix-pull SYSTEM "nix-pull.xml">
-<!ENTITY nix-prefetch-url SYSTEM "nix-prefetch-url.xml">
-<!-- <!ENTITY nix-lang-ref SYSTEM "nix-lang-ref.xml"> -->
-<!ENTITY troubleshooting SYSTEM "troubleshooting.xml">
-<!ENTITY bugs SYSTEM "bugs.xml">
-<!ENTITY version SYSTEM "version.xml">
 ]>
 
 <book>
   <title>Nix User's Guide</title>
 
-  <subtitle>Draft (Version &version;)</subtitle>
+  <subtitle>Draft (Version <xi:include
+  xmlns:xi="http://www.w3.org/2001/XInclude"
+  href="version.txt" parse="text" />)</subtitle>
   
   <bookinfo>
     <author>
@@ -36,52 +19,65 @@
     </author>
     <copyright>
       <year>2004</year>
+      <year>2005</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
   </bookinfo>
-
-  &introduction;
-  &quick-start;
-  &installation;
-  &package-management;
-  &writing-nix-expressions;
-  &build-farm;
+  
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="introduction.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="quick-start.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="installation.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="package-management.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="writing-nix-expressions.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="build-farm.xml" />
 
   <appendix>
     <title>Command Reference</title>
-    <sect1>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="env-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="conf-file.xml" />
+    <sect1 id="sec-nix-env">
       <title>nix-env</title>
-      &nix-env;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-env.xml" />
+    </sect1>
+    <sect1 id="sec-nix-build">
+      <title>nix-build</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-build.xml" />
     </sect1>
     <sect1>
       <title>nix-store</title>
-      &nix-store;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-store.xml" />
     </sect1>
-    <sect1>
+    <sect1 id="sec-nix-instantiate">
       <title>nix-instantiate</title>
-      &nix-instantiate;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-instantiate.xml" />
     </sect1>
     <sect1>
       <title>nix-collect-garbage</title>
-      &nix-collect-garbage;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-collect-garbage.xml" />
+    </sect1>
+    <sect1 id="sec-nix-channel">
+      <title>nix-channel</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-channel.xml" />
     </sect1>
     <sect1>
       <title>nix-push</title>
-      &nix-push;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-push.xml" />
     </sect1>
     <sect1>
       <title>nix-pull</title>
-      &nix-pull;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-pull.xml" />
     </sect1>
     <sect1>
       <title>nix-prefetch-url</title>
-      &nix-prefetch-url;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-prefetch-url.xml" />
     </sect1>
   </appendix>
 
-<!--  &nix-lang-ref; -->
+  <!--  &nix-lang-ref; -->
 
-  &troubleshooting;
-  &bugs;
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="troubleshooting.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="bugs.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="glossary.xml" />
 
 </book>

doc/manual/nix-build.xml

@@ -0,0 +1,74 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-build</refname>
+  <refpurpose>build a Nix expression</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-build</command>
+    <arg><option>--add-drv-link</option></arg>
+    <arg><option>--no-link</option></arg>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>The <command>nix-build</command> command builds the derivations
+described by the Nix expressions in <replaceable>paths</replaceable>.
+If the build succeeds, it places a symlink to the result in the
+current directory.  The symlink is called <filename>result</filename>.
+If there are multiple Nix expressions, or the Nix expressions evaluate
+to multiple derivations, multiple sequentially numbered symlinks are
+created (<filename>result</filename>, <filename>result-2</filename>,
+and so on).</para>
+
+<note><para><command>nix-build</command> is essentially a wrapper
+around <link
+linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>
+(to translate a high-level Nix expression to a low-level store
+derivation) and <link
+linkend="rsec-nix-store-realise"><command>nix-store
+--realise</command></link> (to build the store
+derivation).</para></note>
+
+<warning><para>The result of the build is automatically registered as
+a root of the Nix garbage collector.  This root disappears
+automatically when the <filename>result</filename> symlink is deleted
+or renamed.  So don’t rename the symlink.</para></warning>
+
+</refsection>
+
+
+<refsection><title>Options</title>
+
+<variablelist>
+
+  <varlistentry><term><option>--add-drv-link</option></term>
+  
+    <listitem><para>Add a symlink in the current directory to the
+    store derivation produced by <command>nix-instantiate</command>.
+    The symlink is called <filename>derivation</filename> (which is
+    numbered in the case of multiple derivations).  The derivation is
+    a root of the garbage collector until the symlink is deleted or
+    renamed.</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--no-link</option></term>
+  
+    <listitem><para>Do not create a symlink to the output path.  Note
+    that as a result the output does not become a root of the garbage
+    collector, and so might be deleted by <command>nix-store
+    --gc</command>.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</refsection>
+
+
+</refentry>

doc/manual/nix-channel.xml

@@ -0,0 +1,83 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-channel</refname>
+  <refpurpose>manage Nix channels</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-channel</command>
+    <group choice='req'>
+      <arg choice='plain'><option>--add</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--remove</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--list</option></arg>
+      <arg choice='plain'><option>--update</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>A Nix channel is mechanism that allows you to automatically stay
+up-to-date with a set of pre-built Nix expressions.  A Nix channel is
+just a URL that points to a place that contains a set of Nix
+expressions, as well as a <command>nix-push</command> manifest.  See
+also <xref linkend="sec-channels" />.</para>
+
+<para>This command has the following operations:
+
+<variablelist>
+
+  <varlistentry><term><option>--add</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Adds <replaceable>url</replaceable> to the list of
+    subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--remove</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Removes <replaceable>url</replaceable> from the
+    list of subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--list</option></term>
+
+    <listitem><para>Prints the URLs of all subscribed channels on
+    standard output.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--update</option></term>
+
+    <listitem><para>Downloads the Nix expressions of all subscribed
+    channels, makes the conjunction of these the default for
+    <command>nix-env</command> operations (by calling <command>nix-env
+    -I</command>), and performs a <command>nix-pull</command> on the
+    manifests of all channels to make pre-built binaries
+    available.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+<para>Note that <option>--add</option> and <option>--remove</option>
+do not automatically perform an update.</para>
+
+<para>The list of subscribed channels is stored in
+<filename>~/.nix-channels</filename>.</para>
+
+<para>A channel consists of two elements: a bzipped Tar archive
+containing the Nix expressions, and a manifest created by
+<command>nix-push</command>.  These must be stored under
+<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal> and
+<literal><replaceable>url</replaceable>/MANIFEST</literal>,
+respectively.</para>
+
+</refsection>
+
+</refentry>

doc/manual/nix-collect-garbage.xml

@@ -1,87 +1,29 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-collect-garbage</refname>
-    <refpurpose>remove unreachable store paths</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-collect-garbage</refname>
+  <refpurpose>delete unreachable store paths</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-collect-garbage</command>
-      <group choice='opt'>
-        <arg choice='plain'><option>--print-live</option></arg>
-        <arg choice='plain'><option>--print-dead</option></arg>
-      </group>
-      <arg><option>--min-age</option> <replaceable>age</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-collect-garbage</command>
+    <group choice='opt'>
+      <arg choice='plain'><option>--print-roots</option></arg>
+      <arg choice='plain'><option>--print-live</option></arg>
+      <arg choice='plain'><option>--print-dead</option></arg>
+      <arg choice='plain'><option>--delete</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
+<refsection><title>Description</title>
 
-    <para>
-      The command <command>nix-collect-garbage</command> performs a
-      garbage collection on the Nix store: any paths in the Nix store
-      that are garbage (not reachable from a set of root store
-      expressions) are deleted.
-    </para>
+<para>The command <command>nix-collect-garbage</command> is an
+obsolete wrapper around <link
+linkend="rsec-nix-store-gc"><command>nix-store
+--gc</command></link>.</para>
 
-    <para>
-      The roots of the garbage collector are the store expressions
-      mentioned in the files in the directory
-      <filename><replaceable>prefix</replaceable>/var/nix/gcroots</filename>.
-      By default, the roots are all user environments in
-      <filename><replaceable>prefix</replaceable>/var/nix/profiles</filename>.
-      You can register other store expressions as roots by writing the
-      full path of the store expression to an arbitrary file in the
-      <filename>gcroots</filename> directory (or a subdirectory
-      thereof).
-    </para>
+</refsection>
 
-  </refsection>
-
-  <refsection>
-    <title>Options</title>
-
-    <variablelist>
-
-      <varlistentry>
-        <term><option>--print-live</option> / <option>--print-dead</option></term>
-        <listitem>
-          <para>
-            These options cause the set of live or dead paths to be
-            printed, respectively, rather than performing an actual
-            garbage collector.  They correspond exactly with the
-            sub-operations in <command>nix-store
-            <option>--gc</option></command>.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-      <varlistentry>
-        <term><option>--min-age</option> <replaceable>age</replaceable></term>
-        <listitem>
-          <para>
-            This option corresponds to the <option>--min-age</option>
-            option in <command>nix-store <option>--gc</option></command>.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-    </variablelist>
-
-  </refsection>
-
-  <refsection>
-    <title>Examples</title>
-
-    <para>
-      To delete all unreachable paths, just do:
-    
-      <screen>
-$ nix-collect-garbage</screen>
-
-    </para>
-
-  </refsection>
-    
 </refentry>

doc/manual/nix-instantiate.xml

@@ -1,90 +1,97 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-instantiate</refname>
-    <refpurpose>instantiate store expressions from Nix expressions</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-instantiate</refname>
+  <refpurpose>instantiate store derivations from Nix expressions</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-instantiate</command>
-      &opt-common-syn;
-      <group choice='opt'>
-        <arg choice='plain'><option>--parse-only</option></arg>
-        <arg choice='plain'><option>--eval-only</option></arg>
-      </group>
-      <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-instantiate</command>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
+    <arg><option>--indirect</option></arg>
+    <group choice='opt'>
+      <arg choice='plain'><option>--parse-only</option></arg>
+      <arg choice='plain'><option>--eval-only</option></arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-instantiate</command> generates
-      (low-level) store expressions from (high-level) Nix expressions.
-      It loads and evaluates the Nix expressions in each of
-      <replaceable>files</replaceable>.  Each top-level expression
-      should evaluate to a derivation, a list of derivations, or a set
-      of derivations.  The paths of the resulting store expressions
-      are printed on standard output.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is generally used for testing Nix expression before
-      they are used with <command>nix-env</command>.
-    </para>
+<para>The command <command>nix-instantiate</command> generates <link
+linkend="gloss-derivation">store derivations</link> from (high-level)
+Nix expressions.  It loads and evaluates the Nix expressions in each
+of <replaceable>files</replaceable>.  Each top-level expression should
+evaluate to a derivation, a list of derivations, or a set of
+derivations.  The paths of the resulting store derivations are printed
+on standard output.</para>
 
-  </refsection>
+<para>Most users and developers don’t need to use this command
+(<command>nix-env</command> and <command>nix-build</command> perform
+store derivation instantiation from Nix expressions automatically).
+It is most commonly used for implementing new deployment
+policies.</para>
 
-  <refsection>
-    <title>Options</title>
+<para>See also <xref linkend="sec-common-options" /> for a list of
+common options.</para>
 
-    <variablelist>
+</refsection>
 
-      &opt-common;
 
-      <varlistentry>
-        <term><option>--parse-only</option></term>
-        <listitem>
-          <para>
-            Just parse the input files, and print their abstract
-            syntax trees on standard output in ATerm format.
-          </para>
-        </listitem>
-      </varlistentry>
+<refsection><title>Options</title>
+
+<variablelist>
+
+  <varlistentry>
+    <term><option>--add-root</option> <replaceable>path</replaceable></term>
+    <term><option>--indirect</option></term>
+
+    <listitem><para>See the <link linkend="opt-add-root">corresponding
+    options</link> in <command>nix-store</command>.</para></listitem>
+
+  </varlistentry>
+
+    
+  <varlistentry><term><option>--parse-only</option></term>
+  
+    <listitem><para>Just parse the input files, and print their
+    abstract syntax trees on standard output in ATerm
+    format.</para></listitem>
+    
+  </varlistentry>
       
-      <varlistentry>
-        <term><option>--eval-only</option></term>
-        <listitem>
-          <para>
-            Just parse and evaluate the input files, and print the
-            resulting values on standard output.  No instantiation of
-            store expressions takes place.
-          </para>
-        </listitem>
-      </varlistentry>
+  <varlistentry><term><option>--eval-only</option></term>
+  
+    <listitem><para>Just parse and evaluate the input files, and print
+    the resulting values on standard output.  No instantiation of
+    store derivations takes place.</para></listitem>
+    
+  </varlistentry>
 
-    </variablelist>
+</variablelist>
 
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Examples</title>
 
-    <screen>
-$ nix-instantiate gcc.nix <lineannotation>(instantiate)</lineannotation>
-/nix/store/468abdcb93aa22bb721142615b97698b-d-gcc-3.3.2.store
+<refsection><title>Examples</title>
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(build)</lineannotation>
+<screen>
+$ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation>
+/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(print output path)</lineannotation>
-/nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
+$ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation>
+<replaceable>...</replaceable>
+/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation>
 
-$ ls -l /nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
-dr-xr-xr-x    2 eelco    users         360 2003-12-01 16:12 bin
-dr-xr-xr-x    3 eelco    users          72 2003-12-01 16:12 include
+$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
+dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
 ...</screen>
 
-  </refsection>
-    
+</refsection>
+
+
 </refentry>

doc/manual/nix-prefetch-url.xml

@@ -1,54 +1,69 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-prefetch-url</refname>
-    <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-prefetch-url</refname>
+  <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-prefetch-url</command>
-      <arg choice='plain'><replaceable>url</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-prefetch-url</command>
+    <arg choice='plain'><replaceable>url</replaceable></arg>
+    <arg><replaceable>hash</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-prefetch-url</command> downloads the
-      file referenced by the URL <replaceable>url</replaceable>,
-      prints its MD5 cryptographic hash code, and copies it into the
-      Nix store.  The file name in the store is
-      <filename><replaceable>hash</replaceable>-<replaceable>basename</replaceable></filename>,
-      where <replaceable>basename</replaceable> is everything
-      following the final slash in <replaceable>url</replaceable>.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is just a convenience to Nix expression writers.
-      Often a Nix expressions fetch some source distribution from the
-      network using the <literal>fetchurl</literal> expression
-      contained in <literal>nixpkgs</literal>.  However,
-      <literal>fetchurl</literal> requires an MD5 hash.  If you don't
-      know the hash, you would have to download the file first, and
-      then <literal>fetchurl</literal> would download it again when
-      you build your Nix expression.  Since
-      <literal>fetchurl</literal> uses the same name for the
-      downloaded file as <command>nix-prefetch-url</command>, the
-      redundant download can be avoided.
-    </para>
+<para>The command <command>nix-prefetch-url</command> downloads the
+file referenced by the URL <replaceable>url</replaceable>, prints its
+cryptographic hash, and copies it into the Nix store.  The file name
+in the store is
+<filename><replaceable>hash</replaceable>-<replaceable>baseName</replaceable></filename>,
+where <replaceable>baseName</replaceable> is everything following the
+final slash in <replaceable>url</replaceable>.</para>
 
-  </refsection>
+<para>This command is just a convenience for Nix expression writers.
+Often a Nix expression fetches some source distribution from the
+network using the <literal>fetchurl</literal> expression contained in
+Nixpkgs.  However, <literal>fetchurl</literal> requires a
+cryptographic hash.  If you don't know the hash, you would have to
+download the file first, and then <literal>fetchurl</literal> would
+download it again when you build your Nix expression.  Since
+<literal>fetchurl</literal> uses the same name for the downloaded file
+as <command>nix-prefetch-url</command>, the redundant download can be
+avoided.</para>
 
-  <refsection>
-    <title>Examples</title>
+<para>The environment variable <envar>NIX_HASH_ALGO</envar> specifies
+which hash algorithm to use.  It can be either <literal>md5</literal>,
+<literal>sha1</literal>, or <literal>sha256</literal>.  The default is
+<literal>md5</literal>.</para>
 
-    <screen>
+<para>If <replaceable>hash</replaceable> is specified, then a download
+is not performed if the Nix store already contains a file with the
+same hash and base name.  Otherwise, the file is downloaded, and an
+error if signaled if the actual hash of the file does not match the
+specified hash.</para>
+
+<para>This command prints the hash on standard output.  Additionally,
+if the environment variable <envar>PRINT_PATH</envar> is set, the path
+of the downloaded file in the Nix store is also printed.</para>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<screen>
 $ nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
-...
-file has hash 0bbd1df101bc0294d440471e50feca71
-...</screen>
+0bbd1df101bc0294d440471e50feca71
+
+$ PRINT_PATH=1 nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
+0bbd1df101bc0294d440471e50feca71
+/nix/store/wvyz8ifdn7wyz1p3pqyn0ra45ka2l492-make-3.80.tar.bz2</screen>
+
+</refsection>
 
-  </refsection>
     
 </refentry>

doc/manual/nix-push.xml

@@ -1,138 +1,116 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-push</refname>
-    <refpurpose>push store paths onto a network cache</refpurpose>
-  </refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-push</command>
-      <arg choice='plain'><replaceable>archives-put-url</replaceable></arg>
-      <arg choice='plain'><replaceable>archives-get-url</replaceable></arg>
-      <arg choice='plain'><replaceable>manifest-put-url</replaceable></arg>
-      <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refnamediv>
+  <refname>nix-push</refname>
+  <refpurpose>push store paths onto a network cache</refpurpose>
+</refnamediv>
 
-  <refsection>
-    <title>Description</title>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-push</command>
+    <group choice='req'>
+      <arg choice='req'>
+        <arg choice='plain'><replaceable>archivesPutURL</replaceable></arg>
+        <arg choice='plain'><replaceable>archivesGetURL</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestPutURL</replaceable></arg>
+      </arg>
+      <arg choice='req'>
+        <arg choice='plain'><option>--copy</option></arg>
+        <arg choice='plain'><replaceable>archivesDir</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestFile</replaceable></arg>
+      </arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-    <para>
-      The command <command>nix-push</command> builds a set of store
-      expressions (if necessary), and then packages and uploads all
-      store paths in the resulting closures to a server.  A network
-      cache thus populated can subsequently be used to speed up
-      software deployment on other machines using the
-      <command>nix-pull</command> command.
-    </para>
 
-    <para>
-      <command>nix-push</command> performs the following actions.
+<refsection><title>Description</title>
+
+<para>The command <command>nix-push</command> builds a set of store
+paths (if necessary), and then packages and uploads all store paths in
+the resulting closures to a server.  A network cache thus populated
+can subsequently be used to speed up software deployment on other
+machines using the <command>nix-pull</command> command.</para>
+
+<para><command>nix-push</command> performs the following actions.
       
-      <orderedlist>
+<orderedlist>
 
-        <listitem>
-          <para>
-            The store expressions stored in
-            <replaceable>paths</replaceable> are realised (using
-            <literal>nix-store --realise</literal>).
-          </para>
-        </listitem>
+  <listitem><para>Each path in <replaceable>paths</replaceable> is
+  realised (using <link
+  linkend='rsec-nix-store-realise'><literal>nix-store
+  --realise</literal></link>).</para></listitem>
 
-        <listitem>
-          <para>
-            All paths in the closure of the store expressions stored
-            in <replaceable>paths</replaceable> are determined (using
-            <literal>nix-store --query --requisites
-            --include-successors</literal>).  It should be noted that
-            since the <option>--include-successors</option> flag is
-            used, if you specify a derivation store expression, you
-            get a combined source/binary distribution.  If you only
-            want a binary distribution, you should specify the closure
-            store expression that result from realising these (see
-            below).
-          </para>
-        </listitem>
+  <listitem><para>All paths in the closure of the store expressions
+  stored in <replaceable>paths</replaceable> are determined (using
+  <literal>nix-store --query --requisites
+  --include-outputs</literal>).  It should be noted that since the
+  <option>--include-outputs</option> flag is used, you get a combined
+  source/binary distribution.</para></listitem>
 
-        <listitem>
-          <para>
-            All store paths determined in the previous step are
-            packaged and compressed into a <command>bzip</command>ped
-            NAR archive (extension <filename>.nar.bz2</filename>).
-          </para>
-        </listitem>
+  <listitem><para>All store paths determined in the previous step are
+  packaged and compressed into a <command>bzip</command>ped NAR
+  archive (extension <filename>.nar.bz2</filename>).</para></listitem>
 
-        <listitem>
-          <para>
-            A <emphasis>manifest</emphasis> is created that contains
-            information on the store paths, their eventual URLs in the
-            cache, and cryptographic hashes of the contents of the NAR
-            archives.
-          </para>
-        </listitem>
+  <listitem><para>A <emphasis>manifest</emphasis> is created that
+  contains information on the store paths, their eventual URLs in the
+  cache, and cryptographic hashes of the contents of the NAR
+  archives.</para></listitem>
 
-        <listitem>
-          <para>
-            Each store path is uploaded to the remote directory
-            specified by <replaceable>archives-put-url</replaceable>.
-            HTTP PUT requests are used to do this.  However, before a
-            file <varname>x</varname> is uploaded to
-            <literal><replaceable>archives-put-url</replaceable>/<varname>x</varname></literal>,
-            <command>nix-push</command> first determines whether this
-            upload is unnecessary by issuing a HTTP HEAD request on
-            <literal><replaceable>archives-get-url</replaceable>/<varname>x</varname></literal>.
-            This allows a cache to be shared between many partially
-            overlapping <command>nix-push</command> invocations.
-            (We use two URLs because the upload URL typically
-            refers to a CGI script, while the download URL just refers
-            to a file system directory on the server.)
-          </para>
-        </listitem>
+  <listitem><para>Each store path is uploaded to the remote directory
+  specified by <replaceable>archivesPutURL</replaceable>.  HTTP PUT
+  requests are used to do this.  However, before a file
+  <varname>x</varname> is uploaded to
+  <literal><replaceable>archivesPutURL</replaceable>/<varname>x</varname></literal>,
+  <command>nix-push</command> first determines whether this upload is
+  unnecessary by issuing a HTTP HEAD request on
+  <literal><replaceable>archivesGetURL</replaceable>/<varname>x</varname></literal>.
+  This allows a cache to be shared between many partially overlapping
+  <command>nix-push</command> invocations.  (We use two URLs because
+  the upload URL typically refers to a CGI script, while the download
+  URL just refers to a file system directory on the server.)</para></listitem>
 
-        <listitem>
-          <para>
-            The manifest is uploaded using an HTTP PUT request to
-            <replaceable>manifest-put-url</replaceable>.  The
-            corresponding URL to download the manifest can then be
-            used by <command>nix-pull</command>.
-          </para>
-        </listitem>
+  <listitem><para>The manifest is uploaded using an HTTP PUT request
+  to <replaceable>manifestPutURL</replaceable>.  The corresponding
+  URL to download the manifest can then be used by
+  <command>nix-pull</command>.</para></listitem>
         
-      </orderedlist>
-    </para>
+</orderedlist>
+
+</para>
+
+<para>TODO: <option>--copy</option></para>
             
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Examples</title>
 
-    <para>
-      To upload files there typically is some CGI script on the server
-      side.  This script should be be protected with a password.  The
-      following example uploads the store paths resulting from
-      building the Nix expressions in <filename>foo.nix</filename>,
-      passing appropriate authentication information:
+<refsection><title>Examples</title>
+
+<para>To upload files there typically is some CGI script on the server
+side.  This script should be be protected with a password.  The
+following example uploads the store paths resulting from building the
+Nix expressions in <filename>foo.nix</filename>, passing appropriate
+authentication information:
     
-      <screen>
+<screen>
 $ nix-push \
     http://foo@bar:server.domain/cgi-bin/upload.pl/cache \
     http://server.domain/cache \
     http://foo@bar:server.domain/cgi-bin/upload.pl/MANIFEST \
     $(nix-instantiate foo.nix)</screen>
 
-    This will push both sources and binaries (and any build-time
-    dependencies used in the build, such as compilers).
-    </para>
+This will push both sources and binaries (and any build-time
+dependencies used in the build, such as compilers).</para>
 
-    <para>
-      If we just want to push binaries, not sources and build-time
-      dependencies, we can do:
+<para>If we just want to push binaries, not sources and build-time
+dependencies, we can do:
       
-      <screen>
+<screen>
 $ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
     
-    </para>
+</para>
 
-  </refsection>
+</refsection>
     
 </refentry>

doc/manual/opt-common-syn.xml

@@ -1,3 +1,5 @@
+<nop>
+  
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'><option>--verbose</option></arg>
@@ -17,3 +19,6 @@
 <arg><option>-K</option></arg>
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
+<arg><option>--log-type</option> <replaceable>type</replaceable></arg>
+
+</nop>

doc/manual/opt-common.xml

@@ -1,184 +1,216 @@
-<varlistentry>
-  <term><option>--help</option></term>
+<sect1 id="sec-common-options"><title>Common options</title>
+
+<para>Most Nix commands accept the following command-line options:</para>
+
+<variablelist>
+
+<varlistentry><term><option>--help</option></term>
+  
+  <listitem><para>Prints out a summary of the command syntax and
+  exits.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--version</option></term>
+  
+  <listitem><para>Prints out the Nix version number on standard output
+  and exits.</para></listitem>
+</varlistentry>
+
+
+<varlistentry><term><option>--verbose</option></term>
+  <term><option>-v</option></term>
+
   <listitem>
-    <para>
-      Prints out a summary of the command syntax and exits.
+    
+  <para>Increases the level of verbosity of diagnostic messages
+  printed on standard error.  For each Nix operation, the information
+  printed on standard output is well-defined; any diagnostic
+  information is printed on standard error, never on standard
+  output.</para>
+
+  <para>This option may be specified repeatedly.  Currently, the
+  following verbosity levels exist:</para>
+
+  <variablelist>
+    
+    <varlistentry><term>0</term>
+    <listitem><para>“Errors only”: only print messages
+    explaining why the Nix invocation failed.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>1</term>
+    <listitem><para>“Informational”: print
+    <emphasis>useful</emphasis> messages about what Nix is doing.
+    This is the default.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>2</term>
+    <listitem><para>“Talkative”: print more informational
+    messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>3</term>
+    <listitem><para>“Chatty”: print even more
+    informational messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>4</term>
+    <listitem><para>“Debug”: print debug
+    information.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>5</term>
+    <listitem><para>“Vomit”: print vast amounts of debug
+    information.</para></listitem>
+    </varlistentry>
+    
+  </variablelist>
+
+  </listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--no-build-output</option></term>
+  <term><option>-Q</option></term>
+
+  <listitem><para>By default, output written by builders to standard
+  output and standard error is echoed to the Nix command's standard
+  error.  This option suppresses this behaviour.  Note that the
+  builder's standard output and error are always written to a log file
+  in
+  <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="opt-max-jobs"><term><option>--max-jobs</option></term>
+  <term><option>-j</option></term>
+
+  <listitem><para>Sets the maximum number of build jobs that Nix will
+  perform in parallel to the specified number.  The default is 1.  A
+  higher value is useful on SMP systems or to exploit I/O latency.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-going</option></term>
+  <term><option>-k</option></term>
+
+  <listitem><para>Keep going in case of failed builds, to the
+  greatest extent possible.  That is, if building an input of some
+  derivation fails, Nix will still build the other inputs, but not the
+  derivation itself.  Without this option, Nix stops if any build
+  fails (except for builds of substitutes), possibly killing builds in
+  progress (in case of parallel or distributed builds).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-failed</option></term>
+  <term><option>-K</option></term>
+
+  <listitem><para>Specifies that in case of a build failure, the
+  temporary directory (usually in <filename>/tmp</filename>) in which
+  the build takes place should not be deleted.  The path of the build
+  directory is printed as an informational message.
     </para>
   </listitem>
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--version</option></term>
+<varlistentry><term><option>--fallback</option></term>
+
   <listitem>
-    <para>
-      Prints out the Nix version number on standard output and exits.
-    </para>
+
+  <para>Whenever Nix attempts to build a derivation for which
+  substitutes are known for each output path, but realising the output
+  paths through the substitutes fails, fall back on building the
+  derivation.</para>
+
+  <para>The most common scenario in which this is useful is when we
+  have registered substitutes in order to perform binary distribution
+  from, say, a network repository.  If the repository is down, the
+  realisation of the derivation will fail.  When this option is
+  specified, Nix will build the derivation instead.  Thus,
+  installation from binaries falls back on nstallation from source.
+  This option is not the default since it is generally not desirable
+  for a transient failure in obtaining the substitutes to lead to a
+  full build from source (with the related consumption of
+  resources).</para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--verbose</option> / <option>-v</option></term>
-  <listitem>
-    <para>
-      Increases the level of verbosity of diagnostic messages printed
-      on standard error.  For each Nix operation, the information
-      printed on standard output is well-defined; any diagnostic
-      information is printed on standard error, never on standard
-      output.
-    </para>
+<varlistentry><term><option>--readonly-mode</option></term>
 
-    <para>
-      This option may be specified repeatedly.  Currently, the
-      following verbosity levels exist:
-    </para>
-
-    <variablelist>
-      <varlistentry>
-        <term>0</term>
-        <listitem>
-          <para>
-            <quote>Errors only</quote>: only print messages explaining
-            why the Nix invocation failed.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>1</term>
-        <listitem>
-          <para>
-            <quote>Informational</quote>: print
-            <emphasis>useful</emphasis> messages about what Nix is
-            doing.  This is the default.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>2</term>
-        <listitem>
-          <para>
-            <quote>Talkative</quote>: print more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>3</term>
-        <listitem>
-          <para>
-            <quote>Chatty</quote>: print even more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>4</term>
-        <listitem>
-          <para>
-            <quote>Debug</quote>: print debug information:
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>5</term>
-        <listitem>
-          <para>
-            <quote>Vomit</quote>: print vast amounts of debug
-            information.
-          </para>
-        </listitem>
-      </varlistentry>
-    </variablelist>
-
-  </listitem>
+  <listitem><para>When this option is used, no attempt is made to open
+  the Nix database.  Most Nix operations do need database access, so
+  those operations will fail.</para></listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--no-build-output</option> / <option>-Q</option></term>
+<varlistentry id="opt-log-type"><term><option>--log-type</option>
+<replaceable>type</replaceable></term>
+
   <listitem>
-    <para>
-      By default, output written by builders to standard output and
-      standard error is echoed to the Nix command's standard error.
-      This option suppresses this behaviour.  Note that the builder's
-      standard output and error are always written to a log file in
-      <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.
-    </para>
+
+  <para>This option determines how the output written to standard
+  error is formatted.  Nix’s diagnostic messages are typically
+  <emphasis>nested</emphasis>.  For instance, when tracing Nix
+  expression evaluation (<command>nix-env -vvvvv</command>, messages
+  from subexpressions are nested inside their parent expressions.  Nix
+  builder output is also often nested.  For instance, the Nix Packages
+  generic builder nests the various build tasks (unpack, configure,
+  compile, etc.), and the GNU Make in <literal>stdenv-linux</literal>
+  has been patched to provide nesting for recursive Make
+  invocations.</para>
+
+  <para><replaceable>type</replaceable> can be one of the
+  following:
+
+  <variablelist>
+
+    <varlistentry><term><literal>pretty</literal></term>
+
+      <listitem><para>Pretty-print the output, indicating different
+      nesting levels using spaces.  This is the
+      default.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>escapes</literal></term>
+
+      <listitem><para>Indicate nesting using escape codes that can be
+      interpreted by the <command>log2xml</command> tool in the Nix
+      source distribution.  The resulting XML file can be fed into the
+      <command>log2html.xsl</command> stylesheet to create an HTML
+      file that can be browsed interactively, using Javascript to
+      expand and collapse parts of the output.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>flat</literal></term>
+
+      <listitem><para>Remove all nesting.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>    
+  
+  </para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--max-jobs</option> / <option>-j</option></term>
-  <listitem>
-    <para>
-      Sets the maximum number of build jobs that Nix will perform in
-      parallel to the specified number.  The default is 1.  A higher
-      value is useful on SMP systems or to exploit I/O latency.
-    </para>
-  </listitem>
-</varlistentry>
+</variablelist>
 
 
-<varlistentry>
-  <term><option>--keep-going</option> / <option>-k</option></term>
-  <listitem>
-    <para>
-      Keep going in case of failed builds, to the greatest extent
-      possible.  That is, if building an input of some derivation
-      fails, Nix will still build the other inputs, but not the
-      derivation itself.  Without this option, Nix stops if any build
-      fails (except for builds of substitutes), possibly killing
-      builds in progress (in case of parallel or distributed builds).
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--keep-failed</option> / <option>-K</option></term>
-  <listitem>
-    <para>
-      Specifies that in case of a build failure, the temporary
-      directory (usually in <filename>/tmp</filename>) in which the
-      build takes place should not be deleted.  The path of the build
-      directory is printed as an informational message.
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--fallback</option></term>
-  <listitem>
-    <para>
-      Whenever Nix attempts to realise a derivation for which a
-      closure is already known, but this closure cannot be realised,
-      fall back on normalising the derivation.
-    </para>
-
-    <para>
-      The most common scenario in which this is useful is when we have
-      registered substitutes in order to perform binary distribution
-      from, say, a network repository.  If the repository is down, the
-      realisation of the derivation will fail.  When this option is
-      specified, Nix will build the derivation instead.  Thus,
-      binary installation falls back on a source installation.  This
-      option is not the default since it is generally not desirable
-      for a transient failure in obtaining the substitutes to lead to
-      a full build from source (with the related consumption of
-      resources).
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--readonly-mode</option></term>
-  <listitem>
-    <para>
-      When this option is used, no attempt is made to open the Nix
-      database.  Most Nix operations do need database access, so those
-      operations will fail.
-    </para>
-  </listitem>
-</varlistentry>
+</sect1>

doc/manual/package-management.xml

@@ -2,24 +2,24 @@
 
 <para>This chapter discusses how to do package management with Nix,
 i.e., how to obtain, install, upgrade, and erase components.  This is
-the <quote>user's</quote> perspective of the Nix system — people
+the “user’s” perspective of the Nix system — people
 who want to <emphasis>create</emphasis> components should consult
 <xref linkend='chap-writing-nix-expressions' />.</para>
 
 
 <sect1><title>Basic package management</title>
 
-<para>The main command for package management is
-<command>nix-env</command>.  You can use it to install, upgrade, and
-erase components, and to query what components are installed or are
-available for installation.</para>
+<para>The main command for package management is <link
+linkend="sec-nix-env"><command>nix-env</command></link>.  You can use
+it to install, upgrade, and erase components, and to query what
+components are installed or are available for installation.</para>
 
-<para>In Nix, different users can have different <quote>views</quote>
+<para>In Nix, different users can have different “views”
 on the set of installed applications.  That is, there might be lots of
 applications present on the system (possibly in many different
 versions), but users can have a specific selection of those active —
-where <quote>active</quote> just means that it appears in a directory
-in the user's <envar>PATH</envar>.  Such a view on the set of
+where “active” just means that it appears in a directory
+in the user’s <envar>PATH</envar>.  Such a view on the set of
 installed applications is called a <emphasis>user
 environment</emphasis>, which is just a directory tree consisting of
 symlinks to the files of the active applications.  </para>
@@ -31,11 +31,9 @@ Nix expressions called the Nix Package collection that contains
 components ranging from basic development stuff such as GCC and Glibc,
 to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
-expression based on it, or completely new ones.)  You can download the
-latest version from <ulink
-url='http://catamaran.labs.cs.uu.nl/dist/nix' />.  You probably want
-the latest unstable release; currently the stable releases tend to lag
-behind quite a bit.</para>
+expressions based on it, or completely new ones.)  You can download
+the latest version from <ulink
+url='http://catamaran.labs.cs.uu.nl/dist/nix' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available components in the release:
@@ -52,7 +50,7 @@ bzip2-1.0.2
 ...</screen>
 
 where <literal>nixpkgs-<replaceable>version</replaceable></literal> is
-where you've unpacked the release.</para>
+where you’ve unpacked the release.</para>
 
 <para>It is also possible to see the <emphasis>status</emphasis> of
 available components, i.e., whether they are installed into the user
@@ -72,7 +70,7 @@ component is installed in your current user environment.  The second
 (in which case installing it into your user environment would be a
 very quick operation).  The last one (<literal>S</literal>) indicates
 whether there is a so-called <emphasis>substitute</emphasis> for the
-component, which is Nix's mechanism for doing binary deployment.  It
+component, which is Nix’s mechanism for doing binary deployment.  It
 just means that Nix know that it can fetch a pre-built component from
 somewhere (typically a network server) instead of building it
 locally.</para>
@@ -98,7 +96,7 @@ available somewhere.  This is done using the
 <command>nix-pull</command> command, which must be supplied with a URL
 containing a <emphasis>manifest</emphasis> describing what binaries
 are available.  This URL should correspond to the Nix Packages release
-that you're using.  For instance, if you obtained a release from
+that you’re using.  For instance, if you obtained a release from
 <ulink
 url='http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />,
 then you should do:
@@ -111,7 +109,7 @@ downloading binaries from <systemitem
 class='fqdomainname'>catamaran.labs.cs.uu.nl</systemitem>, instead of
 building them from source.  This might still take a while since all
 dependencies must be downloaded, but on a reasonably fast connection
-such as an DSL line it's on the order of a few minutes.</para>
+such as an DSL line it’s on the order of a few minutes.</para>
 
 <para>Naturally, packages can also be uninstalled:
 
@@ -127,9 +125,9 @@ release of Nix Packages, you can do:
 $ nix-env -f nixpkgs-<replaceable>version</replaceable> -u subversion</screen>
 
 This will <emphasis>only</emphasis> upgrade Subversion if there is a
-<quote>newer</quote> version in the new set of Nix expressions, as
+“newer” version in the new set of Nix expressions, as
 defined by some pretty arbitrary rules regarding ordering of version
-numbers (which generally do what you'd expect of them).  To just
+numbers (which generally do what you’d expect of them).  To just
 unconditionally replace Subversion with whatever version is in the Nix
 expressions, use <parameter>-i</parameter> instead of
 <parameter>-u</parameter>; <parameter>-i</parameter> will remove
@@ -143,7 +141,7 @@ $ nix-env -f nixpkgs-<replaceable>version</replaceable> -u '*'</screen>
 
 </para>
 
-<para>Sometimes it's useful to be able to ask what
+<para>Sometimes it’s useful to be able to ask what
 <command>nix-env</command> would do, without actually doing it.  For
 instance, to find out what packages would be upgraded by
 <literal>nix-env -u '*'</literal>, you can do
@@ -175,30 +173,28 @@ set.</para></footnote></para>
 </sect1>
 
 
-<sect1><title>Profiles</title>
+<sect1 id="sec-profiles"><title>Profiles</title>
 
-<para>Profiles and user environments are Nix's mechanism for
+<para>Profiles and user environments are Nix’s mechanism for
 implementing the ability to allow differens users to have different
 configurations, and to do atomic upgrades and rollbacks.  To
-understand how they work, it's useful to know a bit about how Nix
+understand how they work, it’s useful to know a bit about how Nix
 works.  In Nix, components are stored in unique locations in the
 <emphasis>Nix store</emphasis> (typically,
 <filename>/nix/store</filename>).  For instance, a particular version
 of the Subversion component might be stored in a directory
-<filename>/nix/store/eeeeaf42e56b...-subversion-0.32.1/</filename>,
+<filename>/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/</filename>,
 while another version might be stored in
-<filename>/nix/store/58823d558a6a...-subversion-0.34/</filename>.  The
-long hexadecimal numbers prefixed to the directory names are
-cryptographic hashes<footnote><para>128 bit MD5 hashes, to be
-precise.</para></footnote> of <emphasis>all</emphasis> inputs involved
-in building the component — sources, dependencies, compiler flags, and
-so on.  So if two components differ in any way, they end up in
-different locations in the file system, so they don't interfere with
-each other.  <xref linkend='fig-user-environments'
-/><footnote><para>TODO: the figure isn't entirely up to date.  It
-should show multiple profiles and
-<filename>~/.nix-profile</filename>.</para></footnote> shows a part of
-a typical Nix store.</para>
+<filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>.
+The long strings prefixed to the directory names are cryptographic
+hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in
+a base-32 notation, to be precise.</para></footnote> of
+<emphasis>all</emphasis> inputs involved in building the component —
+sources, dependencies, compiler flags, and so on.  So if two
+components differ in any way, they end up in different locations in
+the file system, so they don’t interfere with each other.  <xref
+linkend='fig-user-environments' /> shows a part of a typical Nix
+store.</para>
 
 <figure id='fig-user-environments'><title>User environments</title>
   <mediaobject>
@@ -208,41 +204,42 @@ a typical Nix store.</para>
   </mediaobject>
 </figure>
 
-<para>Of course, you wouldn't want to type
+<para>Of course, you wouldn’t want to type
 
 <screen>
-$ /nix/store/eeeeaf42e56b...-subversion-0.32.1/bin/svn</screen>
+$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen>
 
 every time you want to run Subversion.  Of course we could set up the
 <envar>PATH</envar> environment variable to include the
 <filename>bin</filename> directory of every component we want to use,
 but this is not very convenient since changing <envar>PATH</envar>
-doesn't take effect for already existing processes.  The solution Nix
+doesn’t take effect for already existing processes.  The solution Nix
 uses is to create directory trees of symlinks to
 <emphasis>activated</emphasis> components.  These are called
 <emphasis>user environments</emphasis> and they are components
 themselves (though automatically generated by
 <command>nix-env</command>), so they too reside in the Nix store.  For
 instance, in <xref linkend='fig-user-environments' /> the user
-environment <filename>/nix/store/068150f63831...-user-env</filename>
-contains a symlink to just Subversion 0.32.1 (arrows in the figure
+environment <filename>/nix/store/5mq2jcn36ldl...-user-env</filename>
+contains a symlink to just Subversion 1.1.2 (arrows in the figure
 indicate symlinks).  This would be what we would obtain if we had done
 
 <screen>
 $ nix-env -i subversion</screen>
 
-on a set of Nix expressions that contained Subversion 0.32.1.</para>
+on a set of Nix expressions that contained Subversion 1.1.2.</para>
 
-<para>This doesn't in itself solve the problem, of course; you
-wouldn't want to type
-<filename>/nix/store/068150f63831...-user-env/bin/svn</filename>
-either.  Therefore there are symlinks outside of the store that point
+<para>This doesn’t in itself solve the problem, of course; you
+wouldn’t want to type
+<filename>/nix/store/0c1p5z4kda11...-user-env/bin/svn</filename>
+either.  That’s why there are symlinks outside of the store that point
 to the user environments in the store; for instance, the symlinks
-<filename>42</filename> and <filename>43</filename> in the example.
-These are called <emphasis>generations</emphasis> since every time you
-perform a <command>nix-env</command> operation, a new user environment
-is generated based on the current one.  For instance, generation 43
-was created from generation 42 when we did
+<filename>default-42-link</filename> and
+<filename>default-43-link</filename> in the example.  These are called
+<emphasis>generations</emphasis> since every time you perform a
+<command>nix-env</command> operation, a new user environment is
+generated based on the current one.  For instance, generation 43 was
+created from generation 42 when we did
 
 <screen>
 $ nix-env -i subversion mozilla</screen>
@@ -251,14 +248,14 @@ on a set of Nix expressions that contained Mozilla and a new version
 of Subversion.</para>
 
 <para>Generations are grouped together into
-<emphasis>profiles</emphasis> so that different users don't interfere
-with each other if they don't want to.  For example:
+<emphasis>profiles</emphasis> so that different users don’t interfere
+with each other if they don’t want to.  For example:
 
 <screen>
 $ ls -l /nix/var/nix/profiles/
 ...
-lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/068150f63831...-user-env
-lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/84c85f89ddbf...-user-env
+lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
+lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
 lrwxrwxrwx  1 eelco ... default -> default-43-link</screen>
 
 This shows a profile called <filename>default</filename>.  The file
@@ -268,7 +265,7 @@ operation, a new user environment and generation link are created
 based on the current one, and finally the <filename>default</filename>
 symlink is made to point at the new generation.  This last step is
 atomic on Unix, which explains how we can do atomic upgrades.  (Note
-that the building/installing of new components doesn't interfere in
+that the building/installing of new components doesn’t interfere in
 any way with old components, since they are stored in different
 locations in the Nix store.)</para>
 
@@ -293,13 +290,13 @@ can also see all available generations:
 $ nix-env --list-generations</screen></para>
 
 <para>Actually, there is another level of indirection not shown in the
-figure above.  You generally wouldn't have
+figure above.  You generally wouldn’t have
 <filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
 in your <envar>PATH</envar>.  Rather, there is a symlink
 <filename>~/.nix-profile</filename> that points to your current
 profile.  This means that you should put
 <filename>~/.nix-profile/bin</filename> in your <envar>PATH</envar>
-(and indeed, that's what the initialisation script
+(and indeed, that’s what the initialisation script
 <filename>/nix/etc/profile.d/nix.sh</filename> does).  This makes it
 easier to switch to a different profile.  You can do that using the
 command <command>nix-env --switch-profile</command>:
@@ -310,7 +307,7 @@ $ nix-env --switch-profile /nix/var/nix/profiles/my-profile
 $ nix-env --switch-profile /nix/var/nix/profiles/default</screen>
 
 These commands switch to the <filename>my-profile</filename> and
-default profile, respectively.  If the profile doesn't exist, it will
+default profile, respectively.  If the profile doesn’t exist, it will
 be created automatically.  You should be careful about storing a
 profile in another location than the <filename>profiles</filename>
 directory, since otherwise it might not be used as a root of the
@@ -337,7 +334,7 @@ This will <emphasis>not</emphasis> change the
 (<option>-u</option>) and uninstall (<option>-e</option>) never
 actually delete components from the system.  All they do (as shown
 above) is to create a new user environment that no longer contains
-symlinks to the <quote>deleted</quote> components.</para>
+symlinks to the “deleted” components.</para>
 
 <para>Of course, since disk space is not infinite, unused components
 should be removed at some point.  You can do this by running the Nix
@@ -346,7 +343,7 @@ not used (directly or indirectly) by any generation of any
 profile.</para>
 
 <para>Note however that as long as old generations reference a
-component, it will not be deleted.  After all, we wouldn't be able to
+component, it will not be deleted.  After all, we wouldn’t be able to
 do a rollback otherwise.  So in order for garbage collection to be
 effective, you should also delete (some) old generations.  Of course,
 this should only be done if you are certain that you will not need to
@@ -370,51 +367,61 @@ $ nix-env --delete-generations 10 11 14</screen>
 garbage collector as follows:
 
 <screen>
-$ nix-collect-garbage</screen>
+$ nix-store --gc</screen>
 
-You can alo first view what files would be deleted:
+If you are feeling uncertain, you can also first view what files would
+be deleted:
 
 <screen>
-$ nix-collect-garbage --print-dead</screen>
+$ nix-store --gc --print-dead</screen>
 
 Likewise, the option <option>--print-live</option> will show the paths
-that <emphasis>won't</emphasis> be deleted.</para>
+that <emphasis>won’t</emphasis> be deleted.</para>
 
-<sect2><title>Garbage collector roots</title>
 
-<para>TODO</para>
+<sect2 id="ssec-gc-roots"><title>Garbage collector roots</title>
 
-<para>The garbage collector uses as roots all store expressions
-mentioned in all files with extension <filename>.gcroot</filename> in
-the directory
-<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>,
-or in any file or directory symlinked to from that directory.  E.g.,
-by default,
-<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>
-contains a symlink to
-<filename><replaceable>prefix</replaceable>/var/nix/profiles/</filename>,
-so all generations of all profiles are also roots of the collector.</para>
+<para>The roots of the garbage collector are all store paths to which
+there are symlinks in the directory
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>.
+For instance, the following command makes the path
+<filename>/nix/store/d718ef...-foo</filename> a root of the collector:
+
+<screen>
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar</screen>
+	
+That is, after this command, the garbage collector will not remove
+<filename>/nix/store/d718ef...-foo</filename> or any of its
+dependencies.</para>
+
+<para>Subdirectories of
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>
+are also searched for symlinks.  Symlinks to non-store paths are
+followed and searched for roots, but symlinks to non-store paths
+<emphasis>inside</emphasis> the paths reached in that way are not
+followed to prevent infinite recursion.</para>
 
 </sect2>
 
 </sect1>
 
 
-<sect1><title>Channels</title>
+<sect1 id="sec-channels"><title>Channels</title>
 
-<para>If you want to stay up to date with a set of packages, it's not
+<para>If you want to stay up to date with a set of packages, it’s not
 very convenient to manually download the latest set of Nix expressions
 for those packages, use <command>nix-pull</command> to register
 pre-built binaries (if available), and upgrade using
-<command>nix-env</command>.  Fortunately, there's a better way:
+<command>nix-env</command>.  Fortunately, there’s a better way:
 <emphasis>Nix channels</emphasis>.</para>
 
 <para>A Nix channel is just a URL that points to a place that contains
-a set of Nix expressions and a manifest.  Using the command
-<command>nix-channel</command> you can automatically stay up to date
-with whatever is available at that URL.</para>
+a set of Nix expressions and a manifest.  Using the command <link
+linkend="sec-nix-channel"><command>nix-channel</command></link> you
+can automatically stay up to date with whatever is available at that
+URL.</para>
 
-<para>You can <quote>subscribe</quote> to a channel using
+<para>You can “subscribe” to a channel using
 <command>nix-channel --add</command>, e.g.,
 
 <screen>
@@ -427,7 +434,7 @@ of the Nix Packages collection.  (Instead of
 stability, but right now is just outdated.)  Subscribing really just
 means that the URL is added to the file
 <filename>~/.nix-channels</filename>.  Right now there is no command
-to <quote>unsubscribe</quote>; you should just edit that file manually
+to “unsubscribe”; you should just edit that file manually
 and delete the offending URL.</para>
 
 <para>To obtain the latest Nix expressions available in a channel, do
@@ -440,7 +447,7 @@ This downloads the Nix expressions in every channel (downloaded from
 and registers any available pre-built binaries in every channel
 (by <command>nix-pull</command>ing
 <literal><replaceable>url</replaceable>/MANIFEST</literal>).  It also
-makes the union of each channel's Nix expressions the default for
+makes the union of each channel’s Nix expressions the default for
 <command>nix-env</command> operations.  Consequently, you can then say
 
 <screen>

doc/manual/quick-start.xml

@@ -88,8 +88,8 @@ $ nix-channel --update
 $ nix-env -u '*'</screen>
 
 The latter command will upgrade each installed component for which
-there is a <quote>newer</quote> version (as determined by comparing
-the version numbers).</para></listitem>
+there is a “newer” version (as determined by comparing the version
+numbers).</para></listitem>
 
 <listitem><para>If you're unhappy with the result of a
 <command>nix-env</command> action (e.g., an upgraded component turned
@@ -106,12 +106,12 @@ actually delete them:
 
 <screen>
 $ nix-env --delete-generations old
-$ nix-collect-garbage</screen>
+$ nix-store --gc</screen>
 
-The first command deletes old <quote>generations</quote> of your
-profile (making rollbacks impossible, but also making the components
-in those old generations available for garbage collection), while the
-second command actually deletes them.</para></listitem>
+The first command deletes old “generations” of your profile (making
+rollbacks impossible, but also making the components in those old
+generations available for garbage collection), while the second
+command actually deletes them.</para></listitem>
 
 </orderedlist>
 

doc/manual/style.css

@@ -102,6 +102,8 @@ pre.screen
 
 .note,.warning
 {
+    margin-top: 1em;
+    margin-bottom: 1em;
     border: 1px solid #6185a0;
     padding: 0px 1em;
     background: #fffff5;
@@ -154,7 +156,7 @@ a:hover { background: #ffffcd; }
                                Special elements:
  ***************************************************************************/
 
-tt
+tt, code
 {
     color: #400000;
 }
@@ -229,4 +231,4 @@ div.epigraph
 table.productionset table.productionset
 {
     font-family: monospace;
-}
+}

doc/manual/troubleshooting.xml

@@ -1,14 +1,71 @@
-<appendix>
-  <title>Troubleshooting</title>
+<appendix><title>Troubleshooting</title>
+
+<para>This section provides solutions for some common problems.</para>
+
+
+<sect1><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
+
+<para>Symptom: Nix operations (in particular the
+<command>nix-store</command> operations <option>--gc</option>,
+<option>--verify</option>, and <option>--clear-substitutes</option> —
+the latter being called by <command>nix-channel --update</command>)
+failing:
+
+<screen>
+$ nix-store --verify
+error: Db::del: Cannot allocate memory</screen>
+
+</para>
+
+<para>Possible solution: make sure that no Nix processes are running,
+then do:
+
+<screen>
+$ cd /nix/var/nix/db
+$ rm __db.00*</screen>
+
+</para>
+
+</sect1>
+
+
+<sect1><title>Collisions in <command>nix-env</command></title>
+
+<para>Symptom: when installing or upgrading, you get an error message such as
+
+<screen>
+$ nix-env -i docbook-xml
+...
+adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
+collission between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
+  and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
+  at /nix/store/...-builder.pl line 62.</screen>
+
+</para>
+
+<para>The cause is that two installed packages in the user environment
+have overlapping filenames (e.g.,
+<filename>xml/dtd/docbook/calstblx.dtd</filename>.  This usually
+happens when you accidentally try to install two versions of the same
+package.  For instance, in the example above, the Nix Packages
+collection contains two versions of <literal>docbook-xml</literal>, so
+<command>nix-env -i</command> will try to install both.  The default
+user environment builder has no way to way to resolve such conflicts,
+so it just gives up.</para>
+
+<para>Solution: remove one of the offending packages from the user
+environment (if already installed) using <command>nix-env
+-u</command>, or specify exactly which version should be installed
+(e.g., <literal>nix-env -i docbook-xml-4.2</literal>).</para>
+
+<para>Alternatively, you can modify the user environment builder
+script (in
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>)
+to implement some conflict resolution policy.  E.g., the script could
+be modified to rename conflicting file names, or to pick one over the
+other.</para>
+
+</sect1>
+
 
-  <para>
-    (Nothing.)
-  </para>
-	
 </appendix>
-
-<!--
-local variables:
-sgml-parent-document: ("book.xml" "appendix")
-end:
--->

doc/manual/writing-nix-expressions.xml

@@ -383,7 +383,7 @@ some fragments of
     that can be built by Nix (since when we fill in the arguments of
     the function, what we get is its body, which is the call to
     <varname>stdenv.mkDerivation</varname> in <xref
-    linkend='ex-hello-nix ' />).</para>
+    linkend='ex-hello-nix' />).</para>
 
   </callout>
 
@@ -447,7 +447,8 @@ following:
 (import pkgs/system/i686-linux.nix).hello</programlisting>
 
 Call it <filename>test.nix</filename>.  You can then build it without
-installing it using the command <command>nix-build</command>:
+installing it using the command <link
+linkend="sec-nix-build"><command>nix-build</command></link>:
 
 <screen>
 $ nix-build ./test.nix
@@ -575,7 +576,7 @@ builder becomes even shorter:
 
 <programlisting>
 . $stdenv/setup
-genericBuilder</programlisting>
+genericBuild</programlisting>
 
 In fact, <varname>mkDerivation</varname> provides a default builder
 that looks exactly like that, so it is actually possible to omit the
@@ -817,7 +818,7 @@ set.</para>
 </simplesect>
 
 
-<simplesect><title>Functions</title>
+<simplesect id="ss-functions"><title>Functions</title>
 
 <para>Functions have the following form:
 
@@ -1103,7 +1104,7 @@ weakest binding).</para>
 </simplesect>
 
 
-<simplesect><title>Derivations</title>
+<simplesect id="ssec-derivation"><title>Derivations</title>
 
 <para>The most important built-in function is
 <function>derivation</function>, which is used to describe a
@@ -1112,7 +1113,7 @@ set, the attributes of which specify the inputs of the build.</para>
 
 <itemizedlist>
 
-  <listitem><para>There must be an attribute named
+  <listitem id="attr-system"><para>There must be an attribute named
   <varname>system</varname> whose value must be a string specifying a
   Nix platform identifier, such as <literal>"i686-linux"</literal> or
   <literal>"powerpc-darwin"</literal><footnote><para>To figure out
@@ -1253,6 +1254,17 @@ command-line argument.  See <xref linkend='sec-standard-environment'
   performed by looking for the hash parts of file names of the
   inputs.</para></listitem>
 
+  <listitem><para>After the build, Nix sets the last-modified
+  timestamp on all files in the build result to 0 (00:00:00 1/1/1970
+  UTC), sets the group to the default group, and sets the mode of the
+  file to 0444 or 0555 (i.e., read-only, with execute permission
+  enabled if the file was originally executable).  Note that possible
+  <literal>setuid</literal> and <literal>setgid</literal> bits are
+  cleared.  Setuid and setgid programs are not currently supported by
+  Nix.  This is because the Nix archives used in deployment have no
+  concept of ownership information, and because it makes the build
+  result dependent on the user performing the build.</para></listitem>
+
 </itemizedlist>
 
 </para>
@@ -1371,7 +1383,7 @@ following:
 </para>
 
 <para>The <filename>setup</filename> script also exports a function
-called <function>genericBuilder</function> that knows how to build
+called <function>genericBuild</function> that knows how to build
 typical Autoconf-style components.  It can be customised to perform
 builds for any type of component.  It is advisable to use
 <function>genericBuild</function> since it provides facilities that

externals/Makefile.am

@@ -1,11 +1,11 @@
 # Berkeley DB
 
-DB = db-4.2.52
+DB = db-4.3.28.NC
 
 $(DB).tar.gz:
 	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.2.52 from"
-	@echo "  http://www.sleepycat.com/update/snapshot/db-4.2.52.tar.gz"
+	@echo "Please download version 4.3.28 from"
+	@echo "  http://downloads.sleepycat.com/db-4.3.28.NC.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -26,20 +26,20 @@ build-db: have-db
 	 ../dist/configure --prefix=$$pfx/inst-bdb \
 	 --enable-cxx --disable-shared --disable-cryptography \
          --disable-replication --disable-verify && \
-	make && \
-	make install)
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-db
 endif
 
 
 # CWI ATerm
 
-ATERM = aterm-2.2
+ATERM = aterm-2.3.1
 
 $(ATERM).tar.gz:
 	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.2 from"
-	@echo "  http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.2.tar.gz"
+	@echo "Please download version 2.3.1 from"
+	@echo "  http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.3.1.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -57,8 +57,8 @@ build-aterm: have-aterm
 	(pfx=`pwd` && \
 	cd $(ATERM) && \
 	CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm && \
-	make && \
-	make install)
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-aterm
 endif
 

make/examples/aterm/aterm/default.nix

@@ -0,0 +1,34 @@
+{sharedLib ? true}:
+
+rec {
+
+  inherit (import ../../../lib) compileC makeLibrary;
+
+  sources = [
+    ./afun.c
+    ./aterm.c
+    ./bafio.c
+    ./byteio.c
+    ./gc.c
+    ./hash.c
+    ./list.c
+    ./make.c
+    ./md5c.c
+    ./memory.c
+    ./tafio.c
+    ./version.c
+  ];
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    forSharedLib = sharedLib;
+  };
+
+  libATerm = makeLibrary {
+    libraryName = "ATerm";
+    objects = map compile sources;
+    inherit sharedLib;
+  };
+
+}

make/examples/aterm/default.nix

@@ -0,0 +1 @@
+import test/default.nix

make/examples/aterm/test/default.nix

@@ -0,0 +1,18 @@
+let {
+
+  inherit (import ../../../lib) compileC link;
+
+  inherit (import ../aterm {}) libATerm;
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    cFlags = "-I../aterm";
+  };
+
+  fib = link {objects = compile ./fib.c; libraries = libATerm;};
+
+  primes = link {objects = compile ./primes.c; libraries = libATerm;};
+  
+  body = [fib primes];
+}

make/examples/default.nix

@@ -0,0 +1,6 @@
+[ (import ./trivial)
+  (import ./simple-header)
+  (import ./not-so-simple-header)
+  (import ./not-so-simple-header-auto)
+  (import ./aterm)
+]

make/examples/not-so-simple-header-auto/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header-auto/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC findIncludes link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = "auto";
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header-auto/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header-auto/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/not-so-simple-header/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header/default.nix

@@ -0,0 +1,14 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = [
+      [./foo/fnord/indirect.h "fnord/indirect.h"]
+      [./bar/hello.h "fnord/../../bar/hello.h"]
+    ];
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {
+    main = ./hello.c;
+    localIncludes = [ [./hello.h "hello.h"] ];
+  };};
+
+  body = [hello];
+}

make/examples/simple-header/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "hello.h"
+
+int main(int argc, char * * argv)
+{
+    printf("Hello " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/trivial/default.nix

@@ -0,0 +1,8 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {main = ./hello.c;};};
+
+  body = [hello];
+}

make/examples/trivial/hello.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(int argc, char * * argv)
+{
+    printf("Hello World\n");
+    return 0;
+}

make/lib/compile-c.sh

@@ -0,0 +1,73 @@
+. $stdenv/setup
+
+mainName=$(basename $main | cut -c34-)
+
+echo "compiling \`$mainName'..."
+
+# Turn $localIncludes into an array.
+localIncludes=($localIncludes)
+
+# Determine how many `..' levels appear in the header file references.
+# E.g., if there is some reference `../../foo.h', then we have to
+# insert two extra levels in the directory structure, so that `a.c' is
+# stored at `dotdot/dotdot/a.c', and a reference from it to
+# `../../foo.h' resolves to `dotdot/dotdot/../../foo.h' == `foo.h'.
+n=0
+maxDepth=0
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    target=${localIncludes[$((n + 1))]}
+
+    # Split the target name into path components using some IFS magic.
+    savedIFS="$IFS"
+    IFS=/
+    components=($target)
+    depth=0
+    for ((m = 0; m < ${#components[*]}; m++)); do
+        c=${components[m]}
+        if test "$c" = ".."; then
+            depth=$((depth + 1))
+        fi
+    done
+    IFS="$savedIFS"
+
+    if test $depth -gt $maxDepth; then
+        maxDepth=$depth;
+    fi
+done
+
+# Create the extra levels in the directory hierarchy.
+prefix=
+for ((n = 0; n < maxDepth; n++)); do
+    prefix="dotdot/$prefix"
+done
+
+# Create symlinks to the header files.
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    source=${localIncludes[n]}
+    target=${localIncludes[$((n + 1))]}
+
+    # Create missing directories.  We use IFS magic to split the path
+    # into path components.
+    savedIFS="$IFS"
+    IFS=/
+    components=($prefix$target)
+    fullPath=(.)
+    for ((m = 0; m < ${#components[*]} - 1; m++)); do
+        fullPath=("${fullPath[@]}" ${components[m]})
+        if ! test -d "${fullPath[*]}"; then
+            mkdir "${fullPath[*]}"
+        fi
+    done
+    IFS="$savedIFS"
+    
+    ln -sf $source $prefix$target
+done
+
+# Create a symlink to the main file.
+if ! test "$(readlink $prefix$mainName)" = $main; then
+    ln -s $main $prefix$mainName
+fi
+
+mkdir $out
+test "$prefix" && cd $prefix
+gcc -Wall $cFlags -c $mainName -o $out/$mainName.o

make/lib/default.nix

@@ -0,0 +1,59 @@
+rec {
+
+  # Should point at your Nixpkgs installation.
+  pkgPath = ./pkgs;
+
+  pkgs = import (pkgPath + /system/all-packages.nix) {};
+
+  stdenv = pkgs.stdenv;
+  
+
+  compileC = {main, localIncludes ? [], cFlags ? "", forSharedLib ? false}:
+  stdenv.mkDerivation {
+    name = "compile-c";
+    builder = ./compile-c.sh;
+    localIncludes =
+      if localIncludes == "auto" then
+        import (findIncludes {
+          main = toString main;
+          hack = __currentTime;
+          inherit cFlags;
+        })
+      else
+        localIncludes;
+    inherit main;
+    cFlags = [
+      cFlags
+      (if forSharedLib then ["-fpic"] else [])
+    ];
+  };
+
+  /*
+  runCommand = {command}: {
+    name = "run-command";
+    builder = ./run-command.sh;
+    inherit command;
+  };
+  */
+
+  findIncludes = {main, hack, cFlags ? ""}: stdenv.mkDerivation {
+    name = "find-includes";
+    builder = ./find-includes.sh;
+    inherit main hack cFlags;
+  };
+  
+  link = {objects, programName ? "program", libraries ? []}: stdenv.mkDerivation {
+    name = "link";
+    builder = ./link.sh;
+    inherit objects programName libraries;
+  };
+
+  makeLibrary = {objects, libraryName ? [], sharedLib ? false}:
+  # assert sharedLib -> fold (obj: x: assert obj.sharedLib && x) false objects
+  stdenv.mkDerivation {
+    name = "library";
+    builder = ./make-library.sh;
+    inherit objects libraryName sharedLib;
+  };
+
+}

make/lib/find-includes.sh

@@ -0,0 +1,20 @@
+. $stdenv/setup
+
+echo "finding includes of \`$(basename $main)'..."
+
+makefile=$NIX_BUILD_TOP/makefile
+
+mainDir=$(dirname $main)
+(cd $mainDir && gcc $cFlags -MM $(basename $main) -MF $makefile) || false
+
+echo "[" >$out
+
+while read line; do
+    line=$(echo "$line" | sed 's/.*://')
+    for i in $line; do
+        fullPath=$(readlink -f $mainDir/$i)
+        echo "  [ $fullPath \"$i\" ]" >>$out
+    done
+done < $makefile
+
+echo "]" >>$out

make/lib/link.sh

@@ -0,0 +1,21 @@
+. $stdenv/setup
+
+shopt -s nullglob
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+libs=
+for i in $libraries; do
+    lib=$(echo $i/*.a; echo $i/*.so)
+    name=$(echo $(basename $lib) | sed -e 's/^lib//' -e 's/.a$//' -e 's/.so$//')
+    libs="$libs -L$(dirname $lib) -l$name" 
+done
+
+echo "linking object files into \`$programName'..."
+
+mkdir $out
+gcc -o $out/$programName $objs $libs

make/lib/make-library.sh

@@ -0,0 +1,28 @@
+. $stdenv/setup
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+echo "archiving object files into library \`$libraryName'..."
+
+ensureDir $out
+
+if test -z "$sharedLib"; then
+
+    outPath=$out/lib${libraryName}.a
+
+    ar crs $outPath $objs
+    ranlib $outPath
+
+else
+
+    outPath=$out/lib${libraryName}.so
+
+    gcc -shared -o $outPath $objs
+
+fi    
+
+    

nix.conf.example

@@ -0,0 +1,49 @@
+### Option `gc-keep-outputs'
+#
+# If `true', the garbage collector will keep the outputs of
+# non-garbage derivations.  If `false' (default), outputs will be
+# deleted unless they are GC roots themselves (or reachable from other
+# roots).
+# 
+# In general, outputs must be registered as roots separately.
+# However, even if the output of a derivation is registered as a root,
+# the collector will still delete store paths that are used only at
+# build time (e.g., the C compiler, or source tarballs downloaded from
+# the network).  To prevent it from doing so, set this option to
+# `true'.
+gc-keep-outputs = false
+
+
+### Option `gc-keep-derivations'
+#
+# If `true' (default), the garbage collector will keep the derivations
+# from which non-garbage store paths were built.  If `false', they
+# will be deleted unless explicitly registered as a root (or reachable
+# from other roots).
+#
+# Keeping derivation around is useful for querying and traceability
+# (e.g., it allows you to ask with what dependencies or options a
+# store path was built), so by default this option is on.  Turn it off
+# to safe a bit of disk space (or a lot if `gc-keep-outputs' is also
+# turned on).
+gc-keep-derivations = true
+
+
+### Option `env-keep-derivations'
+#
+# If `false' (default), derivations are not stored in Nix user
+# environments.  That is, the derivation any build-time-only
+# dependencies may be garbage-collected.
+#
+# If `true', when you add a Nix derivation to a user environment, the
+# path of the derivation is stored in the user environment.  Thus, the
+# derivation will not be garbage-collected until the user environment
+# generation is deleted (`nix-env --delete-generations').  To prevent
+# build-time-only dependencies from being collected, you should also
+# turn on `gc-keep-outputs'.
+#
+# The difference between this option and `gc-keep-derivations' is that
+# this one is `sticky': it applies to any user environment created
+# while this option was enabled, while `gc-keep-derivations' only
+# applies at the moment the garbage collector is run.
+env-keep-derivations = false

nix.spec.in

@@ -14,7 +14,7 @@ Release: 1
 License: GPL
 Group: Software Deployment
 URL: http://www.cs.uu.nl/groups/ST/Trace/Nix
-Source0: %{name}-@version@.tar.gz
+Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
 %define _prefix /nix
 Prefix: %{_prefix}

scripts/Makefile.am

@@ -2,18 +2,17 @@ bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build
 
-noinst_SCRIPTS = nix-profile.sh
+noinst_SCRIPTS = nix-profile.sh generate-patches.pl
 
-nix-pull nix-push: readmanifest.pm
+nix-pull nix-push: readmanifest.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm
+install-exec-local: readmanifest.pm download-using-manifests.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
-# !!! don't overwrite local modifications
-	$(INSTALL_DATA) prebuilts.conf $(DESTDIR)$(sysconfdir)/nix/prebuilts.conf
 
 include ../substitute.mk
 
@@ -21,5 +20,7 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-pull.in nix-push.in nix-profile.sh.in \
   nix-prefetch-url.in nix-install-package.in \
   nix-channel.in \
-  prebuilts.conf readmanifest.pm.in \
-  nix-build.in
+  readmanifest.pm.in \
+  nix-build.in \
+  download-using-manifests.pl.in \
+  generate-patches.pl.in

scripts/copying-collector.pl

@@ -0,0 +1,111 @@
+#! /usr/bin/perl -w
+
+use strict;
+
+my @paths = `nix-store -qR /home/eelco/.nix-profile/bin/firefox`;
+
+my %copyMap;
+my %rewriteMap;
+
+
+my $counter = 0;
+
+foreach my $path (@paths) {
+    chomp $path;
+
+    $path =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$path'";
+    my $hash = $2;
+
+#    my $newHash = "deadbeef" . (sprintf "%024d", $counter++);
+    my $newHash = "deadbeef" . substr($hash, 0, 24);
+    my $newPath = "/home/eelco/chroot/$1/$newHash-$3";
+
+    die unless length $newHash == length $hash;
+
+    $copyMap{$path} = $newPath;
+    $rewriteMap{$hash} = $newHash;
+}
+
+
+my %rewriteMap2;
+
+
+sub rewrite;
+sub rewrite {
+    my $src = shift;
+    my $dst = shift;
+
+    if (-l $dst) {
+
+        my $target = readlink $dst or die;
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+            $target =~ s/$srcHash/$dstHash/g;
+        }
+
+        unlink $dst or die;
+
+        symlink $target, $dst;
+        
+    }
+
+    elsif (-f $dst) {
+
+        print "$dst\n";
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+
+            my @stats = lstat $dst or die;
+            
+            system "sed s/$srcHash/$dstHash/g < '$dst' > '$dst.tmp'";
+            die if $? != 0;
+            rename "$dst.tmp", $dst or die;
+
+            chmod $stats[2], $dst or die;
+        }
+
+    }
+
+    elsif (-d $dst) {
+
+        chmod 0755, $dst;
+        
+        opendir(DIR, "$dst") or die "cannot open `$dst': $!";
+        my @files = readdir DIR;
+        closedir DIR;
+        
+        foreach my $file (@files) {
+            next if $file eq "." || $file eq "..";
+            rewrite "$src/$file", "$dst/$file";
+        }
+    }
+}
+
+
+foreach my $src (keys %copyMap) {
+    my $dst = $copyMap{$src};
+    print "$src -> $dst\n";
+
+    if (!-e $dst) {
+        system "cp -prd $src $dst";
+        die if $? != 0;
+
+        my @refs = `nix-store -q --references $src`;
+
+        %rewriteMap2 = ();
+        foreach my $ref (@refs) {
+            chomp $ref;
+
+            $ref =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$ref'";
+            my $hash = $2;
+
+            $rewriteMap2{$hash} = $rewriteMap{$hash};
+        }
+
+        rewrite $src, $dst;
+    }
+}

scripts/download-using-manifests.pl.in

@@ -0,0 +1,279 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use readmanifest;
+
+my $manifestDir = "@localstatedir@/nix/manifests";
+my $logFile = "@localstatedir@/log/nix/downloads";
+
+open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
+
+
+# Check the arguments.
+die unless scalar @ARGV == 1;
+my $targetPath = $ARGV[0];
+
+my $date = `date` or die;
+chomp $date;
+print LOGFILE "$$ get $targetPath $date\n";
+
+print "\n*** Trying to download/patch `$targetPath'\n";
+
+
+# Load all manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+for my $manifest (glob "$manifestDir/*.nixmanifest") {
+#    print STDERR "reading $manifest\n";
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
+        exit 1;
+    }
+}
+
+
+# Build a graph of all store paths that might contribute to the
+# construction of $targetPath, and the special node "start".  The
+# edges are either patch operations, or downloads of full NAR files.
+# The latter edges only occur between "start" and a store path.
+
+my %graph;
+
+$graph{"start"} = {d => 0, pred => undef, edges => []};
+
+my @queue = ();
+my $queueFront = 0;
+my %done;
+
+sub addToQueue {
+    my $v = shift;
+    return if defined $done{$v};
+    $done{$v} = 1;
+    push @queue, $v;
+}
+
+sub addNode {
+    my $u = shift;
+    $graph{$u} = {d => 999999999999, pred => undef, edges => []}
+        unless defined $graph{$u};
+}
+
+sub addEdge {
+    my $u = shift;
+    my $v = shift;
+    my $w = shift;
+    my $type = shift;
+    my $info = shift;
+    addNode $u;
+    push @{$graph{$u}->{edges}},
+        {weight => $w, start => $u, end => $v, type => $type, info => $info};
+    my $n = scalar @{$graph{$u}->{edges}};
+}
+
+addToQueue $targetPath;
+
+sub isValidPath {
+    my $p = shift;
+    system "@bindir@/nix-store --check-validity '$p' 2> /dev/null";
+    return $? == 0;
+}
+
+sub parseHash {
+    my $hash = shift;
+    if ($hash =~ /^(.+):(.+)$/) {
+        return ($1, $2);
+    } else {
+        return ("md5", $hash);
+    }
+}
+
+while ($queueFront < scalar @queue) {
+    my $u = $queue[$queueFront++];
+#    print "$u\n";
+
+    addNode $u;
+
+    # If the path already exists, it has distance 0 from the "start"
+    # node.
+    if (isValidPath($u)) {
+        addEdge "start", $u, 0, "present", undef;
+    }
+
+    else {
+
+        # Add patch edges.
+        my $patchList = $patches{$u};
+        foreach my $patch (@{$patchList}) {
+            if (isValidPath($patch->{basePath})) {
+                # !!! this should be cached
+                my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
+                my $format = "--base32";
+                $format = "" if $baseHashAlgo eq "md5";
+                my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
+                chomp $hash;
+#                print "  MY HASH is $hash\n";
+                if ($hash ne $baseHash) {
+                    print LOGFILE "$$ rejecting $patch->{basePath}\n";
+                    next;
+                }
+            }
+#            print "  PATCH from $patch->{basePath}\n";
+            addToQueue $patch->{basePath};
+            addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
+        }
+
+        # Add NAR file edges to the start node.
+        my $narFileList = $narFiles{$u};
+        foreach my $narFile (@{$narFileList}) {
+#            print "  NAR from $narFile->{url}\n";
+            addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
+            if ($u eq $targetPath) {
+                print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
+            }
+        }
+
+    }
+}
+
+
+# Run Dijkstra's shortest path algorithm to determine the shortest
+# sequence of download and/or patch actions that will produce
+# $targetPath.
+
+sub byDistance { # sort by distance, reversed
+    return -($graph{$a}->{d} <=> $graph{$b}->{d});
+}
+
+my @todo = keys %graph;
+
+while (scalar @todo > 0) {
+
+    # Remove the closest element from the todo list.
+    @todo = sort byDistance @todo;
+    my $u = pop @todo;
+
+    my $u_ = $graph{$u};
+
+#    print "IN $u $u_->{d}\n";
+
+    foreach my $edge (@{$u_->{edges}}) {
+        my $v_ = $graph{$edge->{end}};
+        if ($v_->{d} > $u_->{d} + $edge->{weight}) {
+            $v_->{d} = $u_->{d} + $edge->{weight};
+            # Store the edge; to edge->start is actually the
+            # predecessor.
+            $v_->{pred} = $edge; 
+#            print "  RELAX $edge->{end} $v_->{d}\n";
+        }
+    }
+}
+
+
+# Retrieve the shortest path from "start" to $targetPath.
+my @path = ();
+my $cur = $targetPath;
+die "don't know how to produce $targetPath\n"
+    unless defined $graph{$targetPath}->{pred};
+while ($cur ne "start") {
+    push @path, $graph{$cur}->{pred};
+    $cur = $graph{$cur}->{pred}->{start};
+}
+
+
+# Traverse the shortest path, perform the actions described by the
+# edges.
+my $curStep = 1;
+my $maxStep = scalar @path;
+
+sub downloadFile {
+    my $url = shift;
+    my ($hashAlgo, $hash) = parseHash(shift);
+    $ENV{"PRINT_PATH"} = 1;
+    $ENV{"QUIET"} = 1;
+    $ENV{"NIX_HASH_ALGO"} = $hashAlgo;
+    my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
+    die "download of `$url' failed" unless $? == 0;
+    chomp $hash2;
+    chomp $path;
+    die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
+    return $path;
+}
+
+while (scalar @path > 0) {
+    my $edge = pop @path;
+    my $u = $edge->{start};
+    my $v = $edge->{end};
+
+    print "\n*** Step $curStep/$maxStep: ";
+
+    if ($edge->{type} eq "present") {
+        print "using already present path `$v'\n";
+        print LOGFILE "$$ present $v\n";
+
+        if ($curStep < $maxStep) {
+            # Since this is not the last step, the path will be used
+            # as a base to one or more patches.  So turn the base path
+            # into a NAR archive, to which we can apply the patch.
+            print "  packing base path...\n";
+            system "@bindir@/nix-store --dump $v > /tmp/nar";
+            die "cannot dump `$v'" if ($? != 0);
+        }
+    }
+
+    elsif ($edge->{type} eq "patch") {
+        my $patch = $edge->{info};
+        print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
+
+        print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n";
+
+        # Download the patch.
+        print "  downloading patch...\n";
+        my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
+
+        # Apply the patch to the NAR archive produced in step 1 (for
+        # the already present path) or a later step (for patch sequences).
+        print "  applying patch...\n";
+        system "@libexecdir@/bspatch /tmp/nar /tmp/nar2 $patchPath";
+        die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
+
+        if ($curStep < $maxStep) {
+            # The archive will be used as the base of the next patch.
+            rename "/tmp/nar2", "/tmp/nar" or die "cannot rename NAR archive: $!";
+        } else {
+            # This was the last patch.  Unpack the final NAR archive
+            # into the target path.
+            print "  unpacking patched archive...\n";
+            system "@bindir@/nix-store --restore $v < /tmp/nar2";
+            die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
+        }
+    }
+
+    elsif ($edge->{type} eq "narfile") {
+        my $narFile = $edge->{info};
+        print "downloading `$narFile->{url}' into `$v'\n";
+
+        print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
+        
+        # Download the archive.
+        print "  downloading archive...\n";
+        my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
+
+        if ($curStep < $maxStep) {
+            # The archive will be used a base to a patch.
+            system "@bunzip2@ < '$narFilePath' > /tmp/nar";
+        } else {
+            # Unpack the archive into the target path.
+            print "  unpacking archive...\n";
+            system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
+            die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
+        }
+    }
+
+    $curStep++;
+}
+
+
+print LOGFILE "$$ success\n";
+close LOGFILE;

scripts/gc-releases.pl

@@ -0,0 +1,75 @@
+#! /usr/bin/perl -w
+
+use strict;
+use readmanifest;
+
+
+# Read the archive directories.
+my @archives = ();
+my %archives;
+
+sub readDir {
+    my $dir = shift;
+    opendir(DIR, "$dir") or die "cannot open `$dir': $!";
+    my @as = readdir DIR;
+    foreach my $archive (@as) {
+        push @archives, $archive;
+        $archives{$archive} = "$dir/$archive";
+    }
+    closedir DIR;
+}
+
+readDir "/mnt/scratchy/eelco/public_html/nix-cache";
+readDir "/mnt/scratchy/eelco/public_html/patches";
+
+print STDERR scalar @archives, "\n";
+
+
+# Read the manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+foreach my $manifest (@ARGV) {
+    print STDERR "loading $manifest\n";
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors, 1) < 3) {
+#        die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n";
+    }
+}
+
+
+# Find the live archives.
+my %usedFiles;
+
+foreach my $narFile (keys %narFiles) {
+    foreach my $file (@{$narFiles{$narFile}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+foreach my $patch (keys %patches) {
+    foreach my $file (@{$patches{$patch}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+
+# Print out the dead archives.
+foreach my $archive (@archives) {
+    next if $archive eq "." || $archive eq "..";
+    if (!defined $usedFiles{$archive}) {
+        print $archives{$archive}, "\n";
+    }
+}

scripts/generate-patches.pl.in

@@ -0,0 +1,376 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use POSIX qw(tmpnam);
+use readmanifest;
+
+die unless scalar @ARGV == 5;
+
+my $hashAlgo = "sha256";
+
+my $cacheDir = $ARGV[0];
+my $patchesDir = $ARGV[1];
+my $patchesURL = $ARGV[2];
+my $srcDir = $ARGV[3];
+my $dstDir = $ARGV[4];
+
+my $tmpdir;
+do { $tmpdir = tmpnam(); }
+until mkdir $tmpdir, 0777;
+
+print "TEMP = $tmpdir\n";
+
+#END { rmdir $tmpdir; }
+
+my %srcNarFiles;
+my %srcPatches;
+my %srcSuccessors;
+
+my %dstNarFiles;
+my %dstPatches;
+my %dstSuccessors;
+
+readManifest "$srcDir/MANIFEST",
+    \%srcNarFiles, \%srcPatches, \%srcSuccessors;
+
+readManifest "$dstDir/MANIFEST",
+    \%dstNarFiles, \%dstPatches, \%dstSuccessors;
+
+
+sub findOutputPaths {
+    my $narFiles = shift;
+    my $successors = shift;
+
+    my %outPaths;
+    
+    foreach my $p (keys %{$narFiles}) {
+
+        # Ignore store expressions.
+        next if ($p =~ /\.store$/);
+        next if ($p =~ /\.drv$/);
+        
+        # Ignore builders (too much ambiguity -- they're all called
+        # `builder.sh').
+        next if ($p =~ /\.sh$/);
+        next if ($p =~ /\.patch$/);
+        
+        # Don't bother including tar files etc.
+        next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
+
+        $outPaths{$p} = 1;
+    }
+
+    return %outPaths;
+}
+
+print "finding src output paths...\n";
+my %srcOutPaths = findOutputPaths \%srcNarFiles, \%srcSuccessors;
+
+print "finding dst output paths...\n";
+my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
+
+
+sub getNameVersion {
+    my $p = shift;
+    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
+    my $name = $1;
+    my $version = $2;
+    $name =~ s/^-//;
+    $version =~ s/^-//;
+    return ($name, $version);
+}
+
+
+# A quick hack to get a measure of the `distance' between two
+# versions: it's just the position of the first character that differs
+# (or 999 if they are the same).
+sub versionDiff {
+    my $s = shift;
+    my $t = shift;
+    my $i;
+    return 999 if $s eq $t;
+    for ($i = 0; $i < length $s; $i++) {
+        return $i if $i >= length $t or
+            substr($s, $i, 1) ne substr($t, $i, 1);
+    }
+    return $i;
+}
+
+
+sub getNarBz2 {
+    my $narFiles = shift;
+    my $storePath = shift;
+    
+    my $narFileList = $$narFiles{$storePath};
+    die "missing store expression $storePath" unless defined $narFileList;
+
+    my $narFile = @{$narFileList}[0];
+    die unless defined $narFile;
+
+    $narFile->{url} =~ /\/([^\/]+)$/;
+    die unless defined $1;
+    return "$cacheDir/$1";
+}
+
+
+sub containsPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $basePath = shift;
+    my $patchList = $$patches{$storePath};
+    return 0 if !defined $patchList;
+    my $found = 0;
+    foreach my $patch (@{$patchList}) {
+        # !!! baseHash might differ
+        return 1 if $patch->{basePath} eq $basePath;
+    }
+    return 0;
+}
+
+
+# Compute the "weighted" number of uses of a path in the build graph.
+sub computeUses {
+    my $narFiles = shift;
+    my $path = shift;
+
+    # Find the deriver of $path.
+    return 1 unless defined $$narFiles{$path};
+    my $deriver = @{$$narFiles{$path}}[0]->{deriver};
+    return 1 unless defined $deriver && $deriver ne "";
+
+#    print "  DERIVER $deriver\n";
+
+    # Optimisation: build the referers graph from the references
+    # graph.
+    my %referers;
+    foreach my $q (keys %{$narFiles}) {
+        my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
+        foreach my $r (@refs) {
+            $referers{$r} = [] unless defined $referers{$r};
+            push @{$referers{$r}}, $q;
+        }
+    }
+
+    # Determine the shortest path from $deriver to all other reachable
+    # paths in the `referers' graph.
+
+    my %dist;
+    $dist{$deriver} = 0;
+
+    my @queue = ($deriver);
+    my $pos = 0;
+    
+    while ($pos < scalar @queue) {
+        my $p = $queue[$pos];
+        $pos++;
+
+        foreach my $q (@{$referers{$p}}) {
+            if (!defined $dist{$q}) {
+                $dist{$q} = $dist{$p} + 1;
+#                print "    $q $dist{$q}\n";
+                push @queue, $q;
+            }
+        }
+    }
+
+    my $wuse = 1.0;
+    foreach my $user (keys %dist) {
+        next if $user eq $deriver;
+#        print "    $user $dist{$user}\n";
+        $wuse += 1.0 / 2.0**$dist{$user};
+    }
+
+#    print "    XXX $path $wuse\n";
+    
+    return $wuse;
+}
+
+
+# For each output path in the destination, see if we need to / can
+# create a patch.
+
+print "creating patches...\n";
+
+foreach my $p (keys %dstOutPaths) {
+
+    # If exactly the same path already exists in the source, skip it.
+    next if defined $srcOutPaths{$p};
+    
+    print "  $p\n";
+
+    # If not, then we should find the paths in the source that are
+    # `most' likely to be present on a system that wants to install
+    # this path.
+
+    (my $name, my $version) = getNameVersion $p;
+
+    my @closest = ();
+    my $closestVersion;
+    my $minDist = -1; # actually, larger means closer
+
+    # Find all source paths with the same name.
+
+    foreach my $q (keys %srcOutPaths) {
+        (my $name2, my $version2) = getNameVersion $q;
+        if ($name eq $name2) {
+
+            # If the sizes differ too much, then skip.  This
+            # disambiguates between, e.g., a real component and a
+            # wrapper component (cf. Firefox in Nixpkgs).
+            my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
+            my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
+            my $ratio = $srcSize / $dstSize;
+            $ratio = 1 / $ratio if $ratio < 1;
+#            print "  SIZE $srcSize $dstSize $ratio $q\n";
+
+            if ($ratio >= 3) {
+                print "  SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
+                next;
+            }
+
+            # If the numbers of weighted uses differ too much, then
+            # skip.  This disambiguates between, e.g., the bootstrap
+            # GCC and the final GCC in Nixpkgs.
+            my $srcUses = computeUses \%srcNarFiles, $q;
+            my $dstUses = computeUses \%dstNarFiles, $p;
+            $ratio = $srcUses / $dstUses;
+            $ratio = 1 / $ratio if $ratio < 1;
+            print "  USE $srcUses $dstUses $ratio $q\n";
+            
+#            if ($ratio >= 2) {
+#                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
+#                next;
+#            }
+
+            # If there are multiple matching names, include the ones
+            # with the closest version numbers.
+            my $dist = versionDiff $version, $version2;
+            if ($dist > $minDist) {
+                $minDist = $dist;
+                @closest = ($q);
+                $closestVersion = $version2;
+            } elsif ($dist == $minDist) {
+                push @closest, $q;
+            }
+        }
+    }
+
+    if (scalar(@closest) == 0) {
+        print "  NO BASE: $p\n";
+        next;
+    }
+
+    foreach my $closest (@closest) {
+
+        # Generate a patch between $closest and $p.
+        print "  $p <- $closest\n";
+
+        # If the patch already exists, skip it.
+        if (containsPatch(\%srcPatches, $p, $closest) ||
+            containsPatch(\%dstPatches, $p, $closest))
+        {
+            print "    skipping, already exists\n";
+            next;
+        }
+
+#        next;
+        
+        my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
+        my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
+        
+        system("@bunzip2@ < $srcNarBz2 > $tmpdir/A") == 0
+            or die "cannot unpack $srcNarBz2";
+
+        system("@bunzip2@ < $dstNarBz2 > $tmpdir/B") == 0
+            or die "cannot unpack $dstNarBz2";
+
+        system("@libexecdir@/bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
+            or die "cannot compute binary diff";
+
+        my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/A` or die;
+        chomp $baseHash;
+
+        my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/B` or die;
+        chomp $narHash;
+
+        my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/DIFF` or die;
+        chomp $narDiffHash;
+
+        my $narDiffSize = (stat "$tmpdir/DIFF")[7];
+        my $dstNarBz2Size = (stat $dstNarBz2)[7];
+
+        if ($narDiffSize >= $dstNarBz2Size) {
+            print "    rejecting; patch bigger than full archive\n";
+            next;
+        }
+    
+        my $finalName =
+            "$narDiffHash.nar-bsdiff";
+
+        print "    size $narDiffSize; full size $dstNarBz2Size\n";
+        
+        if (-e "$patchesDir/$finalName") {
+            print "    not copying, already exists\n";
+        }
+
+        else {
+
+            system("cp '$tmpdir/DIFF' '$patchesDir/$finalName.tmp'") == 0
+                or die "cannot copy diff";
+            
+            rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
+                or die "cannot rename $patchesDir/$finalName.tmp";
+            
+        }
+        
+        # Add the patch to the manifest.
+        addPatch \%dstPatches, $p,
+            { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
+            , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
+            , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
+            }, 0;
+    }
+}
+
+
+# Add in any potentially useful patches in the source (namely, those
+# patches that produce either paths in the destination or paths that
+# can be used as the base for other useful patches).
+
+my $changed;
+do {
+    # !!! we repeat this to reach the transitive closure; inefficient
+    $changed = 0;
+
+    foreach my $p (keys %srcPatches) {
+        my $patchList = $srcPatches{$p};
+
+        my $include = 0;
+
+        # Is path $p included in the destination?  If so, include
+        # patches that produce it.
+        $include = 1 if (defined $dstNarFiles{$p});
+
+        # Is path $p a path that serves as a base for paths in the
+        # destination?  If so, include patches that produce it.
+        foreach my $q (keys %dstPatches) {
+            foreach my $patch (@{$dstPatches{$q}}) {
+                # !!! check baseHash
+                $include = 1 if ($p eq $patch->{basePath});
+            }
+        }
+
+        if ($include) {
+            foreach my $patch (@{$patchList}) {
+                $changed = 1 if addPatch \%dstPatches, $p, $patch;
+            }
+        }            
+        
+    }
+    
+} while $changed;
+
+
+# Rewrite the manifest of the destination (with the new patches).
+writeManifest "$dstDir/MANIFEST",
+    \%dstNarFiles, \%dstPatches, \%dstSuccessors;

scripts/nix-build.in

@@ -8,34 +8,52 @@ if test -z "$nixExpr"; then
 fi
 
 extraArgs=
-noLink=
+addDrvLink=0
+addOutLink=1
 
+
+trap 'rm -f ./.nix-build-tmp-*' EXIT
+
+
+# Process the arguments.
 for i in "$@"; do
     case "$i" in
-        --no-link)
-            noLink=1
+    
+        --add-drv-link)
+            addDrvLink=1
             ;;
+            
+        --no-link)
+            addOutLink=0
+            ;;
+            
         -*)
             extraArgs="$extraArgs $i"
             ;;
+            
         *)
-            storeExprs=$(nix-instantiate "$i")
+            # Instantiate the Nix expression.
+            prefix=
+            if test "$addDrvLink" = 0; then prefix=.nix-build-tmp-; fi
+            storeExprs=$(@bindir@/nix-instantiate \
+                --add-root ./${prefix}derivation --indirect \
+                "$i")
+                
             for j in $storeExprs; do
-                echo "store expression is $j" >&2
+                echo "store expression is $(readlink "$j")" >&2
             done
-            outPaths=$(nix-store -qnfv $extraArgs $storeExprs)
+
+            # Build the resulting store derivation.
+            prefix=
+            if test "$addOutLink" = 0; then prefix=.nix-build-tmp-; fi
+            outPaths=$(@bindir@/nix-store \
+                --add-root ./${prefix}result --indirect \
+                -rv $extraArgs $storeExprs)
+                
             for j in $outPaths; do
-                echo "$j"
-                if test -z "$noLink"; then
-                    if test -L result; then
-                        rm result
-                    elif test -e result; then
-                        echo "cannot remove \`result' (not a symlink)"
-                        exit 1
-                    fi
-                    ln -s "$j" result
-                fi
+                echo "$(readlink "$j")"
             done
+            
             ;;
     esac
 done

scripts/nix-channel.in

@@ -48,11 +48,28 @@ sub addChannel {
 }
 
 
+# Remove a channel from the file $channelsList;
+sub removeChannel {
+    my $url = shift;
+    my @left = ();
+    readChannels;
+    foreach my $url2 (@channels) {
+        push @left, $url2 if $url ne $url2;
+    }
+    @channels = @left;
+    writeChannels;
+}
+
+
 # Fetch Nix expressions and pull cache manifests from the subscribed
 # channels.
 sub update {
     readChannels;
 
+    # Get rid of all the old substitutes.
+    system "@bindir@/nix-store --clear-substitutes";
+    die "cannot clear substitutes" if ($? != 0);
+
     # Pull cache manifests.
     foreach my $url (@channels) {
         print "pulling cache manifest from `$url'\n";
@@ -64,69 +81,94 @@ sub update {
     # expressions.
 
     my $nixExpr = "[";
-    
     foreach my $url (@channels) {
         my $fullURL = "$url/nixexprs.tar.bz2";
-        my $hash = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`
-            or die "cannot fetch `$fullURL'";
-        chomp $hash;
-        # !!! escaping
-        $nixExpr .= "((import @datadir@/nix/corepkgs/fetchurl) " .
-            "{url = $fullURL; md5 = \"$hash\"; system = \"@system@\";}) "
+        $ENV{"PRINT_PATH"} = 1;
+        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
+        die "cannot fetch `$fullURL'" if $? != 0;
+        chomp $path;
+        $nixExpr .= $path . " ";
     }
-
     $nixExpr .= "]";
 
     $nixExpr =
         "(import @datadir@/nix/corepkgs/channels/unpack.nix) " .
         "{inputs = $nixExpr; system = \"@system@\";}";
 
-    # Instantiate the Nix expression.
-    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
-        or die "cannot instantiate Nix expression";
-    chomp $storeExpr;
-
-    # Register the store expression as a root of the garbage
-    # collector.
+    # Figure out a name for the GC root.
     my $userName = getpwuid($<);
     die "who ARE you? go away" unless defined $userName;
 
-    my $rootFile = "$rootsDir/$userName.gcroot";
-    my $tmpRootFile = "$rootsDir/$userName-tmp.gcroot";
+    my $rootFile = "$rootsDir/$userName";
     
-    open ROOT, ">$tmpRootFile" or die "cannot create `$tmpRootFile': $!";
-    print ROOT "$storeExpr";
-    close ROOT;
+    # Instantiate the Nix expression.
+    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate --add-root '$rootFile'.tmp -`
+        or die "cannot instantiate Nix expression";
+    chomp $storeExpr;
 
-    # Realise the store expression.
-    my $outPath = `nix-store -qnf '$storeExpr'`
+    # Build the resulting derivation.
+    my $outPath = `nix-store --add-root '$rootFile' -r '$storeExpr'`
         or die "cannot realise store expression";
     chomp $outPath;
 
+    unlink "$rootFile.tmp";
+
     # Make it the default Nix expression for `nix-env'.
     system "@bindir@/nix-env --import '$outPath'";
     die "cannot pull set default Nix expression to `$outPath'" if ($? != 0);
-
-    rename $tmpRootFile, $rootFile or die "cannot rename `$tmpRootFile' to `$rootFile': $!";
 }
 
 
+sub usageError {
+    print STDERR <<EOF;
+Usage:
+  nix-channel --add URL
+  nix-channel --remove URL
+  nix-channel --list
+  nix-channel --update
+EOF
+    exit 1;
+}
+
+
+usageError if scalar @ARGV == 0;
+
+
 while (scalar @ARGV) {
     my $arg = shift @ARGV;
 
     if ($arg eq "--add") {
-        die "syntax: nix-channel --add URL" if (scalar @ARGV != 1);
+        usageError if scalar @ARGV != 1;
         addChannel (shift @ARGV);
         last;
     }
 
-    elsif ($arg eq "--update") {
-        die "syntax: nix-channel --update" if (scalar @ARGV != 0);
-        update;
+    if ($arg eq "--remove") {
+        usageError if scalar @ARGV != 1;
+        removeChannel (shift @ARGV);
         last;
     }
 
+    if ($arg eq "--list") {
+        usageError if scalar @ARGV != 0;
+        readChannels;
+        foreach my $url (@channels) {
+            print "$url\n";
+        }
+        last;
+    }
+
+    elsif ($arg eq "--update") {
+        usageError if scalar @ARGV != 0;
+        update;
+        last;
+    }
+    
+    elsif ($arg eq "--help") {
+        usageError;
+    }
+
     else {
-        die "unknown argument `$arg'";
+        die "unknown argument `$arg'; try `--help'";
     }
 }

scripts/nix-collect-garbage.in

@@ -1,90 +1,2 @@
-#! @perl@ -w
-
-use strict;
-use IPC::Open2;
-
-my $rootsDir = "@localstatedir@/nix/gcroots";
-my $storeDir = "@storedir@";
-
-my %alive;
-
-my $gcOper = "--delete";
-my $minAge = 0;
-
-my @roots = ();
-
-
-# Parse the command line.
-for (my $i = 0; $i < scalar @ARGV; $i++) {
-    my $arg = $ARGV[$i];
-    if ($arg eq "--delete" || $arg eq "--print-live" || $arg eq "--print-dead") {
-        $gcOper = $arg;
-    }
-    elsif ($arg eq "--min-age") {
-        $i++;
-        $minAge = undef;
-        $minAge = $ARGV[$i];
-        die "invalid minimum age" unless defined $minAge && $minAge =~ /^\d*$/;
-    }
-    else { die "unknown argument `$arg'" };
-}
-
-
-# Read all GC roots from the given file.
-sub readRoots {
-    my $fileName = shift;
-    open ROOT, "<$fileName" or die "cannot open `$fileName': $!";
-    while (<ROOT>) {
-        chomp;
-        foreach my $root (split ' ') {
-            die "bad root `$root' in file `$fileName'"
-                unless $root =~ /^\S+$/;
-            push @roots, $root;
-        }
-    }
-    close ROOT;
-}
-
-
-# Recursively finds all *.gcroot files in the given directory.
-sub findRoots;
-sub findRoots {
-    my $followSymlinks = shift;
-    my $dir = shift;
-
-    opendir(DIR, $dir) or die "cannot open directory `$dir': $!";
-    my @names = readdir DIR or die "cannot read directory `$dir': $!";
-    closedir DIR;
-
-    foreach my $name (@names) {
-        next if $name eq "." || $name eq "..";
-        $name = $dir . "/" . $name;
-        if ($name =~ /.gcroot$/ && -f $name) {
-            readRoots $name;
-        }
-        elsif (-d $name) {
-            if ($followSymlinks || !-l $name) {
-                findRoots 0, $name;
-            }
-        }
-    }
-    
-}
-
-
-# Find GC roots, starting at $rootsDir.
-findRoots 1, $rootsDir;
-
-
-# Run the collector with the roots we found.
-my $pid = open2(">&1", \*WRITE, "@bindir@/nix-store --gc $gcOper --min-age $minAge")
-    or die "cannot run `nix-store --gc'";
-
-foreach my $root (@roots) {
-    print WRITE "$root\n";
-}
-
-close WRITE;
-
-waitpid $pid, 0;
-$? == 0 or die "`nix-store --gc' failed";
+#! @shell@ -e
+exec @bindir@/nix-store --gc "$@"

scripts/nix-install-package.in

@@ -3,35 +3,49 @@
 use strict;
 use POSIX qw(tmpnam);
 
-my $pkgfile = $ARGV[0];
-die unless defined $pkgfile;
+my $pkgFile = $ARGV[0];
+die unless defined $pkgFile;
 
-my $tmpdir;
-do { $tmpdir = tmpnam(); }
-until mkdir $tmpdir, 0777;
 
-# !!! remove tmpdir on exit
+# Re-execute in a terminal, if necessary, so that if we're executed
+# from a web browser, the user gets to see us.
+if (!defined $ENV{"NIX_HAVE_TERMINAL"}) {
+    $ENV{"NIX_HAVE_TERMINAL"} = "1";
+    $ENV{"LD_LIBRARY_PATH"} = "";
+    exec("xterm", "-e", "@shell@", "-c", "@bindir@/nix-install-package '$pkgFile' || read");
+    die "cannot execute `xterm'";
+}
 
-print "Unpacking $pkgfile in $tmpdir...\n";
-system "bunzip2 < $pkgfile | (cd $tmpdir && tar xf -)";
-die if $?;
 
-print "This package contains the following derivations:\n";
-system "@bindir@/nix-env -qasf $tmpdir/default.nix";
-die if $?;
+# Read and parse the package file.
+open PKGFILE, "<$pkgFile" or die "cannot open `$pkgFile': $!";
+my $contents = <PKGFILE>;
+close PKGFILE;
 
-print "Do you wish to install these (Y/N)? ";
+$contents =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ or die "invalid package contents";
+my $version = $1;
+my $manifestURL = $2;
+my $drvName = $3;
+my $system = $4;
+my $drvPath = $5;
+my $outPath = $6;
+
+die "invalid package version `$version'" unless $version eq "NIXPKG1";
+
+
+# Ask confirmation.
+print "Do you want to install `$drvName' (Y/N)? ";
 my $reply = <STDIN>;
 chomp $reply;
-exit if (!($reply eq "y"));
+exit if $reply ne "y" && $reply ne "Y";
 
-print "Pulling caches...\n";
-system "@bindir@/nix-pull `cat $tmpdir/caches`";
-die if $?;
+print "\nPulling manifests...\n";
+system "@bindir@/nix-pull '$manifestURL'";
+die if $? != 0;
 
-print "Installing package...\n";
-system "@bindir@/nix-env -if $tmpdir/default.nix '*'";
-die if $?;
+print "\nInstalling package...\n";
+system "@bindir@/nix-env -i '$outPath'";
+die if $? != 0;
 
-print "Installation succeeded! Press Enter to continue.\n";
+print "\nInstallation succeeded! Press Enter to continue.\n";
 <STDIN>;

scripts/nix-prefetch-url.in

@@ -1,49 +1,66 @@
 #! @shell@ -e
 
 url=$1
+expHash=$2
+
+hashType=$NIX_HASH_ALGO
+if test -z "$hashType"; then
+    hashType=md5
+fi
+
+hashFormat=
+if test "$hashType" != "md5"; then
+    hashFormat=--base32
+fi
 
 if test -z "$url"; then
-    echo "syntax: nix-prefetch-url URL" >&2
+    echo "syntax: nix-prefetch-url URL [EXPECTED-HASH]" >&2
     exit 1
 fi
 
-# !!! race
-tmpPath1=@storedir@/nix-prefetch-url-$$
+name=$(basename "$url")
+if test -z "$name"; then echo "invalid url"; exit 1; fi
 
-# Test whether we have write permission in the store.  If not, fetch
-# to /tmp and don't copy to the store.  This is a hack to make this
-# script at least work somewhat in setuid installations.
-if ! touch $tmpPath1 2> /dev/null; then
-    echo "(cannot write to the store, result won't be cached)" >&2
-    dummyMode=1
-    tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
+
+# If the hash was given, a file with that hash may already be in the
+# store.
+if test -n "$expHash"; then
+    finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$expHash" "$name")
+    if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
+        finalPath=
+    fi
+    hash=$expHash
 fi
 
-# Perform the checkout.
-@curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
 
-# Compute the hash.
-hash=$(@bindir@/nix-hash --flat $tmpPath1)
-echo "hash is $hash" >&2
+# If we don't know the hash or a file with that hash doesn't exist,
+# download the file and add it to the store.
+if test -z "$finalPath"; then
 
-# Rename it so that the fetchurl builder can find it.
-if test "$dummyMode" != 1; then
-    tmpPath2=@storedir@/nix-prefetch-url-$hash
-    test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
+    tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
+    tmpFile=$tmpPath/$name
+    mkdir $tmpPath
+
+    # Perform the download.
+    @curl@ --fail --location --max-redirs 20 "$url" > $tmpFile
+
+    # Compute the hash.
+    hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
+    if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
+
+    # Add the downloaded file to the Nix store.
+    finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+
+    if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+
+    if test -n "$expHash" -a "$expHash" != "$hash"; then
+        echo "hash mismatch for URL \`$url'"
+        exit 1
+    fi
 fi
 
-# Create a Nix expression that does a fetchurl.
-storeExpr=$( \
-    echo "(import @datadir@/nix/corepkgs/fetchurl) \
-        {url = $url; md5 = \"$hash\"; system = \"@system@\";}" \
-    | @bindir@/nix-instantiate -)
 
-# Realise it.
-finalPath=$(@bindir@/nix-store -qnB --force-realise $storeExpr)
-    
-echo "path is $finalPath" >&2
-
-rm -rf $tmpPath1 $tmpPath2 || true
+if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
 
 echo $hash
 

scripts/nix-pull.in

@@ -10,84 +10,92 @@ do { $tmpdir = tmpnam(); }
 until mkdir $tmpdir, 0777;
 
 my $manifest = "$tmpdir/manifest";
-my $confFile = "@sysconfdir@/nix/prebuilts.conf";
 
-#END { unlink $manifest; rmdir $tmpdir; }
+END { unlink $manifest; rmdir $tmpdir; }
+
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
+$libexecDir = "@libexecdir@" unless defined $libexecDir;
+
+my $stateDir = $ENV{"NIX_STATE_DIR"};
+$stateDir = "@localstatedir@/nix" unless defined $stateDir;
+
+
+# Prevent access problems in shared-stored installations.
+umask 0022;
 
 
 # Obtain URLs either from the command line or from a configuration file.
-my %storePaths2urls;
-my %urls2hashes;
+my %narFiles;
+my %patches;
 my %successors;
-sub doURL {
+
+sub processURL {
     my $url = shift;
-    processURL $manifest, $url, \%storePaths2urls, \%urls2hashes, \%successors;
+
+    $url =~ s/\/$//;
+    print "obtaining list of Nix archives at $url...\n";
+
+    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
+           "'$url' > '$manifest'") == 0
+           or die "curl failed: $?";
+
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
+    }
+
+    my $baseName = "unnamed";
+    if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
+        $baseName = $1;
+    }
+
+    my $hash = `$binDir/nix-hash --flat '$manifest'`
+        or die "cannot hash `$manifest'";
+    chomp $hash;
+    
+    my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
+    
+    system("mv -f '$manifest' '$finalPath'") == 0
+        or die "cannot move `$manifest' to `$finalPath";
 }
-if (scalar @ARGV > 0) {
-    while (@ARGV) {
-        my $url = shift @ARGV;
-	doURL $url;
-    }
-} else {
-    open CONFFILE, "<$confFile";
-    while (<CONFFILE>) {
-        chomp;
-        if (/^\s*(\S+)\s*(\#.*)?$/) {
-            my $url = $1;
-	    doURL $url;
-        }
-    }
-    close CONFFILE;
+
+while (@ARGV) {
+    my $url = shift @ARGV;
+    processURL $url;
 }
 
 
-my $size = scalar (keys %storePaths2urls);
+my $size = scalar (keys %narFiles);
 print "$size store paths in manifest\n";
 
 
-# Instantiate a store expression that builds the substitute program
-# (the program that fetches URLs and unpacks them into the store).
-my $nixExpr =
-    "(import @datadir@/nix/corepkgs/nix-pull) " .
-    "{system = \"@system@\";}";
-
-print STDERR "instantiating store expression...\n";
-my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
-    or die "cannot instantiate Nix expression";
-chomp $storeExpr;
-
-
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
 
-my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --substitute")
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --register-substitutes")
     or die "cannot run nix-store";
 
 close READ;
 
-foreach my $storePath (keys %storePaths2urls) {
-    print WRITE "$storePath\n";
-    print WRITE "$storeExpr\n";
-    print WRITE "/fetch\n";
-    print WRITE "2\n";
-    print WRITE "$storePaths2urls{$storePath}\n";
-    print WRITE "$urls2hashes{$storePaths2urls{$storePath}}\n";
+foreach my $storePath (keys %narFiles) {
+    my $narFileList = $narFiles{$storePath};
+    foreach my $narFile (@{$narFileList}) {
+        print WRITE "$storePath\n";
+        print WRITE "$narFile->{deriver}\n";
+        print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
+        print WRITE "0\n";
+        my @references = split " ", $narFile->{references};
+        my $count = scalar @references;
+        print WRITE "$count\n";
+        foreach my $reference (@references) {
+            print WRITE "$reference\n";
+        }
+    }
 }
 
 close WRITE;
 
 waitpid $pid, 0;
 $? == 0 or die "nix-store failed";
-
-
-# Register all successors.
-print STDERR "registering successors...\n";
-my @sucs = %successors;
-while (scalar @sucs > 0) {
-    my $n = scalar @sucs;
-    if ($n > 256) { $n = 256 };
-    my @sucs2 = @sucs[0..$n - 1];
-    @sucs = @sucs[$n..scalar @sucs - 1];
-    system "@bindir@/nix-store --successor @sucs2";
-    if ($?) { die "`nix-store --successor' failed"; }
-}

scripts/nix-push.in

@@ -1,7 +1,11 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
 
 use strict;
+use IPC::Open2;
 use POSIX qw(tmpnam);
+use readmanifest;
+
+my $hashAlgo = "sha256";
 
 my $tmpdir;
 do { $tmpdir = tmpnam(); }
@@ -16,35 +20,65 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $dataDir = $ENV{"NIX_DATA_DIR"};
+$dataDir = "@datadir@" unless defined $dataDir;
+
 
 # Parse the command line.
-my $archives_put_url = shift @ARGV;
-my $archives_get_url = shift @ARGV;
-my $manifest_put_url = shift @ARGV;
+my $localCopy;
+my $localArchivesDir;
+my $localManifestFile;
+
+my $archivesPutURL;
+my $archivesGetURL;
+my $manifestPutURL;
+
+if ($ARGV[0] eq "--copy") {
+    die "syntax: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 1;
+    shift @ARGV;
+    $localArchivesDir = shift @ARGV;
+    $localManifestFile = shift @ARGV;
+}
+else {
+    die "syntax: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL " .
+        "MANIFEST_PUT_URL PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 0;
+    $archivesPutURL = shift @ARGV;
+    $archivesGetURL = shift @ARGV;
+    $manifestPutURL = shift @ARGV;
+}
 
 
-# From the given store expressions, determine the requisite store
-# paths.
-my %storepaths;
+# From the given store paths, determine the set of requisite store
+# paths, i.e, the paths required to realise them.
+my %storePaths;
 
-foreach my $storeexpr (@ARGV) {
-    die unless $storeexpr =~ /^\//;
+foreach my $path (@ARGV) {
+    die unless $path =~ /^\//;
 
     # Get all paths referenced by the normalisation of the given 
     # Nix expression.
-    system "@bindir@/nix-store --realise $storeexpr > /dev/null";
-    die if ($?);
-
-    open PATHS, "@bindir@/nix-store --query --requisites --include-successors $storeexpr 2> /dev/null |" or die;
-    while (<PATHS>) {
+    my $pid = open2(\*READ, \*WRITE,
+        "$binDir/nix-store --query --requisites --force-realise " .
+        "--include-outputs '$path'") or die;
+    close WRITE;
+    
+    while (<READ>) {
         chomp;
         die "bad: $_" unless /^\//;
-        $storepaths{$_} = "";
+        $storePaths{$_} = "";
     }
-    close PATHS;
+    close READ;
+    
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
-my @storepaths = keys %storepaths;
+my @storePaths = keys %storePaths;
 
 
 # For each path, create a Nix expression that turns the path into
@@ -52,14 +86,13 @@ my @storepaths = keys %storepaths;
 open NIX, ">$nixfile";
 print NIX "[";
 
-foreach my $storepath (@storepaths) {
-    die unless ($storepath =~ /\/[0-9a-z]{32}.*$/);
+foreach my $storePath (@storePaths) {
+    die unless ($storePath =~ /\/[0-9a-z]{32}.*$/);
 
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
-        "((import @datadir@/nix/corepkgs/nar/nar.nix) " .
-        # !!! $storepath should be represented as a closure
-        "{path = \"$storepath\"; system = \"@system@\";}) ";
+        "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
+        "{path = \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
     
     print NIX $nixexpr;
 }
@@ -69,112 +102,158 @@ close NIX;
 
 
 # Instantiate store expressions from the Nix expression.
-my @storeexprs;
+my @storeExprs;
 print STDERR "instantiating store expressions...\n";
-open STOREEXPRS, "@bindir@/nix-instantiate $nixfile |" or die "cannot run nix-instantiate";
-while (<STOREEXPRS>) {
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-instantiate $nixfile")
+    or die "cannot run nix-instantiate";
+close WRITE;
+while (<READ>) {
     chomp;
     die unless /^\//;
-    push @storeexprs, $_;
+    push @storeExprs, $_;
 }
-close STOREEXPRS;
+close READ;
+
+waitpid $pid, 0;
+$? == 0 or die "nix-instantiate failed";
 
 
 # Realise the store expressions.
 print STDERR "creating archives...\n";
 
-my @narpaths;
+my @narPaths;
 
-my @tmp = @storeexprs;
+my @tmp = @storeExprs;
 while (scalar @tmp > 0) {
     my $n = scalar @tmp;
     if ($n > 256) { $n = 256 };
     my @tmp2 = @tmp[0..$n - 1];
     @tmp = @tmp[$n..scalar @tmp - 1];
 
-    system "@bindir@/nix-store --realise @tmp2 > /dev/null";
-    if ($?) { die "`nix-store --realise' failed"; }
-
-    open NARPATHS, "@bindir@/nix-store --query --list @tmp2 |" or die "cannot run nix";
-    while (<NARPATHS>) {
+    my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --realise @tmp2")
+        or die "cannot run nix-store";
+    close WRITE;
+    while (<READ>) {
         chomp;
         die unless (/^\//);
-        push @narpaths, "$_";
+        push @narPaths, "$_";
     }
-    close NARPATHS;
+    close READ;
+
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
 
 # Create the manifest.
 print STDERR "creating manifest...\n";
 
-open MANIFEST, ">$manifest";
+my %narFiles;
+my %patches;
 
-my @nararchives;
-for (my $n = 0; $n < scalar @storepaths; $n++) {
-    my $storepath = $storepaths[$n];
-    my $nardir = $narpaths[$n];
+my @narArchives;
+for (my $n = 0; $n < scalar @storePaths; $n++) {
+    my $storePath = $storePaths[$n];
+    my $narDir = $narPaths[$n];
     
-    $storepath =~ /\/([^\/]*)$/;
+    $storePath =~ /\/([^\/]*)$/;
     my $basename = $1;
     defined $basename or die;
 
-    my $narname = "$basename.nar.bz2";
+    open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
+    my $narbz2Hash = <HASH>;
+    chomp $narbz2Hash;
+    $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
 
-    my $narfile = "$nardir/$narname";
-    (-f $narfile) or die "narfile for $storepath not found";
-    push @nararchives, $narfile;
+    open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
+    my $narHash = <HASH>;
+    chomp $narHash;
+    $narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
+    
+    my $narName = "$narbz2Hash.nar.bz2";
 
-    open MD5, "$nardir/md5" or die "cannot open hash";
-    my $hash = <MD5>;
-    chomp $hash;
-    $hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
-    close MD5;
+    my $narFile = "$narDir/$narName";
+    (-f $narFile) or die "narfile for $storePath not found";
+    push @narArchives, $narFile;
 
-    print MANIFEST "{\n";
-    print MANIFEST "  StorePath: $storepath\n";
-    print MANIFEST "  NarURL: $archives_get_url/$narname\n";
-    print MANIFEST "  MD5: $hash\n";
+    my $narbz2Size = (stat $narFile)[7];
 
-    if ($storepath =~ /\.store$/) {
-        open PREDS, "@bindir@/nix-store --query --predecessors $storepath |" or die "cannot run nix";
-        while (<PREDS>) {
-            chomp;
-            die unless (/^\//);
-            my $pred = $_;
-            # Only include predecessors that are themselves being
-            # pushed.
-            if (defined $storepaths{$pred}) {
-                print MANIFEST "  SuccOf: $pred\n";
-            }
-        }
-        close PREDS;
+    my $references = `$binDir/nix-store --query --references '$storePath'`;
+    die "cannot query references for `$storePath'" if $? != 0;
+    $references = join(" ", split(" ", $references));
+
+    my $deriver = `$binDir/nix-store --query --deriver '$storePath'`;
+    die "cannot query deriver for `$storePath'" if $? != 0;
+    chomp $deriver;
+    $deriver = "" if $deriver eq "unknown-deriver";
+
+    my $url;
+    if ($localCopy) {
+        $url = "file://$localArchivesDir/$narName";
+    } else {
+        $url = "$archivesGetURL/$narName";
     }
-
-    print MANIFEST "}\n";
+    $narFiles{$storePath} = [
+        { url => $url
+        , hash => "$hashAlgo:$narbz2Hash"
+        , size => $narbz2Size
+        , narHash => "$hashAlgo:$narHash"
+        , references => $references
+        , deriver => $deriver
+        }
+    ];
 }
 
-close MANIFEST;
+writeManifest $manifest, \%narFiles, \%patches;
+
+
+sub copyFile {
+    my $src = shift;
+    my $dst = shift;
+    system("cp '$src' '$dst.tmp'") == 0 or die "cannot copy file";
+    rename("$dst.tmp", "$dst") or die "cannot rename file";
+}
 
 
 # Upload the archives.
 print STDERR "uploading archives...\n";
-foreach my $nararchive (@nararchives) {
 
-    $nararchive =~ /\/([^\/]*)$/;
+sub archiveExists {
+    my $name = shift;
+    print STDERR "  HEAD on $archivesGetURL/$name\n";
+    return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
+}
+
+foreach my $narArchive (@narArchives) {
+
+    $narArchive =~ /\/([^\/]*)$/;
     my $basename = $1;
 
-    if (system("$curl --head $archives_get_url/$basename > /dev/null") != 0) {
-        print STDERR "  $nararchive\n";
-        system("$curl --show-error --upload-file " .
-               "'$nararchive' '$archives_put_url/$basename' > /dev/null") == 0 or
-            die "curl failed on $nararchive: $?";
+    if ($localCopy) {
+        if (! -f "$localArchivesDir/$basename") {
+            print STDERR "  $narArchive\n";
+            copyFile $narArchive, "$localArchivesDir/$basename";
+        }
+    }
+    else {
+        if (!archiveExists("$basename")) {
+            print STDERR "  $narArchive\n";
+            system("$curl --show-error --upload-file " .
+                   "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
+                   die "curl failed on $narArchive: $?";
+        }
     }
 }
 
 
 # Upload the manifest.
 print STDERR "uploading manifest...\n";
-system("$curl  --show-error --upload-file " .
-       "'$manifest' '$manifest_put_url' > /dev/null") == 0 or
-    die "curl failed on $manifest: $?";
+if ($localCopy) {
+    copyFile $manifest, $localManifestFile;
+} else {
+    system("$curl  --show-error --upload-file " .
+           "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
+           die "curl failed on $manifest: $?";
+}

scripts/optimise-store.pl

@@ -0,0 +1,69 @@
+#! /usr/bin/perl -w
+
+use strict;
+
+{ my $ofh = select STDOUT;
+  $| = 1;
+  select $ofh;
+}
+
+#my @paths = ("/nix/store/caef3a49150506d233f474322a824e50-glibc-2.3.3", "/nix/store/a8a9d585d1ad4b1bc911be7743b3b996-glibc-2.3.3");
+my @paths = ("/nix/store");
+
+my $tmpfile = "/tmp/nix-optimise-hash-list";
+#my $tmpfile = "/data/nix-optimise-hash-list";
+
+system("find @paths -type f -print0 | xargs -0 md5sum -- > $tmpfile") == 0
+    or die "cannot hash store files";
+
+system("sort $tmpfile > $tmpfile.sorted") == 0
+    or die "cannot sort list";
+
+open LIST, "<$tmpfile.sorted" or die;
+
+my $prevFile;
+my $prevHash;
+
+my $totalSpace = 0;
+my $savedSpace = 0;
+
+my $files = 0;
+
+while (<LIST>) {
+#    print "D";
+    /^([0-9a-f]*)\s+(.*)$/ or die;
+    my $curFile = $2;
+    my $curHash = $1;
+
+#    print "A";
+    my $fileSize = (stat $curFile)[7];
+#    print "B";
+#    my $fileSize = 1;
+    $totalSpace += $fileSize;
+
+    if (defined $prevHash && $curHash eq $prevHash) {
+        
+#        print "$curFile = $prevFile\n";
+
+        $savedSpace += $fileSize;
+        
+    } else {
+        $prevFile = $curFile;
+        $prevHash = $curHash;
+    }
+
+    print "." if ($files++ % 100 == 0);
+    #print ".";
+
+#    print "C";
+}
+
+print "\n";
+
+print "total space = $totalSpace\n";
+print "saved space = $savedSpace\n";
+my $savings = ($savedSpace / $totalSpace) * 100.0;
+print "savings = $savings %\n";
+
+
+close LIST;

scripts/readmanifest.pm.in

@@ -1,27 +1,64 @@
 use strict;
 
-sub processURL {
+
+sub addPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $patch = shift;
+    my $allowConflicts = shift;
+
+    $$patches{$storePath} = []
+        unless defined $$patches{$storePath};
+
+    my $patchList = $$patches{$storePath};
+
+    my $found = 0;
+    foreach my $patch2 (@{$patchList}) {
+        if ($patch2->{url} eq $patch->{url}) {
+            if ($patch2->{hash} eq $patch->{hash}) {
+                $found = 1 if ($patch2->{basePath} eq $patch->{basePath});
+            } else {
+                die "conflicting hashes for URL $patch->{url}, " .
+                    "namely $patch2->{hash} and $patch->{hash}"
+                    unless $allowConflicts;
+            }
+        }
+    }
+    
+    push @{$patchList}, $patch if !$found;
+
+    return !$found;
+}
+
+
+sub readManifest {
     my $manifest = shift;
-    my $url = shift;
-    my $storePaths2urls = shift;
-    my $urls2hashes = shift;
+    my $narFiles = shift;
+    my $patches = shift;
     my $successors = shift;
+    my $allowConflicts = shift;
+    $allowConflicts = 0 unless defined $allowConflicts;
 
-    $url =~ s/\/$//;
-    print "obtaining list of Nix archives at $url...\n";
-
-    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
-           "'$url' > '$manifest'") == 0
-           or die "curl failed: $?";
-        
-    open MANIFEST, "<$manifest";
+    open MANIFEST, "<$manifest"
+        or die "cannot open `$manifest': $!";
 
     my $inside = 0;
+    my $type;
+
+    my $manifestVersion = 2;
 
     my $storePath;
-    my $narurl;
+    my $url;
     my $hash;
+    my $size;
     my @preds;
+    my $basePath;
+    my $baseHash;
+    my $patchType;
+    my $narHash;
+    my $references;
+    my $deriver;
+    my $hashAlgo;
 
     while (<MANIFEST>) {
         chomp;
@@ -29,43 +66,150 @@ sub processURL {
         next if (/^$/);
 
         if (!$inside) {
-            if (/^\{$/) { 
+
+            if (/^\s*(\w*)\s*\{$/) {
+                $type = $1;
+                $type = "narfile" if $type eq "";
                 $inside = 1;
                 undef $storePath;
-                undef $narurl;
+                undef $url;
                 undef $hash;
+                undef $size;
                 @preds = ();
+                undef $narHash;
+                undef $basePath;
+                undef $baseHash;
+                undef $patchType;
+                $references = "";
+                $deriver = "";
+                $hashAlgo = "md5";
 	    }
-            else { die "bad line: $_"; }
+
         } else {
+            
             if (/^\}$/) {
                 $inside = 0;
 
-		$$storePaths2urls{$storePath} = $narurl;
-		$$urls2hashes{$narurl} = $hash;
+                if ($type eq "narfile") {
 
-                foreach my $p (@preds) {
-		    $$successors{$p} = $storePath;
+                    $$narFiles{$storePath} = []
+                        unless defined $$narFiles{$storePath};
+
+                    my $narFileList = $$narFiles{$storePath};
+
+                    my $found = 0;
+                    foreach my $narFile (@{$narFileList}) {
+                        if ($narFile->{url} eq $url) {
+                            if ($narFile->{hash} eq $hash) {
+                                $found = 1;
+                            } else {
+                                die "conflicting hashes for URL $url, " .
+                                    "namely $narFile->{hash} and $hash"
+                                    unless $allowConflicts;
+                            }
+                        }
+                    }
+                    if (!$found) {
+                        push @{$narFileList},
+                            { url => $url, hash => $hash, size => $size
+                            , narHash => $narHash, references => $references
+                            , deriver => $deriver, hashAlgo => $hashAlgo
+                            };
+                    }
+                
+                    foreach my $p (@preds) {
+                        $$successors{$p} = $storePath;
+                    }
+
+                }
+
+                elsif ($type eq "patch") {
+                    addPatch $patches, $storePath,
+                        { url => $url, hash => $hash, size => $size
+                        , basePath => $basePath, baseHash => $baseHash
+                        , narHash => $narHash, patchType => $patchType
+                        , hashAlgo => $hashAlgo
+                        }, $allowConflicts;
                 }
 
             }
-            elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
-                $storePath = $1;
-            }
-            elsif (/^\s*NarURL:\s*(\S+)\s*$/) {
-                $narurl = $1;
-	    }
-            elsif (/^\s*MD5:\s*(\S+)\s*$/) {
-                $hash = $1;
-            }
-            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
-                push @preds, $1;
-            }
-            else { die "bad line: $_"; }
+            
+            elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; }
+            elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; }
+            elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; }
+            elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; }
+            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { push @preds, $1; }
+            elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
+            elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
+            elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
+            elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+            elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
+            elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
+            elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
+
+            # Compatibility;
+            elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
+            elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; }
+
         }
     }
 
     close MANIFEST;
+
+    return $manifestVersion;
+}
+
+
+sub writeManifest
+{
+    my $manifest = shift;
+    my $narFiles = shift;
+    my $patches = shift;
+
+    open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
+
+    print MANIFEST "version {\n";
+    print MANIFEST "  ManifestVersion: 3\n";
+    print MANIFEST "}\n";
+
+    foreach my $storePath (keys %{$narFiles}) {
+        my $narFileList = $$narFiles{$storePath};
+        foreach my $narFile (@{$narFileList}) {
+            print MANIFEST "{\n";
+            print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  NarURL: $narFile->{url}\n";
+            print MANIFEST "  Hash: $narFile->{hash}\n";
+            print MANIFEST "  NarHash: $narFile->{narHash}\n";
+            print MANIFEST "  Size: $narFile->{size}\n";
+            print MANIFEST "  References: $narFile->{references}\n"
+                if defined $narFile->{references} && $narFile->{references} ne "";
+            print MANIFEST "  Deriver: $narFile->{deriver}\n"
+                if defined $narFile->{deriver} && $narFile->{deriver} ne "";
+            print MANIFEST "}\n";
+        }
+    }
+    
+    foreach my $storePath (keys %{$patches}) {
+        my $patchList = $$patches{$storePath};
+        foreach my $patch (@{$patchList}) {
+            print MANIFEST "patch {\n";
+            print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  NarURL: $patch->{url}\n";
+            print MANIFEST "  Hash: $patch->{hash}\n";
+            print MANIFEST "  NarHash: $patch->{narHash}\n";
+            print MANIFEST "  Size: $patch->{size}\n";
+            print MANIFEST "  BasePath: $patch->{basePath}\n";
+            print MANIFEST "  BaseHash: $patch->{baseHash}\n";
+            print MANIFEST "  Type: $patch->{patchType}\n";
+            print MANIFEST "}\n";
+        }
+    }
+    
+    
+    close MANIFEST;
+
+    rename("$manifest.tmp", $manifest)
+        or die "cannot rename $manifest.tmp: $!";
 }
 
 

scripts/remove-patches.pl

@@ -0,0 +1,19 @@
+#! /usr/bin/perl -w -I/home/eelco/nix/scripts
+
+use strict;
+use readmanifest;
+
+for my $p (@ARGV) {
+
+    my %narFiles;
+    my %patches;
+    my %successors;
+
+    readManifest $p,
+        \%narFiles, \%patches, \%successors;
+
+    %patches = ();
+    
+    writeManifest $p,
+        \%narFiles, \%patches, \%successors;
+}

scripts/update-manifest.pl

@@ -0,0 +1,53 @@
+#! /usr/bin/perl -w -I.
+
+use strict;
+use readmanifest;
+
+die unless scalar @ARGV == 2;
+
+my $cache = $ARGV[0];
+my $manifest = $ARGV[1];
+my %narFiles;
+my %patches;
+my %successors;
+
+readManifest $manifest, \%narFiles, \%patches, \%successors;
+
+foreach my $storePath (keys %narFiles) {
+    my $narFileList = $narFiles{$storePath};
+
+    foreach my $narFile (@{$narFileList}) {
+        if (!defined $narFile->{size} or
+            !defined $narFile->{narHash})
+        {
+            $narFile->{url} =~ /\/([^\/]+)$/;
+            die unless defined $1;
+            my $fn = "$cache/$1";
+            
+            my @info = stat $fn or die;
+            $narFile->{size} = $info[7];
+
+            my $narHash;
+            my $hashFile = "$fn.NARHASH";
+            if (-e $hashFile) {
+                open HASH, "<$hashFile" or die;
+                $narHash = <HASH>;
+                close HASH;
+            } else {
+                print "$fn\n";
+                $narHash = `bunzip2 < '$fn' | nix-hash --flat /dev/stdin` or die;
+                open HASH, ">$hashFile" or die;
+                print HASH $narHash;
+                close HASH;
+            }
+            chomp $narHash;
+            $narFile->{narHash} = $narHash;
+        }
+    }
+}
+
+if (! -e "$manifest.backup") {
+    system "mv --reply=no '$manifest' '$manifest.backup'";
+}
+
+writeManifest $manifest, \%narFiles, \%patches, \%successors;

src/Makefile.am

@@ -1,5 +1,5 @@
 SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
- libexpr nix-instantiate nix-env log2xml
+ libexpr nix-instantiate nix-env log2xml bsdiff-4.2
 
 EXTRA_DIST = aterm-helper.pl
 

src/aterm-helper.pl

@@ -56,6 +56,7 @@ while (<STDIN>) {
         my $unpack = "";
         my $n = 1;
         foreach my $type (@types) {
+            my $realType = $type;
             $args .= ", ";
             if ($type eq "string") {
 #                $args .= "(ATerm) ATmakeAppl0(ATmakeAFun((char *) e$n, 0, ATtrue))";
@@ -83,6 +84,9 @@ while (<STDIN>) {
                 $unpack .= "    e$n = (ATermList) ATgetArgument(e, $m);\n";
             } elsif ($type eq "ATermBlob") {
                 $unpack .= "    e$n = (ATermBlob) ATgetArgument(e, $m);\n";
+            } elsif ($realType eq "string") {
+                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
+                $unpack .= "    if (ATgetType(e$n) != AT_APPL) return false;\n";
             } else {
                 $unpack .= "    e$n = ATgetArgument(e, $m);\n";
             }
@@ -96,12 +100,18 @@ while (<STDIN>) {
         print IMPL "AFun sym$funname = 0;\n";
         
         print HEADER "static inline $result make$funname($formals) {\n";
-        print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
+        if ($arity <= 6) {
+            print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
+        } else {
+            $args =~ s/^,//;
+            print HEADER "    ATerm array[$arity] = {$args};\n";
+            print HEADER "    return (ATerm) ATmakeApplArray(sym$funname, array);\n";
+        }
         print HEADER "}\n\n";
 
         print HEADER "#ifdef __cplusplus\n";
         print HEADER "static inline bool match$funname(ATerm e$formals2) {\n";
-        print HEADER "    if (ATgetType(e) != AT_APPL || ATgetAFun(e) != sym$funname) return false;\n";
+        print HEADER "    if (ATgetType(e) != AT_APPL || (AFun) ATgetAFun(e) != sym$funname) return false;\n";
         print HEADER "$unpack";
         print HEADER "    return true;\n";
         print HEADER "}\n";

src/boost/shared_ptr.hpp

@@ -324,7 +324,7 @@ public:
 // Tasteless as this may seem, making all members public allows member templates
 // to work in the absence of member template friends. (Matthew Langston)
 
-#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+#if 0
 
 private:
 

src/boost/weak_ptr.hpp

@@ -154,7 +154,7 @@ public:
 // Tasteless as this may seem, making all members public allows member templates
 // to work in the absence of member template friends. (Matthew Langston)
 
-#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+#if 0
 
 private:
 

src/bsdiff-4.2/LICENSE

@@ -0,0 +1,121 @@
+BSD Protection License
+February 2002
+
+Preamble
+--------
+
+The Berkeley Software Distribution ("BSD") license has proven very effective
+over the years at allowing for a wide spread of work throughout both
+commercial and non-commercial products.  For programmers whose primary
+intention is to improve the general quality of available software, it is
+arguable that there is no better license than the BSD license, as it
+permits improvements to be used wherever they will help, without idealogical
+or metallic constraint.
+
+This is of particular value to those who produce reference implementations
+of proposed standards: The case of TCP/IP clearly illustrates that freely
+and universally available implementations leads the rapid acceptance of
+standards -- often even being used instead of a de jure standard (eg, OSI
+network models).
+
+With the rapid proliferation of software licensed under the GNU General
+Public License, however, the continued success of this role is called into
+question.  Given that the inclusion of a few lines of "GPL-tainted" work
+into a larger body of work will result in restricted distribution -- and
+given that further work will likely build upon the "tainted" portions,
+making them difficult to remove at a future date -- there are inevitable
+circumstances where authors would, in order to protect their goal of
+providing for the widespread usage of their work, wish to guard against
+such "GPL-taint".
+
+In addition, one can imagine that companies which operate by producing and
+selling (possibly closed-source) code would wish to protect themselves
+against the rise of a GPL-licensed competitor.  While under existing
+licenses this would mean not releasing their code under any form of open
+license, if a license existed under which they could incorporate any
+improvements back into their own (commercial) products then they might be
+far more willing to provide for non-closed distribution.
+
+For the above reasons, we put forth this "BSD Protection License": A
+license designed to retain the freedom granted by the BSD license to use
+licensed works in a wide variety of settings, both non-commercial and
+commercial, while protecting the work from having future contributors
+restrict that freedom.
+
+The precise terms and conditions for copying, distribution, and
+modification follow.
+
+BSD PROTECTION LICENSE
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION, AND MODIFICATION
+----------------------------------------------------------------
+
+0. Definitions.
+   a) "Program", below, refers to any program or work distributed under
+      the terms of this license.
+   b) A "work based on the Program", below, refers to either the Program
+      or any derivative work under copyright law.
+   c) "Modification", below, refers to the act of creating derivative works.
+   d) "You", below, refers to each licensee.
+
+1. Scope.
+   This license governs the copying, distribution, and modification of the
+   Program.  Other activities are outside the scope of this license; The
+   act of running the Program is not restricted, and the output from the
+   Program is covered only if its contents constitute a work based on the
+   Program.
+
+2. Verbatim copies.
+   You may copy and distribute verbatim copies of the Program as you
+   receive it, in any medium, provided that you conspicuously and
+   appropriately publish on each copy an appropriate copyright notice; keep
+   intact all the notices that refer to this License and to the absence of
+   any warranty; and give any other recipients of the Program a copy of this
+   License along with the Program.
+
+3. Modification and redistribution under closed license.
+   You may modify your copy or copies of the Program, and distribute
+   the resulting derivative works, provided that you meet the
+   following conditions:
+   a) The copyright notice and disclaimer on the Program must be reproduced
+      and included in the source code, documentation, and/or other materials
+      provided in a manner in which such notices are normally distributed.
+   b) The derivative work must be clearly identified as such, in order that
+      it may not be confused with the original work.
+   c) The license under which the derivative work is distributed must
+      expressly prohibit the distribution of further derivative works.
+
+4. Modification and redistribution under open license.
+   You may modify your copy or copies of the Program, and distribute
+   the resulting derivative works, provided that you meet the
+   following conditions:
+   a) The copyright notice and disclaimer on the Program must be reproduced
+      and included in the source code, documentation, and/or other materials
+      provided in a manner in which such notices are normally distributed.
+   b) You must clearly indicate the nature and date of any changes made
+      to the Program.  The full details need not necessarily be included in
+      the individual modified files, provided that each modified file is
+      clearly marked as such and instructions are included on where the
+      full details of the modifications may be found.
+   c) You must cause any work that you distribute or publish, that in whole
+      or in part contains or is derived from the Program or any part
+      thereof, to be licensed as a whole at no charge to all third
+      parties under the terms of this License.
+
+5. Implied acceptance.
+   You may not copy or distribute the Program or any derivative works except
+   as expressly provided under this license.  Consequently, any such action
+   will be taken as implied acceptance of the terms of this license.
+
+6. NO WARRANTY.
+   THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+   INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+   AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL
+   THE COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+   REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE FOR ANY DIRECT,
+   INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+   ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING, BUT
+   NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+   TORT, EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+   POSSIBILITY OF SUCH DAMAGES.

src/bsdiff-4.2/Makefile.am

@@ -0,0 +1,7 @@
+libexec_PROGRAMS = bsdiff bspatch
+
+bsdiff_SOURCES = bsdiff.c
+
+bspatch_SOURCES = bspatch.c
+
+AM_CFLAGS = -O3 -DBZIP2=\"$(bzip2)\"

src/bsdiff-4.2/Makefile.old

@@ -0,0 +1,18 @@
+CFLAGS		+=	-O3
+.ifdef BZIP2
+CFLAGS		+=	-DBZIP2=\"${BZIP2}\"
+.endif
+
+PREFIX		?=	/usr/local
+INSTALL_PROGRAM	?=	${INSTALL} -c -s -m 555
+INSTALL_MAN	?=	${INSTALL} -c -m 444
+
+all:		bsdiff bspatch
+bsdiff:		bsdiff.c
+bspatch:	bspatch.c
+
+install:
+	${INSTALL_PROGRAM} bsdiff bspatch ${PREFIX}/bin
+.ifndef WITHOUT_MAN
+	${INSTALL_MAN} bsdiff.1 bspatch.1 ${PREFIX}/man/man1
+.endif

src/bsdiff-4.2/bsdiff.1

@@ -0,0 +1,36 @@
+.Dd May 18, 2003
+.Dt BSDIFF 1
+.Os FreeBSD
+.Sh NAME
+.Nm bsdiff
+.Nd generate a patch between two binary files
+.Sh SYNOPSIS
+.Nm
+.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
+.Sh DESCRIPTION
+.Nm
+compares
+.Ao Ar oldfile Ac
+to
+.Ao Ar newfile Ac
+and writes to
+.Ao Ar patchfile Ac
+a binary patch suitable for use by bspatch(1).
+When
+.Ao Ar oldfile Ac
+and
+.Ao Ar newfile Ac
+are two versions of an executable program, the
+patches produced are on average a factor of five smaller
+than those produced by any other binary patch tool known
+to the author.
+.Pp
+.Nm
+uses memory equal to 17 times the size of 
+.Ao Ar oldfile Ac ,
+and requires
+an absolute minimum working set size of 8 times the size of oldfile.
+.Sh SEE ALSO
+.Xr bspatch 1
+.Sh AUTHORS
+.An Colin Percival Aq cperciva@daemonology.net

src/bsdiff-4.2/bsdiff.c

@@ -0,0 +1,385 @@
+/*
+  bsdiff.c -- Binary patch generator.
+
+  Copyright 2003 Colin Percival
+
+  For the terms under which this work may be distributed, please see
+  the adjoining file "LICENSE".
+*/
+
+#ifndef BZIP2
+#define BZIP2 "/usr/bin/bzip2"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <err.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+#define MIN(x,y) (((x)<(y)) ? (x) : (y))
+
+void split(off_t *I,off_t *V,off_t start,off_t len,off_t h)
+{
+	off_t i,j,k,x,tmp,jj,kk;
+
+	if(len<16) {
+		for(k=start;k<start+len;k+=j) {
+			j=1;x=V[I[k]+h];
+			for(i=1;k+i<start+len;i++) {
+				if(V[I[k+i]+h]<x) {
+					x=V[I[k+i]+h];
+					j=0;
+				};
+				if(V[I[k+i]+h]==x) {
+					tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp;
+					j++;
+				};
+			};
+			for(i=0;i<j;i++) V[I[k+i]]=k+j-1;
+			if(j==1) I[k]=-1;
+		};
+		return;
+	};
+
+	x=V[I[start+len/2]+h];
+	jj=0;kk=0;
+	for(i=start;i<start+len;i++) {
+		if(V[I[i]+h]<x) jj++;
+		if(V[I[i]+h]==x) kk++;
+	};
+	jj+=start;kk+=jj;
+
+	i=start;j=0;k=0;
+	while(i<jj) {
+		if(V[I[i]+h]<x) {
+			i++;
+		} else if(V[I[i]+h]==x) {
+			tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp;
+			j++;
+		} else {
+			tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp;
+			k++;
+		};
+	};
+
+	while(jj+j<kk) {
+		if(V[I[jj+j]+h]==x) {
+			j++;
+		} else {
+			tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp;
+			k++;
+		};
+	};
+
+	if(jj>start) split(I,V,start,jj-start,h);
+
+	for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1;
+	if(jj==kk-1) I[jj]=-1;
+
+	if(start+len>kk) split(I,V,kk,start+len-kk,h);
+}
+
+void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize)
+{
+	off_t buckets[256];
+	off_t i,h,len;
+
+	for(i=0;i<256;i++) buckets[i]=0;
+	for(i=0;i<oldsize;i++) buckets[old[i]]++;
+	for(i=1;i<256;i++) buckets[i]+=buckets[i-1];
+	for(i=255;i>0;i--) buckets[i]=buckets[i-1];
+	buckets[0]=0;
+
+	for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i;
+	I[0]=oldsize;
+	for(i=0;i<oldsize;i++) V[i]=buckets[old[i]];
+	V[oldsize]=0;
+	for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1;
+	I[0]=-1;
+
+	for(h=1;I[0]!=-(oldsize+1);h+=h) {
+		len=0;
+		for(i=0;i<oldsize+1;) {
+			if(I[i]<0) {
+				len-=I[i];
+				i-=I[i];
+			} else {
+				if(len) I[i-len]=-len;
+				len=V[I[i]]+1-i;
+				split(I,V,i,len,h);
+				i+=len;
+				len=0;
+			};
+		};
+		if(len) I[i-len]=-len;
+	};
+
+	for(i=0;i<oldsize+1;i++) I[V[i]]=i;
+}
+
+off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize)
+{
+	off_t i;
+
+	for(i=0;(i<oldsize)&&(i<newsize);i++)
+		if(old[i]!=new[i]) break;
+
+	return i;
+}
+
+off_t search(off_t *I,u_char *old,off_t oldsize,
+		u_char *new,off_t newsize,off_t st,off_t en,off_t *pos)
+{
+	off_t x,y;
+
+	if(en-st<2) {
+		x=matchlen(old+I[st],oldsize-I[st],new,newsize);
+		y=matchlen(old+I[en],oldsize-I[en],new,newsize);
+
+		if(x>y) {
+			*pos=I[st];
+			return x;
+		} else {
+			*pos=I[en];
+			return y;
+		}
+	};
+
+	x=st+(en-st)/2;
+	if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) {
+		return search(I,old,oldsize,new,newsize,x,en,pos);
+	} else {
+		return search(I,old,oldsize,new,newsize,st,x,pos);
+	};
+}
+
+void offtout(off_t x,u_char *buf)
+{
+	off_t y;
+
+	if(x<0) y=-x; else y=x;
+
+		buf[0]=y%256;y-=buf[0];
+	y=y/256;buf[1]=y%256;y-=buf[1];
+	y=y/256;buf[2]=y%256;y-=buf[2];
+	y=y/256;buf[3]=y%256;y-=buf[3];
+	y=y/256;buf[4]=y%256;y-=buf[4];
+	y=y/256;buf[5]=y%256;y-=buf[5];
+	y=y/256;buf[6]=y%256;y-=buf[6];
+	y=y/256;buf[7]=y%256;
+
+	if(x<0) buf[7]|=0x80;
+}
+
+int main(int argc,char *argv[])
+{
+	int fd,p[2];
+	pid_t pid;
+	u_char *old,*new;
+	off_t oldsize,newsize;
+	off_t *I,*V;
+
+	off_t scan,pos,len;
+	off_t lastscan,lastpos,lastoffset;
+	off_t oldscore,scsc;
+
+	off_t s,Sf,lenf,Sb,lenb;
+	off_t overlap,Ss,lens;
+	off_t i;
+
+	off_t dblen,eblen;
+	u_char *db,*eb;
+
+	u_char buf[8];
+	u_char header[32];
+
+	if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
+
+	/* Allocate oldsize+1 bytes instead of oldsize bytes to ensure
+		that we never try to malloc(0) and get a NULL pointer */
+	if(((fd=open(argv[1],O_RDONLY,0))<0) ||
+		((oldsize=lseek(fd,0,SEEK_END))==-1) ||
+		((old=malloc(oldsize+1))==NULL) ||
+		(lseek(fd,0,SEEK_SET)!=0) ||
+		(read(fd,old,oldsize)!=oldsize) ||
+		(close(fd)==-1)) err(1,"%s",argv[1]);
+
+	if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) ||
+		((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL);
+
+	qsufsort(I,V,old,oldsize);
+
+	free(V);
+
+	/* Allocate newsize+1 bytes instead of newsize bytes to ensure
+		that we never try to malloc(0) and get a NULL pointer */
+	if(((fd=open(argv[2],O_RDONLY,0))<0) ||
+		((newsize=lseek(fd,0,SEEK_END))==-1) ||
+		((new=malloc(newsize+1))==NULL) ||
+		(lseek(fd,0,SEEK_SET)!=0) ||
+		(read(fd,new,newsize)!=newsize) ||
+		(close(fd)==-1)) err(1,"%s",argv[2]);
+
+	if(((db=malloc(newsize+1))==NULL) ||
+		((eb=malloc(newsize+1))==NULL)) err(1,NULL);
+	dblen=0;
+	eblen=0;
+
+	if((fd=open(argv[3],O_CREAT|O_TRUNC|O_WRONLY,0666))<0)
+		err(1,"%s",argv[3]);
+
+	/* Header is
+		0	8	 "BSDIFF40"
+		8	8	length of bzip2ed ctrl block
+		16	8	length of bzip2ed diff block
+		24	8	length of new file */
+	/* File is
+		0	32	Header
+		32	??	Bzip2ed ctrl block
+		??	??	Bzip2ed diff block
+		??	??	Bzip2ed extra block */
+	memcpy(header,"BSDIFF40",8);
+	memset(header+8,0,24);
+	if(write(fd,header,32)!=32) err(1,"%s",argv[3]);
+
+	if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
+	if(pid==0) {
+		if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
+			(dup2(p[0],0)==-1) || (close(fd)==-1) ||
+			(close(p[0])==-1) || (close(p[1])==-1))
+			err(1,NULL);
+		execl(BZIP2,BZIP2,"-zc",NULL);
+		err(1,"%s",BZIP2);
+	};
+	if(close(p[0])==-1) err(1,NULL);
+
+	scan=0;len=0;
+	lastscan=0;lastpos=0;lastoffset=0;
+	while(scan<newsize) {
+		oldscore=0;
+
+		for(scsc=scan+=len;scan<newsize;scan++) {
+			len=search(I,old,oldsize,new+scan,newsize-scan,
+					0,oldsize,&pos);
+
+			for(;scsc<scan+len;scsc++)
+			if((scsc+lastoffset<oldsize) &&
+				(old[scsc+lastoffset] == new[scsc]))
+				oldscore++;
+
+			if(((len==oldscore) && (len!=0)) || 
+				(len>oldscore+8)) break;
+
+			if((scan+lastoffset<oldsize) &&
+				(old[scan+lastoffset] == new[scan]))
+				oldscore--;
+		};
+
+		if((len!=oldscore) || (scan==newsize)) {
+			s=0;Sf=0;lenf=0;
+			for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
+				if(old[lastpos+i]==new[lastscan+i]) s++;
+				i++;
+				if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
+			};
+
+			lenb=0;
+			if(scan<newsize) {
+				s=0;Sb=0;
+				for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
+					if(old[pos-i]==new[scan-i]) s++;
+					if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
+				};
+			};
+
+			if(lastscan+lenf>scan-lenb) {
+				overlap=(lastscan+lenf)-(scan-lenb);
+				s=0;Ss=0;lens=0;
+				for(i=0;i<overlap;i++) {
+					if(new[lastscan+lenf-overlap+i]==
+					   old[lastpos+lenf-overlap+i]) s++;
+					if(new[scan-lenb+i]==
+					   old[pos-lenb+i]) s--;
+					if(s>Ss) { Ss=s; lens=i+1; };
+				};
+
+				lenf+=lens-overlap;
+				lenb-=lens;
+			};
+
+			for(i=0;i<lenf;i++)
+				db[dblen+i]=new[lastscan+i]-old[lastpos+i];
+			for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
+				eb[eblen+i]=new[lastscan+lenf+i];
+
+			dblen+=lenf;
+			eblen+=(scan-lenb)-(lastscan+lenf);
+
+			offtout(lenf,buf);
+			if(write(p[1],buf,8)!=8) err(1,NULL);
+			offtout((scan-lenb)-(lastscan+lenf),buf);
+			if(write(p[1],buf,8)!=8) err(1,NULL);
+			offtout((pos-lenb)-(lastpos+lenf),buf);
+			if(write(p[1],buf,8)!=8) err(1,NULL);
+
+			lastscan=scan-lenb;
+			lastpos=pos-lenb;
+			lastoffset=pos-scan;
+		};
+	};
+
+	if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
+
+	if((len=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
+	offtout(len-32,buf);
+	if((lseek(fd,8,SEEK_SET)!=8) || (write(fd,buf,8)!=8))
+		err(1,"%s",argv[3]);
+	offtout(newsize,buf);
+	if((lseek(fd,24,SEEK_SET)!=24) || (write(fd,buf,8)!=8))
+		err(1,"%s",argv[3]);
+
+	if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
+	if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
+	if(pid==0) {
+		if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
+			(dup2(p[0],0)==-1) || (close(fd)==-1) ||
+			(close(p[0])==-1) || (close(p[1])==-1))
+			err(1,NULL);
+		execl(BZIP2,BZIP2,"-zc",NULL);
+		err(1,"%s",BZIP2);
+	};
+	if(close(p[0])==-1) err(1,NULL);
+	if(write(p[1],db,dblen)!=dblen) err(1,NULL);
+	if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
+
+	if((newsize=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
+	offtout(newsize-len,buf);
+	if((lseek(fd,16,SEEK_SET)!=16) || (write(fd,buf,8)!=8))
+		err(1,"%s",argv[3]);
+
+	if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
+	if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
+	if(pid==0) {
+		if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
+			(dup2(p[0],0)==-1) || (close(fd)==-1) ||
+			(close(p[0])==-1) || (close(p[1])==-1))
+			err(1,NULL);
+		execl(BZIP2,BZIP2,"-zc",NULL);
+		err(1,"%s",BZIP2);
+	};
+	if(close(p[0])==-1) err(1,NULL);
+	if(write(p[1],eb,eblen)!=eblen) err(1,NULL);
+	if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
+	if(close(fd)==-1) err(1,"%s",argv[3]);
+
+	free(db);
+	free(eb);
+	free(I);
+	free(old);
+	free(new);
+
+	return 0;
+}

src/bsdiff-4.2/bspatch.1

@@ -0,0 +1,32 @@
+.Dd May 18, 2003
+.Dt BSPATCH 1
+.Os FreeBSD
+.Sh NAME
+.Nm bspatch
+.Nd apply a patch built with bsdiff(1)
+.Sh SYNOPSIS
+.Nm
+.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
+.Sh DESCRIPTION
+.Nm
+generates
+.Ao Ar newfile Ac
+from
+.Ao Ar oldfile Ac
+and
+.Ao Ar patchfile Ac
+where
+.Ao Ar patchfile Ac
+is a binary patch built by bsdiff(1).
+.Pp
+.Nm
+uses memory equal to the size of 
+.Ao Ar oldfile Ac
+plus the size of 
+.Ao Ar newfile Ac ,
+but can tolerate a very small working set without a dramatic loss
+of performance.
+.Sh SEE ALSO
+.Xr bsdiff 1
+.Sh AUTHORS
+.An Colin Percival Aq cperciva@daemonology.net

src/bsdiff-4.2/bspatch.c

@@ -0,0 +1,216 @@
+/*
+  .c -- Binary patcher
+
+  Copyright 2003,2004 Colin Percival
+
+  For the terms under which this work may be distributed, please see
+  the adjoining file "LICENSE".
+*/
+
+#ifndef BZIP2
+#define BZIP2 "/usr/bin/bzip2"
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+
+ssize_t loopread(int d,void *buf,size_t nbytes)
+{
+	ssize_t ptr,lenread;
+
+	for(ptr=0;ptr<nbytes;ptr+=lenread) {
+		lenread=read(d,buf+ptr,nbytes-ptr);
+		if(lenread==0) return ptr;
+		if(lenread==-1) return -1;
+	};
+	return ptr;
+}
+
+int bz2read(int fd,off_t offset,off_t len,char * fname,pid_t * pids)
+{
+	int p0[2],p1[2];
+	u_char * data;
+
+	if((pipe(p0)==-1) || (pipe(p1)==-1)) err(1,NULL);
+
+	if((pids[0]=fork())==-1) err(1,NULL);
+	if(pids[0]==0) {
+		if(close(0) || close(1) || close(p0[0]) ||
+			close(p1[0]) || close(p1[1])) err(1,NULL);
+		if((data=malloc(len+1))==NULL) err(1,NULL);
+		if((pread(fd,data,len,offset)!=len) || close(fd))
+			err(1,"%s",fname);
+		if((write(p0[1],data,len)!=len) || close(p0[1]))
+			err(1,NULL);
+		free(data);
+		_exit(0);
+	};
+
+	if((pids[1]=fork())==-1) err(1,NULL);
+	if(pids[1]==0) {
+		if(close(0) || close(1) || close(p0[1]) ||
+			close(p1[0])) err(1,NULL);
+		if((dup2(p0[0],0)==-1) || close(p0[0])) err(1,NULL);
+		if((dup2(p1[1],1)==-1) || close(p1[1])) err(1,NULL);
+		if(close(fd)==-1) err(1,"%s",fname);
+
+		execl(BZIP2,BZIP2,"-dc",NULL);
+		err(1,"%s",BZIP2);
+	};
+
+	if(close(p0[0]) || close(p0[1]) || close(p1[1])) err(1,NULL);
+
+	return p1[0];
+}
+
+off_t offtin(u_char *buf)
+{
+	off_t y;
+
+	y=buf[7]&0x7F;
+	y=y*256;y+=buf[6];
+	y=y*256;y+=buf[5];
+	y=y*256;y+=buf[4];
+	y=y*256;y+=buf[3];
+	y=y*256;y+=buf[2];
+	y=y*256;y+=buf[1];
+	y=y*256;y+=buf[0];
+
+	if(buf[7]&0x80) y=-y;
+
+	return y;
+}
+
+int main(int argc,char * argv[])
+{
+	int fd,ctrlpipe,diffpipe,extrapipe;
+	pid_t pids[6];
+	ssize_t patchsize,oldsize,newsize;
+	ssize_t bzctrllen,bzdatalen;
+	u_char header[32],buf[8];
+	int version=0;
+	u_char *old, *new;
+	off_t oldpos,newpos;
+	off_t ctrl[3];
+	off_t lenread;
+	off_t i;
+
+	if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
+
+	if(((fd=open(argv[3],O_RDONLY,0))<0) ||
+		((patchsize=lseek(fd,0,SEEK_END))==-1) ||
+		(lseek(fd,0,SEEK_SET)!=0)) err(1,"%s",argv[3]);
+	if(patchsize<32) errx(1,"Corrupt patch\n");
+
+	/*
+	  Ok, this is going to be messy.  There are two different patch
+	formats which we need to support.
+
+	  The old format (pre-4.0) is:
+		0	8	"QSUFDIFF" or "BSDIFF30"
+		8	8	X
+		16	8	Y
+		24	8	sizeof(newfile)
+		32	X	bzip2(control block)
+		32+X	Y	bzip2(data block)
+	with control block a set of pairs (x,y) meaning "seek forward
+	in oldfile by y bytes, and add the next x bytes to x bytes from
+	the data block".
+
+	  The new format (4.0) is:
+		0	8	"BSDIFF40"
+		8	8	X
+		16	8	Y
+		24	8	sizeof(newfile)
+		32	X	bzip2(control block)
+		32+X	Y	bzip2(diff block)
+		32+X+Y	???	bzip2(extra block)
+	with control block a set of triples (x,y,z) meaning "add x bytes
+	from oldfile to x bytes from the diff block; copy y bytes from the
+	extra block; seek forwards in oldfile by z bytes".
+	*/
+
+	if(read(fd,header,32)!=32) err(1,"%s",argv[3]);
+	if(memcmp(header,"QSUFDIFF",8)==0) version=1;
+	if(memcmp(header,"BSDIFF30",8)==0) version=1;
+	if(memcmp(header,"BSDIFF40",8)==0) version=2;
+
+	if(!version) errx(1,"Corrupt patch\n");
+
+	bzctrllen=offtin(header+8);
+	bzdatalen=offtin(header+16);
+	newsize=offtin(header+24);
+	if((bzctrllen<0) || (bzdatalen<0) || (newsize<0) ||
+		((version==1) && (32+bzctrllen+bzdatalen!=patchsize)))
+		errx(1,"Corrupt patch\n");
+
+	ctrlpipe=bz2read(fd,32,bzctrllen,argv[3],pids);
+	diffpipe=bz2read(fd,32+bzctrllen,bzdatalen,argv[3],pids+2);
+	if(version==2) {
+		extrapipe=bz2read(fd,32+bzctrllen+bzdatalen,
+			patchsize-(32+bzctrllen+bzdatalen),argv[3],pids+4);
+	};
+
+	if(close(fd)==-1) err(1,"%s",argv[3]);
+	if(((fd=open(argv[1],O_RDONLY,0))<0) ||
+		((oldsize=lseek(fd,0,SEEK_END))==-1) ||
+		((old=malloc(oldsize+1))==NULL) ||
+		(lseek(fd,0,SEEK_SET)!=0) ||
+		(read(fd,old,oldsize)!=oldsize) ||
+		(close(fd)==-1)) err(1,"%s",argv[1]);
+	if((new=malloc(newsize+1))==NULL) err(1,NULL);
+
+	oldpos=0;newpos=0;
+	while(newpos<newsize) {
+		for(i=0;i<=version;i++) {
+			if((lenread=loopread(ctrlpipe,buf,8))<0) err(1,NULL);
+			if(lenread<8) errx(1,"Corrupt patch\n");
+			ctrl[i]=offtin(buf);
+		};
+
+		if(version==1) oldpos+=ctrl[1];
+
+		if(newpos+ctrl[0]>newsize) errx(1,"Corrupt patch\n");
+		if((lenread=loopread(diffpipe,new+newpos,ctrl[0]))<0)
+			err(1,NULL);
+		if(lenread!=ctrl[0]) errx(1,"Corrupt patch\n");
+		for(i=0;i<ctrl[0];i++)
+			if((oldpos+i>=0) && (oldpos+i<oldsize))
+				new[newpos+i]+=old[oldpos+i];
+		newpos+=ctrl[0];
+		oldpos+=ctrl[0];
+
+		if(version==2) {
+			if(newpos+ctrl[1]>newsize) errx(1,"Corrupt patch\n");
+			if((lenread=loopread(extrapipe,new+newpos,ctrl[1]))<0)
+				err(1,NULL);
+			if(lenread!=ctrl[1]) errx(1,"Corrupt patch\n");
+
+			newpos+=ctrl[1];
+			oldpos+=ctrl[2];
+		};
+	};
+
+	if(loopread(ctrlpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
+	if(loopread(diffpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
+	if(version==2)
+		if(loopread(extrapipe,buf,1)!=0) errx(1,"Corrupt patch\n");
+
+	if(close(ctrlpipe) || close(diffpipe) || 
+		((version==2) && close(extrapipe)))
+		err(1,NULL);
+	for(i=0;i<(version+1)*2;i++) waitpid(pids[i],NULL,0);
+
+	if(((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) ||
+		(write(fd,new,newsize)!=newsize) || (close(fd)==-1))
+		err(1,"%s",argv[2]);
+
+	free(new);
+	free(old);
+
+	return 0;
+}

src/libexpr/Makefile.am

@@ -29,7 +29,7 @@ lexer-tab.c lexer-tab.h: lexer.l
 nixexpr-ast.cc nixexpr-ast.hh: ../aterm-helper.pl nixexpr-ast.def
 	$(perl) ../aterm-helper.pl nixexpr-ast.hh nixexpr-ast.cc < nixexpr-ast.def
 
-nixexpr.hh: nixexpr-ast.hh
+nixexpr.cc nixexpr.hh: nixexpr-ast.hh
 
 
 CLEANFILES =

src/libexpr/nixexpr.cc

@@ -1,5 +1,5 @@
 #include "nixexpr.hh"
-#include "storeexpr.hh"
+#include "derivations.hh"
 
 
 #include "nixexpr-ast.hh"
@@ -7,6 +7,7 @@
 
 
 ATermMap::ATermMap(unsigned int initialSize, unsigned int maxLoadPct)
+    : table(0)
 {
     this->maxLoadPct = maxLoadPct;
     table = ATtableCreate(initialSize, maxLoadPct);
@@ -16,6 +17,36 @@ ATermMap::ATermMap(unsigned int initialSize, unsigned int maxLoadPct)
 
 ATermMap::ATermMap(const ATermMap & map)
     : table(0)
+{
+    copy(map);
+}
+
+
+ATermMap::~ATermMap()
+{
+    free();
+}
+
+
+ATermMap & ATermMap::operator = (const ATermMap & map)
+{
+    if (this == &map) return *this;
+    free();
+    copy(map);
+    return *this;
+}
+
+
+void ATermMap::free()
+{
+    if (table) {
+        ATtableDestroy(table);
+        table = 0;
+    }
+}
+
+
+void ATermMap::copy(const ATermMap & map)
 {
     ATermList keys = map.keys();
 
@@ -28,12 +59,6 @@ ATermMap::ATermMap(const ATermMap & map)
 }
 
 
-ATermMap::~ATermMap()
-{
-    if (table) ATtableDestroy(table);
-}
-
-
 void ATermMap::set(ATerm key, ATerm value)
 {
     return ATtablePut(table, key, value);

src/libexpr/nixexpr.hh

@@ -29,6 +29,8 @@ public:
     ATermMap(const ATermMap & map);
     ~ATermMap();
 
+    ATermMap & ATermMap::operator = (const ATermMap & map);
+        
     void set(ATerm key, ATerm value);
     void set(const string & key, ATerm value);
 
@@ -46,6 +48,9 @@ public:
 
 private:
     void add(const ATermMap & map, ATermList & keys);
+
+    void free();
+    void copy(const ATermMap & map);
 };
 
 
@@ -89,11 +94,5 @@ void checkVarDefs(const ATermMap & def, Expr e);
 /* Create an expression representing a boolean. */
 Expr makeBool(bool b);
 
-/* Create an expression representing a string. */
-Expr makeString(const string & s);
-
-/* Create an expression representing a path. */
-Expr makePath(const Path & path);
-
 
 #endif /* !__NIXEXPR_H */