* Sync with thesis: do not include store location in the hash

computation (in the intensional model).

Makefile.am

@@ -1,5 +1,6 @@
 SUBDIRS = externals src scripts corepkgs doc misc tests
-EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh svn-revision
+EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
+  svn-revision nix.conf.example
 
 include ./substitute.mk
 
@@ -12,6 +13,11 @@ relname:
 	echo -n $(distdir) > relname
 
 install-data-local: init-state
+	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
+	$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix
+	if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
+		$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
+	fi
 
 if INIT_STATE
 if SETUID_HACK
@@ -22,8 +28,10 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/db
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix/drvs
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles

NEWS

@@ -1,4 +1,174 @@
-Version 0.7
+Version 0.9
+
+* Unpacking of patch sequences is much faster now by not doing
+  redundant unpacking and repacking of intermediate paths.
+
+
+Version 0.8 (April 11, 2005)
+
+NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).  As a
+result, `nix-pull' manifests and channels built for Nix 0.7 and below
+will now work anymore.  However, the Nix expression language has not
+changed, so you can still build from source.  Also, existing user
+environments continue to work.  Nix 0.8 will automatically upgrade the
+database schema of previous installations when it is first run.
+
+If you get the error message
+
+  you have an old-style manifest `/nix/var/nix/manifests/[...]';
+  please delete it
+
+you should delete previously downloaded manifests:
+
+  $ rm /nix/var/nix/manifests/*
+
+If `nix-channel' gives the error message
+
+  manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST'
+  is too old (i.e., for Nix <= 0.7)
+
+then you should unsubscribe from the offending channel (`nix-channel
+--remove URL'; leave out `/MANIFEST'), and subscribe to the same URL,
+with `channels' replaced by `channels-v3' (e.g.,
+http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable).
+
+Nix 0.8 has the following improvements:
+
+* The cryptographic hashes used in store paths are now 160 bits long,
+  but encoded in base-32 so that they are still only 32 characters
+  long (e.g., /nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0).
+  (This is actually a 160 bit truncation of a SHA-256 hash.)
+
+* Big cleanups and simplifications of the basic store semantics.  The
+  notion of "closure store expressions" is gone (and so is the notion
+  of "successors"); the file system references of a store path are now
+  just stored in the database.
+
+  For instance, given any store path, you can query its closure:
+
+    $ nix-store -qR $(which firefox)
+    ... lots of paths ...
+
+  Also, Nix now remembers for each store path the derivation that
+  built it (the "deriver"):
+
+    $ nix-store -qR $(which firefox)
+    /nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv
+
+  So to see the build-time dependencies, you can do
+  
+    $ nix-store -qR $(nix-store -qd $(which firefox))
+
+  or, in a nicer format:
+
+    $ nix-store -q --tree $(nix-store -qd $(which firefox))
+
+  File system references are also stored in reverse.  For instance,
+  you can query all paths that directly or indirectly use a certain
+  Glibc:
+
+    $ nix-store -q --referers-closure \
+        /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+
+* The concept of fixed-output derivations has been formalised.
+  Previously, functions such as `fetchurl' in Nixpkgs used a hack
+  (namely, explicitly specifying a store path hash) to prevent changes
+  to, say, the URL of the file from propagating upwards through the
+  dependency graph, causing rebuilds of everything.  This can now be
+  done cleanly by specifying the `outputHash' and `outputHashAlgo'
+  attributes.  Nix itself checks that the content of the output has
+  the specified hash.  (This is important for maintaining certain
+  invariants necessary for future work on secure shared stores.)
+
+* One-click installation :-)  It is now possible to install any
+  top-level component in Nixpkgs directly, through the web - see,
+  e.g., http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/.  All you
+  have to do is associate `/nix/bin/nix-install-package' with the MIME
+  type `application/nix-package' (or the extension `.nixpkg'), and
+  clicking on a package link will cause it to be installed, with all
+  appropriate dependencies.  If you just want to install some specific
+  application, this is easier than subscribing to a channel.
+
+* `nix-store -r PATHS' now builds all the derivations PATHS in
+  parallel.  Previously it did them sequentially (though exploiting
+  possible parallelism between subderivations).  This is nice for
+  build farms.
+
+* `nix-channel' has new operations `--list' and `--remove'.
+
+* New ways of installing components into user environments:
+
+  - Copy from another user environment:
+
+      $ nix-env -i --from-profile .../other-profile firefox
+
+  - Install a store derivation directly (bypassing the Nix expression
+    language entirely):
+
+      $ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv
+
+    (This is used to implement `nix-install-package', which is
+    therefore immune to evolution in the Nix expression language.)
+
+  - Install an already built store path directly:
+
+      $ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1
+
+  - Install the result of a Nix expression specified as a command-line
+    argument:
+
+      $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'
+
+    The difference with the normal installation mode is that `-E' does
+    not use the `name' attributes of derivations.  Therefore, this can
+    be used to disambiguate multiple derivations with the same name.
+
+* A hash of the contents of a store path is now stored in the database
+  after a succesful build.  This allows you to check whether store
+  paths have been tampered with: `nix-store --verify --check-contents'.
+    
+* Implemented a concurrent garbage collector.  It is now always safe
+  to run the garbage collector, even if other Nix operations are
+  happening simultaneously.
+
+  However, there can still be GC races if you use `nix-instantiate'
+  and `nix-store -r' directly to build things.  To prevent races, use
+  the `--add-root' flag of those commands.
+
+* The garbage collector now finally deletes paths in the right order
+  (i.e., topologically sorted under the `references' relation), thus
+  making it safe to interrupt the collector without risking a store
+  that violates the closure invariant.
+
+* Likewise, the substitute mechanism now downloads files in the right
+  order, thus preserving the closure invariant at all times.
+
+* The result of `nix-build' is now registered as a root of the garbage
+  collector.  If the `./result' link is deleted, the GC root
+  disappears automatically.
+
+* The behaviour of the garbage collector can be changed globally by
+  setting options in `/nix/etc/nix/nix.conf'.
+
+  - `gc-keep-derivations' specifies whether deriver links should be
+    followed when searching for live paths.
+
+  - `gc-keep-outputs' specifies whether outputs of derivations should
+    be followed when searching for live paths.
+  
+  - `env-keep-derivations' specifies whether user environments should
+    store the paths of derivations when they are added (thus keeping
+    the derivations alive).
+
+* New `nix-env' query flags `--drv-path' and `--out-path'.
+
+* `fetchurl' allows SHA-1 and SHA-256 in addition to MD5.  Just
+  specify the attribute `sha1' or `sha256' instead of `md5'.
+
+* Manual updates.
+
+  
+Version 0.7 (January 12, 2005)
 
 * Binary patching.  When upgrading components using pre-built binaries
   (through nix-pull / nix-channel), Nix can automatically download and
@@ -20,7 +190,7 @@ Version 0.7
   dependencies are revealed.
 
 
-Version 0.6
+Version 0.6 (November 14, 2004)
 
 Major changes include the following:
 

README

@@ -1,5 +1,9 @@
-                             *** Nix ***
-
 For installation and usage instructions, please read the manual, which
 can be found in `docs/manual/manual.html', and additionally at the Nix
 website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
+
+
+Acknowledgments
+
+This product includes software developed by the OpenSSL Project for
+use in the OpenSSL Toolkit (http://www.OpenSSL.org/)

blacklisting/check-env.pl

@@ -0,0 +1,252 @@
+#! /usr/bin/perl -w -I /home/eelco/.nix-profile/lib/site_perl
+
+use strict;
+use XML::LibXML;
+#use XML::Simple;
+
+my $blacklistFN = shift @ARGV;
+die unless defined $blacklistFN;
+my $userEnv = shift @ARGV;
+die unless defined $userEnv;
+
+
+# Read the blacklist.
+my $parser = XML::LibXML->new();
+my $blacklist = $parser->parse_file($blacklistFN)->getDocumentElement;
+
+#print $blacklist->toString() , "\n";
+
+
+# Get all the elements of the user environment.
+my $userEnvElems = `nix-store --query --references '$userEnv'`;
+die "cannot query user environment elements" if $? != 0;
+my @userEnvElems = split ' ', $userEnvElems;
+
+
+my %storePathHashes;
+
+
+sub getElemNodes {
+    my $node = shift;
+    my @elems = ();
+    foreach my $node ($node->getChildNodes) {
+        push @elems, $node if $node->nodeType == XML_ELEMENT_NODE;
+    }
+    return @elems;
+}
+
+
+my %referencesCache;
+sub getReferences {
+    my $path = shift;
+    return $referencesCache{$path} if defined $referencesCache{$path};
+    
+    my $references = `nix-store --query --references '$path'`;
+    die "cannot query references" if $? != 0;
+    $referencesCache{$path} = [split ' ', $references];
+    
+    return $referencesCache{$path};
+}
+
+
+my %attrsCache;
+sub getAttr {
+    my $path = shift;
+    my $name = shift;
+    my $key = "$path/$name";
+    return $referencesCache{$key} if defined $referencesCache{$key};
+
+    my $value = `nix-store --query --binding '$name' '$path' 2> /dev/null`;
+    $value = "" if $? != 0; # !!!
+    chomp $value;
+    $referencesCache{$key} = $value;
+
+    return $value;
+}
+
+
+sub evalCondition;
+
+
+sub traverse {
+    my $done = shift;
+    my $set = shift;
+    my $path = shift;
+    my $stopCondition = shift;
+
+    return if defined $done->{$path};
+    $done->{$path} = 1;
+    $set->{$path} = 1;
+
+#    print "  in $path\n";
+
+    if (!evalCondition({$path => 1}, $stopCondition)) {
+#        print "  STOPPING in $path\n";
+        return;
+    }
+
+    # Get the requisites of the deriver.
+
+    foreach my $reference (@{getReferences $path}) {
+        traverse($done, $set, $reference, $stopCondition);
+    }
+}
+
+
+sub evalSet {
+    my $inSet = shift;
+    my $expr = shift;
+    my $name = $expr->getName;
+    
+    if ($name eq "traverse") {
+        my $stopCondition = (getElemNodes $expr)[0];
+        my $done = { };
+        my $set = { };
+        foreach my $path (keys %{$inSet}) {
+            traverse($done, $set, $path, $stopCondition);
+        }
+        return $set;
+    }
+
+    else {
+        die "unknown element `$name'";
+    }
+}
+
+
+# Function for evaluating conditions.
+sub evalCondition {
+    my $storePaths = shift;
+    my $condition = shift;
+    my $elemName = $condition->getName;
+    
+    if ($elemName eq "containsSource") {
+        my $hash = $condition->attributes->getNamedItem("hash")->getValue;
+        foreach my $path (keys %{$storePathHashes{$hash}}) {
+            return 1 if defined $storePaths->{$path};
+        }
+        return 0;
+    }
+
+    elsif ($elemName eq "hasName") { 
+        my $nameRE = $condition->attributes->getNamedItem("name")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            return 1 if $path =~ /$nameRE/;
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "hasAttr") { 
+        my $name = $condition->attributes->getNamedItem("name")->getValue;
+        my $valueRE = $condition->attributes->getNamedItem("value")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            if ($path =~ /\.drv$/) {
+                my $value = getAttr($path, $name);
+#                print "    $path $name $value\n";
+                return 1 if $value =~ /$valueRE/;
+            }
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "and") {
+        my $result = 1;
+        foreach my $node (getElemNodes $condition) {
+            $result &= evalCondition($storePaths, $node);
+        }
+        return $result;
+    }
+
+    elsif ($elemName eq "not") {
+        return !evalCondition($storePaths, (getElemNodes $condition)[0]);
+    }
+    
+    elsif ($elemName eq "within") {
+        my @elems = getElemNodes $condition;
+        my $set = evalSet($storePaths, $elems[0]);
+        return evalCondition($set, $elems[1]);
+    }
+
+    elsif ($elemName eq "true") {
+        return 1;
+    }
+
+    elsif ($elemName eq "false") {
+        return 0;
+    }
+
+    else {
+        die "unknown element `$elemName'";
+    }
+}
+
+
+sub evalOr {
+    my $storePaths = shift;
+    my $nodes = shift;
+
+    my $result = 0;
+    foreach my $node (@{$nodes}) {
+        $result |= evalCondition($storePaths, $node);
+    }
+    
+    return $result;
+}
+
+
+# Iterate over all elements, check them.
+foreach my $userEnvElem (@userEnvElems) {
+
+    # Get the deriver of this path.
+    my $deriver = `nix-store --query --deriver '$userEnvElem'`;
+    die "cannot query deriver" if $? != 0;
+    chomp $deriver;
+
+    if ($deriver eq "unknown-deriver") {
+#        print "  deriver unknown, cannot check sources\n";
+        next;
+    }
+
+    print "CHECKING $userEnvElem\n";
+
+
+    # Get the requisites of the deriver.
+#    my $requisites = `nix-store --query --requisites --include-outputs '$deriver'`;
+#    die "cannot query requisites" if $? != 0;
+#    my @requisites = split ' ', $requisites;
+
+
+    # Get the hashes of the requisites.
+#    my $hashes = `nix-store --query --hash @requisites`;
+#    die "cannot query hashes" if $? != 0;
+#    my @hashes = split ' ', $hashes;
+#    for (my $i = 0; $i < scalar @requisites; $i++) {
+#        die unless $i < scalar @hashes;
+#        my $hash = $hashes[$i];
+#        $storePathHashes{$hash} = {} unless defined $storePathHashes{$hash};
+#        my $r = $storePathHashes{$hash}; # !!! fix
+#        $$r{$requisites[$i]} = 1;
+#    }
+
+
+    # Evaluate each blacklist item.
+    foreach my $item ($blacklist->getChildrenByTagName("item")) {
+        my $itemId = $item->getAttributeNode("id")->getValue;
+#        print "  CHECKING FOR $itemId\n";
+
+        my $condition = ($item->getChildrenByTagName("condition"))[0];
+        die unless $condition;
+
+        # Evaluate the condition.
+        my @elems = getElemNodes $condition;
+        if (evalOr({$deriver => 1}, \@elems)) {
+            # Oops, condition triggered.
+            my $reason = ($item->getChildrenByTagName("reason"))[0]->getChildNodes->to_literal;
+            $reason =~ s/\s+/ /g;
+            $reason =~ s/^\s+//g;
+
+            print "    VULNERABLE TO `$itemId': $reason\n";
+        }
+    }
+}
+

configure.ac

@@ -1,7 +1,7 @@
-AC_INIT(nix, "0.7")
+AC_INIT(nix, "0.9")
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
-AM_INIT_AUTOMAKE
+AM_INIT_AUTOMAKE([dist-bzip2])
 
 # Change to `1' to produce a `stable' release (i.e., the `preREVISION'
 # suffix is not added).
@@ -81,17 +81,19 @@ AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 NEED_PROG(perl, perl)
+NEED_PROG(tar, tar)
+                
+NEED_PROG(cat, cat)
+AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
+  [path of cat, mkdir, etc.]),
+  coreutils=$withval, coreutils=$(dirname $cat))
+AC_SUBST(coreutils)
 
 AC_ARG_WITH(docbook-catalog, AC_HELP_STRING([--with-docbook-catalog=PATH],
   [path of the DocBook XML DTD]),
   docbookcatalog=$withval, docbookcatalog=/docbook-dtd-missing)
 AC_SUBST(docbookcatalog)
 
-AC_ARG_WITH(docbook-ebnf-catalog, AC_HELP_STRING([--with-docbook-ebnf-catalog=PATH],
-  [path of the DocBook XML EBNF module DTD]),
-  docbookebnfcatalog=$withval, docbookcatalog=/docbook-ebnf-dtd-missing)
-AC_SUBST(docbookebnfcatalog)
-
 AC_ARG_WITH(docbook-xsl, AC_HELP_STRING([--with-docbook-xsl=PATH],
   [path of the DocBook XSL stylesheets]),
   docbookxsl=$withval, docbookxsl=/docbook-xsl-missing)
@@ -192,7 +194,6 @@ AC_CONFIG_FILES([Makefile
    src/bsdiff-4.2/Makefile
    scripts/Makefile
    corepkgs/Makefile
-   corepkgs/fetchurl/Makefile
    corepkgs/nar/Makefile
    corepkgs/buildenv/Makefile
    corepkgs/channels/Makefile

corepkgs/Makefile.am

@@ -1 +1 @@
-SUBDIRS = fetchurl nar buildenv channels
+SUBDIRS = nar buildenv channels

corepkgs/buildenv/builder.pl.in

@@ -25,6 +25,7 @@ sub createLinks {
         
 	if ($srcFile =~ /\/propagated-build-inputs$/ ||
             $srcFile =~ /\/nix-support$/ ||
+            $srcFile =~ /\/perllocal.pod$/ ||
             $srcFile =~ /\/log$/)
         {
             # Do nothing.
@@ -72,13 +73,27 @@ sub createLinks {
 
 my %done;
 
+sub addPkg;
 sub addPkg {
     my $pkgDir = shift;
 
     return if (defined $done{$pkgDir});
     $done{$pkgDir} = 1;
 
+    print "adding $pkgDir\n";
     createLinks("$pkgDir", "$out");
+
+    my $propagatedFN = "$pkgDir/nix-support/propagated-build-inputs";
+    if (-e $propagatedFN) {
+        open PROP, "<$propagatedFN" or die;
+        my $propagated = <PROP>;
+        close PROP;
+        my @propagated = split ' ', $propagated;
+        foreach my $p (@propagated) {
+            addPkg $p;
+        }
+    }
+        
 }
 
 
@@ -86,7 +101,6 @@ my @args = split ' ', $ENV{"derivations"};
 
 while (scalar @args > 0) {
     my $drvPath = shift @args;
-    print "adding $drvPath\n";
     addPkg($drvPath);
 }
 

corepkgs/channels/unpack.sh.in

@@ -1,9 +1,7 @@
 #! @shell@ -e
 
-export PATH=/bin:/usr/bin # !!! impure
-
-mkdir $out
-mkdir $out/tmp
+@coreutils@/mkdir $out
+@coreutils@/mkdir $out/tmp
 cd $out/tmp
 
 expr=$out/default.nix
@@ -12,8 +10,8 @@ echo '[' > $expr
 nr=0
 for i in $inputs; do
     echo "unpacking $i"
-    @bunzip2@ < $i | tar xvf -
-    mv * ../$nr # !!! hacky
+    @bunzip2@ < $i | @tar@ xf -
+    @coreutils@/mv * ../$nr # !!! hacky
     echo "(import ./$nr)" >> $expr
     nr=$(($nr + 1))
 done
@@ -21,4 +19,4 @@ done
 echo ']' >> $expr
 
 cd ..
-rmdir tmp
+@coreutils@/rmdir tmp

corepkgs/fetchurl/Makefile.am

@@ -1,11 +0,0 @@
-all-local: builder.sh
-
-install-exec-local:
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-
-include ../../substitute.mk
-
-EXTRA_DIST = default.nix builder.sh.in

corepkgs/fetchurl/builder.sh.in

@@ -1,19 +0,0 @@
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-echo "downloading $url into $out"
-
-prefetch=@storedir@/nix-prefetch-url-$md5
-if test -f "$prefetch"; then
-    echo "using prefetched $prefetch";
-    mv $prefetch $out
-else
-    @curl@ --fail --location --max-redirs 20 "$url" > "$out"
-fi
-
-actual=$(@bindir@/nix-hash --flat $out)
-if test "$actual" != "$md5"; then
-    echo "hash is $actual, expected $md5"
-    exit 1
-fi

corepkgs/fetchurl/default.nix

@@ -1,8 +0,0 @@
-{system, url, md5}:
-
-derivation {
-  name = baseNameOf (toString url);
-  builder = ./builder.sh;
-  id = md5;
-  inherit system url md5;
-}

corepkgs/nar/Makefile.am

@@ -1,13 +1,11 @@
-all-local: nar.sh unnar.sh
+all-local: nar.sh
 
 install-exec-local:
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_DATA) nar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_PROGRAM) nar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_DATA) unnar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_PROGRAM) unnar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
 
 include ../../substitute.mk
 
-EXTRA_DIST = nar.nix nar.sh.in unnar.nix unnar.sh.in
+EXTRA_DIST = nar.nix nar.sh.in

corepkgs/nar/nar.nix

@@ -1,6 +1,5 @@
-{system, path}: derivation {
+{system, path, hashAlgo}: derivation {
   name = "nar";
   builder = ./nar.sh;
-  system = system;
-  path = path;
+  inherit system path hashAlgo;
 }

corepkgs/nar/nar.sh.in

@@ -1,19 +1,14 @@
 #! @shell@ -e
 
-# !!! impure; fix this
-export PATH=/bin:/usr/bin
-
 echo "packing $path into $out..."
-mkdir $out
-dst=$out/$(basename $path).nar.bz2
+@coreutils@/mkdir $out
+dst=$out/tmp.nar.bz2
 @bindir@/nix-store --dump "$path" > tmp
 
 @bzip2@ < tmp > $dst
 
-narHash=$(md5sum -b tmp | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $narHash > $out/nar-hash
+@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
 
-narbz2Hash=$(md5sum -b $dst | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $narbz2Hash > $out/narbz2-hash
+@bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
+
+@coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2

corepkgs/nar/unnar.nix

@@ -1,7 +0,0 @@
-{system, narFile, outPath}: derivation {
-  name = "unnar";
-  builder = ./unnar.sh;
-  system = system;
-  narFile = narFile;
-  outPath = outPath;
-}

corepkgs/nar/unnar.sh.in

@@ -1,4 +0,0 @@
-#! @shell@ -e
-
-echo "unpacking $narFile to $out..."
-@bunzip2@ < $narFile | @bindir@/nix-store --restore "$out"

doc/manual/Makefile.am

@@ -1,37 +1,41 @@
-ENV = SGML_CATALOG_FILES=$(docbookcatalog):$(docbookebnfcatalog)
+ENV = SGML_CATALOG_FILES=$(docbookcatalog)
 
 XMLLINT = $(ENV) $(xmllint) $(xmlflags) --catalogs
 XSLTPROC = $(ENV) $(xsltproc) $(xmlflags) --catalogs \
  --param section.autolabel 1 \
  --param section.label.includes.component.label 1 \
  --param html.stylesheet \'style.css\' \
- --param xref.with.number.and.title 0
+ --param xref.with.number.and.title 1 \
+ --param toc.section.depth 3
 
-man1_MANS = nix-env.1 nix-store.1 nix-instantiate.1 \
+man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
- nix-prefetch-url.1
+ nix-prefetch-url.1 nix-channel.1
 
 FIGURES = figures/user-environments.png
 
-SOURCES = manual.xml introduction.xml installation.xml \
+MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  package-management.xml writing-nix-expressions.xml \
  build-farm.xml \
  $(man1_MANS:.1=.xml) \
  troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
- quick-start.xml nix-lang-ref.xml style.css images
+ env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
+ conf-file.xml \
+ style.css images
 
-manual.is-valid: $(SOURCES) version.xml
-	$(XMLLINT) --noout --valid manual.xml
+manual.is-valid: $(MANUAL_SRCS) version.txt
+	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --valid -
 	touch $@
 
-version.xml:
-	echo -n $(VERSION) > version.xml
+version.txt:
+	echo -n $(VERSION) > version.txt
 
-man $(MANS): $(SOURCES) manual.is-valid
-	$(XSLTPROC) $(docbookxsl)/manpages/docbook.xsl manual.xml
+man $(MANS): $(MANUAL_SRCS) manual.is-valid
+	$(XSLTPROC) --nonet --xinclude $(docbookxsl)/manpages/docbook.xsl manual.xml
 
-manual.html: $(SOURCES) manual.is-valid images
-	$(XSLTPROC) --output manual.html $(docbookxsl)/html/docbook.xsl manual.xml
+manual.html: $(MANUAL_SRCS) manual.is-valid images
+	$(XSLTPROC) --nonet --xinclude --output manual.html \
+	  $(docbookxsl)/html/docbook.xsl manual.xml
 
 all-local: manual.html
 
@@ -50,8 +54,8 @@ images:
 	cp $(docbookxsl)/images/callouts/*.png images/callouts
 	chmod +w -R images
 
-KEEP = manual.html manual.is-valid version.xml $(MANS)
+KEEP = manual.html manual.is-valid version.txt $(MANS)
 
-EXTRA_DIST = $(SOURCES) $(FIGURES) $(KEEP)
+EXTRA_DIST = $(MANUAL_SRCS) $(FIGURES) $(KEEP)
 
 DISTCLEANFILES = $(KEEP)

doc/manual/bugs.xml

@@ -2,90 +2,25 @@
 
 <itemizedlist>
 
-    <listitem>
-      <para>
-        The man-pages generated from the DocBook documentation are ugly.
-      </para>
-    </listitem>
+<listitem><para>The man-pages generated from the DocBook documentation
+are ugly.</para></listitem>
 
-    <listitem>
-      <para>
-        Generations properly form a tree.  E.g., if after switching to
-        generation 39, we perform an installation action, a generation
-        43 is created which is a descendant of 39, not 42.  So a
-        rollback from 43 ought to go back to 39.  This is not
-        currently implemented; generations form a linear sequence.
-      </para>
-    </listitem>
+<listitem><para>Generations properly form a tree.  E.g., if after
+switching to generation 39, we perform an installation action, a
+generation 43 is created which is a descendant of 39, not 42.  So a
+rollback from 43 ought to go back to 39.  This is not currently
+implemented; generations form a linear sequence.</para></listitem>
 
-    <listitem>
-      <para>
-        Unify the concepts of successors and substitutes into a
-        general notion of <emphasis>equivalent expressions</emphasis>.
-        Expressions are equivalent if they have the same target paths
-        with the same identifiers.  However, even though they are
-        functionally equivalent, they may differ stronly with respect
-        to their <emphasis>performance characteristics</emphasis>.
-        For example, realising a closure expression is more efficient
-        that realising the derivation expression from which it was
-        produced.  On the other hand, distributing sources may be more
-        efficient (storage- or bandwidth-wise) than distributing
-        binaries.  So we need to be able to attach weigths or
-        priorities or performance annotations to expressions; Nix can
-        then choose the most efficient expression dependent on the
-        context.
-      </para>
-    </listitem>
-
-    <listitem>
-      <para>
-        <emphasis>Build management.</emphasis>  In principle it is already
-        possible to do build management using Nix (by writing builders that
-        perform appropriate build steps), but the Nix expression language is
-        not yet powerful enough to make this pleasant (?).  The language should
-        be extended with features from the <ulink
-          url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
-        Another interesting idea is to write a <command>make</command>
-        implementation that uses Nix as a back-end to support <ulink
-          url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink> 
-        build files.
-      </para>
-    </listitem>
-
-    <listitem>
-      <para>
-        There are race conditions between the garbage collector and
-        other Nix tools.  For instance, when we run
-        <command>nix-env</command> to build and install a derivation
-        and run the garbage collector at the same time, the garbage
-        collector may kick in exactly between the build and
-        installation steps, i.e., before the newly built derivation
-        has become reachable from a root of the garbage collector.
-      </para>
-
-      <para>
-        One solution would be for these programs to properly register
-        temporary roots for the collector.  Another would be to use
-        stop-the-world garbage collection: if any tool is running, the
-        garbage collector blocks, and vice versa.  These solutions do
-        not solve the situation where multiple tools are involved,
-        e.g.,
-
-        <screen>
-$ nix-store -r $(nix-instantiate foo.nix)</screen>
-
-        since even if <command>nix-instantiate</command> where to
-        register a temporary root, it would be released by the time
-        <command>nix-store</command> is started.  A solution would be
-        to write the intermediate value to a file that is used as a
-        root to the collector, e.g.,
-        
-        <screen>
-$ nix-instantiate foo.nix > /nix/var/nix/roots/bla
-$ nix-store -r $(cat /nix/var/nix/roots/bla)</screen>
-
-      </para>
-    </listitem>
+<listitem><para><emphasis>Build management.</emphasis> In principle it
+is already possible to do build management using Nix (by writing
+builders that perform appropriate build steps), but the Nix expression
+language is not yet powerful enough to make this pleasant (?).  The
+language should be extended with features from the <ulink
+url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
+Another interesting idea is to write a <command>make</command>
+implementation that uses Nix as a back-end to support <ulink
+url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink>
+build files.</para></listitem>
 
 <listitem><para>For security, <command>nix-push</command> manifests
 should be digitally signed, and <command>nix-pull</command> should
@@ -94,15 +29,18 @@ need to be signed, since the manifest contains cryptographic hashes of
 these files (and <filename>fetchurl.nix</filename> checks
 them).</para></listitem>
 
-<listitem><para>We should switch away from MD5, since it has been
-more-or-less cracked.  We don't currently depend very much on the
-collision-resistance of MD5, but we will once we start sharing build
-results between users.</para></listitem>
-
 <listitem><para>It would be useful to have an option in
 <command>nix-env --delete-generations</command> to remove non-current
 generations older than a certain age.</para></listitem>
 
+<listitem><para>There should be a flexible way to change the user
+environment builder.  Currently, you have to replace
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>,
+which is hard-coded into <command>nix-env</command>.  Also, the
+default builder should be more powerful.  For instance, there should
+be some way to specify priorities to resolve
+collisions.</para></listitem>
+
 </itemizedlist>
 
 </appendix>

doc/manual/build-farm.xml

@@ -65,7 +65,10 @@ will call whenever it wants to build a derivation.  The build hook
 will perform it in the usual way if possible, or it can accept it, in
 which case it is responsible for somehow getting the inputs of the
 build to another machine, doing the build there, and getting the
-results back.</para>
+results back.  The details of the build hook protocol are described in
+the documentation of the <link
+linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar>
+variable</link>.</para>
 
 <example id='ex-remote-systems'><title>Remote machine configuration:
 <filename>remote-systems.conf</filename></title>

doc/manual/conf-file.xml

@@ -0,0 +1,82 @@
+<sect1 id="sec-conf-file"><title>Nix configuration file</title>
+
+<para>A number of persistent settings of Nix are stored in the file
+<filename><replaceable>prefix</replaceable>/etc/nix/nix.conf</filename>.
+This file is a list of <literal><replaceable>name</replaceable> =
+<replaceable>value</replaceable></literal> pairs, one per line.
+Comments start with a <literal>#</literal> character.  An example
+configuration file is shown in <xref linkend="ex-nix-conf" />.</para>
+
+<example id='ex-nix-conf'><title>Nix configuration file</title>
+
+<programlisting>
+gc-keep-outputs = true       # Nice for developers
+gc-keep-derivations = true   # Idem
+env-keep-derivations = false
+</programlisting>
+</example>
+
+<para>The following variables are currently available: 
+
+<variablelist>
+
+  <varlistentry id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
+
+    <listitem><para>If <literal>true</literal>, the garbage collector
+    will keep the outputs of non-garbage derivations.  If
+    <literal>false</literal> (default), outputs will be deleted unless
+    they are GC roots themselves (or reachable from other roots).</para>
+ 
+    <para>In general, outputs must be registered as roots separately.
+    However, even if the output of a derivation is registered as a
+    root, the collector will still delete store paths that are used
+    only at build time (e.g., the C compiler, or source tarballs
+    downloaded from the network).  To prevent it from doing so, set
+    this option to <literal>true</literal>.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>true</literal> (default), the garbage
+    collector will keep the derivations from which non-garbage store
+    paths were built.  If <literal>false</literal>, they will be
+    deleted unless explicitly registered as a root (or reachable from
+    other roots).</para>
+
+    <para>Keeping derivation around is useful for querying and
+    traceability (e.g., it allows you to ask with what dependencies or
+    options a store path was built), so by default this option is on.
+    Turn it off to safe a bit of disk space (or a lot if
+    <literal>gc-keep-outputs</literal> is also turned on).</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><literal>env-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>false</literal> (default), derivations
+    are not stored in Nix user environments.  That is, the derivation
+    any build-time-only dependencies may be garbage-collected.</para>
+
+    <para>If <literal>true</literal>, when you add a Nix derivation to
+    a user environment, the path of the derivation is stored in the
+    user environment.  Thus, the derivation will not be
+    garbage-collected until the user environment generation is deleted
+    (<command>nix-env --delete-generations</command>).  To prevent
+    build-time-only dependencies from being collected, you should also
+    turn on <literal>gc-keep-outputs</literal>.</para>
+
+    <para>The difference between this option and
+    <literal>gc-keep-derivations</literal> is that this one is
+    “sticky”: it applies to any user environment created while this
+    option was enabled, while <literal>gc-keep-derivations</literal>
+    only applies at the moment the garbage collector is
+    run.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+</sect1>

doc/manual/env-common.xml

@@ -0,0 +1,274 @@
+<sect1 id="sec-common-env"><title>Common environment variables</title>
+
+<para>Most Nix commands interpret the following environment variables:</para>
+
+<variablelist>
+
+  
+<varlistentry><term><envar>NIX_ROOT</envar></term>
+
+  <listitem><para>If <envar>NIX_ROOT</envar> is set, the Nix command
+  will on startup perform a <function>chroot()</function> to the
+  specified directory.  This is useful in certain bootstrapping
+  situations (e.g., when installing a Nix installation onto a hard
+  disk from CD-ROM).</para></listitem>
+
+</varlistentry>
+
+  
+<varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term>
+
+  <listitem>
+
+  <para>Normally, the Nix store directory (typically
+  <filename>/nix/store</filename>) is not allowed to contain any
+  symlink components.  This is to prevent “impure” builds.  Builders
+  sometimes “canonicalise” paths by resolving all symlink components.
+  Thus, builds on different machines (with
+  <filename>/nix/store</filename> resolving to different locations)
+  could yield different results.  This is generally not a problem,
+  except when builds are deployed to machines where
+  <filename>/nix/store</filename> resolves differently.  If you are
+  sure that you’re not going to do that, you can set
+  <envar>NIX_IGNORE_SYMLINK_STORE</envar> to <envar>1</envar>.</para>
+
+  <para>Note that if you’re symlinking the Nix store so that you can
+  put it on another file system than the root file system, on Linux
+  you’re better off using <literal>bind</literal> mount points, e.g.,
+
+  <screen>
+$ mkdir /nix   
+$ mount -o bind /mnt/otherdisk/nix /nix</screen>
+
+  Consult the <citerefentry><refentrytitle>mount</refentrytitle>
+  <manvolnum>8</manvolnum></citerefentry> manual page for details.</para>
+
+  </listitem>
+
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STORE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix store (default
+  <filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DATA_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix static data
+  directory (default
+  <filename><replaceable>prefix</replaceable>/share</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix log directory
+  (default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STATE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix state directory
+  (default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DB_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix database (default
+  <filename><replaceable>$NIX_STATE_DIR</replaceable>/db</filename>, i.e.,
+  <filename><replaceable>prefix</replaceable>/var/nix/db</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_CONF_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix configuration
+  directory (default
+  <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_TYPE</envar></term>
+
+  <listitem><para>Equivalent to the <link
+  linkend="opt-log-type"><option>--log-type</option>
+  option</link>.</para></listitem>
+
+</varlistentry>
+  
+
+<varlistentry><term><envar>TMPDIR</envar></term>
+
+  <listitem><para>Use the specified directory to store temporary
+  files.  In particular, this includes temporary build directories;
+  these can take up substantial amounts of disk space.  The default is
+  <filename>/tmp</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term>
+
+  <listitem>
+
+  <para>Specifies the location of the <emphasis>build hook</emphasis>,
+  which is a program (typically some script) that Nix will call
+  whenever it wants to build a derivation.  This is used to implement
+  distributed builds (see <xref linkend="sec-distributed-builds"
+  />).  The protocol by which the calling Nix process and the build
+  hook communicate is as follows.</para>
+
+  <para>The build hook is called with the following command-line
+  arguments:
+
+  <orderedlist>
+
+    <listitem><para>A boolean value <literal>0</literal> or
+    <literal>1</literal> specifying whether Nix can locally execute
+    more builds, as per the <link
+    linkend="opt-max-jobs"><option>--max-jobs</option> option</link>.
+    The purpose of this argument is to allow the hook to not have to
+    maintain bookkeeping for the local machine.</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the local machine
+    (e.g., <literal>i686-linux</literal>).</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the derivation,
+    i.e., its <link linkend="attr-system"><varname>system</varname>
+    attribute</link>.</para></listitem>
+
+    <listitem><para>The store path of the derivation.</para></listitem>
+
+  </orderedlist>
+
+  </para>
+
+  <para>On the basis of this information, and whatever persistent
+  state the build hook keeps about other machines and their current
+  load, it has to decide what to do with the build.  It should print
+  out on file descriptor 3 one of the following responses (terminated
+  by a newline, <literal>"\n"</literal>):
+
+  <variablelist>
+
+    <varlistentry><term><literal>decline</literal></term>
+
+      <listitem><para>The build hook is not willing or able to perform
+      the build; the calling Nix process should do the build itself,
+      if possible.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>postpone</literal></term>
+
+      <listitem><para>The build hook cannot perform the build now, but
+      can do so in the future (e.g., because all available build slots
+      on remote machines are in use).  The calling Nix process should
+      postpone this build until at least one currently running build
+      has terminated.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>accept</literal></term>
+
+      <listitem><para>The build hook has accepted the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the build hook accepts the build, it is possible that it is
+  no longer necessary to do the build because some other process has
+  performed the build in the meantime.  To prevent races, the hook
+  must read from file descriptor 4 a single line that tells it whether
+  to continue:
+
+  <variablelist>
+
+    <varlistentry><term><literal>cancel</literal></term>
+
+      <listitem><para>The build has already been done, so the hook
+      should exit.</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><literal>okay</literal></term>
+
+      <listitem><para>The hook should proceed with the build.  At this
+      point, the calling Nix process has acquired locks on the output
+      path, so no other Nix process will perform the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the hook has been told to proceed, Nix will store in the
+  hook’s current directory a number of text files that contain
+  information about the derivation:
+
+  <variablelist>
+
+    <varlistentry><term><filename>inputs</filename></term>
+
+      <listitem><para>The set of store paths that are inputs to the
+      build process (one per line).  These have to be copied
+      <emphasis>to</emphasis> the remote machine (in addition to the
+      store derivation itself).</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><filename>outputs</filename></term>
+
+      <listitem><para>The set of store paths that are outputs of the
+      derivation (one per line).  These have to be copied
+      <emphasis>from</emphasis> the remote machine if the build
+      succeeds.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><filename>references</filename></term>
+
+      <listitem><para>The reference graph of the inputs, in the format
+      accepted by the command <command>nix-store
+      --register-validity</command>.  It is necessary to run this
+      command on the remote machine after copying the inputs to inform
+      Nix on the remote machine that the inputs are valid
+      paths.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>The hook should copy the inputs to the remote machine,
+  register the validity of the inputs, perform the remote build, and
+  copy the outputs back to the local machine.  An exit code other than
+  <literal>0</literal> indicates that the hook has failed.</para>
+
+  </listitem>
+
+
+</varlistentry>
+
+
+</variablelist>
+
+</sect1>

doc/manual/glossary.xml

@@ -0,0 +1,163 @@
+<appendix><title>Glossary</title>
+
+<glosslist>
+
+
+<glossentry id="gloss-derivation"><glossterm>derivation</glossterm>
+
+  <glossdef><para>A description of a build action.  The result of a
+  derivation is a store object.  Derivations are typically specified
+  in Nix expressions using the <link
+  linkend="ssec-derivation"><function>derivation</function>
+  primitive</link>.  These are translated into low-level
+  <emphasis>store derivations</emphasis> (implicitly by
+  <command>nix-env</command> and <command>nix-build</command>, or
+  explicitly by <command>nix-instantiate</command>).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store</glossterm>
+
+  <glossdef><para>The location in the file system where store objects
+  live.  Typically <filename>/nix/store</filename>.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store path</glossterm>
+
+  <glossdef><para>The location in the file system of a store object,
+  i.e., an immediate child of the Nix store
+  directory.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store object</glossterm>
+
+  <glossdef><para>A file that is an immediate child of the Nix store
+  directory.  These can be regular files, but also entire directory
+  trees.  Store objects can be sources (objects copied from outside of
+  the store), derivation outputs (objects produced by running a build
+  action), or derivations (files describing a build
+  action).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-substitute"><glossterm>substitute</glossterm>
+
+  <glossdef><para>A substitute is a command invocation stored in the
+  Nix database that describes how to build a store object, bypassing
+  normal the build mechanism (i.e., derivations).  Typically, the
+  substitute builds the store object by downloading a pre-built
+  version of the store object from some server.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>purity</glossterm>
+
+  <glossdef><para>The assumption that equal Nix derivations when run
+  always produce the same output.  This cannot be guaranteed in
+  general (e.g., a builder can rely on external inputs such as the
+  network or the system time) but the Nix model assumes
+  it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>Nix expression</glossterm>
+
+  <glossdef><para>A high-level description of software components and
+  compositions thereof.  Deploying software using Nix entails writing
+  Nix expressions for your components.  Nix expressions are translated
+  to derivations that are stored in the Nix store.  These derivations
+  can then be built.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-reference"><glossterm>reference</glossterm>
+
+  <glossdef><para>A store path <varname>P</varname> is said to have a
+  reference to a store path <varname>Q</varname> if the store object
+  at <varname>P</varname> contains the path <varname>Q</varname>
+  somewhere.  This implies than an execution involving
+  <varname>P</varname> potentially needs <varname>Q</varname> to be
+  present.  The <emphasis>references</emphasis> of a store path are
+  the set of store paths to which it has a reference.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-closure"><glossterm>closure</glossterm>
+
+  <glossdef><para>The closure of a store path is the set of store
+  paths that are directly or indirectly “reachable” from that store
+  path; that is, it’s the closure of the path under the <link
+  linkend="gloss-reference">references</link> relation.  For instance,
+  if the store object at path <varname>P</varname> contains a
+  reference to path <varname>Q</varname>, then <varname>Q</varname> is
+  in the closure of <varname>P</varname>.  For correct deployment it
+  is necessary to deploy whole closures, since otherwise at runtime
+  files could be missing.  The command <command>nix-store
+  -qR</command> prints out closures of store paths.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-output-path"><glossterm>output path</glossterm>
+
+  <glossdef><para>A store path produced by a derivation.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-deriver"><glossterm>deriver</glossterm>
+
+  <glossdef><para>The deriver of an <link
+  linkend="gloss-output-path">output path</link> is the store
+  derivation that built it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-validity"><glossterm>validity</glossterm>
+
+  <glossdef><para>A store path is considered
+  <emphasis>valid</emphasis> if it exists in the file system, is
+  listed in the Nix database as being valid, and if all paths in its
+  closure are also valid.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-user-env"><glossterm>user environment</glossterm>
+
+  <glossdef><para>An automatically generated store object that
+  consists of a set of symlinks to “active” applications, i.e., other
+  store paths.  These are generated automatically by <link
+  linkend="sec-nix-env"><command>nix-env</command></link>.  See <xref
+  linkend="sec-profiles" />.</para>
+
+  </glossdef>
+  
+</glossentry>
+
+
+<glossentry id="gloss-profile"><glossterm>profile</glossterm>
+
+  <glossdef><para>A symlink to the current <link
+  linkend="gloss-user-env">user environment</link> of a user, e.g.,
+  <filename>/nix/var/nix/profiles/default</filename>.</para></glossdef>
+
+</glossentry>
+
+
+
+</glosslist>
+
+
+</appendix>

doc/manual/installation.xml

@@ -5,8 +5,8 @@
 
 <para>The easiest way to obtain Nix is to download a <ulink
 url='http://www.cs.uu.nl/groups/ST/Trace/Nix'>source
-distribution</ulink>.  RPMs for Red Hat 9 are also available.  These
-distributions are generated automatically.</para>
+distribution</ulink>.  RPMs for Red Hat, SuSE, and Fedore Core are
+also available.</para>
 
 <para>Alternatively, the most recent sources of Nix can be obtained
 from its <ulink

doc/manual/introduction.xml

@@ -130,11 +130,7 @@ collection.  It also discusses some advanced topics, such as setting
 up a Nix-based build farm, and doing service deployment using
 Nix.</para>
 
-<warning><para>This manual is a work in progress.  It's quite likely
-to be incomplete, inconsistent with the current implementation, or
-simply wrong.</para></warning>
-
-<note><para>Some background information on Nix can be found in two
+<note><para>Some background information on Nix can be found in three
 papers.  The ICSE 2004 paper <ulink
 url='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'><citetitle>Imposing
 a Memory Management Discipline on Software
@@ -145,6 +141,10 @@ different versions and variants of packages.  The LISA 2004 paper
 url='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'><citetitle>Nix:
 A Safe and Policy-Free System for Software
 Deployment</citetitle></ulink> gives a more general discussion of Nix
-from a system-administration perspective.</para></note>
+from a system-administration perspective.  The CBSE 2005 paper <ulink
+url='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'><citetitle>Efficient
+Upgrading in a Purely Functional Component Deployment Model
+</citetitle></ulink> is about transparent patch deployment in
+Nix.</para></note>
 
 </chapter>

doc/manual/manual.xml

@@ -3,31 +3,14 @@
     PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
     "http://www.docbook.org/xml/4.3/docbook-xml-4.3.zip"
 [
-<!ENTITY introduction SYSTEM "introduction.xml">
-<!ENTITY quick-start SYSTEM "quick-start.xml">
-<!ENTITY installation SYSTEM "installation.xml">
-<!ENTITY package-management SYSTEM "package-management.xml">
-<!ENTITY writing-nix-expressions SYSTEM "writing-nix-expressions.xml">
-<!ENTITY build-farm SYSTEM "build-farm.xml">
-<!ENTITY opt-common SYSTEM "opt-common.xml">
-<!ENTITY opt-common-syn SYSTEM "opt-common-syn.xml">
-<!ENTITY nix-env SYSTEM "nix-env.xml">
-<!ENTITY nix-store SYSTEM "nix-store.xml">
-<!ENTITY nix-instantiate SYSTEM "nix-instantiate.xml">
-<!ENTITY nix-collect-garbage SYSTEM "nix-collect-garbage.xml">
-<!ENTITY nix-push SYSTEM "nix-push.xml">
-<!ENTITY nix-pull SYSTEM "nix-pull.xml">
-<!ENTITY nix-prefetch-url SYSTEM "nix-prefetch-url.xml">
-<!-- <!ENTITY nix-lang-ref SYSTEM "nix-lang-ref.xml"> -->
-<!ENTITY troubleshooting SYSTEM "troubleshooting.xml">
-<!ENTITY bugs SYSTEM "bugs.xml">
-<!ENTITY version SYSTEM "version.xml">
 ]>
 
 <book>
   <title>Nix User's Guide</title>
 
-  <subtitle>Draft (Version &version;)</subtitle>
+  <subtitle>Draft (Version <xi:include
+  xmlns:xi="http://www.w3.org/2001/XInclude"
+  href="version.txt" parse="text" />)</subtitle>
   
   <bookinfo>
     <author>
@@ -36,52 +19,65 @@
     </author>
     <copyright>
       <year>2004</year>
+      <year>2005</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
   </bookinfo>
-
-  &introduction;
-  &quick-start;
-  &installation;
-  &package-management;
-  &writing-nix-expressions;
-  &build-farm;
+  
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="introduction.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="quick-start.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="installation.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="package-management.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="writing-nix-expressions.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="build-farm.xml" />
 
   <appendix>
     <title>Command Reference</title>
-    <sect1>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="env-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="conf-file.xml" />
+    <sect1 id="sec-nix-env">
       <title>nix-env</title>
-      &nix-env;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-env.xml" />
+    </sect1>
+    <sect1 id="sec-nix-build">
+      <title>nix-build</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-build.xml" />
     </sect1>
     <sect1>
       <title>nix-store</title>
-      &nix-store;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-store.xml" />
     </sect1>
-    <sect1>
+    <sect1 id="sec-nix-instantiate">
       <title>nix-instantiate</title>
-      &nix-instantiate;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-instantiate.xml" />
     </sect1>
     <sect1>
       <title>nix-collect-garbage</title>
-      &nix-collect-garbage;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-collect-garbage.xml" />
+    </sect1>
+    <sect1 id="sec-nix-channel">
+      <title>nix-channel</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-channel.xml" />
     </sect1>
     <sect1>
       <title>nix-push</title>
-      &nix-push;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-push.xml" />
     </sect1>
     <sect1>
       <title>nix-pull</title>
-      &nix-pull;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-pull.xml" />
     </sect1>
     <sect1>
       <title>nix-prefetch-url</title>
-      &nix-prefetch-url;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-prefetch-url.xml" />
     </sect1>
   </appendix>
 
-<!--  &nix-lang-ref; -->
+  <!--  &nix-lang-ref; -->
 
-  &troubleshooting;
-  &bugs;
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="troubleshooting.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="bugs.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="glossary.xml" />
 
 </book>

doc/manual/nix-build.xml

@@ -0,0 +1,74 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-build</refname>
+  <refpurpose>build a Nix expression</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-build</command>
+    <arg><option>--add-drv-link</option></arg>
+    <arg><option>--no-link</option></arg>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>The <command>nix-build</command> command builds the derivations
+described by the Nix expressions in <replaceable>paths</replaceable>.
+If the build succeeds, it places a symlink to the result in the
+current directory.  The symlink is called <filename>result</filename>.
+If there are multiple Nix expressions, or the Nix expressions evaluate
+to multiple derivations, multiple sequentially numbered symlinks are
+created (<filename>result</filename>, <filename>result-2</filename>,
+and so on).</para>
+
+<note><para><command>nix-build</command> is essentially a wrapper
+around <link
+linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>
+(to translate a high-level Nix expression to a low-level store
+derivation) and <link
+linkend="rsec-nix-store-realise"><command>nix-store
+--realise</command></link> (to build the store
+derivation).</para></note>
+
+<warning><para>The result of the build is automatically registered as
+a root of the Nix garbage collector.  This root disappears
+automatically when the <filename>result</filename> symlink is deleted
+or renamed.  So don’t rename the symlink.</para></warning>
+
+</refsection>
+
+
+<refsection><title>Options</title>
+
+<variablelist>
+
+  <varlistentry><term><option>--add-drv-link</option></term>
+  
+    <listitem><para>Add a symlink in the current directory to the
+    store derivation produced by <command>nix-instantiate</command>.
+    The symlink is called <filename>derivation</filename> (which is
+    numbered in the case of multiple derivations).  The derivation is
+    a root of the garbage collector until the symlink is deleted or
+    renamed.</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--no-link</option></term>
+  
+    <listitem><para>Do not create a symlink to the output path.  Note
+    that as a result the output does not become a root of the garbage
+    collector, and so might be deleted by <command>nix-store
+    --gc</command>.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</refsection>
+
+
+</refentry>

doc/manual/nix-channel.xml

@@ -0,0 +1,83 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-channel</refname>
+  <refpurpose>manage Nix channels</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-channel</command>
+    <group choice='req'>
+      <arg choice='plain'><option>--add</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--remove</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--list</option></arg>
+      <arg choice='plain'><option>--update</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>A Nix channel is mechanism that allows you to automatically stay
+up-to-date with a set of pre-built Nix expressions.  A Nix channel is
+just a URL that points to a place that contains a set of Nix
+expressions, as well as a <command>nix-push</command> manifest.  See
+also <xref linkend="sec-channels" />.</para>
+
+<para>This command has the following operations:
+
+<variablelist>
+
+  <varlistentry><term><option>--add</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Adds <replaceable>url</replaceable> to the list of
+    subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--remove</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Removes <replaceable>url</replaceable> from the
+    list of subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--list</option></term>
+
+    <listitem><para>Prints the URLs of all subscribed channels on
+    standard output.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--update</option></term>
+
+    <listitem><para>Downloads the Nix expressions of all subscribed
+    channels, makes the conjunction of these the default for
+    <command>nix-env</command> operations (by calling <command>nix-env
+    -I</command>), and performs a <command>nix-pull</command> on the
+    manifests of all channels to make pre-built binaries
+    available.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+<para>Note that <option>--add</option> and <option>--remove</option>
+do not automatically perform an update.</para>
+
+<para>The list of subscribed channels is stored in
+<filename>~/.nix-channels</filename>.</para>
+
+<para>A channel consists of two elements: a bzipped Tar archive
+containing the Nix expressions, and a manifest created by
+<command>nix-push</command>.  These must be stored under
+<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal> and
+<literal><replaceable>url</replaceable>/MANIFEST</literal>,
+respectively.</para>
+
+</refsection>
+
+</refentry>

doc/manual/nix-collect-garbage.xml

@@ -1,87 +1,29 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-collect-garbage</refname>
-    <refpurpose>remove unreachable store paths</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-collect-garbage</refname>
+  <refpurpose>delete unreachable store paths</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-collect-garbage</command>
-      <group choice='opt'>
-        <arg choice='plain'><option>--print-live</option></arg>
-        <arg choice='plain'><option>--print-dead</option></arg>
-      </group>
-      <arg><option>--min-age</option> <replaceable>age</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-collect-garbage</command>
+    <group choice='opt'>
+      <arg choice='plain'><option>--print-roots</option></arg>
+      <arg choice='plain'><option>--print-live</option></arg>
+      <arg choice='plain'><option>--print-dead</option></arg>
+      <arg choice='plain'><option>--delete</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
+<refsection><title>Description</title>
 
-    <para>
-      The command <command>nix-collect-garbage</command> performs a
-      garbage collection on the Nix store: any paths in the Nix store
-      that are garbage (not reachable from a set of root store
-      expressions) are deleted.
-    </para>
+<para>The command <command>nix-collect-garbage</command> is an
+obsolete wrapper around <link
+linkend="rsec-nix-store-gc"><command>nix-store
+--gc</command></link>.</para>
 
-    <para>
-      The roots of the garbage collector are the store expressions
-      mentioned in the files in the directory
-      <filename><replaceable>prefix</replaceable>/var/nix/gcroots</filename>.
-      By default, the roots are all user environments in
-      <filename><replaceable>prefix</replaceable>/var/nix/profiles</filename>.
-      You can register other store expressions as roots by writing the
-      full path of the store expression to an arbitrary file in the
-      <filename>gcroots</filename> directory (or a subdirectory
-      thereof).
-    </para>
+</refsection>
 
-  </refsection>
-
-  <refsection>
-    <title>Options</title>
-
-    <variablelist>
-
-      <varlistentry>
-        <term><option>--print-live</option> / <option>--print-dead</option></term>
-        <listitem>
-          <para>
-            These options cause the set of live or dead paths to be
-            printed, respectively, rather than performing an actual
-            garbage collector.  They correspond exactly with the
-            sub-operations in <command>nix-store
-            <option>--gc</option></command>.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-      <varlistentry>
-        <term><option>--min-age</option> <replaceable>age</replaceable></term>
-        <listitem>
-          <para>
-            This option corresponds to the <option>--min-age</option>
-            option in <command>nix-store <option>--gc</option></command>.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-    </variablelist>
-
-  </refsection>
-
-  <refsection>
-    <title>Examples</title>
-
-    <para>
-      To delete all unreachable paths, just do:
-    
-      <screen>
-$ nix-collect-garbage</screen>
-
-    </para>
-
-  </refsection>
-    
 </refentry>

doc/manual/nix-instantiate.xml

@@ -1,90 +1,97 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-instantiate</refname>
-    <refpurpose>instantiate store expressions from Nix expressions</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-instantiate</refname>
+  <refpurpose>instantiate store derivations from Nix expressions</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-instantiate</command>
-      &opt-common-syn;
-      <group choice='opt'>
-        <arg choice='plain'><option>--parse-only</option></arg>
-        <arg choice='plain'><option>--eval-only</option></arg>
-      </group>
-      <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-instantiate</command>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
+    <arg><option>--indirect</option></arg>
+    <group choice='opt'>
+      <arg choice='plain'><option>--parse-only</option></arg>
+      <arg choice='plain'><option>--eval-only</option></arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-instantiate</command> generates
-      (low-level) store expressions from (high-level) Nix expressions.
-      It loads and evaluates the Nix expressions in each of
-      <replaceable>files</replaceable>.  Each top-level expression
-      should evaluate to a derivation, a list of derivations, or a set
-      of derivations.  The paths of the resulting store expressions
-      are printed on standard output.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is generally used for testing Nix expression before
-      they are used with <command>nix-env</command>.
-    </para>
+<para>The command <command>nix-instantiate</command> generates <link
+linkend="gloss-derivation">store derivations</link> from (high-level)
+Nix expressions.  It loads and evaluates the Nix expressions in each
+of <replaceable>files</replaceable>.  Each top-level expression should
+evaluate to a derivation, a list of derivations, or a set of
+derivations.  The paths of the resulting store derivations are printed
+on standard output.</para>
 
-  </refsection>
+<para>Most users and developers don’t need to use this command
+(<command>nix-env</command> and <command>nix-build</command> perform
+store derivation instantiation from Nix expressions automatically).
+It is most commonly used for implementing new deployment
+policies.</para>
 
-  <refsection>
-    <title>Options</title>
+<para>See also <xref linkend="sec-common-options" /> for a list of
+common options.</para>
 
-    <variablelist>
+</refsection>
 
-      &opt-common;
 
-      <varlistentry>
-        <term><option>--parse-only</option></term>
-        <listitem>
-          <para>
-            Just parse the input files, and print their abstract
-            syntax trees on standard output in ATerm format.
-          </para>
-        </listitem>
-      </varlistentry>
+<refsection><title>Options</title>
+
+<variablelist>
+
+  <varlistentry>
+    <term><option>--add-root</option> <replaceable>path</replaceable></term>
+    <term><option>--indirect</option></term>
+
+    <listitem><para>See the <link linkend="opt-add-root">corresponding
+    options</link> in <command>nix-store</command>.</para></listitem>
+
+  </varlistentry>
+
+    
+  <varlistentry><term><option>--parse-only</option></term>
+  
+    <listitem><para>Just parse the input files, and print their
+    abstract syntax trees on standard output in ATerm
+    format.</para></listitem>
+    
+  </varlistentry>
       
-      <varlistentry>
-        <term><option>--eval-only</option></term>
-        <listitem>
-          <para>
-            Just parse and evaluate the input files, and print the
-            resulting values on standard output.  No instantiation of
-            store expressions takes place.
-          </para>
-        </listitem>
-      </varlistentry>
+  <varlistentry><term><option>--eval-only</option></term>
+  
+    <listitem><para>Just parse and evaluate the input files, and print
+    the resulting values on standard output.  No instantiation of
+    store derivations takes place.</para></listitem>
+    
+  </varlistentry>
 
-    </variablelist>
+</variablelist>
 
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Examples</title>
 
-    <screen>
-$ nix-instantiate gcc.nix <lineannotation>(instantiate)</lineannotation>
-/nix/store/468abdcb93aa22bb721142615b97698b-d-gcc-3.3.2.store
+<refsection><title>Examples</title>
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(build)</lineannotation>
+<screen>
+$ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation>
+/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(print output path)</lineannotation>
-/nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
+$ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation>
+<replaceable>...</replaceable>
+/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation>
 
-$ ls -l /nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
-dr-xr-xr-x    2 eelco    users         360 2003-12-01 16:12 bin
-dr-xr-xr-x    3 eelco    users          72 2003-12-01 16:12 include
+$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
+dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
 ...</screen>
 
-  </refsection>
-    
+</refsection>
+
+
 </refentry>

doc/manual/nix-prefetch-url.xml

@@ -1,54 +1,69 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-prefetch-url</refname>
-    <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-prefetch-url</refname>
+  <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-prefetch-url</command>
-      <arg choice='plain'><replaceable>url</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-prefetch-url</command>
+    <arg choice='plain'><replaceable>url</replaceable></arg>
+    <arg><replaceable>hash</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-prefetch-url</command> downloads the
-      file referenced by the URL <replaceable>url</replaceable>,
-      prints its MD5 cryptographic hash code, and copies it into the
-      Nix store.  The file name in the store is
-      <filename><replaceable>hash</replaceable>-<replaceable>basename</replaceable></filename>,
-      where <replaceable>basename</replaceable> is everything
-      following the final slash in <replaceable>url</replaceable>.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is just a convenience to Nix expression writers.
-      Often a Nix expressions fetch some source distribution from the
-      network using the <literal>fetchurl</literal> expression
-      contained in <literal>nixpkgs</literal>.  However,
-      <literal>fetchurl</literal> requires an MD5 hash.  If you don't
-      know the hash, you would have to download the file first, and
-      then <literal>fetchurl</literal> would download it again when
-      you build your Nix expression.  Since
-      <literal>fetchurl</literal> uses the same name for the
-      downloaded file as <command>nix-prefetch-url</command>, the
-      redundant download can be avoided.
-    </para>
+<para>The command <command>nix-prefetch-url</command> downloads the
+file referenced by the URL <replaceable>url</replaceable>, prints its
+cryptographic hash, and copies it into the Nix store.  The file name
+in the store is
+<filename><replaceable>hash</replaceable>-<replaceable>baseName</replaceable></filename>,
+where <replaceable>baseName</replaceable> is everything following the
+final slash in <replaceable>url</replaceable>.</para>
 
-  </refsection>
+<para>This command is just a convenience for Nix expression writers.
+Often a Nix expression fetches some source distribution from the
+network using the <literal>fetchurl</literal> expression contained in
+Nixpkgs.  However, <literal>fetchurl</literal> requires a
+cryptographic hash.  If you don't know the hash, you would have to
+download the file first, and then <literal>fetchurl</literal> would
+download it again when you build your Nix expression.  Since
+<literal>fetchurl</literal> uses the same name for the downloaded file
+as <command>nix-prefetch-url</command>, the redundant download can be
+avoided.</para>
 
-  <refsection>
-    <title>Examples</title>
+<para>The environment variable <envar>NIX_HASH_ALGO</envar> specifies
+which hash algorithm to use.  It can be either <literal>md5</literal>,
+<literal>sha1</literal>, or <literal>sha256</literal>.  The default is
+<literal>md5</literal>.</para>
 
-    <screen>
+<para>If <replaceable>hash</replaceable> is specified, then a download
+is not performed if the Nix store already contains a file with the
+same hash and base name.  Otherwise, the file is downloaded, and an
+error if signaled if the actual hash of the file does not match the
+specified hash.</para>
+
+<para>This command prints the hash on standard output.  Additionally,
+if the environment variable <envar>PRINT_PATH</envar> is set, the path
+of the downloaded file in the Nix store is also printed.</para>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<screen>
 $ nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
-...
-file has hash 0bbd1df101bc0294d440471e50feca71
-...</screen>
+0bbd1df101bc0294d440471e50feca71
+
+$ PRINT_PATH=1 nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
+0bbd1df101bc0294d440471e50feca71
+/nix/store/wvyz8ifdn7wyz1p3pqyn0ra45ka2l492-make-3.80.tar.bz2</screen>
+
+</refsection>
 
-  </refsection>
     
 </refentry>

doc/manual/nix-push.xml

@@ -1,138 +1,116 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-push</refname>
-    <refpurpose>push store paths onto a network cache</refpurpose>
-  </refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-push</command>
-      <arg choice='plain'><replaceable>archives-put-url</replaceable></arg>
-      <arg choice='plain'><replaceable>archives-get-url</replaceable></arg>
-      <arg choice='plain'><replaceable>manifest-put-url</replaceable></arg>
-      <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refnamediv>
+  <refname>nix-push</refname>
+  <refpurpose>push store paths onto a network cache</refpurpose>
+</refnamediv>
 
-  <refsection>
-    <title>Description</title>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-push</command>
+    <group choice='req'>
+      <arg choice='req'>
+        <arg choice='plain'><replaceable>archivesPutURL</replaceable></arg>
+        <arg choice='plain'><replaceable>archivesGetURL</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestPutURL</replaceable></arg>
+      </arg>
+      <arg choice='req'>
+        <arg choice='plain'><option>--copy</option></arg>
+        <arg choice='plain'><replaceable>archivesDir</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestFile</replaceable></arg>
+      </arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-    <para>
-      The command <command>nix-push</command> builds a set of store
-      expressions (if necessary), and then packages and uploads all
-      store paths in the resulting closures to a server.  A network
-      cache thus populated can subsequently be used to speed up
-      software deployment on other machines using the
-      <command>nix-pull</command> command.
-    </para>
 
-    <para>
-      <command>nix-push</command> performs the following actions.
+<refsection><title>Description</title>
+
+<para>The command <command>nix-push</command> builds a set of store
+paths (if necessary), and then packages and uploads all store paths in
+the resulting closures to a server.  A network cache thus populated
+can subsequently be used to speed up software deployment on other
+machines using the <command>nix-pull</command> command.</para>
+
+<para><command>nix-push</command> performs the following actions.
       
-      <orderedlist>
+<orderedlist>
 
-        <listitem>
-          <para>
-            The store expressions stored in
-            <replaceable>paths</replaceable> are realised (using
-            <literal>nix-store --realise</literal>).
-          </para>
-        </listitem>
+  <listitem><para>Each path in <replaceable>paths</replaceable> is
+  realised (using <link
+  linkend='rsec-nix-store-realise'><literal>nix-store
+  --realise</literal></link>).</para></listitem>
 
-        <listitem>
-          <para>
-            All paths in the closure of the store expressions stored
-            in <replaceable>paths</replaceable> are determined (using
-            <literal>nix-store --query --requisites
-            --include-successors</literal>).  It should be noted that
-            since the <option>--include-successors</option> flag is
-            used, if you specify a derivation store expression, you
-            get a combined source/binary distribution.  If you only
-            want a binary distribution, you should specify the closure
-            store expression that result from realising these (see
-            below).
-          </para>
-        </listitem>
+  <listitem><para>All paths in the closure of the store expressions
+  stored in <replaceable>paths</replaceable> are determined (using
+  <literal>nix-store --query --requisites
+  --include-outputs</literal>).  It should be noted that since the
+  <option>--include-outputs</option> flag is used, you get a combined
+  source/binary distribution.</para></listitem>
 
-        <listitem>
-          <para>
-            All store paths determined in the previous step are
-            packaged and compressed into a <command>bzip</command>ped
-            NAR archive (extension <filename>.nar.bz2</filename>).
-          </para>
-        </listitem>
+  <listitem><para>All store paths determined in the previous step are
+  packaged and compressed into a <command>bzip</command>ped NAR
+  archive (extension <filename>.nar.bz2</filename>).</para></listitem>
 
-        <listitem>
-          <para>
-            A <emphasis>manifest</emphasis> is created that contains
-            information on the store paths, their eventual URLs in the
-            cache, and cryptographic hashes of the contents of the NAR
-            archives.
-          </para>
-        </listitem>
+  <listitem><para>A <emphasis>manifest</emphasis> is created that
+  contains information on the store paths, their eventual URLs in the
+  cache, and cryptographic hashes of the contents of the NAR
+  archives.</para></listitem>
 
-        <listitem>
-          <para>
-            Each store path is uploaded to the remote directory
-            specified by <replaceable>archives-put-url</replaceable>.
-            HTTP PUT requests are used to do this.  However, before a
-            file <varname>x</varname> is uploaded to
-            <literal><replaceable>archives-put-url</replaceable>/<varname>x</varname></literal>,
-            <command>nix-push</command> first determines whether this
-            upload is unnecessary by issuing a HTTP HEAD request on
-            <literal><replaceable>archives-get-url</replaceable>/<varname>x</varname></literal>.
-            This allows a cache to be shared between many partially
-            overlapping <command>nix-push</command> invocations.
-            (We use two URLs because the upload URL typically
-            refers to a CGI script, while the download URL just refers
-            to a file system directory on the server.)
-          </para>
-        </listitem>
+  <listitem><para>Each store path is uploaded to the remote directory
+  specified by <replaceable>archivesPutURL</replaceable>.  HTTP PUT
+  requests are used to do this.  However, before a file
+  <varname>x</varname> is uploaded to
+  <literal><replaceable>archivesPutURL</replaceable>/<varname>x</varname></literal>,
+  <command>nix-push</command> first determines whether this upload is
+  unnecessary by issuing a HTTP HEAD request on
+  <literal><replaceable>archivesGetURL</replaceable>/<varname>x</varname></literal>.
+  This allows a cache to be shared between many partially overlapping
+  <command>nix-push</command> invocations.  (We use two URLs because
+  the upload URL typically refers to a CGI script, while the download
+  URL just refers to a file system directory on the server.)</para></listitem>
 
-        <listitem>
-          <para>
-            The manifest is uploaded using an HTTP PUT request to
-            <replaceable>manifest-put-url</replaceable>.  The
-            corresponding URL to download the manifest can then be
-            used by <command>nix-pull</command>.
-          </para>
-        </listitem>
+  <listitem><para>The manifest is uploaded using an HTTP PUT request
+  to <replaceable>manifestPutURL</replaceable>.  The corresponding
+  URL to download the manifest can then be used by
+  <command>nix-pull</command>.</para></listitem>
         
-      </orderedlist>
-    </para>
+</orderedlist>
+
+</para>
+
+<para>TODO: <option>--copy</option></para>
             
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Examples</title>
 
-    <para>
-      To upload files there typically is some CGI script on the server
-      side.  This script should be be protected with a password.  The
-      following example uploads the store paths resulting from
-      building the Nix expressions in <filename>foo.nix</filename>,
-      passing appropriate authentication information:
+<refsection><title>Examples</title>
+
+<para>To upload files there typically is some CGI script on the server
+side.  This script should be be protected with a password.  The
+following example uploads the store paths resulting from building the
+Nix expressions in <filename>foo.nix</filename>, passing appropriate
+authentication information:
     
-      <screen>
+<screen>
 $ nix-push \
     http://foo@bar:server.domain/cgi-bin/upload.pl/cache \
     http://server.domain/cache \
     http://foo@bar:server.domain/cgi-bin/upload.pl/MANIFEST \
     $(nix-instantiate foo.nix)</screen>
 
-    This will push both sources and binaries (and any build-time
-    dependencies used in the build, such as compilers).
-    </para>
+This will push both sources and binaries (and any build-time
+dependencies used in the build, such as compilers).</para>
 
-    <para>
-      If we just want to push binaries, not sources and build-time
-      dependencies, we can do:
+<para>If we just want to push binaries, not sources and build-time
+dependencies, we can do:
       
-      <screen>
+<screen>
 $ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
     
-    </para>
+</para>
 
-  </refsection>
+</refsection>
     
 </refentry>

doc/manual/opt-common-syn.xml

@@ -1,3 +1,5 @@
+<nop>
+  
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'><option>--verbose</option></arg>
@@ -17,3 +19,6 @@
 <arg><option>-K</option></arg>
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
+<arg><option>--log-type</option> <replaceable>type</replaceable></arg>
+
+</nop>

doc/manual/opt-common.xml

@@ -1,184 +1,216 @@
-<varlistentry>
-  <term><option>--help</option></term>
+<sect1 id="sec-common-options"><title>Common options</title>
+
+<para>Most Nix commands accept the following command-line options:</para>
+
+<variablelist>
+
+<varlistentry><term><option>--help</option></term>
+  
+  <listitem><para>Prints out a summary of the command syntax and
+  exits.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--version</option></term>
+  
+  <listitem><para>Prints out the Nix version number on standard output
+  and exits.</para></listitem>
+</varlistentry>
+
+
+<varlistentry><term><option>--verbose</option></term>
+  <term><option>-v</option></term>
+
   <listitem>
-    <para>
-      Prints out a summary of the command syntax and exits.
+    
+  <para>Increases the level of verbosity of diagnostic messages
+  printed on standard error.  For each Nix operation, the information
+  printed on standard output is well-defined; any diagnostic
+  information is printed on standard error, never on standard
+  output.</para>
+
+  <para>This option may be specified repeatedly.  Currently, the
+  following verbosity levels exist:</para>
+
+  <variablelist>
+    
+    <varlistentry><term>0</term>
+    <listitem><para>“Errors only”: only print messages
+    explaining why the Nix invocation failed.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>1</term>
+    <listitem><para>“Informational”: print
+    <emphasis>useful</emphasis> messages about what Nix is doing.
+    This is the default.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>2</term>
+    <listitem><para>“Talkative”: print more informational
+    messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>3</term>
+    <listitem><para>“Chatty”: print even more
+    informational messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>4</term>
+    <listitem><para>“Debug”: print debug
+    information.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>5</term>
+    <listitem><para>“Vomit”: print vast amounts of debug
+    information.</para></listitem>
+    </varlistentry>
+    
+  </variablelist>
+
+  </listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--no-build-output</option></term>
+  <term><option>-Q</option></term>
+
+  <listitem><para>By default, output written by builders to standard
+  output and standard error is echoed to the Nix command's standard
+  error.  This option suppresses this behaviour.  Note that the
+  builder's standard output and error are always written to a log file
+  in
+  <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="opt-max-jobs"><term><option>--max-jobs</option></term>
+  <term><option>-j</option></term>
+
+  <listitem><para>Sets the maximum number of build jobs that Nix will
+  perform in parallel to the specified number.  The default is 1.  A
+  higher value is useful on SMP systems or to exploit I/O latency.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-going</option></term>
+  <term><option>-k</option></term>
+
+  <listitem><para>Keep going in case of failed builds, to the
+  greatest extent possible.  That is, if building an input of some
+  derivation fails, Nix will still build the other inputs, but not the
+  derivation itself.  Without this option, Nix stops if any build
+  fails (except for builds of substitutes), possibly killing builds in
+  progress (in case of parallel or distributed builds).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-failed</option></term>
+  <term><option>-K</option></term>
+
+  <listitem><para>Specifies that in case of a build failure, the
+  temporary directory (usually in <filename>/tmp</filename>) in which
+  the build takes place should not be deleted.  The path of the build
+  directory is printed as an informational message.
     </para>
   </listitem>
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--version</option></term>
+<varlistentry><term><option>--fallback</option></term>
+
   <listitem>
-    <para>
-      Prints out the Nix version number on standard output and exits.
-    </para>
+
+  <para>Whenever Nix attempts to build a derivation for which
+  substitutes are known for each output path, but realising the output
+  paths through the substitutes fails, fall back on building the
+  derivation.</para>
+
+  <para>The most common scenario in which this is useful is when we
+  have registered substitutes in order to perform binary distribution
+  from, say, a network repository.  If the repository is down, the
+  realisation of the derivation will fail.  When this option is
+  specified, Nix will build the derivation instead.  Thus,
+  installation from binaries falls back on nstallation from source.
+  This option is not the default since it is generally not desirable
+  for a transient failure in obtaining the substitutes to lead to a
+  full build from source (with the related consumption of
+  resources).</para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--verbose</option> / <option>-v</option></term>
-  <listitem>
-    <para>
-      Increases the level of verbosity of diagnostic messages printed
-      on standard error.  For each Nix operation, the information
-      printed on standard output is well-defined; any diagnostic
-      information is printed on standard error, never on standard
-      output.
-    </para>
+<varlistentry><term><option>--readonly-mode</option></term>
 
-    <para>
-      This option may be specified repeatedly.  Currently, the
-      following verbosity levels exist:
-    </para>
-
-    <variablelist>
-      <varlistentry>
-        <term>0</term>
-        <listitem>
-          <para>
-            <quote>Errors only</quote>: only print messages explaining
-            why the Nix invocation failed.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>1</term>
-        <listitem>
-          <para>
-            <quote>Informational</quote>: print
-            <emphasis>useful</emphasis> messages about what Nix is
-            doing.  This is the default.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>2</term>
-        <listitem>
-          <para>
-            <quote>Talkative</quote>: print more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>3</term>
-        <listitem>
-          <para>
-            <quote>Chatty</quote>: print even more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>4</term>
-        <listitem>
-          <para>
-            <quote>Debug</quote>: print debug information:
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>5</term>
-        <listitem>
-          <para>
-            <quote>Vomit</quote>: print vast amounts of debug
-            information.
-          </para>
-        </listitem>
-      </varlistentry>
-    </variablelist>
-
-  </listitem>
+  <listitem><para>When this option is used, no attempt is made to open
+  the Nix database.  Most Nix operations do need database access, so
+  those operations will fail.</para></listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--no-build-output</option> / <option>-Q</option></term>
+<varlistentry id="opt-log-type"><term><option>--log-type</option>
+<replaceable>type</replaceable></term>
+
   <listitem>
-    <para>
-      By default, output written by builders to standard output and
-      standard error is echoed to the Nix command's standard error.
-      This option suppresses this behaviour.  Note that the builder's
-      standard output and error are always written to a log file in
-      <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.
-    </para>
+
+  <para>This option determines how the output written to standard
+  error is formatted.  Nix’s diagnostic messages are typically
+  <emphasis>nested</emphasis>.  For instance, when tracing Nix
+  expression evaluation (<command>nix-env -vvvvv</command>, messages
+  from subexpressions are nested inside their parent expressions.  Nix
+  builder output is also often nested.  For instance, the Nix Packages
+  generic builder nests the various build tasks (unpack, configure,
+  compile, etc.), and the GNU Make in <literal>stdenv-linux</literal>
+  has been patched to provide nesting for recursive Make
+  invocations.</para>
+
+  <para><replaceable>type</replaceable> can be one of the
+  following:
+
+  <variablelist>
+
+    <varlistentry><term><literal>pretty</literal></term>
+
+      <listitem><para>Pretty-print the output, indicating different
+      nesting levels using spaces.  This is the
+      default.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>escapes</literal></term>
+
+      <listitem><para>Indicate nesting using escape codes that can be
+      interpreted by the <command>log2xml</command> tool in the Nix
+      source distribution.  The resulting XML file can be fed into the
+      <command>log2html.xsl</command> stylesheet to create an HTML
+      file that can be browsed interactively, using Javascript to
+      expand and collapse parts of the output.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>flat</literal></term>
+
+      <listitem><para>Remove all nesting.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>    
+  
+  </para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--max-jobs</option> / <option>-j</option></term>
-  <listitem>
-    <para>
-      Sets the maximum number of build jobs that Nix will perform in
-      parallel to the specified number.  The default is 1.  A higher
-      value is useful on SMP systems or to exploit I/O latency.
-    </para>
-  </listitem>
-</varlistentry>
+</variablelist>
 
 
-<varlistentry>
-  <term><option>--keep-going</option> / <option>-k</option></term>
-  <listitem>
-    <para>
-      Keep going in case of failed builds, to the greatest extent
-      possible.  That is, if building an input of some derivation
-      fails, Nix will still build the other inputs, but not the
-      derivation itself.  Without this option, Nix stops if any build
-      fails (except for builds of substitutes), possibly killing
-      builds in progress (in case of parallel or distributed builds).
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--keep-failed</option> / <option>-K</option></term>
-  <listitem>
-    <para>
-      Specifies that in case of a build failure, the temporary
-      directory (usually in <filename>/tmp</filename>) in which the
-      build takes place should not be deleted.  The path of the build
-      directory is printed as an informational message.
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--fallback</option></term>
-  <listitem>
-    <para>
-      Whenever Nix attempts to realise a derivation for which a
-      closure is already known, but this closure cannot be realised,
-      fall back on normalising the derivation.
-    </para>
-
-    <para>
-      The most common scenario in which this is useful is when we have
-      registered substitutes in order to perform binary distribution
-      from, say, a network repository.  If the repository is down, the
-      realisation of the derivation will fail.  When this option is
-      specified, Nix will build the derivation instead.  Thus,
-      binary installation falls back on a source installation.  This
-      option is not the default since it is generally not desirable
-      for a transient failure in obtaining the substitutes to lead to
-      a full build from source (with the related consumption of
-      resources).
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--readonly-mode</option></term>
-  <listitem>
-    <para>
-      When this option is used, no attempt is made to open the Nix
-      database.  Most Nix operations do need database access, so those
-      operations will fail.
-    </para>
-  </listitem>
-</varlistentry>
+</sect1>

doc/manual/package-management.xml

@@ -2,24 +2,24 @@
 
 <para>This chapter discusses how to do package management with Nix,
 i.e., how to obtain, install, upgrade, and erase components.  This is
-the <quote>user's</quote> perspective of the Nix system — people
+the “user’s” perspective of the Nix system — people
 who want to <emphasis>create</emphasis> components should consult
 <xref linkend='chap-writing-nix-expressions' />.</para>
 
 
 <sect1><title>Basic package management</title>
 
-<para>The main command for package management is
-<command>nix-env</command>.  You can use it to install, upgrade, and
-erase components, and to query what components are installed or are
-available for installation.</para>
+<para>The main command for package management is <link
+linkend="sec-nix-env"><command>nix-env</command></link>.  You can use
+it to install, upgrade, and erase components, and to query what
+components are installed or are available for installation.</para>
 
-<para>In Nix, different users can have different <quote>views</quote>
+<para>In Nix, different users can have different “views”
 on the set of installed applications.  That is, there might be lots of
 applications present on the system (possibly in many different
 versions), but users can have a specific selection of those active —
-where <quote>active</quote> just means that it appears in a directory
-in the user's <envar>PATH</envar>.  Such a view on the set of
+where “active” just means that it appears in a directory
+in the user’s <envar>PATH</envar>.  Such a view on the set of
 installed applications is called a <emphasis>user
 environment</emphasis>, which is just a directory tree consisting of
 symlinks to the files of the active applications.  </para>
@@ -31,11 +31,9 @@ Nix expressions called the Nix Package collection that contains
 components ranging from basic development stuff such as GCC and Glibc,
 to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
-expression based on it, or completely new ones.)  You can download the
-latest version from <ulink
-url='http://catamaran.labs.cs.uu.nl/dist/nix' />.  You probably want
-the latest unstable release; currently the stable releases tend to lag
-behind quite a bit.</para>
+expressions based on it, or completely new ones.)  You can download
+the latest version from <ulink
+url='http://catamaran.labs.cs.uu.nl/dist/nix' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available components in the release:
@@ -52,7 +50,7 @@ bzip2-1.0.2
 ...</screen>
 
 where <literal>nixpkgs-<replaceable>version</replaceable></literal> is
-where you've unpacked the release.</para>
+where you’ve unpacked the release.</para>
 
 <para>It is also possible to see the <emphasis>status</emphasis> of
 available components, i.e., whether they are installed into the user
@@ -72,7 +70,7 @@ component is installed in your current user environment.  The second
 (in which case installing it into your user environment would be a
 very quick operation).  The last one (<literal>S</literal>) indicates
 whether there is a so-called <emphasis>substitute</emphasis> for the
-component, which is Nix's mechanism for doing binary deployment.  It
+component, which is Nix’s mechanism for doing binary deployment.  It
 just means that Nix know that it can fetch a pre-built component from
 somewhere (typically a network server) instead of building it
 locally.</para>
@@ -98,7 +96,7 @@ available somewhere.  This is done using the
 <command>nix-pull</command> command, which must be supplied with a URL
 containing a <emphasis>manifest</emphasis> describing what binaries
 are available.  This URL should correspond to the Nix Packages release
-that you're using.  For instance, if you obtained a release from
+that you’re using.  For instance, if you obtained a release from
 <ulink
 url='http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />,
 then you should do:
@@ -111,7 +109,7 @@ downloading binaries from <systemitem
 class='fqdomainname'>catamaran.labs.cs.uu.nl</systemitem>, instead of
 building them from source.  This might still take a while since all
 dependencies must be downloaded, but on a reasonably fast connection
-such as an DSL line it's on the order of a few minutes.</para>
+such as an DSL line it’s on the order of a few minutes.</para>
 
 <para>Naturally, packages can also be uninstalled:
 
@@ -127,9 +125,9 @@ release of Nix Packages, you can do:
 $ nix-env -f nixpkgs-<replaceable>version</replaceable> -u subversion</screen>
 
 This will <emphasis>only</emphasis> upgrade Subversion if there is a
-<quote>newer</quote> version in the new set of Nix expressions, as
+“newer” version in the new set of Nix expressions, as
 defined by some pretty arbitrary rules regarding ordering of version
-numbers (which generally do what you'd expect of them).  To just
+numbers (which generally do what you’d expect of them).  To just
 unconditionally replace Subversion with whatever version is in the Nix
 expressions, use <parameter>-i</parameter> instead of
 <parameter>-u</parameter>; <parameter>-i</parameter> will remove
@@ -143,7 +141,7 @@ $ nix-env -f nixpkgs-<replaceable>version</replaceable> -u '*'</screen>
 
 </para>
 
-<para>Sometimes it's useful to be able to ask what
+<para>Sometimes it’s useful to be able to ask what
 <command>nix-env</command> would do, without actually doing it.  For
 instance, to find out what packages would be upgraded by
 <literal>nix-env -u '*'</literal>, you can do
@@ -175,30 +173,28 @@ set.</para></footnote></para>
 </sect1>
 
 
-<sect1><title>Profiles</title>
+<sect1 id="sec-profiles"><title>Profiles</title>
 
-<para>Profiles and user environments are Nix's mechanism for
+<para>Profiles and user environments are Nix’s mechanism for
 implementing the ability to allow differens users to have different
 configurations, and to do atomic upgrades and rollbacks.  To
-understand how they work, it's useful to know a bit about how Nix
+understand how they work, it’s useful to know a bit about how Nix
 works.  In Nix, components are stored in unique locations in the
 <emphasis>Nix store</emphasis> (typically,
 <filename>/nix/store</filename>).  For instance, a particular version
 of the Subversion component might be stored in a directory
-<filename>/nix/store/eeeeaf42e56b...-subversion-0.32.1/</filename>,
+<filename>/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/</filename>,
 while another version might be stored in
-<filename>/nix/store/58823d558a6a...-subversion-0.34/</filename>.  The
-long hexadecimal numbers prefixed to the directory names are
-cryptographic hashes<footnote><para>128 bit MD5 hashes, to be
-precise.</para></footnote> of <emphasis>all</emphasis> inputs involved
-in building the component — sources, dependencies, compiler flags, and
-so on.  So if two components differ in any way, they end up in
-different locations in the file system, so they don't interfere with
-each other.  <xref linkend='fig-user-environments'
-/><footnote><para>TODO: the figure isn't entirely up to date.  It
-should show multiple profiles and
-<filename>~/.nix-profile</filename>.</para></footnote> shows a part of
-a typical Nix store.</para>
+<filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>.
+The long strings prefixed to the directory names are cryptographic
+hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in
+a base-32 notation, to be precise.</para></footnote> of
+<emphasis>all</emphasis> inputs involved in building the component —
+sources, dependencies, compiler flags, and so on.  So if two
+components differ in any way, they end up in different locations in
+the file system, so they don’t interfere with each other.  <xref
+linkend='fig-user-environments' /> shows a part of a typical Nix
+store.</para>
 
 <figure id='fig-user-environments'><title>User environments</title>
   <mediaobject>
@@ -208,41 +204,42 @@ a typical Nix store.</para>
   </mediaobject>
 </figure>
 
-<para>Of course, you wouldn't want to type
+<para>Of course, you wouldn’t want to type
 
 <screen>
-$ /nix/store/eeeeaf42e56b...-subversion-0.32.1/bin/svn</screen>
+$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen>
 
 every time you want to run Subversion.  Of course we could set up the
 <envar>PATH</envar> environment variable to include the
 <filename>bin</filename> directory of every component we want to use,
 but this is not very convenient since changing <envar>PATH</envar>
-doesn't take effect for already existing processes.  The solution Nix
+doesn’t take effect for already existing processes.  The solution Nix
 uses is to create directory trees of symlinks to
 <emphasis>activated</emphasis> components.  These are called
 <emphasis>user environments</emphasis> and they are components
 themselves (though automatically generated by
 <command>nix-env</command>), so they too reside in the Nix store.  For
 instance, in <xref linkend='fig-user-environments' /> the user
-environment <filename>/nix/store/068150f63831...-user-env</filename>
-contains a symlink to just Subversion 0.32.1 (arrows in the figure
+environment <filename>/nix/store/5mq2jcn36ldl...-user-env</filename>
+contains a symlink to just Subversion 1.1.2 (arrows in the figure
 indicate symlinks).  This would be what we would obtain if we had done
 
 <screen>
 $ nix-env -i subversion</screen>
 
-on a set of Nix expressions that contained Subversion 0.32.1.</para>
+on a set of Nix expressions that contained Subversion 1.1.2.</para>
 
-<para>This doesn't in itself solve the problem, of course; you
-wouldn't want to type
-<filename>/nix/store/068150f63831...-user-env/bin/svn</filename>
-either.  Therefore there are symlinks outside of the store that point
+<para>This doesn’t in itself solve the problem, of course; you
+wouldn’t want to type
+<filename>/nix/store/0c1p5z4kda11...-user-env/bin/svn</filename>
+either.  That’s why there are symlinks outside of the store that point
 to the user environments in the store; for instance, the symlinks
-<filename>42</filename> and <filename>43</filename> in the example.
-These are called <emphasis>generations</emphasis> since every time you
-perform a <command>nix-env</command> operation, a new user environment
-is generated based on the current one.  For instance, generation 43
-was created from generation 42 when we did
+<filename>default-42-link</filename> and
+<filename>default-43-link</filename> in the example.  These are called
+<emphasis>generations</emphasis> since every time you perform a
+<command>nix-env</command> operation, a new user environment is
+generated based on the current one.  For instance, generation 43 was
+created from generation 42 when we did
 
 <screen>
 $ nix-env -i subversion mozilla</screen>
@@ -251,14 +248,14 @@ on a set of Nix expressions that contained Mozilla and a new version
 of Subversion.</para>
 
 <para>Generations are grouped together into
-<emphasis>profiles</emphasis> so that different users don't interfere
-with each other if they don't want to.  For example:
+<emphasis>profiles</emphasis> so that different users don’t interfere
+with each other if they don’t want to.  For example:
 
 <screen>
 $ ls -l /nix/var/nix/profiles/
 ...
-lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/068150f63831...-user-env
-lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/84c85f89ddbf...-user-env
+lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
+lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
 lrwxrwxrwx  1 eelco ... default -> default-43-link</screen>
 
 This shows a profile called <filename>default</filename>.  The file
@@ -268,7 +265,7 @@ operation, a new user environment and generation link are created
 based on the current one, and finally the <filename>default</filename>
 symlink is made to point at the new generation.  This last step is
 atomic on Unix, which explains how we can do atomic upgrades.  (Note
-that the building/installing of new components doesn't interfere in
+that the building/installing of new components doesn’t interfere in
 any way with old components, since they are stored in different
 locations in the Nix store.)</para>
 
@@ -293,13 +290,13 @@ can also see all available generations:
 $ nix-env --list-generations</screen></para>
 
 <para>Actually, there is another level of indirection not shown in the
-figure above.  You generally wouldn't have
+figure above.  You generally wouldn’t have
 <filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
 in your <envar>PATH</envar>.  Rather, there is a symlink
 <filename>~/.nix-profile</filename> that points to your current
 profile.  This means that you should put
 <filename>~/.nix-profile/bin</filename> in your <envar>PATH</envar>
-(and indeed, that's what the initialisation script
+(and indeed, that’s what the initialisation script
 <filename>/nix/etc/profile.d/nix.sh</filename> does).  This makes it
 easier to switch to a different profile.  You can do that using the
 command <command>nix-env --switch-profile</command>:
@@ -310,7 +307,7 @@ $ nix-env --switch-profile /nix/var/nix/profiles/my-profile
 $ nix-env --switch-profile /nix/var/nix/profiles/default</screen>
 
 These commands switch to the <filename>my-profile</filename> and
-default profile, respectively.  If the profile doesn't exist, it will
+default profile, respectively.  If the profile doesn’t exist, it will
 be created automatically.  You should be careful about storing a
 profile in another location than the <filename>profiles</filename>
 directory, since otherwise it might not be used as a root of the
@@ -337,7 +334,7 @@ This will <emphasis>not</emphasis> change the
 (<option>-u</option>) and uninstall (<option>-e</option>) never
 actually delete components from the system.  All they do (as shown
 above) is to create a new user environment that no longer contains
-symlinks to the <quote>deleted</quote> components.</para>
+symlinks to the “deleted” components.</para>
 
 <para>Of course, since disk space is not infinite, unused components
 should be removed at some point.  You can do this by running the Nix
@@ -346,7 +343,7 @@ not used (directly or indirectly) by any generation of any
 profile.</para>
 
 <para>Note however that as long as old generations reference a
-component, it will not be deleted.  After all, we wouldn't be able to
+component, it will not be deleted.  After all, we wouldn’t be able to
 do a rollback otherwise.  So in order for garbage collection to be
 effective, you should also delete (some) old generations.  Of course,
 this should only be done if you are certain that you will not need to
@@ -370,51 +367,61 @@ $ nix-env --delete-generations 10 11 14</screen>
 garbage collector as follows:
 
 <screen>
-$ nix-collect-garbage</screen>
+$ nix-store --gc</screen>
 
-You can alo first view what files would be deleted:
+If you are feeling uncertain, you can also first view what files would
+be deleted:
 
 <screen>
-$ nix-collect-garbage --print-dead</screen>
+$ nix-store --gc --print-dead</screen>
 
 Likewise, the option <option>--print-live</option> will show the paths
-that <emphasis>won't</emphasis> be deleted.</para>
+that <emphasis>won’t</emphasis> be deleted.</para>
 
-<sect2><title>Garbage collector roots</title>
 
-<para>TODO</para>
+<sect2 id="ssec-gc-roots"><title>Garbage collector roots</title>
 
-<para>The garbage collector uses as roots all store expressions
-mentioned in all files with extension <filename>.gcroot</filename> in
-the directory
-<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>,
-or in any file or directory symlinked to from that directory.  E.g.,
-by default,
-<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>
-contains a symlink to
-<filename><replaceable>prefix</replaceable>/var/nix/profiles/</filename>,
-so all generations of all profiles are also roots of the collector.</para>
+<para>The roots of the garbage collector are all store paths to which
+there are symlinks in the directory
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>.
+For instance, the following command makes the path
+<filename>/nix/store/d718ef...-foo</filename> a root of the collector:
+
+<screen>
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar</screen>
+	
+That is, after this command, the garbage collector will not remove
+<filename>/nix/store/d718ef...-foo</filename> or any of its
+dependencies.</para>
+
+<para>Subdirectories of
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>
+are also searched for symlinks.  Symlinks to non-store paths are
+followed and searched for roots, but symlinks to non-store paths
+<emphasis>inside</emphasis> the paths reached in that way are not
+followed to prevent infinite recursion.</para>
 
 </sect2>
 
 </sect1>
 
 
-<sect1><title>Channels</title>
+<sect1 id="sec-channels"><title>Channels</title>
 
-<para>If you want to stay up to date with a set of packages, it's not
+<para>If you want to stay up to date with a set of packages, it’s not
 very convenient to manually download the latest set of Nix expressions
 for those packages, use <command>nix-pull</command> to register
 pre-built binaries (if available), and upgrade using
-<command>nix-env</command>.  Fortunately, there's a better way:
+<command>nix-env</command>.  Fortunately, there’s a better way:
 <emphasis>Nix channels</emphasis>.</para>
 
 <para>A Nix channel is just a URL that points to a place that contains
-a set of Nix expressions and a manifest.  Using the command
-<command>nix-channel</command> you can automatically stay up to date
-with whatever is available at that URL.</para>
+a set of Nix expressions and a manifest.  Using the command <link
+linkend="sec-nix-channel"><command>nix-channel</command></link> you
+can automatically stay up to date with whatever is available at that
+URL.</para>
 
-<para>You can <quote>subscribe</quote> to a channel using
+<para>You can “subscribe” to a channel using
 <command>nix-channel --add</command>, e.g.,
 
 <screen>
@@ -427,7 +434,7 @@ of the Nix Packages collection.  (Instead of
 stability, but right now is just outdated.)  Subscribing really just
 means that the URL is added to the file
 <filename>~/.nix-channels</filename>.  Right now there is no command
-to <quote>unsubscribe</quote>; you should just edit that file manually
+to “unsubscribe”; you should just edit that file manually
 and delete the offending URL.</para>
 
 <para>To obtain the latest Nix expressions available in a channel, do
@@ -440,7 +447,7 @@ This downloads the Nix expressions in every channel (downloaded from
 and registers any available pre-built binaries in every channel
 (by <command>nix-pull</command>ing
 <literal><replaceable>url</replaceable>/MANIFEST</literal>).  It also
-makes the union of each channel's Nix expressions the default for
+makes the union of each channel’s Nix expressions the default for
 <command>nix-env</command> operations.  Consequently, you can then say
 
 <screen>

doc/manual/quick-start.xml

@@ -88,8 +88,8 @@ $ nix-channel --update
 $ nix-env -u '*'</screen>
 
 The latter command will upgrade each installed component for which
-there is a <quote>newer</quote> version (as determined by comparing
-the version numbers).</para></listitem>
+there is a “newer” version (as determined by comparing the version
+numbers).</para></listitem>
 
 <listitem><para>If you're unhappy with the result of a
 <command>nix-env</command> action (e.g., an upgraded component turned
@@ -106,12 +106,12 @@ actually delete them:
 
 <screen>
 $ nix-env --delete-generations old
-$ nix-collect-garbage</screen>
+$ nix-store --gc</screen>
 
-The first command deletes old <quote>generations</quote> of your
-profile (making rollbacks impossible, but also making the components
-in those old generations available for garbage collection), while the
-second command actually deletes them.</para></listitem>
+The first command deletes old “generations” of your profile (making
+rollbacks impossible, but also making the components in those old
+generations available for garbage collection), while the second
+command actually deletes them.</para></listitem>
 
 </orderedlist>
 

doc/manual/style.css

@@ -102,6 +102,8 @@ pre.screen
 
 .note,.warning
 {
+    margin-top: 1em;
+    margin-bottom: 1em;
     border: 1px solid #6185a0;
     padding: 0px 1em;
     background: #fffff5;
@@ -154,7 +156,7 @@ a:hover { background: #ffffcd; }
                                Special elements:
  ***************************************************************************/
 
-tt
+tt, code
 {
     color: #400000;
 }
@@ -229,4 +231,4 @@ div.epigraph
 table.productionset table.productionset
 {
     font-family: monospace;
-}
+}

doc/manual/troubleshooting.xml

@@ -1,14 +1,71 @@
-<appendix>
-  <title>Troubleshooting</title>
+<appendix><title>Troubleshooting</title>
+
+<para>This section provides solutions for some common problems.</para>
+
+
+<sect1><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
+
+<para>Symptom: Nix operations (in particular the
+<command>nix-store</command> operations <option>--gc</option>,
+<option>--verify</option>, and <option>--clear-substitutes</option> —
+the latter being called by <command>nix-channel --update</command>)
+failing:
+
+<screen>
+$ nix-store --verify
+error: Db::del: Cannot allocate memory</screen>
+
+</para>
+
+<para>Possible solution: make sure that no Nix processes are running,
+then do:
+
+<screen>
+$ cd /nix/var/nix/db
+$ rm __db.00*</screen>
+
+</para>
+
+</sect1>
+
+
+<sect1><title>Collisions in <command>nix-env</command></title>
+
+<para>Symptom: when installing or upgrading, you get an error message such as
+
+<screen>
+$ nix-env -i docbook-xml
+...
+adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
+collission between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
+  and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
+  at /nix/store/...-builder.pl line 62.</screen>
+
+</para>
+
+<para>The cause is that two installed packages in the user environment
+have overlapping filenames (e.g.,
+<filename>xml/dtd/docbook/calstblx.dtd</filename>.  This usually
+happens when you accidentally try to install two versions of the same
+package.  For instance, in the example above, the Nix Packages
+collection contains two versions of <literal>docbook-xml</literal>, so
+<command>nix-env -i</command> will try to install both.  The default
+user environment builder has no way to way to resolve such conflicts,
+so it just gives up.</para>
+
+<para>Solution: remove one of the offending packages from the user
+environment (if already installed) using <command>nix-env
+-u</command>, or specify exactly which version should be installed
+(e.g., <literal>nix-env -i docbook-xml-4.2</literal>).</para>
+
+<para>Alternatively, you can modify the user environment builder
+script (in
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>)
+to implement some conflict resolution policy.  E.g., the script could
+be modified to rename conflicting file names, or to pick one over the
+other.</para>
+
+</sect1>
+
 
-  <para>
-    (Nothing.)
-  </para>
-	
 </appendix>
-
-<!--
-local variables:
-sgml-parent-document: ("book.xml" "appendix")
-end:
--->

doc/manual/writing-nix-expressions.xml

@@ -383,7 +383,7 @@ some fragments of
     that can be built by Nix (since when we fill in the arguments of
     the function, what we get is its body, which is the call to
     <varname>stdenv.mkDerivation</varname> in <xref
-    linkend='ex-hello-nix ' />).</para>
+    linkend='ex-hello-nix' />).</para>
 
   </callout>
 
@@ -447,7 +447,8 @@ following:
 (import pkgs/system/i686-linux.nix).hello</programlisting>
 
 Call it <filename>test.nix</filename>.  You can then build it without
-installing it using the command <command>nix-build</command>:
+installing it using the command <link
+linkend="sec-nix-build"><command>nix-build</command></link>:
 
 <screen>
 $ nix-build ./test.nix
@@ -817,7 +818,7 @@ set.</para>
 </simplesect>
 
 
-<simplesect><title>Functions</title>
+<simplesect id="ss-functions"><title>Functions</title>
 
 <para>Functions have the following form:
 
@@ -1103,7 +1104,7 @@ weakest binding).</para>
 </simplesect>
 
 
-<simplesect><title>Derivations</title>
+<simplesect id="ssec-derivation"><title>Derivations</title>
 
 <para>The most important built-in function is
 <function>derivation</function>, which is used to describe a
@@ -1112,7 +1113,7 @@ set, the attributes of which specify the inputs of the build.</para>
 
 <itemizedlist>
 
-  <listitem><para>There must be an attribute named
+  <listitem id="attr-system"><para>There must be an attribute named
   <varname>system</varname> whose value must be a string specifying a
   Nix platform identifier, such as <literal>"i686-linux"</literal> or
   <literal>"powerpc-darwin"</literal><footnote><para>To figure out

externals/Makefile.am

@@ -1,11 +1,11 @@
 # Berkeley DB
 
-DB = db-4.2.52
+DB = db-4.3.28.NC
 
 $(DB).tar.gz:
 	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.2.52 from"
-	@echo "  http://www.sleepycat.com/update/snapshot/db-4.2.52.tar.gz"
+	@echo "Please download version 4.3.28 from"
+	@echo "  http://downloads.sleepycat.com/db-4.3.28.NC.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -26,8 +26,8 @@ build-db: have-db
 	 ../dist/configure --prefix=$$pfx/inst-bdb \
 	 --enable-cxx --disable-shared --disable-cryptography \
          --disable-replication --disable-verify && \
-	make && \
-	make install)
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-db
 endif
 
@@ -57,8 +57,8 @@ build-aterm: have-aterm
 	(pfx=`pwd` && \
 	cd $(ATERM) && \
 	CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm && \
-	make && \
-	make install)
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-aterm
 endif
 

make/examples/aterm/aterm/default.nix

@@ -0,0 +1,34 @@
+{sharedLib ? true}:
+
+rec {
+
+  inherit (import ../../../lib) compileC makeLibrary;
+
+  sources = [
+    ./afun.c
+    ./aterm.c
+    ./bafio.c
+    ./byteio.c
+    ./gc.c
+    ./hash.c
+    ./list.c
+    ./make.c
+    ./md5c.c
+    ./memory.c
+    ./tafio.c
+    ./version.c
+  ];
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    forSharedLib = sharedLib;
+  };
+
+  libATerm = makeLibrary {
+    libraryName = "ATerm";
+    objects = map compile sources;
+    inherit sharedLib;
+  };
+
+}

make/examples/aterm/default.nix

@@ -0,0 +1 @@
+import test/default.nix

make/examples/aterm/test/default.nix

@@ -0,0 +1,18 @@
+let {
+
+  inherit (import ../../../lib) compileC link;
+
+  inherit (import ../aterm {}) libATerm;
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    cFlags = "-I../aterm";
+  };
+
+  fib = link {objects = compile ./fib.c; libraries = libATerm;};
+
+  primes = link {objects = compile ./primes.c; libraries = libATerm;};
+  
+  body = [fib primes];
+}

make/examples/default.nix

@@ -0,0 +1,6 @@
+[ (import ./trivial)
+  (import ./simple-header)
+  (import ./not-so-simple-header)
+  (import ./not-so-simple-header-auto)
+  (import ./aterm)
+]

make/examples/not-so-simple-header-auto/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header-auto/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC findIncludes link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = "auto";
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header-auto/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header-auto/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/not-so-simple-header/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header/default.nix

@@ -0,0 +1,14 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = [
+      [./foo/fnord/indirect.h "fnord/indirect.h"]
+      [./bar/hello.h "fnord/../../bar/hello.h"]
+    ];
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {
+    main = ./hello.c;
+    localIncludes = [ [./hello.h "hello.h"] ];
+  };};
+
+  body = [hello];
+}

make/examples/simple-header/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "hello.h"
+
+int main(int argc, char * * argv)
+{
+    printf("Hello " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/trivial/default.nix

@@ -0,0 +1,8 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {main = ./hello.c;};};
+
+  body = [hello];
+}

make/examples/trivial/hello.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(int argc, char * * argv)
+{
+    printf("Hello World\n");
+    return 0;
+}

make/lib/compile-c.sh

@@ -0,0 +1,73 @@
+. $stdenv/setup
+
+mainName=$(basename $main | cut -c34-)
+
+echo "compiling \`$mainName'..."
+
+# Turn $localIncludes into an array.
+localIncludes=($localIncludes)
+
+# Determine how many `..' levels appear in the header file references.
+# E.g., if there is some reference `../../foo.h', then we have to
+# insert two extra levels in the directory structure, so that `a.c' is
+# stored at `dotdot/dotdot/a.c', and a reference from it to
+# `../../foo.h' resolves to `dotdot/dotdot/../../foo.h' == `foo.h'.
+n=0
+maxDepth=0
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    target=${localIncludes[$((n + 1))]}
+
+    # Split the target name into path components using some IFS magic.
+    savedIFS="$IFS"
+    IFS=/
+    components=($target)
+    depth=0
+    for ((m = 0; m < ${#components[*]}; m++)); do
+        c=${components[m]}
+        if test "$c" = ".."; then
+            depth=$((depth + 1))
+        fi
+    done
+    IFS="$savedIFS"
+
+    if test $depth -gt $maxDepth; then
+        maxDepth=$depth;
+    fi
+done
+
+# Create the extra levels in the directory hierarchy.
+prefix=
+for ((n = 0; n < maxDepth; n++)); do
+    prefix="dotdot/$prefix"
+done
+
+# Create symlinks to the header files.
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    source=${localIncludes[n]}
+    target=${localIncludes[$((n + 1))]}
+
+    # Create missing directories.  We use IFS magic to split the path
+    # into path components.
+    savedIFS="$IFS"
+    IFS=/
+    components=($prefix$target)
+    fullPath=(.)
+    for ((m = 0; m < ${#components[*]} - 1; m++)); do
+        fullPath=("${fullPath[@]}" ${components[m]})
+        if ! test -d "${fullPath[*]}"; then
+            mkdir "${fullPath[*]}"
+        fi
+    done
+    IFS="$savedIFS"
+    
+    ln -sf $source $prefix$target
+done
+
+# Create a symlink to the main file.
+if ! test "$(readlink $prefix$mainName)" = $main; then
+    ln -s $main $prefix$mainName
+fi
+
+mkdir $out
+test "$prefix" && cd $prefix
+gcc -Wall $cFlags -c $mainName -o $out/$mainName.o

make/lib/default.nix

@@ -0,0 +1,59 @@
+rec {
+
+  # Should point at your Nixpkgs installation.
+  pkgPath = ./pkgs;
+
+  pkgs = import (pkgPath + /system/all-packages.nix) {};
+
+  stdenv = pkgs.stdenv;
+  
+
+  compileC = {main, localIncludes ? [], cFlags ? "", forSharedLib ? false}:
+  stdenv.mkDerivation {
+    name = "compile-c";
+    builder = ./compile-c.sh;
+    localIncludes =
+      if localIncludes == "auto" then
+        import (findIncludes {
+          main = toString main;
+          hack = __currentTime;
+          inherit cFlags;
+        })
+      else
+        localIncludes;
+    inherit main;
+    cFlags = [
+      cFlags
+      (if forSharedLib then ["-fpic"] else [])
+    ];
+  };
+
+  /*
+  runCommand = {command}: {
+    name = "run-command";
+    builder = ./run-command.sh;
+    inherit command;
+  };
+  */
+
+  findIncludes = {main, hack, cFlags ? ""}: stdenv.mkDerivation {
+    name = "find-includes";
+    builder = ./find-includes.sh;
+    inherit main hack cFlags;
+  };
+  
+  link = {objects, programName ? "program", libraries ? []}: stdenv.mkDerivation {
+    name = "link";
+    builder = ./link.sh;
+    inherit objects programName libraries;
+  };
+
+  makeLibrary = {objects, libraryName ? [], sharedLib ? false}:
+  # assert sharedLib -> fold (obj: x: assert obj.sharedLib && x) false objects
+  stdenv.mkDerivation {
+    name = "library";
+    builder = ./make-library.sh;
+    inherit objects libraryName sharedLib;
+  };
+
+}

make/lib/find-includes.sh

@@ -0,0 +1,20 @@
+. $stdenv/setup
+
+echo "finding includes of \`$(basename $main)'..."
+
+makefile=$NIX_BUILD_TOP/makefile
+
+mainDir=$(dirname $main)
+(cd $mainDir && gcc $cFlags -MM $(basename $main) -MF $makefile) || false
+
+echo "[" >$out
+
+while read line; do
+    line=$(echo "$line" | sed 's/.*://')
+    for i in $line; do
+        fullPath=$(readlink -f $mainDir/$i)
+        echo "  [ $fullPath \"$i\" ]" >>$out
+    done
+done < $makefile
+
+echo "]" >>$out

make/lib/link.sh

@@ -0,0 +1,21 @@
+. $stdenv/setup
+
+shopt -s nullglob
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+libs=
+for i in $libraries; do
+    lib=$(echo $i/*.a; echo $i/*.so)
+    name=$(echo $(basename $lib) | sed -e 's/^lib//' -e 's/.a$//' -e 's/.so$//')
+    libs="$libs -L$(dirname $lib) -l$name" 
+done
+
+echo "linking object files into \`$programName'..."
+
+mkdir $out
+gcc -o $out/$programName $objs $libs

make/lib/make-library.sh

@@ -0,0 +1,28 @@
+. $stdenv/setup
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+echo "archiving object files into library \`$libraryName'..."
+
+ensureDir $out
+
+if test -z "$sharedLib"; then
+
+    outPath=$out/lib${libraryName}.a
+
+    ar crs $outPath $objs
+    ranlib $outPath
+
+else
+
+    outPath=$out/lib${libraryName}.so
+
+    gcc -shared -o $outPath $objs
+
+fi    
+
+    

nix.conf.example

@@ -0,0 +1,49 @@
+### Option `gc-keep-outputs'
+#
+# If `true', the garbage collector will keep the outputs of
+# non-garbage derivations.  If `false' (default), outputs will be
+# deleted unless they are GC roots themselves (or reachable from other
+# roots).
+# 
+# In general, outputs must be registered as roots separately.
+# However, even if the output of a derivation is registered as a root,
+# the collector will still delete store paths that are used only at
+# build time (e.g., the C compiler, or source tarballs downloaded from
+# the network).  To prevent it from doing so, set this option to
+# `true'.
+gc-keep-outputs = false
+
+
+### Option `gc-keep-derivations'
+#
+# If `true' (default), the garbage collector will keep the derivations
+# from which non-garbage store paths were built.  If `false', they
+# will be deleted unless explicitly registered as a root (or reachable
+# from other roots).
+#
+# Keeping derivation around is useful for querying and traceability
+# (e.g., it allows you to ask with what dependencies or options a
+# store path was built), so by default this option is on.  Turn it off
+# to safe a bit of disk space (or a lot if `gc-keep-outputs' is also
+# turned on).
+gc-keep-derivations = true
+
+
+### Option `env-keep-derivations'
+#
+# If `false' (default), derivations are not stored in Nix user
+# environments.  That is, the derivation any build-time-only
+# dependencies may be garbage-collected.
+#
+# If `true', when you add a Nix derivation to a user environment, the
+# path of the derivation is stored in the user environment.  Thus, the
+# derivation will not be garbage-collected until the user environment
+# generation is deleted (`nix-env --delete-generations').  To prevent
+# build-time-only dependencies from being collected, you should also
+# turn on `gc-keep-outputs'.
+#
+# The difference between this option and `gc-keep-derivations' is that
+# this one is `sticky': it applies to any user environment created
+# while this option was enabled, while `gc-keep-derivations' only
+# applies at the moment the garbage collector is run.
+env-keep-derivations = false

nix.spec.in

@@ -14,7 +14,7 @@ Release: 1
 License: GPL
 Group: Software Deployment
 URL: http://www.cs.uu.nl/groups/ST/Trace/Nix
-Source0: %{name}-@version@.tar.gz
+Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
 %define _prefix /nix
 Prefix: %{_prefix}

scripts/Makefile.am

@@ -2,7 +2,7 @@ bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build
 
-noinst_SCRIPTS = nix-profile.sh
+noinst_SCRIPTS = nix-profile.sh generate-patches.pl
 
 nix-pull nix-push: readmanifest.pm download-using-manifests.pl
 
@@ -22,4 +22,5 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-channel.in \
   readmanifest.pm.in \
   nix-build.in \
-  download-using-manifests.pl.in
+  download-using-manifests.pl.in \
+  generate-patches.pl.in

scripts/copying-collector.pl

@@ -0,0 +1,111 @@
+#! /usr/bin/perl -w
+
+use strict;
+
+my @paths = `nix-store -qR /home/eelco/.nix-profile/bin/firefox`;
+
+my %copyMap;
+my %rewriteMap;
+
+
+my $counter = 0;
+
+foreach my $path (@paths) {
+    chomp $path;
+
+    $path =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$path'";
+    my $hash = $2;
+
+#    my $newHash = "deadbeef" . (sprintf "%024d", $counter++);
+    my $newHash = "deadbeef" . substr($hash, 0, 24);
+    my $newPath = "/home/eelco/chroot/$1/$newHash-$3";
+
+    die unless length $newHash == length $hash;
+
+    $copyMap{$path} = $newPath;
+    $rewriteMap{$hash} = $newHash;
+}
+
+
+my %rewriteMap2;
+
+
+sub rewrite;
+sub rewrite {
+    my $src = shift;
+    my $dst = shift;
+
+    if (-l $dst) {
+
+        my $target = readlink $dst or die;
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+            $target =~ s/$srcHash/$dstHash/g;
+        }
+
+        unlink $dst or die;
+
+        symlink $target, $dst;
+        
+    }
+
+    elsif (-f $dst) {
+
+        print "$dst\n";
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+
+            my @stats = lstat $dst or die;
+            
+            system "sed s/$srcHash/$dstHash/g < '$dst' > '$dst.tmp'";
+            die if $? != 0;
+            rename "$dst.tmp", $dst or die;
+
+            chmod $stats[2], $dst or die;
+        }
+
+    }
+
+    elsif (-d $dst) {
+
+        chmod 0755, $dst;
+        
+        opendir(DIR, "$dst") or die "cannot open `$dst': $!";
+        my @files = readdir DIR;
+        closedir DIR;
+        
+        foreach my $file (@files) {
+            next if $file eq "." || $file eq "..";
+            rewrite "$src/$file", "$dst/$file";
+        }
+    }
+}
+
+
+foreach my $src (keys %copyMap) {
+    my $dst = $copyMap{$src};
+    print "$src -> $dst\n";
+
+    if (!-e $dst) {
+        system "cp -prd $src $dst";
+        die if $? != 0;
+
+        my @refs = `nix-store -q --references $src`;
+
+        %rewriteMap2 = ();
+        foreach my $ref (@refs) {
+            chomp $ref;
+
+            $ref =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$ref'";
+            my $hash = $2;
+
+            $rewriteMap2{$hash} = $rewriteMap{$hash};
+        }
+
+        rewrite $src, $dst;
+    }
+}

scripts/download-using-manifests.pl.in

@@ -13,7 +13,7 @@ open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
 die unless scalar @ARGV == 1;
 my $targetPath = $ARGV[0];
 
-my $date = `date`;
+my $date = `date` or die;
 chomp $date;
 print LOGFILE "$$ get $targetPath $date\n";
 
@@ -27,7 +27,10 @@ my %successors;
 
 for my $manifest (glob "$manifestDir/*.nixmanifest") {
 #    print STDERR "reading $manifest\n";
-    readManifest $manifest, \%narFiles, \%patches, \%successors;
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
+        exit 1;
+    }
 }
 
 
@@ -73,10 +76,19 @@ addToQueue $targetPath;
 
 sub isValidPath {
     my $p = shift;
-    system "nix-store --isvalid '$p' 2> /dev/null";
+    system "@bindir@/nix-store --check-validity '$p' 2> /dev/null";
     return $? == 0;
 }
 
+sub parseHash {
+    my $hash = shift;
+    if ($hash =~ /^(.+):(.+)$/) {
+        return ($1, $2);
+    } else {
+        return ("md5", $hash);
+    }
+}
+
 while ($queueFront < scalar @queue) {
     my $u = $queue[$queueFront++];
 #    print "$u\n";
@@ -96,10 +108,13 @@ while ($queueFront < scalar @queue) {
         foreach my $patch (@{$patchList}) {
             if (isValidPath($patch->{basePath})) {
                 # !!! this should be cached
-                my $hash = `nix-hash "$patch->{basePath}"`;
+                my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
+                my $format = "--base32";
+                $format = "" if $baseHashAlgo eq "md5";
+                my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
                 chomp $hash;
 #                print "  MY HASH is $hash\n";
-                if ($hash ne $patch->{baseHash}) {
+                if ($hash ne $baseHash) {
                     print LOGFILE "$$ rejecting $patch->{basePath}\n";
                     next;
                 }
@@ -174,13 +189,15 @@ my $maxStep = scalar @path;
 
 sub downloadFile {
     my $url = shift;
-    my $hash = shift;
+    my ($hashAlgo, $hash) = parseHash(shift);
     $ENV{"PRINT_PATH"} = 1;
     $ENV{"QUIET"} = 1;
-    my ($hash2, $path) = `nix-prefetch-url '$url' '$hash'`;
+    $ENV{"NIX_HASH_ALGO"} = $hashAlgo;
+    my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
+    die "download of `$url' failed" unless $? == 0;
     chomp $hash2;
     chomp $path;
-    die "hash mismatch" if $hash ne $hash2;
+    die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
     return $path;
 }
 
@@ -190,11 +207,19 @@ while (scalar @path > 0) {
     my $v = $edge->{end};
 
     print "\n*** Step $curStep/$maxStep: ";
-    $curStep++;
 
     if ($edge->{type} eq "present") {
         print "using already present path `$v'\n";
         print LOGFILE "$$ present $v\n";
+
+        if ($curStep < $maxStep) {
+            # Since this is not the last step, the path will be used
+            # as a base to one or more patches.  So turn the base path
+            # into a NAR archive, to which we can apply the patch.
+            print "  packing base path...\n";
+            system "@bindir@/nix-store --dump $v > /tmp/nar";
+            die "cannot dump `$v'" if ($? != 0);
+        }
     }
 
     elsif ($edge->{type} eq "patch") {
@@ -207,21 +232,22 @@ while (scalar @path > 0) {
         print "  downloading patch...\n";
         my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
 
-        # Turn the base path into a NAR archive, to which we can
-        # actually apply the patch.
-        print "  packing base path...\n";
-        system "nix-store --dump $patch->{basePath} > /tmp/nar";
-        die "cannot dump `$patch->{basePath}'" if ($? != 0);
-
-        # Apply the patch.
+        # Apply the patch to the NAR archive produced in step 1 (for
+        # the already present path) or a later step (for patch sequences).
         print "  applying patch...\n";
         system "@libexecdir@/bspatch /tmp/nar /tmp/nar2 $patchPath";
         die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
 
-        # Unpack the resulting NAR archive into the target path.
-        print "  unpacking patched archive...\n";
-        system "nix-store --restore $v < /tmp/nar2";
-        die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
+        if ($curStep < $maxStep) {
+            # The archive will be used as the base of the next patch.
+            rename "/tmp/nar2", "/tmp/nar" or die "cannot rename NAR archive: $!";
+        } else {
+            # This was the last patch.  Unpack the final NAR archive
+            # into the target path.
+            print "  unpacking patched archive...\n";
+            system "@bindir@/nix-store --restore $v < /tmp/nar2";
+            die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
+        }
     }
 
     elsif ($edge->{type} eq "narfile") {
@@ -234,11 +260,18 @@ while (scalar @path > 0) {
         print "  downloading archive...\n";
         my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
 
-        # Unpack the archive into the target path.
-        print "  unpacking archive...\n";
-        system "bunzip2 < '$narFilePath' | nix-store --restore '$v'";
-        die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
+        if ($curStep < $maxStep) {
+            # The archive will be used a base to a patch.
+            system "@bunzip2@ < '$narFilePath' > /tmp/nar";
+        } else {
+            # Unpack the archive into the target path.
+            print "  unpacking archive...\n";
+            system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
+            die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
+        }
     }
+
+    $curStep++;
 }
 
 

scripts/gc-releases.pl

@@ -0,0 +1,75 @@
+#! /usr/bin/perl -w
+
+use strict;
+use readmanifest;
+
+
+# Read the archive directories.
+my @archives = ();
+my %archives;
+
+sub readDir {
+    my $dir = shift;
+    opendir(DIR, "$dir") or die "cannot open `$dir': $!";
+    my @as = readdir DIR;
+    foreach my $archive (@as) {
+        push @archives, $archive;
+        $archives{$archive} = "$dir/$archive";
+    }
+    closedir DIR;
+}
+
+readDir "/mnt/scratchy/eelco/public_html/nix-cache";
+readDir "/mnt/scratchy/eelco/public_html/patches";
+
+print STDERR scalar @archives, "\n";
+
+
+# Read the manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+foreach my $manifest (@ARGV) {
+    print STDERR "loading $manifest\n";
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors, 1) < 3) {
+#        die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n";
+    }
+}
+
+
+# Find the live archives.
+my %usedFiles;
+
+foreach my $narFile (keys %narFiles) {
+    foreach my $file (@{$narFiles{$narFile}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+foreach my $patch (keys %patches) {
+    foreach my $file (@{$patches{$patch}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+
+# Print out the dead archives.
+foreach my $archive (@archives) {
+    next if $archive eq "." || $archive eq "..";
+    if (!defined $usedFiles{$archive}) {
+        print $archives{$archive}, "\n";
+    }
+}

scripts/generate-patches.pl.in

@@ -1,4 +1,4 @@
-#! /usr/bin/perl -w -I/home/eelco/nix/scripts
+#! @perl@ -w -I@libexecdir@/nix
 
 use strict;
 use POSIX qw(tmpnam);
@@ -6,6 +6,8 @@ use readmanifest;
 
 die unless scalar @ARGV == 5;
 
+my $hashAlgo = "sha256";
+
 my $cacheDir = $ARGV[0];
 my $patchesDir = $ARGV[1];
 my $patchesURL = $ARGV[2];
@@ -45,6 +47,7 @@ sub findOutputPaths {
 
         # Ignore store expressions.
         next if ($p =~ /\.store$/);
+        next if ($p =~ /\.drv$/);
         
         # Ignore builders (too much ambiguity -- they're all called
         # `builder.sh').
@@ -69,7 +72,7 @@ my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
 
 sub getNameVersion {
     my $p = shift;
-    $p =~ /\/[0-9a-f]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
+    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
     my $name = $1;
     my $version = $2;
     $name =~ s/^-//;
@@ -125,6 +128,64 @@ sub containsPatch {
 }
 
 
+# Compute the "weighted" number of uses of a path in the build graph.
+sub computeUses {
+    my $narFiles = shift;
+    my $path = shift;
+
+    # Find the deriver of $path.
+    return 1 unless defined $$narFiles{$path};
+    my $deriver = @{$$narFiles{$path}}[0]->{deriver};
+    return 1 unless defined $deriver && $deriver ne "";
+
+#    print "  DERIVER $deriver\n";
+
+    # Optimisation: build the referers graph from the references
+    # graph.
+    my %referers;
+    foreach my $q (keys %{$narFiles}) {
+        my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
+        foreach my $r (@refs) {
+            $referers{$r} = [] unless defined $referers{$r};
+            push @{$referers{$r}}, $q;
+        }
+    }
+
+    # Determine the shortest path from $deriver to all other reachable
+    # paths in the `referers' graph.
+
+    my %dist;
+    $dist{$deriver} = 0;
+
+    my @queue = ($deriver);
+    my $pos = 0;
+    
+    while ($pos < scalar @queue) {
+        my $p = $queue[$pos];
+        $pos++;
+
+        foreach my $q (@{$referers{$p}}) {
+            if (!defined $dist{$q}) {
+                $dist{$q} = $dist{$p} + 1;
+#                print "    $q $dist{$q}\n";
+                push @queue, $q;
+            }
+        }
+    }
+
+    my $wuse = 1.0;
+    foreach my $user (keys %dist) {
+        next if $user eq $deriver;
+#        print "    $user $dist{$user}\n";
+        $wuse += 1.0 / 2.0**$dist{$user};
+    }
+
+#    print "    XXX $path $wuse\n";
+    
+    return $wuse;
+}
+
+
 # For each output path in the destination, see if we need to / can
 # create a patch.
 
@@ -135,9 +196,9 @@ foreach my $p (keys %dstOutPaths) {
     # If exactly the same path already exists in the source, skip it.
     next if defined $srcOutPaths{$p};
     
-#    print "  $p\n";
+    print "  $p\n";
 
-    # If not, then we should find the path in the source that is
+    # If not, then we should find the paths in the source that are
     # `most' likely to be present on a system that wants to install
     # this path.
 
@@ -152,6 +213,37 @@ foreach my $p (keys %dstOutPaths) {
     foreach my $q (keys %srcOutPaths) {
         (my $name2, my $version2) = getNameVersion $q;
         if ($name eq $name2) {
+
+            # If the sizes differ too much, then skip.  This
+            # disambiguates between, e.g., a real component and a
+            # wrapper component (cf. Firefox in Nixpkgs).
+            my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
+            my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
+            my $ratio = $srcSize / $dstSize;
+            $ratio = 1 / $ratio if $ratio < 1;
+#            print "  SIZE $srcSize $dstSize $ratio $q\n";
+
+            if ($ratio >= 3) {
+                print "  SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
+                next;
+            }
+
+            # If the numbers of weighted uses differ too much, then
+            # skip.  This disambiguates between, e.g., the bootstrap
+            # GCC and the final GCC in Nixpkgs.
+            my $srcUses = computeUses \%srcNarFiles, $q;
+            my $dstUses = computeUses \%dstNarFiles, $p;
+            $ratio = $srcUses / $dstUses;
+            $ratio = 1 / $ratio if $ratio < 1;
+            print "  USE $srcUses $dstUses $ratio $q\n";
+            
+#            if ($ratio >= 2) {
+#                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
+#                next;
+#            }
+
+            # If there are multiple matching names, include the ones
+            # with the closest version numbers.
             my $dist = versionDiff $version, $version2;
             if ($dist > $minDist) {
                 $minDist = $dist;
@@ -186,22 +278,22 @@ foreach my $p (keys %dstOutPaths) {
         my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
         my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
         
-        system("bunzip2 < $srcNarBz2 > $tmpdir/A") == 0
+        system("@bunzip2@ < $srcNarBz2 > $tmpdir/A") == 0
             or die "cannot unpack $srcNarBz2";
 
-        system("bunzip2 < $dstNarBz2 > $tmpdir/B") == 0
+        system("@bunzip2@ < $dstNarBz2 > $tmpdir/B") == 0
             or die "cannot unpack $dstNarBz2";
 
-        system("bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
+        system("@libexecdir@/bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
             or die "cannot compute binary diff";
 
-        my $baseHash = `nix-hash --flat $tmpdir/A` or die;
+        my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/A` or die;
         chomp $baseHash;
 
-        my $narHash = `nix-hash --flat $tmpdir/B` or die;
+        my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/B` or die;
         chomp $narHash;
 
-        my $narDiffHash = `nix-hash --flat $tmpdir/DIFF` or die;
+        my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/DIFF` or die;
         chomp $narDiffHash;
 
         my $narDiffSize = (stat "$tmpdir/DIFF")[7];
@@ -213,7 +305,7 @@ foreach my $p (keys %dstOutPaths) {
         }
     
         my $finalName =
-            "$narDiffHash-$name-$closestVersion-to-$version.nar-bsdiff";
+            "$narDiffHash.nar-bsdiff";
 
         print "    size $narDiffSize; full size $dstNarBz2Size\n";
         
@@ -233,11 +325,10 @@ foreach my $p (keys %dstOutPaths) {
         
         # Add the patch to the manifest.
         addPatch \%dstPatches, $p,
-            { url => "$patchesURL/$finalName", hash => $narDiffHash
-            , size => $narDiffSize
-            , basePath => $closest, baseHash => $baseHash
-            , narHash => $narHash, patchType => "nar-bsdiff"
-            };
+            { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
+            , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
+            , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
+            }, 0;
     }
 }
 

scripts/nix-build.in

@@ -8,34 +8,52 @@ if test -z "$nixExpr"; then
 fi
 
 extraArgs=
-noLink=
+addDrvLink=0
+addOutLink=1
 
+
+trap 'rm -f ./.nix-build-tmp-*' EXIT
+
+
+# Process the arguments.
 for i in "$@"; do
     case "$i" in
-        --no-link)
-            noLink=1
+    
+        --add-drv-link)
+            addDrvLink=1
             ;;
+            
+        --no-link)
+            addOutLink=0
+            ;;
+            
         -*)
             extraArgs="$extraArgs $i"
             ;;
+            
         *)
-            storeExprs=$(nix-instantiate "$i")
+            # Instantiate the Nix expression.
+            prefix=
+            if test "$addDrvLink" = 0; then prefix=.nix-build-tmp-; fi
+            storeExprs=$(@bindir@/nix-instantiate \
+                --add-root ./${prefix}derivation --indirect \
+                "$i")
+                
             for j in $storeExprs; do
-                echo "store expression is $j" >&2
+                echo "store expression is $(readlink "$j")" >&2
             done
-            outPaths=$(nix-store -qnfv $extraArgs $storeExprs)
+
+            # Build the resulting store derivation.
+            prefix=
+            if test "$addOutLink" = 0; then prefix=.nix-build-tmp-; fi
+            outPaths=$(@bindir@/nix-store \
+                --add-root ./${prefix}result --indirect \
+                -rv $extraArgs $storeExprs)
+                
             for j in $outPaths; do
-                echo "$j"
-                if test -z "$noLink"; then
-                    if test -L result; then
-                        rm result
-                    elif test -e result; then
-                        echo "cannot remove \`result' (not a symlink)"
-                        exit 1
-                    fi
-                    ln -s "$j" result
-                fi
+                echo "$(readlink "$j")"
             done
+            
             ;;
     esac
 done

scripts/nix-channel.in

@@ -48,6 +48,19 @@ sub addChannel {
 }
 
 
+# Remove a channel from the file $channelsList;
+sub removeChannel {
+    my $url = shift;
+    my @left = ();
+    readChannels;
+    foreach my $url2 (@channels) {
+        push @left, $url2 if $url ne $url2;
+    }
+    @channels = @left;
+    writeChannels;
+}
+
+
 # Fetch Nix expressions and pull cache manifests from the subscribed
 # channels.
 sub update {
@@ -68,69 +81,94 @@ sub update {
     # expressions.
 
     my $nixExpr = "[";
-    
     foreach my $url (@channels) {
         my $fullURL = "$url/nixexprs.tar.bz2";
-        my $hash = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`
-            or die "cannot fetch `$fullURL'";
-        chomp $hash;
-        # !!! escaping
-        $nixExpr .= "((import @datadir@/nix/corepkgs/fetchurl) " .
-            "{url = $fullURL; md5 = \"$hash\"; system = \"@system@\";}) "
+        $ENV{"PRINT_PATH"} = 1;
+        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
+        die "cannot fetch `$fullURL'" if $? != 0;
+        chomp $path;
+        $nixExpr .= $path . " ";
     }
-
     $nixExpr .= "]";
 
     $nixExpr =
         "(import @datadir@/nix/corepkgs/channels/unpack.nix) " .
         "{inputs = $nixExpr; system = \"@system@\";}";
 
-    # Instantiate the Nix expression.
-    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
-        or die "cannot instantiate Nix expression";
-    chomp $storeExpr;
-
-    # Register the store expression as a root of the garbage
-    # collector.
+    # Figure out a name for the GC root.
     my $userName = getpwuid($<);
     die "who ARE you? go away" unless defined $userName;
 
-    my $rootFile = "$rootsDir/$userName.gcroot";
-    my $tmpRootFile = "$rootsDir/$userName-tmp.gcroot";
+    my $rootFile = "$rootsDir/$userName";
     
-    open ROOT, ">$tmpRootFile" or die "cannot create `$tmpRootFile': $!";
-    print ROOT "$storeExpr";
-    close ROOT;
+    # Instantiate the Nix expression.
+    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate --add-root '$rootFile'.tmp -`
+        or die "cannot instantiate Nix expression";
+    chomp $storeExpr;
 
-    # Realise the store expression.
-    my $outPath = `nix-store -qnf '$storeExpr'`
+    # Build the resulting derivation.
+    my $outPath = `nix-store --add-root '$rootFile' -r '$storeExpr'`
         or die "cannot realise store expression";
     chomp $outPath;
 
+    unlink "$rootFile.tmp";
+
     # Make it the default Nix expression for `nix-env'.
     system "@bindir@/nix-env --import '$outPath'";
     die "cannot pull set default Nix expression to `$outPath'" if ($? != 0);
-
-    rename $tmpRootFile, $rootFile or die "cannot rename `$tmpRootFile' to `$rootFile': $!";
 }
 
 
+sub usageError {
+    print STDERR <<EOF;
+Usage:
+  nix-channel --add URL
+  nix-channel --remove URL
+  nix-channel --list
+  nix-channel --update
+EOF
+    exit 1;
+}
+
+
+usageError if scalar @ARGV == 0;
+
+
 while (scalar @ARGV) {
     my $arg = shift @ARGV;
 
     if ($arg eq "--add") {
-        die "syntax: nix-channel --add URL" if (scalar @ARGV != 1);
+        usageError if scalar @ARGV != 1;
         addChannel (shift @ARGV);
         last;
     }
 
-    elsif ($arg eq "--update") {
-        die "syntax: nix-channel --update" if (scalar @ARGV != 0);
-        update;
+    if ($arg eq "--remove") {
+        usageError if scalar @ARGV != 1;
+        removeChannel (shift @ARGV);
         last;
     }
 
+    if ($arg eq "--list") {
+        usageError if scalar @ARGV != 0;
+        readChannels;
+        foreach my $url (@channels) {
+            print "$url\n";
+        }
+        last;
+    }
+
+    elsif ($arg eq "--update") {
+        usageError if scalar @ARGV != 0;
+        update;
+        last;
+    }
+    
+    elsif ($arg eq "--help") {
+        usageError;
+    }
+
     else {
-        die "unknown argument `$arg'";
+        die "unknown argument `$arg'; try `--help'";
     }
 }

scripts/nix-collect-garbage.in

@@ -1,90 +1,2 @@
-#! @perl@ -w
-
-use strict;
-use IPC::Open2;
-
-my $rootsDir = "@localstatedir@/nix/gcroots";
-my $storeDir = "@storedir@";
-
-my %alive;
-
-my $gcOper = "--delete";
-my $minAge = 0;
-
-my @roots = ();
-
-
-# Parse the command line.
-for (my $i = 0; $i < scalar @ARGV; $i++) {
-    my $arg = $ARGV[$i];
-    if ($arg eq "--delete" || $arg eq "--print-live" || $arg eq "--print-dead") {
-        $gcOper = $arg;
-    }
-    elsif ($arg eq "--min-age") {
-        $i++;
-        $minAge = undef;
-        $minAge = $ARGV[$i];
-        die "invalid minimum age" unless defined $minAge && $minAge =~ /^\d*$/;
-    }
-    else { die "unknown argument `$arg'" };
-}
-
-
-# Read all GC roots from the given file.
-sub readRoots {
-    my $fileName = shift;
-    open ROOT, "<$fileName" or die "cannot open `$fileName': $!";
-    while (<ROOT>) {
-        chomp;
-        foreach my $root (split ' ') {
-            die "bad root `$root' in file `$fileName'"
-                unless $root =~ /^\S+$/;
-            push @roots, $root;
-        }
-    }
-    close ROOT;
-}
-
-
-# Recursively finds all *.gcroot files in the given directory.
-sub findRoots;
-sub findRoots {
-    my $followSymlinks = shift;
-    my $dir = shift;
-
-    opendir(DIR, $dir) or die "cannot open directory `$dir': $!";
-    my @names = readdir DIR or die "cannot read directory `$dir': $!";
-    closedir DIR;
-
-    foreach my $name (@names) {
-        next if $name eq "." || $name eq "..";
-        $name = $dir . "/" . $name;
-        if ($name =~ /.gcroot$/ && -f $name) {
-            readRoots $name;
-        }
-        elsif (-d $name) {
-            if ($followSymlinks || !-l $name) {
-                findRoots 0, $name;
-            }
-        }
-    }
-    
-}
-
-
-# Find GC roots, starting at $rootsDir.
-findRoots 1, $rootsDir;
-
-
-# Run the collector with the roots we found.
-my $pid = open2(">&1", \*WRITE, "@bindir@/nix-store --gc $gcOper --min-age $minAge")
-    or die "cannot run `nix-store --gc'";
-
-foreach my $root (@roots) {
-    print WRITE "$root\n";
-}
-
-close WRITE;
-
-waitpid $pid, 0;
-$? == 0 or die "`nix-store --gc' failed";
+#! @shell@ -e
+exec @bindir@/nix-store --gc "$@"

scripts/nix-install-package.in

@@ -3,35 +3,49 @@
 use strict;
 use POSIX qw(tmpnam);
 
-my $pkgfile = $ARGV[0];
-die unless defined $pkgfile;
+my $pkgFile = $ARGV[0];
+die unless defined $pkgFile;
 
-my $tmpdir;
-do { $tmpdir = tmpnam(); }
-until mkdir $tmpdir, 0777;
 
-# !!! remove tmpdir on exit
+# Re-execute in a terminal, if necessary, so that if we're executed
+# from a web browser, the user gets to see us.
+if (!defined $ENV{"NIX_HAVE_TERMINAL"}) {
+    $ENV{"NIX_HAVE_TERMINAL"} = "1";
+    $ENV{"LD_LIBRARY_PATH"} = "";
+    exec("xterm", "-e", "@shell@", "-c", "@bindir@/nix-install-package '$pkgFile' || read");
+    die "cannot execute `xterm'";
+}
 
-print "Unpacking $pkgfile in $tmpdir...\n";
-system "bunzip2 < $pkgfile | (cd $tmpdir && tar xf -)";
-die if $?;
 
-print "This package contains the following derivations:\n";
-system "@bindir@/nix-env -qasf $tmpdir/default.nix";
-die if $?;
+# Read and parse the package file.
+open PKGFILE, "<$pkgFile" or die "cannot open `$pkgFile': $!";
+my $contents = <PKGFILE>;
+close PKGFILE;
 
-print "Do you wish to install these (Y/N)? ";
+$contents =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ or die "invalid package contents";
+my $version = $1;
+my $manifestURL = $2;
+my $drvName = $3;
+my $system = $4;
+my $drvPath = $5;
+my $outPath = $6;
+
+die "invalid package version `$version'" unless $version eq "NIXPKG1";
+
+
+# Ask confirmation.
+print "Do you want to install `$drvName' (Y/N)? ";
 my $reply = <STDIN>;
 chomp $reply;
-exit if (!($reply eq "y"));
+exit if $reply ne "y" && $reply ne "Y";
 
-print "Pulling caches...\n";
-system "@bindir@/nix-pull `cat $tmpdir/caches`";
-die if $?;
+print "\nPulling manifests...\n";
+system "@bindir@/nix-pull '$manifestURL'";
+die if $? != 0;
 
-print "Installing package...\n";
-system "@bindir@/nix-env -if $tmpdir/default.nix '*'";
-die if $?;
+print "\nInstalling package...\n";
+system "@bindir@/nix-env -i '$outPath'";
+die if $? != 0;
 
-print "Installation succeeded! Press Enter to continue.\n";
+print "\nInstallation succeeded! Press Enter to continue.\n";
 <STDIN>;

scripts/nix-prefetch-url.in

@@ -1,58 +1,67 @@
 #! @shell@ -e
 
 url=$1
-hash=$2
+expHash=$2
+
+hashType=$NIX_HASH_ALGO
+if test -z "$hashType"; then
+    hashType=md5
+fi
+
+hashFormat=
+if test "$hashType" != "md5"; then
+    hashFormat=--base32
+fi
 
 if test -z "$url"; then
-    echo "syntax: nix-prefetch-url URL" >&2
+    echo "syntax: nix-prefetch-url URL [EXPECTED-HASH]" >&2
     exit 1
 fi
 
-# Determine the hash, unless it was given.
-if test -z "$hash"; then
+name=$(basename "$url")
+if test -z "$name"; then echo "invalid url"; exit 1; fi
 
-    # !!! race
-    tmpPath1=@storedir@/nix-prefetch-url-$$
 
-    # Test whether we have write permission in the store.  If not,
-    # fetch to /tmp and don't copy to the store.  This is a hack to
-    # make this script at least work somewhat in setuid installations.
-    if ! touch $tmpPath1 2> /dev/null; then
-        echo "(cannot write to the store, result won't be cached)" >&2
-        dummyMode=1
-        tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
+# If the hash was given, a file with that hash may already be in the
+# store.
+if test -n "$expHash"; then
+    finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$expHash" "$name")
+    if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
+        finalPath=
     fi
+    hash=$expHash
+fi
 
-    # Perform the checkout.
-    @curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
+
+# If we don't know the hash or a file with that hash doesn't exist,
+# download the file and add it to the store.
+if test -z "$finalPath"; then
+
+    tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
+    tmpFile=$tmpPath/$name
+    mkdir $tmpPath
+
+    # Perform the download.
+    @curl@ --fail --location --max-redirs 20 "$url" > $tmpFile
 
     # Compute the hash.
-    hash=$(@bindir@/nix-hash --flat $tmpPath1)
+    hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
     if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
 
-    # Rename it so that the fetchurl builder can find it.
-    if test "$dummyMode" != 1; then
-        tmpPath2=@storedir@/nix-prefetch-url-$hash
-        test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
+    # Add the downloaded file to the Nix store.
+    finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+
+    if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+
+    if test -n "$expHash" -a "$expHash" != "$hash"; then
+        echo "hash mismatch for URL \`$url'"
+        exit 1
     fi
-
 fi
 
-# Create a Nix expression that does a fetchurl.
-storeExpr=$( \
-    echo "(import @datadir@/nix/corepkgs/fetchurl) \
-        {url = $url; md5 = \"$hash\"; system = \"@system@\";}" \
-    | @bindir@/nix-instantiate -)
 
-# Realise it.
-finalPath=$(@bindir@/nix-store -qnB --force-realise $storeExpr)
-    
 if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
 
-if test -n "$tmpPath1" -o -n "$tmpPath2"; then
-    rm -rf $tmpPath1 $tmpPath2 || true
-fi
-
 echo $hash
 
 if test -n "$PRINT_PATH"; then

scripts/nix-pull.in

@@ -13,6 +13,19 @@ my $manifest = "$tmpdir/manifest";
 
 END { unlink $manifest; rmdir $tmpdir; }
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
+$libexecDir = "@libexecdir@" unless defined $libexecDir;
+
+my $stateDir = $ENV{"NIX_STATE_DIR"};
+$stateDir = "@localstatedir@/nix" unless defined $stateDir;
+
+
+# Prevent access problems in shared-stored installations.
+umask 0022;
+
 
 # Obtain URLs either from the command line or from a configuration file.
 my %narFiles;
@@ -29,20 +42,22 @@ sub processURL {
            "'$url' > '$manifest'") == 0
            or die "curl failed: $?";
 
-    readManifest $manifest, \%narFiles, \%patches, \%successors;
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
+    }
 
     my $baseName = "unnamed";
     if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
         $baseName = $1;
     }
 
-    my $hash = `@bindir@/nix-hash --flat '$manifest'`
+    my $hash = `$binDir/nix-hash --flat '$manifest'`
         or die "cannot hash `$manifest'";
     chomp $hash;
     
-    my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest";
+    my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
     
-    system("mv '$manifest' '$finalPath'") == 0
+    system("mv -f '$manifest' '$finalPath'") == 0
         or die "cannot move `$manifest' to `$finalPath";
 }
 
@@ -59,7 +74,7 @@ print "$size store paths in manifest\n";
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
 
-my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --substitute")
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --register-substitutes")
     or die "cannot run nix-store";
 
 close READ;
@@ -68,8 +83,15 @@ foreach my $storePath (keys %narFiles) {
     my $narFileList = $narFiles{$storePath};
     foreach my $narFile (@{$narFileList}) {
         print WRITE "$storePath\n";
-        print WRITE "@libexecdir@/nix/download-using-manifests.pl\n";
+        print WRITE "$narFile->{deriver}\n";
+        print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
         print WRITE "0\n";
+        my @references = split " ", $narFile->{references};
+        my $count = scalar @references;
+        print WRITE "$count\n";
+        foreach my $reference (@references) {
+            print WRITE "$reference\n";
+        }
     }
 }
 
@@ -77,16 +99,3 @@ close WRITE;
 
 waitpid $pid, 0;
 $? == 0 or die "nix-store failed";
-
-
-# Register all successors.
-print STDERR "registering successors...\n";
-my @sucs = %successors;
-while (scalar @sucs > 0) {
-    my $n = scalar @sucs;
-    if ($n > 256) { $n = 256 };
-    my @sucs2 = @sucs[0..$n - 1];
-    @sucs = @sucs[$n..scalar @sucs - 1];
-    system "@bindir@/nix-store --successor @sucs2";
-    if ($?) { die "`nix-store --successor' failed"; }
-}

scripts/nix-push.in

@@ -1,9 +1,12 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
 
 use strict;
+use IPC::Open2;
 use POSIX qw(tmpnam);
 use readmanifest;
 
+my $hashAlgo = "sha256";
+
 my $tmpdir;
 do { $tmpdir = tmpnam(); }
 until mkdir $tmpdir, 0777;
@@ -17,32 +20,62 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $dataDir = $ENV{"NIX_DATA_DIR"};
+$dataDir = "@datadir@" unless defined $dataDir;
+
 
 # Parse the command line.
-my $archives_put_url = shift @ARGV;
-my $archives_get_url = shift @ARGV;
-my $manifest_put_url = shift @ARGV;
+my $localCopy;
+my $localArchivesDir;
+my $localManifestFile;
+
+my $archivesPutURL;
+my $archivesGetURL;
+my $manifestPutURL;
+
+if ($ARGV[0] eq "--copy") {
+    die "syntax: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 1;
+    shift @ARGV;
+    $localArchivesDir = shift @ARGV;
+    $localManifestFile = shift @ARGV;
+}
+else {
+    die "syntax: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL " .
+        "MANIFEST_PUT_URL PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 0;
+    $archivesPutURL = shift @ARGV;
+    $archivesGetURL = shift @ARGV;
+    $manifestPutURL = shift @ARGV;
+}
 
 
-# From the given store expressions, determine the requisite store
-# paths.
+# From the given store paths, determine the set of requisite store
+# paths, i.e, the paths required to realise them.
 my %storePaths;
 
-foreach my $storeexpr (@ARGV) {
-    die unless $storeexpr =~ /^\//;
+foreach my $path (@ARGV) {
+    die unless $path =~ /^\//;
 
     # Get all paths referenced by the normalisation of the given 
     # Nix expression.
-    system "@bindir@/nix-store --realise $storeexpr > /dev/null";
-    die if ($?);
-
-    open PATHS, "@bindir@/nix-store --query --requisites --include-successors $storeexpr 2> /dev/null |" or die;
-    while (<PATHS>) {
+    my $pid = open2(\*READ, \*WRITE,
+        "$binDir/nix-store --query --requisites --force-realise " .
+        "--include-outputs '$path'") or die;
+    close WRITE;
+    
+    while (<READ>) {
         chomp;
         die "bad: $_" unless /^\//;
         $storePaths{$_} = "";
     }
-    close PATHS;
+    close READ;
+    
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
 my @storePaths = keys %storePaths;
@@ -58,9 +91,8 @@ foreach my $storePath (@storePaths) {
 
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
-        "((import @datadir@/nix/corepkgs/nar/nar.nix) " .
-        # !!! $storePath should be represented as a closure
-        "{path = \"$storePath\"; system = \"@system@\";}) ";
+        "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
+        "{path = \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
     
     print NIX $nixexpr;
 }
@@ -70,39 +102,46 @@ close NIX;
 
 
 # Instantiate store expressions from the Nix expression.
-my @storeexprs;
+my @storeExprs;
 print STDERR "instantiating store expressions...\n";
-open STOREEXPRS, "@bindir@/nix-instantiate $nixfile |" or die "cannot run nix-instantiate";
-while (<STOREEXPRS>) {
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-instantiate $nixfile")
+    or die "cannot run nix-instantiate";
+close WRITE;
+while (<READ>) {
     chomp;
     die unless /^\//;
-    push @storeexprs, $_;
+    push @storeExprs, $_;
 }
-close STOREEXPRS;
+close READ;
+
+waitpid $pid, 0;
+$? == 0 or die "nix-instantiate failed";
 
 
 # Realise the store expressions.
 print STDERR "creating archives...\n";
 
-my @narpaths;
+my @narPaths;
 
-my @tmp = @storeexprs;
+my @tmp = @storeExprs;
 while (scalar @tmp > 0) {
     my $n = scalar @tmp;
     if ($n > 256) { $n = 256 };
     my @tmp2 = @tmp[0..$n - 1];
     @tmp = @tmp[$n..scalar @tmp - 1];
 
-    system "@bindir@/nix-store --realise @tmp2 > /dev/null";
-    if ($?) { die "`nix-store --realise' failed"; }
-
-    open NARPATHS, "@bindir@/nix-store --query --list @tmp2 |" or die "cannot run nix";
-    while (<NARPATHS>) {
+    my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --realise @tmp2")
+        or die "cannot run nix-store";
+    close WRITE;
+    while (<READ>) {
         chomp;
         die unless (/^\//);
-        push @narpaths, "$_";
+        push @narPaths, "$_";
     }
-    close NARPATHS;
+    close READ;
+
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
 
@@ -111,83 +150,110 @@ print STDERR "creating manifest...\n";
 
 my %narFiles;
 my %patches;
-my %successors;
 
-my @nararchives;
+my @narArchives;
 for (my $n = 0; $n < scalar @storePaths; $n++) {
     my $storePath = $storePaths[$n];
-    my $nardir = $narpaths[$n];
+    my $narDir = $narPaths[$n];
     
     $storePath =~ /\/([^\/]*)$/;
     my $basename = $1;
     defined $basename or die;
 
-    my $narname = "$basename.nar.bz2";
-
-    my $narfile = "$nardir/$narname";
-    (-f $narfile) or die "narfile for $storePath not found";
-    push @nararchives, $narfile;
-
-    open MD5, "$nardir/narbz2-hash" or die "cannot open narbz2-hash";
-    my $narbz2Hash = <MD5>;
+    open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
+    my $narbz2Hash = <HASH>;
     chomp $narbz2Hash;
-    $narbz2Hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
-    close MD5;
+    $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
 
-    open MD5, "$nardir/nar-hash" or die "cannot open nar-hash";
-    my $narHash = <MD5>;
+    open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
+    my $narHash = <HASH>;
     chomp $narHash;
-    $narHash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
-    close MD5;
+    $narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
     
-    my $narbz2Size = (stat $narfile)[7];
+    my $narName = "$narbz2Hash.nar.bz2";
 
+    my $narFile = "$narDir/$narName";
+    (-f $narFile) or die "narfile for $storePath not found";
+    push @narArchives, $narFile;
+
+    my $narbz2Size = (stat $narFile)[7];
+
+    my $references = `$binDir/nix-store --query --references '$storePath'`;
+    die "cannot query references for `$storePath'" if $? != 0;
+    $references = join(" ", split(" ", $references));
+
+    my $deriver = `$binDir/nix-store --query --deriver '$storePath'`;
+    die "cannot query deriver for `$storePath'" if $? != 0;
+    chomp $deriver;
+    $deriver = "" if $deriver eq "unknown-deriver";
+
+    my $url;
+    if ($localCopy) {
+        $url = "file://$localArchivesDir/$narName";
+    } else {
+        $url = "$archivesGetURL/$narName";
+    }
     $narFiles{$storePath} = [
-        { url => $archives_get_url/$narname
-        , hash => $narbz2Hash
+        { url => $url
+        , hash => "$hashAlgo:$narbz2Hash"
         , size => $narbz2Size
-        , narHash => $narHash
+        , narHash => "$hashAlgo:$narHash"
+        , references => $references
+        , deriver => $deriver
         }
     ];
-                            
-    if ($storePath =~ /\.store$/) {
-        open PREDS, "@bindir@/nix-store --query --predecessors $storePath |" or die "cannot run nix";
-        while (<PREDS>) {
-            chomp;
-            die unless (/^\//);
-            my $pred = $_;
-            # Only include predecessors that are themselves being
-            # pushed.
-            if (defined $storePaths{$pred}) {
-                $successors{$pred} = $storePath;
-            }
-        }
-        close PREDS;
-    }
-
 }
 
-writeManifest $manifest, \%narFiles, \%patches, \%successors;
+writeManifest $manifest, \%narFiles, \%patches;
+
+
+sub copyFile {
+    my $src = shift;
+    my $dst = shift;
+    system("cp '$src' '$dst.tmp'") == 0 or die "cannot copy file";
+    rename("$dst.tmp", "$dst") or die "cannot rename file";
+}
 
 
 # Upload the archives.
 print STDERR "uploading archives...\n";
-foreach my $nararchive (@nararchives) {
 
-    $nararchive =~ /\/([^\/]*)$/;
+sub archiveExists {
+    my $name = shift;
+    print STDERR "  HEAD on $archivesGetURL/$name\n";
+    return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
+}
+
+foreach my $narArchive (@narArchives) {
+
+    $narArchive =~ /\/([^\/]*)$/;
     my $basename = $1;
 
-    if (system("$curl --head $archives_get_url/$basename > /dev/null") != 0) {
-        print STDERR "  $nararchive\n";
-        system("$curl --show-error --upload-file " .
-               "'$nararchive' '$archives_put_url/$basename' > /dev/null") == 0 or
-            die "curl failed on $nararchive: $?";
+    if ($localCopy) {
+        if (! -f "$localArchivesDir/$basename") {
+            print STDERR "  $narArchive\n";
+            copyFile $narArchive, "$localArchivesDir/$basename";
+        }
+    }
+    else {
+        if (!archiveExists("$basename")) {
+            print STDERR "  $narArchive\n";
+            system("$curl --show-error --upload-file " .
+                   "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
+                   die "curl failed on $narArchive: $?";
+        }
     }
 }
 
 
 # Upload the manifest.
 print STDERR "uploading manifest...\n";
-system("$curl  --show-error --upload-file " .
-       "'$manifest' '$manifest_put_url' > /dev/null") == 0 or
-    die "curl failed on $manifest: $?";
+if ($localCopy) {
+    copyFile $manifest, $localManifestFile;
+} else {
+    system("$curl  --show-error --upload-file " .
+           "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
+           die "curl failed on $manifest: $?";
+}

scripts/readmanifest.pm.in

@@ -5,6 +5,7 @@ sub addPatch {
     my $patches = shift;
     my $storePath = shift;
     my $patch = shift;
+    my $allowConflicts = shift;
 
     $$patches{$storePath} = []
         unless defined $$patches{$storePath};
@@ -18,7 +19,8 @@ sub addPatch {
                 $found = 1 if ($patch2->{basePath} eq $patch->{basePath});
             } else {
                 die "conflicting hashes for URL $patch->{url}, " .
-                    "namely $patch2->{hash} and $patch->{hash}";
+                    "namely $patch2->{hash} and $patch->{hash}"
+                    unless $allowConflicts;
             }
         }
     }
@@ -34,12 +36,17 @@ sub readManifest {
     my $narFiles = shift;
     my $patches = shift;
     my $successors = shift;
+    my $allowConflicts = shift;
+    $allowConflicts = 0 unless defined $allowConflicts;
 
-    open MANIFEST, "<$manifest";
+    open MANIFEST, "<$manifest"
+        or die "cannot open `$manifest': $!";
 
     my $inside = 0;
     my $type;
 
+    my $manifestVersion = 2;
+
     my $storePath;
     my $url;
     my $hash;
@@ -49,6 +56,9 @@ sub readManifest {
     my $baseHash;
     my $patchType;
     my $narHash;
+    my $references;
+    my $deriver;
+    my $hashAlgo;
 
     while (<MANIFEST>) {
         chomp;
@@ -70,6 +80,9 @@ sub readManifest {
                 undef $basePath;
                 undef $baseHash;
                 undef $patchType;
+                $references = "";
+                $deriver = "";
+                $hashAlgo = "md5";
 	    }
 
         } else {
@@ -91,14 +104,16 @@ sub readManifest {
                                 $found = 1;
                             } else {
                                 die "conflicting hashes for URL $url, " .
-                                    "namely $narFile->{hash} and $hash";
+                                    "namely $narFile->{hash} and $hash"
+                                    unless $allowConflicts;
                             }
                         }
                     }
                     if (!$found) {
                         push @{$narFileList},
                             { url => $url, hash => $hash, size => $size
-                            , narHash => $narHash
+                            , narHash => $narHash, references => $references
+                            , deriver => $deriver, hashAlgo => $hashAlgo
                             };
                     }
                 
@@ -113,7 +128,8 @@ sub readManifest {
                         { url => $url, hash => $hash, size => $size
                         , basePath => $basePath, baseHash => $baseHash
                         , narHash => $narHash, patchType => $patchType
-                        };
+                        , hashAlgo => $hashAlgo
+                        }, $allowConflicts;
                 }
 
             }
@@ -127,15 +143,20 @@ sub readManifest {
             elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
             elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
             elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+            elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
+            elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
+            elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
 
             # Compatibility;
             elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
-            elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = $1; }
+            elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; }
 
         }
     }
 
     close MANIFEST;
+
+    return $manifestVersion;
 }
 
 
@@ -144,24 +165,26 @@ sub writeManifest
     my $manifest = shift;
     my $narFiles = shift;
     my $patches = shift;
-    my $successors = shift;
 
     open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
 
+    print MANIFEST "version {\n";
+    print MANIFEST "  ManifestVersion: 3\n";
+    print MANIFEST "}\n";
+
     foreach my $storePath (keys %{$narFiles}) {
         my $narFileList = $$narFiles{$storePath};
         foreach my $narFile (@{$narFileList}) {
             print MANIFEST "{\n";
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $narFile->{url}\n";
-            print MANIFEST "  MD5: $narFile->{hash}\n";
+            print MANIFEST "  Hash: $narFile->{hash}\n";
             print MANIFEST "  NarHash: $narFile->{narHash}\n";
             print MANIFEST "  Size: $narFile->{size}\n";
-            foreach my $p (keys %{$successors}) { # !!! quadratic
-                if ($$successors{$p} eq $storePath) {
-                    print MANIFEST "  SuccOf: $p\n";
-                }
-            }
+            print MANIFEST "  References: $narFile->{references}\n"
+                if defined $narFile->{references} && $narFile->{references} ne "";
+            print MANIFEST "  Deriver: $narFile->{deriver}\n"
+                if defined $narFile->{deriver} && $narFile->{deriver} ne "";
             print MANIFEST "}\n";
         }
     }
@@ -172,7 +195,7 @@ sub writeManifest
             print MANIFEST "patch {\n";
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $patch->{url}\n";
-            print MANIFEST "  MD5: $patch->{hash}\n";
+            print MANIFEST "  Hash: $patch->{hash}\n";
             print MANIFEST "  NarHash: $patch->{narHash}\n";
             print MANIFEST "  Size: $patch->{size}\n";
             print MANIFEST "  BasePath: $patch->{basePath}\n";

src/aterm-helper.pl

@@ -56,6 +56,7 @@ while (<STDIN>) {
         my $unpack = "";
         my $n = 1;
         foreach my $type (@types) {
+            my $realType = $type;
             $args .= ", ";
             if ($type eq "string") {
 #                $args .= "(ATerm) ATmakeAppl0(ATmakeAFun((char *) e$n, 0, ATtrue))";
@@ -83,6 +84,9 @@ while (<STDIN>) {
                 $unpack .= "    e$n = (ATermList) ATgetArgument(e, $m);\n";
             } elsif ($type eq "ATermBlob") {
                 $unpack .= "    e$n = (ATermBlob) ATgetArgument(e, $m);\n";
+            } elsif ($realType eq "string") {
+                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
+                $unpack .= "    if (ATgetType(e$n) != AT_APPL) return false;\n";
             } else {
                 $unpack .= "    e$n = ATgetArgument(e, $m);\n";
             }
@@ -96,12 +100,18 @@ while (<STDIN>) {
         print IMPL "AFun sym$funname = 0;\n";
         
         print HEADER "static inline $result make$funname($formals) {\n";
-        print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
+        if ($arity <= 6) {
+            print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
+        } else {
+            $args =~ s/^,//;
+            print HEADER "    ATerm array[$arity] = {$args};\n";
+            print HEADER "    return (ATerm) ATmakeApplArray(sym$funname, array);\n";
+        }
         print HEADER "}\n\n";
 
         print HEADER "#ifdef __cplusplus\n";
         print HEADER "static inline bool match$funname(ATerm e$formals2) {\n";
-        print HEADER "    if (ATgetType(e) != AT_APPL || ATgetAFun(e) != sym$funname) return false;\n";
+        print HEADER "    if (ATgetType(e) != AT_APPL || (AFun) ATgetAFun(e) != sym$funname) return false;\n";
         print HEADER "$unpack";
         print HEADER "    return true;\n";
         print HEADER "}\n";

src/boost/shared_ptr.hpp

@@ -324,7 +324,7 @@ public:
 // Tasteless as this may seem, making all members public allows member templates
 // to work in the absence of member template friends. (Matthew Langston)
 
-#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+#if 0
 
 private:
 

src/boost/weak_ptr.hpp

@@ -154,7 +154,7 @@ public:
 // Tasteless as this may seem, making all members public allows member templates
 // to work in the absence of member template friends. (Matthew Langston)
 
-#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+#if 0
 
 private:
 

src/libexpr/Makefile.am

@@ -29,7 +29,7 @@ lexer-tab.c lexer-tab.h: lexer.l
 nixexpr-ast.cc nixexpr-ast.hh: ../aterm-helper.pl nixexpr-ast.def
 	$(perl) ../aterm-helper.pl nixexpr-ast.hh nixexpr-ast.cc < nixexpr-ast.def
 
-nixexpr.hh: nixexpr-ast.hh
+nixexpr.cc nixexpr.hh: nixexpr-ast.hh
 
 
 CLEANFILES =

src/libexpr/nixexpr.cc

@@ -1,5 +1,5 @@
 #include "nixexpr.hh"
-#include "storeexpr.hh"
+#include "derivations.hh"
 
 
 #include "nixexpr-ast.hh"
@@ -7,6 +7,7 @@
 
 
 ATermMap::ATermMap(unsigned int initialSize, unsigned int maxLoadPct)
+    : table(0)
 {
     this->maxLoadPct = maxLoadPct;
     table = ATtableCreate(initialSize, maxLoadPct);
@@ -16,6 +17,36 @@ ATermMap::ATermMap(unsigned int initialSize, unsigned int maxLoadPct)
 
 ATermMap::ATermMap(const ATermMap & map)
     : table(0)
+{
+    copy(map);
+}
+
+
+ATermMap::~ATermMap()
+{
+    free();
+}
+
+
+ATermMap & ATermMap::operator = (const ATermMap & map)
+{
+    if (this == &map) return *this;
+    free();
+    copy(map);
+    return *this;
+}
+
+
+void ATermMap::free()
+{
+    if (table) {
+        ATtableDestroy(table);
+        table = 0;
+    }
+}
+
+
+void ATermMap::copy(const ATermMap & map)
 {
     ATermList keys = map.keys();
 
@@ -28,12 +59,6 @@ ATermMap::ATermMap(const ATermMap & map)
 }
 
 
-ATermMap::~ATermMap()
-{
-    if (table) ATtableDestroy(table);
-}
-
-
 void ATermMap::set(ATerm key, ATerm value)
 {
     return ATtablePut(table, key, value);

src/libexpr/nixexpr.hh

@@ -29,6 +29,8 @@ public:
     ATermMap(const ATermMap & map);
     ~ATermMap();
 
+    ATermMap & ATermMap::operator = (const ATermMap & map);
+        
     void set(ATerm key, ATerm value);
     void set(const string & key, ATerm value);
 
@@ -46,6 +48,9 @@ public:
 
 private:
     void add(const ATermMap & map, ATermList & keys);
+
+    void free();
+    void copy(const ATermMap & map);
 };
 
 
@@ -89,11 +94,5 @@ void checkVarDefs(const ATermMap & def, Expr e);
 /* Create an expression representing a boolean. */
 Expr makeBool(bool b);
 
-/* Create an expression representing a string. */
-Expr makeString(const string & s);
-
-/* Create an expression representing a path. */
-Expr makePath(const Path & path);
-
 
 #endif /* !__NIXEXPR_H */

src/libexpr/parser.cc

@@ -75,6 +75,65 @@ int yyparse(yyscan_t scanner, ParseData * data);
 }
 
 
+static void checkAttrs(ATermMap & names, ATermList bnds)
+{
+    for (ATermIterator i(bnds); i; ++i) {
+        ATerm name;
+        Expr e;
+        ATerm pos;
+        if (!matchBind(*i, name, e, pos)) abort(); /* can't happen */
+        if (names.get(name))
+            throw Error(format("duplicate attribute `%1%' at %2%")
+                % aterm2String(name) % showPos(pos));
+        names.set(name, name);
+    }
+}
+
+
+static void checkAttrSets(ATerm e)
+{
+    ATermList formals;
+    ATerm body, pos;
+    if (matchFunction(e, formals, body, pos)) {
+        ATermMap names;
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm name;
+            Expr deflt;
+            if (!matchNoDefFormal(*i, name) &&
+                !matchDefFormal(*i, name, deflt))
+                abort();
+            if (names.get(name))
+                throw Error(format("duplicate formal function argument `%1%' at %2%")
+                    % aterm2String(name) % showPos(pos));
+            names.set(name, name);
+        }
+    }
+
+    ATermList bnds;
+    if (matchAttrs(e, bnds)) {
+        ATermMap names;
+        checkAttrs(names, bnds);
+    }
+    
+    ATermList rbnds, nrbnds;
+    if (matchRec(e, rbnds, nrbnds)) {
+        ATermMap names;
+        checkAttrs(names, rbnds);
+        checkAttrs(names, nrbnds);
+    }
+    
+    if (ATgetType(e) == AT_APPL) {
+        int arity = ATgetArity(ATgetAFun(e));
+        for (int i = 0; i < arity; ++i)
+            checkAttrSets(ATgetArgument(e, i));
+    }
+
+    else if (ATgetType(e) == AT_LIST)
+        for (ATermIterator i((ATermList) e); i; ++i)
+            checkAttrSets(*i);
+}
+
+
 static Expr parse(EvalState & state,
     const char * text, const Path & path,
     const Path & basePath)
@@ -96,6 +155,8 @@ static Expr parse(EvalState & state,
     } catch (Error & e) {
         throw Error(format("%1%, in `%2%'") % e.msg() % path);
     }
+    
+    checkAttrSets(data.result);
 
     return data.result;
 }

src/libexpr/primops.cc

@@ -1,6 +1,7 @@
-#include "normalise.hh"
+#include "build.hh"
 #include "eval.hh"
 #include "globals.hh"
+#include "misc.hh"
 #include "nixexpr-ast.hh"
 
 
@@ -8,80 +9,95 @@
    argument. */ 
 static Expr primImport(EvalState & state, const ATermVector & args)
 {
-    ATerm path;
-    Expr fn = evalExpr(state, args[0]);
-    if (!matchPath(fn, path))
-        throw Error("path expected");
-    return evalFile(state, aterm2String(path));
-}
+    ATermList es;
+    Path path;
 
+    Expr arg = evalExpr(state, args[0]), arg2;
+    
+    if (matchPath(arg, arg2))
+        path = aterm2String(arg2);
 
-static PathSet storeExprRootsCached(EvalState & state, const Path & nePath)
-{
-    DrvRoots::iterator i = state.drvRoots.find(nePath);
-    if (i != state.drvRoots.end())
-        return i->second;
-    else {
-        PathSet paths = storeExprRoots(nePath);
-        state.drvRoots[nePath] = paths;
-        return paths;
-    }
-}
+    else if (matchAttrs(arg, es)) {
+        Expr a = queryAttr(arg, "type");
 
+        /* If it is a derivation, we have to realise it and load the
+           Nix expression created at the derivation's output path. */
+        if (a && evalString(state, a) == "derivation") {
+            a = queryAttr(arg, "drvPath");
+            if (!a) throw Error("bad derivation in import");
+            Path drvPath = evalPath(state, a);
 
-static Hash hashDerivation(EvalState & state, StoreExpr ne)
-{
-    if (ne.type == StoreExpr::neDerivation) {
-	PathSet inputs2;
-        for (PathSet::iterator i = ne.derivation.inputs.begin();
-             i != ne.derivation.inputs.end(); i++)
-        {
-            DrvHashes::iterator j = state.drvHashes.find(*i);
-            if (j == state.drvHashes.end())
-                throw Error(format("don't know expression `%1%'") % (string) *i);
-            inputs2.insert(j->second);
+            buildDerivations(singleton<PathSet>(drvPath));
+ 
+            a = queryAttr(arg, "outPath");
+            if (!a) throw Error("bad derivation in import");
+            path = evalPath(state, a);
         }
-	ne.derivation.inputs = inputs2;
     }
-    return hashTerm(unparseStoreExpr(ne));
+
+    if (path == "")
+        throw Error("path or derivation expected in import");
+    
+    return evalFile(state, path);
 }
 
 
-static Path copyAtom(EvalState & state, const Path & srcPath)
+/* Returns the hash of a derivation modulo fixed-output
+   subderivations.  A fixed-output derivation is a derivation with one
+   output (`out') for which an expected hash and hash algorithm are
+   specified (using the `outputHash' and `outputHashAlgo'
+   attributes).  We don't want changes to such derivations to
+   propagate upwards through the dependency graph, changing output
+   paths everywhere.
+
+   For instance, if we change the url in a call to the `fetchurl'
+   function, we do not want to rebuild everything depending on it
+   (after all, (the hash of) the file being downloaded is unchanged).
+   So the *output paths* should not change.  On the other hand, the
+   *derivation store expression paths* should change to reflect the
+   new dependency graph.
+
+   That's what this function does: it returns a hash which is just the
+   of the derivation ATerm, except that any input store expression
+   paths have been replaced by the result of a recursive call to this
+   function, and that for fixed-output derivations we return
+   (basically) its outputHash. */
+static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 {
-    /* !!! should be cached */
-    Path dstPath(addToStore(srcPath));
+    /* Return a fixed hash for fixed-output derivations. */
+    if (drv.outputs.size() == 1) {
+        DerivationOutputs::const_iterator i = drv.outputs.begin();
+        if (i->first == "out" &&
+            i->second.hash != "")
+        {
+            return hashString(htSHA256, "fixed:out:"
+                + i->second.hashAlgo + ":"
+                + i->second.hash /* !!! + ":"
+                                    + i->second.path */);
+        }
+    }
 
-    ClosureElem elem;
-    StoreExpr ne;
-    ne.type = StoreExpr::neClosure;
-    ne.closure.roots.insert(dstPath);
-    ne.closure.elems[dstPath] = elem;
-
-    Hash drvHash = hashDerivation(state, ne);
-    Path drvPath = writeTerm(unparseStoreExpr(ne), "");
-    state.drvHashes[drvPath] = drvHash;
-
-    state.drvRoots[drvPath] = ne.closure.roots;
-
-    printMsg(lvlChatty, format("copied `%1%' -> closure `%2%'")
-        % srcPath % drvPath);
-    return drvPath;
+    /* For other derivations, replace the inputs paths with recursive
+       calls to this function.*/
+    DerivationInputs inputs2;
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+    {
+        Hash h = state.drvHashes[i->first];
+        if (h.type == htUnknown) {
+            Derivation drv2 = derivationFromPath(i->first);
+            h = hashDerivationModulo(state, drv2);
+            state.drvHashes[i->first] = h;
+        }
+        inputs2[printHash(h)] = i->second;
+    }
+    drv.inputDrvs = inputs2;
+    
+    return hashTerm(unparseDerivation(drv));
 }
 
 
-static string addInput(EvalState & state, 
-    Path & nePath, StoreExpr & ne)
-{
-    PathSet paths = storeExprRootsCached(state, nePath);
-    if (paths.size() != 1) abort();
-    Path path = *(paths.begin());
-    ne.derivation.inputs.insert(nePath);
-    return path;
-}
-
-
-static void processBinding(EvalState & state, Expr e, StoreExpr & ne,
+static void processBinding(EvalState & state, Expr e, Derivation & drv,
     Strings & ss)
 {
     e = evalExpr(state, e);
@@ -104,37 +120,61 @@ static void processBinding(EvalState & state, Expr e, StoreExpr & ne,
 
     else if (matchAttrs(e, es)) {
         Expr a = queryAttr(e, "type");
+        
         if (a && evalString(state, a) == "derivation") {
             a = queryAttr(e, "drvPath");
             if (!a) throw Error("derivation name missing");
             Path drvPath = evalPath(state, a);
 
-            a = queryAttr(e, "drvHash");
-            if (!a) throw Error("derivation hash missing");
-            Hash drvHash = parseHash(evalString(state, a));
+            a = queryAttr(e, "outPath");
+            if (!a) throw Error("output path missing");
+            /* !!! supports only single output path */
+            Path outPath = evalPath(state, a);
+
+            drv.inputDrvs[drvPath] = singleton<StringSet>("out");
+            ss.push_back(outPath);
+        }
+
+        else if (a && evalString(state, a) == "storePath") {
 
             a = queryAttr(e, "outPath");
             if (!a) throw Error("output path missing");
-            PathSet drvRoots;
-            drvRoots.insert(evalPath(state, a));
-            
-            state.drvHashes[drvPath] = drvHash;
-            state.drvRoots[drvPath] = drvRoots;
+            /* !!! supports only single output path */
+            Path outPath = evalPath(state, a);
 
-            ss.push_back(addInput(state, drvPath, ne));
-        } else
-            throw Error("invalid derivation attribute");
+            drv.inputSrcs.insert(outPath);
+            ss.push_back(outPath);
+        }
+
+        else throw Error("invalid derivation attribute");
     }
 
     else if (matchPath(e, s)) {
-        Path drvPath = copyAtom(state, aterm2String(s));
-        ss.push_back(addInput(state, drvPath, ne));
+        Path srcPath(canonPath(aterm2String(s)));
+
+        if (isStorePath(srcPath)) {
+            printMsg(lvlChatty, format("using store path `%1%' as source")
+                % srcPath);
+            drv.inputSrcs.insert(srcPath);
+            ss.push_back(srcPath);
+        }
+
+        else {
+            if (isDerivation(srcPath))
+                throw Error(format("file names are not allowed to end in `%1%'")
+                    % drvExtension);
+            Path dstPath(addToStore(srcPath));
+            printMsg(lvlChatty, format("copied source `%1%' -> `%2%'")
+                % srcPath % dstPath);
+            drv.inputSrcs.insert(dstPath);
+            ss.push_back(dstPath);
+        }
     }
     
     else if (matchList(e, es)) {
         for (ATermIterator i(es); i; ++i) {
             startNest(nest, lvlVomit, format("processing list element"));
-	    processBinding(state, evalExpr(state, *i), ne, ss);
+	    processBinding(state, evalExpr(state, *i), drv, ss);
         }
     }
 
@@ -142,7 +182,7 @@ static void processBinding(EvalState & state, Expr e, StoreExpr & ne,
 
     else if (matchSubPath(e, e1, e2)) {
         Strings ss2;
-        processBinding(state, evalExpr(state, e1), ne, ss2);
+        processBinding(state, evalExpr(state, e1), drv, ss2);
         if (ss2.size() != 1)
             throw Error("left-hand side of `~' operator cannot be a list");
         e2 = evalExpr(state, e2);
@@ -174,22 +214,21 @@ static string concatStrings(const Strings & ss)
    derivation; `drvPath' containing the path of the Nix expression;
    and `type' set to `derivation' to indicate that this is a
    derivation. */
-static Expr primDerivation(EvalState & state, const ATermVector & _args)
+static Expr primDerivationStrict(EvalState & state, const ATermVector & args)
 {
     startNest(nest, lvlVomit, "evaluating derivation");
 
     ATermMap attrs;
-    Expr args = _args[0];
-    args = evalExpr(state, args);
-    queryAllAttrs(args, attrs, true);
+    queryAllAttrs(evalExpr(state, args[0]), attrs, true);
 
     /* Build the derivation expression by processing the attributes. */
-    StoreExpr ne;
-    ne.type = StoreExpr::neDerivation;
+    Derivation drv;
 
     string drvName;
-    Hash outHash;
-    bool outHashGiven = false;
+    
+    string outputHash;
+    string outputHashAlgo;
+    bool outputHashRecursive = false;
 
     for (ATermIterator i(attrs.keys()); i; ++i) {
         string key = aterm2String(*i);
@@ -201,7 +240,7 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
 
         Strings ss;
         try {
-            processBinding(state, value, ne, ss);
+            processBinding(state, value, drv, ss);
         } catch (Error & e) {
             throw Error(format("while processing the derivation attribute `%1%' at %2%:\n%3%")
                 % key % showPos(pos) % e.msg());
@@ -211,70 +250,119 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
            command-line arguments to the builder. */
         if (key == "args") {
             for (Strings::iterator i = ss.begin(); i != ss.end(); ++i)
-                ne.derivation.args.push_back(*i);
+                drv.args.push_back(*i);
         }
 
         /* All other attributes are passed to the builder through the
            environment. */
         else {
             string s = concatStrings(ss);
-            ne.derivation.env[key] = s;
-            if (key == "builder") ne.derivation.builder = s;
-            else if (key == "system") ne.derivation.platform = s;
+            drv.env[key] = s;
+            if (key == "builder") drv.builder = s;
+            else if (key == "system") drv.platform = s;
             else if (key == "name") drvName = s;
-            else if (key == "id") { 
-                outHash = parseHash(s);
-                outHashGiven = true;
+            else if (key == "outputHash") outputHash = s;
+            else if (key == "outputHashAlgo") outputHashAlgo = s;
+            else if (key == "outputHashMode") {
+                if (s == "recursive") outputHashRecursive = true; 
+                else if (s == "flat") outputHashRecursive = false;
+                else throw Error(format("invalid value `%1%' for `outputHashMode' attribute") % s);
             }
         }
     }
     
     /* Do we have all required attributes? */
-    if (ne.derivation.builder == "")
+    if (drv.builder == "")
         throw Error("required attribute `builder' missing");
-    if (ne.derivation.platform == "")
+    if (drv.platform == "")
         throw Error("required attribute `system' missing");
     if (drvName == "")
         throw Error("required attribute `name' missing");
 
+    /* If an output hash was given, check it. */
+    if (outputHash == "")
+        outputHashAlgo = "";
+    else {
+        HashType ht = parseHashType(outputHashAlgo);
+        if (ht == htUnknown)
+            throw Error(format("unknown hash algorithm `%1%'") % outputHashAlgo);
+        Hash h;
+        if (outputHash.size() == Hash(ht).hashSize * 2)
+            /* hexadecimal representation */
+            h = parseHash(ht, outputHash);
+        else
+            /* base-32 representation */
+            h = parseHash32(ht, outputHash);
+        string s = outputHash;
+        outputHash = printHash(h);
+        if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
+    }
+
     /* Check the derivation name.  It shouldn't contain whitespace,
        but we are conservative here: we check whether only
        alphanumerics and some other characters appear. */
-    string validChars = "+-._?=";
-    for (string::iterator i = drvName.begin(); i != drvName.end(); ++i)
-        if (!((*i >= 'A' && *i <= 'Z') ||
-              (*i >= 'a' && *i <= 'z') ||
-              (*i >= '0' && *i <= '9') ||
-              validChars.find(*i) != string::npos))
-        {
-            throw Error(format("invalid character `%1%' in derivation name `%2%'")
-                % *i % drvName);
-        }
+    checkStoreName(drvName);
+    if (isDerivation(drvName))
+        throw Error(format("derivation names are not allowed to end in `%1%'")
+            % drvExtension);
+
+    /* !!! the name should not end in the derivation extension (.drv).
+       Likewise for sources. */
+
+    /* Construct the "masked" derivation store expression, which is
+       the final one except that in the list of outputs, the output
+       paths are empty, and the corresponding environment variables
+       have an empty value.  This ensures that changes in the set of
+       output names do get reflected in the hash. */
+    drv.env["out"] = "";
+    drv.outputs["out"] =
+        DerivationOutput("", outputHashAlgo, outputHash);
         
-    /* Determine the output path by hashing the Nix expression with no
-       outputs to produce a unique but deterministic path name for
-       this derivation. */
-    if (!outHashGiven) outHash = hashDerivation(state, ne);
-    Path outPath = canonPath(nixStore + "/" + 
-        ((string) outHash).c_str() + "-" + drvName);
-    ne.derivation.env["out"] = outPath;
-    ne.derivation.outputs.insert(outPath);
+    /* Use the masked derivation expression to compute the output
+       path. */
+    /* XXX */
+    Path outPath;
+    PathHash outPathHash;
+    makeStorePath(hashDerivationModulo(state, drv), drvName, outPath, outPathHash);
+
+    /* Construct the final derivation store expression. */
+    drv.env["out"] = outPath;
+    drv.outputs["out"] =
+        DerivationOutput(outPath, outputHashAlgo, outputHash);
 
     /* Write the resulting term into the Nix store directory. */
-    Hash drvHash = outHashGiven
-        ? hashString((string) outHash + outPath)
-        : hashDerivation(state, ne);
-    Path drvPath = writeTerm(unparseStoreExpr(ne), "-d-" + drvName);
+    Path drvPath = writeDerivation(drv, drvName);
 
     printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'")
         % drvName % drvPath);
 
-    attrs.set("outPath", makeAttrRHS(makePath(toATerm(outPath)), makeNoPos()));
-    attrs.set("drvPath", makeAttrRHS(makePath(toATerm(drvPath)), makeNoPos()));
-    attrs.set("drvHash",
-        makeAttrRHS(makeStr(toATerm((string) drvHash)), makeNoPos()));
+    /* Optimisation, but required in read-only mode! because in that
+       case we don't actually write store expressions, so we can't
+       read them later. */
+    state.drvHashes[drvPath] = hashDerivationModulo(state, drv);
+
+    /* !!! assumes a single output */
+    ATermMap outAttrs;
+    outAttrs.set("outPath", makeAttrRHS(makePath(toATerm(outPath)), makeNoPos()));
+    outAttrs.set("drvPath", makeAttrRHS(makePath(toATerm(drvPath)), makeNoPos()));
+
+    return makeAttrs(outAttrs);
+}
+
+
+static Expr primDerivationLazy(EvalState & state, const ATermVector & args)
+{
+    Expr eAttrs = evalExpr(state, args[0]);
+    ATermMap attrs;
+    queryAllAttrs(eAttrs, attrs, true);
+
     attrs.set("type", makeAttrRHS(makeStr(toATerm("derivation")), makeNoPos()));
 
+    Expr drvStrict = makeCall(makeVar(toATerm("derivation!")), eAttrs);
+
+    attrs.set("outPath", makeAttrRHS(makeSelect(drvStrict, toATerm("outPath")), makeNoPos()));
+    attrs.set("drvPath", makeAttrRHS(makeSelect(drvStrict, toATerm("drvPath")), makeNoPos()));
+    
     return makeAttrs(attrs);
 }
 
@@ -294,7 +382,7 @@ static Expr primToString(EvalState & state, const ATermVector & args)
     ATerm s;
     if (matchStr(arg, s) || matchPath(arg, s) || matchUri(arg, s))
         return makeStr(s);
-    else throw Error("cannot coerce value to string");
+    throw Error("cannot coerce value to string");
 }
 
 
@@ -312,21 +400,21 @@ static Expr primFalse(EvalState & state, const ATermVector & args)
 
 
 /* Return the null value. */
-Expr primNull(EvalState & state, const ATermVector & args)
+static Expr primNull(EvalState & state, const ATermVector & args)
 {
     return makeNull();
 }
 
 
 /* Determine whether the argument is the null value. */
-Expr primIsNull(EvalState & state, const ATermVector & args)
+static Expr primIsNull(EvalState & state, const ATermVector & args)
 {
     return makeBool(matchNull(evalExpr(state, args[0])));
 }
 
 
 /* Apply a function to every element of a list. */
-Expr primMap(EvalState & state, const ATermVector & args)
+static Expr primMap(EvalState & state, const ATermVector & args)
 {
     Expr fun = evalExpr(state, args[0]);
     Expr list = evalExpr(state, args[1]);
@@ -343,17 +431,56 @@ Expr primMap(EvalState & state, const ATermVector & args)
 }
 
 
+/* Return a string constant representing the current platform.  Note!
+   that differs between platforms, so Nix expressions using
+   `__currentSystem' can evaluate to different values on different
+   platforms. */
+static Expr primCurrentSystem(EvalState & state, const ATermVector & args)
+{
+    return makeStr(toATerm(thisSystem));
+}
+
+
+static Expr primCurrentTime(EvalState & state, const ATermVector & args)
+{
+    return ATmake("Int(<int>)", time(0));
+}
+
+
+static Expr primRemoveAttrs(EvalState & state, const ATermVector & args)
+{
+    ATermMap attrs;
+    queryAllAttrs(evalExpr(state, args[0]), attrs, true);
+    
+    ATermList list;
+    if (!matchList(evalExpr(state, args[1]), list))
+        throw Error("`removeAttrs' expects a list as its second argument");
+
+    for (ATermIterator i(list); i; ++i)
+        /* It's not an error for *i not to exist. */
+        attrs.remove(evalString(state, *i));
+
+    return makeAttrs(attrs);
+}
+
+
 void EvalState::addPrimOps()
 {
     addPrimOp("true", 0, primTrue);
     addPrimOp("false", 0, primFalse);
     addPrimOp("null", 0, primNull);
+    addPrimOp("__currentSystem", 0, primCurrentSystem);
+    addPrimOp("__currentTime", 0, primCurrentTime);
 
     addPrimOp("import", 1, primImport);
-    addPrimOp("derivation", 1, primDerivation);
+    addPrimOp("derivation!", 1, primDerivationStrict);
+    addPrimOp("derivation", 1, primDerivationLazy);
     addPrimOp("baseNameOf", 1, primBaseNameOf);
     addPrimOp("toString", 1, primToString);
     addPrimOp("isNull", 1, primIsNull);
 
     addPrimOp("map", 2, primMap);
+    addPrimOp("removeAttrs", 2, primRemoveAttrs);
 }
+
+

src/libmain/Makefile.am

@@ -7,5 +7,6 @@ AM_CXXFLAGS = \
  -DNIX_DATA_DIR=\"$(datadir)\" \
  -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \
  -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \
+ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_VERSION=\"$(VERSION)\" \
  -I.. ${aterm_include} -I../libutil -I../libstore

src/libmain/shared.cc

@@ -13,6 +13,7 @@ extern "C" {
 }
 
 #include "globals.hh"
+#include "gc.hh"
 #include "shared.hh"
 
 #include "config.h"
@@ -30,6 +31,28 @@ void sigintHandler(int signo)
 }
 
 
+Path makeRootName(const Path & gcRoot, int & counter)
+{
+    counter++;
+    if (counter == 1)
+        return gcRoot;
+    else
+        return (format("%1%-%2%") % gcRoot % counter).str();
+}
+
+
+void printGCWarning()
+{
+    static bool warned = false;
+    if (!warned) {
+        printMsg(lvlInfo,
+            "warning: you did not specify `--add-root'; "
+            "the result might be removed by the garbage collector");
+        warned = true;
+    }
+}
+
+
 void setLogType(string lt)
 {
     if (lt == "pretty") logType = ltPretty;
@@ -55,7 +78,16 @@ void checkStoreNotSymlink(Path path)
 }
 
 
-void initStoreExprHelpers();
+struct RemoveTempRoots 
+{
+    ~RemoveTempRoots()
+    {
+        removeTempRoots();
+    }
+};
+
+
+void initDerivationsHelpers();
 
 
 /* Initialize and reorder arguments, then call the actual argument
@@ -69,11 +101,12 @@ static void initAndRun(int argc, char * * argv)
     }
     
     /* Setup Nix paths. */
-    nixStore = getEnv("NIX_STORE_DIR", canonPath(NIX_STORE_DIR));
-    nixDataDir = getEnv("NIX_DATA_DIR", canonPath(NIX_DATA_DIR));
-    nixLogDir = getEnv("NIX_LOG_DIR", canonPath(NIX_LOG_DIR));
-    nixStateDir = getEnv("NIX_STATE_DIR", canonPath(NIX_STATE_DIR));
+    nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
+    nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
+    nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
+    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
     nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
+    nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
 
     /* Check that the store directory and its parent are not
        symlinks. */
@@ -105,7 +138,22 @@ static void initAndRun(int argc, char * * argv)
     if (lt != "") setLogType(lt);
 
     /* ATerm stuff.  !!! find a better place to put this */
-    initStoreExprHelpers();
+    initDerivationsHelpers();
+
+    /* Random number generator needed by makeRandomStorePath(); !!!
+       improve. */
+    srand(time(0));
+
+    /* Set the trust ID to the user name. */
+    currentTrustId = getEnv("NIX_USER_ID"); /* !!! dangerous? */
+    if (currentTrustId == "") {
+        SwitchToOriginalUser sw;
+        uid_t uid = geteuid();
+        struct passwd * pw = getpwuid(uid);
+        if (!pw) throw Error(format("unknown user ID %1%, go away") % uid);
+        currentTrustId = pw->pw_name;
+    }
+    printMsg(lvlError, format("trust ID is `%1%'") % currentTrustId);
     
     /* Put the arguments in a vector. */
     Strings args, remaining;
@@ -171,6 +219,10 @@ static void initAndRun(int argc, char * * argv)
         else remaining.push_back(arg);
     }
 
+    /* Automatically clean up the temporary roots file when we
+       exit. */
+    RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
+
     run(remaining);
 }
 

src/libmain/shared.hh

@@ -17,6 +17,10 @@ void run(Strings args);
 /* Should print a help message to stdout and return. */
 void printHelp();
 
+/* Ugh.  No better place to put this. */
+Path makeRootName(const Path & gcRoot, int & counter);
+void printGCWarning();
+
 
 extern string programId;
 

src/libstore/Makefile.am

@@ -1,18 +1,18 @@
 noinst_LIBRARIES = libstore.a
 
 libstore_a_SOURCES = \
- store.cc store.hh storeexpr.cc storeexpr.hh \
- normalise.cc misc.cc normalise.hh \
+ store.cc store.hh derivations.cc derivations.hh \
+ build.cc build.hh misc.cc misc.hh \
  globals.cc globals.hh db.cc db.hh \
  references.cc references.hh pathlocks.cc pathlocks.hh \
- gc.cc gc.hh storeexpr-ast.hh
+ gc.cc gc.hh derivations-ast.hh
 
-EXTRA_DIST = storeexpr-ast.def storeexpr-ast.cc
+EXTRA_DIST = derivations-ast.def derivations-ast.cc
 
 AM_CXXFLAGS = -Wall \
  -I.. ${bdb_include} ${aterm_include} -I../libutil
 
-storeexpr-ast.cc storeexpr-ast.hh: ../aterm-helper.pl storeexpr-ast.def
-	$(perl) ../aterm-helper.pl storeexpr-ast.hh storeexpr-ast.cc < storeexpr-ast.def
+derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
+	$(perl) ../aterm-helper.pl derivations-ast.hh derivations-ast.cc < derivations-ast.def
 
-storeexpr.cc: storeexpr-ast.hh
+derivations.cc store.cc: derivations-ast.hh

src/libstore/build.hh

@@ -0,0 +1,21 @@
+#ifndef __BUILD_H
+#define __BUILD_H
+
+#include "derivations.hh"
+
+
+/* Ensure that the output paths of the derivation are valid.  If they
+   are already valid, this is a no-op.  Otherwise, validity can
+   be reached in two ways.  First, if the output paths have
+   substitutes, then those can be used.  Second, the output paths can
+   be created by running the builder, after recursively building any
+   sub-derivations. */
+void buildDerivations(const PathSet & drvPaths);
+
+
+/* Ensure that a path is valid.  If it is not currently valid, it may
+   be made valid by running a substitute (if defined for the path). */
+void ensurePath(const Path & storePath);
+
+
+#endif /* !__BUILD_H */

src/libstore/db.cc

@@ -33,11 +33,9 @@ Transaction::Transaction()
 
 
 Transaction::Transaction(Database & db)
+    : txn(0)
 {
-    db.requireEnv();
-    try {
-        db.env->txn_begin(0, &txn, 0);
-    } catch (DbException e) { rethrow(e); }
+    begin(db);
 }
 
 
@@ -47,6 +45,16 @@ Transaction::~Transaction()
 }
 
 
+void Transaction::begin(Database & db)
+{
+    assert(txn == 0);
+    db.requireEnv();
+    try {
+        db.env->txn_begin(0, &txn, 0);
+    } catch (DbException e) { rethrow(e); }
+}
+
+
 void Transaction::commit()
 {
     if (!txn) throw Error("commit called on null transaction");
@@ -82,7 +90,7 @@ void Transaction::moveTo(Transaction & t)
 void Database::requireEnv()
 {
     checkInterrupt();
-    if (!env)throw Error("database environment is not open "
+    if (!env) throw Error("database environment is not open "
         "(maybe you don't have sufficient permission?)");
 }
 
@@ -157,127 +165,152 @@ static int my_fsync(int fd)
 }
 
 
-void Database::open(const string & path)
+void Database::open2(const string & path, bool removeOldEnv)
 {
     if (env) throw Error(format("environment already open"));
 
-    try {
-
-        debug(format("opening database environment"));
+    debug(format("opening database environment"));
 
 
-        /* Create the database environment object. */
-        DbEnv * env = 0; /* !!! close on error */
-        env = new DbEnv(0);
-
-        /* Smaller log files. */
-        env->set_lg_bsize(32 * 1024); /* default */
-        env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */
-
-        /* Write the log, but don't sync.  This protects transactions
-           against application crashes, but if the system crashes,
-           some transactions may be undone.  An acceptable risk, I
-           think. */
-        env->set_flags(DB_TXN_WRITE_NOSYNC | DB_LOG_AUTOREMOVE, 1);
-
-        /* Increase the locking limits.  If you ever get `Dbc::get:
-           Cannot allocate memory' or similar, especially while
-           running `nix-store --verify', just increase the following
-           number, then run db_recover on the database to remove the
-           existing DB environment (since changes only take effect on
-           new environments). */
-        env->set_lk_max_locks(4000);
-        env->set_lk_max_lockers(4000);
-        env->set_lk_max_objects(4000);
-        env->set_lk_detect(DB_LOCK_DEFAULT);
-
-        /* Dangerous, probably, but from the docs it *seems* that BDB
-           shouldn't sync when DB_TXN_WRITE_NOSYNC is used, but it
-           still fsync()s sometimes. */
-        db_env_set_func_fsync(my_fsync);
+    /* Create the database environment object. */
+    DbEnv * env = 0; /* !!! close on error */
+    env = new DbEnv(0);
+    
+    /* Smaller log files. */
+    env->set_lg_bsize(32 * 1024); /* default */
+    env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */
+    
+    /* Write the log, but don't sync.  This protects transactions
+       against application crashes, but if the system crashes, some
+       transactions may be undone.  An acceptable risk, I think. */
+    env->set_flags(DB_TXN_WRITE_NOSYNC | DB_LOG_AUTOREMOVE, 1);
+    
+    /* Increase the locking limits.  If you ever get `Dbc::get: Cannot
+       allocate memory' or similar, especially while running
+       `nix-store --verify', just increase the following number, then
+       run db_recover on the database to remove the existing DB
+       environment (since changes only take effect on new
+       environments). */
+    env->set_lk_max_locks(10000);
+    env->set_lk_max_lockers(10000);
+    env->set_lk_max_objects(10000);
+    env->set_lk_detect(DB_LOCK_DEFAULT);
+    
+    /* Dangerous, probably, but from the docs it *seems* that BDB
+       shouldn't sync when DB_TXN_WRITE_NOSYNC is used, but it still
+       fsync()s sometimes. */
+    db_env_set_func_fsync(my_fsync);
         
 
-        /* The following code provides automatic recovery of the
-           database environment.  Recovery is necessary when a process
-           dies while it has the database open.  To detect this,
-           processes atomically increment a counter when they open the
-           database, and decrement it when they close it.  If we see
-           that counter is > 0 but no processes are accessing the
-           database---determined by attempting to obtain a write lock
-           on a lock file on which all accessors have a read lock---we
-           must run recovery.  Note that this also ensures that we
-           only run recovery when there are no other accessors (which
-           could cause database corruption). */
+    /* The following code provides automatic recovery of the database
+       environment.  Recovery is necessary when a process dies while
+       it has the database open.  To detect this, processes atomically
+       increment a counter when they open the database, and decrement
+       it when they close it.  If we see that counter is > 0 but no
+       processes are accessing the database---determined by attempting
+       to obtain a write lock on a lock file on which all accessors
+       have a read lock---we must run recovery.  Note that this also
+       ensures that we only run recovery when there are no other
+       accessors (which could cause database corruption). */
 
-        /* !!! close fdAccessors / fdLock on exception */
+    /* !!! close fdAccessors / fdLock on exception */
 
-        /* Open the accessor count file. */
-        string accessorsPath = path + "/accessor_count";
-        fdAccessors = ::open(accessorsPath.c_str(), O_RDWR | O_CREAT, 0666);
-        if (fdAccessors == -1)
-            if (errno == EACCES)
-                throw DbNoPermission(
-                    format("permission denied to database in `%1%'") % accessorsPath);
-            else
-                throw SysError(format("opening file `%1%'") % accessorsPath);
-
-        /* Open the lock file. */
-        string lockPath = path + "/access_lock";
-        fdLock = ::open(lockPath.c_str(), O_RDWR | O_CREAT, 0666);
-        if (fdLock == -1)
-            throw SysError(format("opening lock file `%1%'") % lockPath);
-
-        /* Try to acquire a write lock. */
-        debug(format("attempting write lock on `%1%'") % lockPath);
-        if (lockFile(fdLock, ltWrite, false)) { /* don't wait */
-
-            debug(format("write lock granted"));
-
-            /* We have a write lock, which means that there are no
-               other readers or writers. */
-
-            int n = getAccessorCount(fdAccessors);
-
-            if (n != 0) {
-                printMsg(lvlTalkative,
-                    format("accessor count is %1%, running recovery") % n);
-
-                /* Open the environment after running recovery. */
-                openEnv(env, path, DB_RECOVER);
-            }
+    /* Open the accessor count file. */
+    string accessorsPath = path + "/accessor_count";
+    fdAccessors = ::open(accessorsPath.c_str(), O_RDWR | O_CREAT, 0666);
+    if (fdAccessors == -1)
+        if (errno == EACCES)
+            throw DbNoPermission(
+                format("permission denied to database in `%1%'") % accessorsPath);
+        else    
+            throw SysError(format("opening file `%1%'") % accessorsPath);
+    
+    /* Open the lock file. */
+    string lockPath = path + "/access_lock";
+    fdLock = ::open(lockPath.c_str(), O_RDWR | O_CREAT, 0666);
+    if (fdLock == -1)
+        throw SysError(format("opening lock file `%1%'") % lockPath);
+    
+    /* Try to acquire a write lock. */
+    debug(format("attempting write lock on `%1%'") % lockPath);
+    if (lockFile(fdLock, ltWrite, false)) { /* don't wait */
+        
+        debug(format("write lock granted"));
+        
+        /* We have a write lock, which means that there are no other
+           readers or writers. */
+        
+        if (removeOldEnv) {
+            printMsg(lvlError, "removing old Berkeley DB database environment...");
+            env->remove(path.c_str(), DB_FORCE);
+            return;
+        }
+        
+        int n = getAccessorCount(fdAccessors);
+        
+        if (n != 0) {
+            printMsg(lvlTalkative,
+                format("accessor count is %1%, running recovery") % n);
             
-            else 
-                /* Open the environment normally. */
-                openEnv(env, path, 0);
-
-            setAccessorCount(fdAccessors, 1);
-
-            /* Downgrade to a read lock. */
-            debug(format("downgrading to read lock on `%1%'") % lockPath);
-            lockFile(fdLock, ltRead, true);
-
-        } else {
-            /* There are other accessors. */ 
-            debug(format("write lock refused"));
-
-            /* Acquire a read lock. */
-            debug(format("acquiring read lock on `%1%'") % lockPath);
-            lockFile(fdLock, ltRead, true); /* wait indefinitely */
-
-            /* Increment the accessor count. */
-            lockFile(fdAccessors, ltWrite, true);
-            int n = getAccessorCount(fdAccessors) + 1;
-            setAccessorCount(fdAccessors, n);
-            debug(format("incremented accessor count to %1%") % n);
-            lockFile(fdAccessors, ltNone, true);
-
+            /* Open the environment after running recovery. */
+            openEnv(env, path, DB_RECOVER);
+        }
+            
+        else 
             /* Open the environment normally. */
             openEnv(env, path, 0);
-        }
 
-        this->env = env;
+        setAccessorCount(fdAccessors, 1);
+
+        /* Downgrade to a read lock. */
+        debug(format("downgrading to read lock on `%1%'") % lockPath);
+        lockFile(fdLock, ltRead, true);
+
+    } else {
+        /* There are other accessors. */ 
+        debug(format("write lock refused"));
+
+        /* Acquire a read lock. */
+        debug(format("acquiring read lock on `%1%'") % lockPath);
+        lockFile(fdLock, ltRead, true); /* wait indefinitely */
         
-    } catch (DbException e) { rethrow(e); }
+        /* Increment the accessor count. */
+        lockFile(fdAccessors, ltWrite, true);
+        int n = getAccessorCount(fdAccessors) + 1;
+        setAccessorCount(fdAccessors, n);
+        debug(format("incremented accessor count to %1%") % n);
+        lockFile(fdAccessors, ltNone, true);
+        
+        /* Open the environment normally. */
+        openEnv(env, path, 0);
+    }
+    
+    this->env = env;
+}       
+
+
+void Database::open(const string & path)
+{
+    try {
+
+        open2(path, false);
+        
+    } catch (DbException e) {
+        
+        if (e.get_errno() == DB_VERSION_MISMATCH) {
+            /* Remove the environment while we are holding the global
+               lock.  If things go wrong there, we bail out.  !!!
+               there is some leakage here op DbEnv and lock
+               handles. */
+            open2(path, true);
+
+            /* Try again. */
+            open2(path, false);
+        }
+        else
+            rethrow(e);
+    }
+    
 }
 
 

src/libstore/db.hh

@@ -27,6 +27,7 @@ public:
     Transaction(Database & _db);
     ~Transaction();
 
+    void begin(Database & db);
     void abort();
     void commit();
 
@@ -57,6 +58,8 @@ private:
 
     Db * getDb(TableId table);
 
+    void open2(const string & path, bool removeOldEnv);
+    
 public:
     Database();
     ~Database();

src/libstore/derivations-ast.def

@@ -0,0 +1,10 @@
+init initDerivationsHelpers
+
+Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
+
+| string string | ATerm | EnvBinding |
+| string ATermList | ATerm | DerivationInput |
+| string string string string | ATerm | DerivationOutput |
+
+Closure | ATermList ATermList | ATerm | OldClosure |
+| string ATermList | ATerm | OldClosureElem |

src/libstore/derivations.cc

@@ -0,0 +1,169 @@
+#include "derivations.hh"
+#include "globals.hh"
+#include "store.hh"
+
+#include "derivations-ast.hh"
+#include "derivations-ast.cc"
+
+
+Hash hashTerm(ATerm t)
+{
+    return hashString(htSHA256, atPrint(t));
+}
+
+
+Path writeDerivation(const Derivation & drv, const string & name)
+{
+    PathSet references;
+    references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
+    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+        references.insert(i->first);
+    /* Note that the outputs of a derivation are *not* references
+       (that can be missing (of course) and should not necessarily be
+       held during a garbage collection). */
+    return addTextToStore(name + drvExtension,
+        atPrint(unparseDerivation(drv)), references);
+}
+
+
+static void checkPath(const string & s)
+{
+    if (s.size() == 0 || s[0] != '/')
+        throw Error(format("bad path `%1%' in derivation") % s);
+}
+    
+
+static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
+{
+    for (ATermIterator i(paths); i; ++i) {
+        if (ATgetType(*i) != AT_APPL)
+            throw badTerm("not a path", *i);
+        string s = aterm2String(*i);
+        if (arePaths) checkPath(s);
+        out.insert(s);
+    }
+}
+
+
+void throwBadDrv(ATerm t)
+{
+    throw badTerm("not a valid derivation", t);
+}
+
+
+Derivation parseDerivation(ATerm t)
+{
+    Derivation drv;
+    ATermList outs, inDrvs, inSrcs, args, bnds;
+    ATerm builder, platform;
+
+    if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds))
+        throwBadDrv(t);
+
+    for (ATermIterator i(outs); i; ++i) {
+        ATerm id, eqClass, hashAlgo, hash;
+        if (!matchDerivationOutput(*i, id, eqClass, hashAlgo, hash))
+            throwBadDrv(t);
+        DerivationOutput out;
+        out.eqClass = aterm2String(eqClass);
+        // !!!        checkPath(out.path);
+        out.hashAlgo = aterm2String(hashAlgo);
+        out.hash = aterm2String(hash);
+        drv.outputs[aterm2String(id)] = out;
+    }
+
+    for (ATermIterator i(inDrvs); i; ++i) {
+        ATerm drvPath;
+        ATermList ids;
+        if (!matchDerivationInput(*i, drvPath, ids))
+            throwBadDrv(t);
+        Path drvPath2 = aterm2String(drvPath);
+        checkPath(drvPath2);
+        StringSet ids2;
+        parseStrings(ids, ids2, false);
+        drv.inputDrvs[drvPath2] = ids2;
+    }
+    
+    parseStrings(inSrcs, drv.inputSrcs, true);
+
+    drv.builder = aterm2String(builder);
+    drv.platform = aterm2String(platform);
+    
+    for (ATermIterator i(args); i; ++i) {
+        if (ATgetType(*i) != AT_APPL)
+            throw badTerm("string expected", *i);
+        drv.args.push_back(aterm2String(*i));
+    }
+
+    for (ATermIterator i(bnds); i; ++i) {
+        ATerm s1, s2;
+        if (!matchEnvBinding(*i, s1, s2))
+            throw badTerm("tuple of strings expected", *i);
+        drv.env[aterm2String(s1)] = aterm2String(s2);
+    }
+
+    return drv;
+}
+
+
+static ATermList unparseStrings(const StringSet & paths)
+{
+    ATermList l = ATempty;
+    for (PathSet::const_iterator i = paths.begin();
+         i != paths.end(); ++i)
+        l = ATinsert(l, toATerm(*i));
+    return ATreverse(l);
+}
+
+
+ATerm unparseDerivation(const Derivation & drv)
+{
+    ATermList outputs = ATempty;
+    for (DerivationOutputs::const_iterator i = drv.outputs.begin();
+         i != drv.outputs.end(); ++i)
+        outputs = ATinsert(outputs,
+            makeDerivationOutput(
+                toATerm(i->first),
+                toATerm(i->second.eqClass),
+                toATerm(i->second.hashAlgo),
+                toATerm(i->second.hash)));
+
+    ATermList inDrvs = ATempty;
+    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+        inDrvs = ATinsert(inDrvs,
+            makeDerivationInput(
+                toATerm(i->first),
+                unparseStrings(i->second)));
+    
+    ATermList args = ATempty;
+    for (Strings::const_iterator i = drv.args.begin();
+         i != drv.args.end(); ++i)
+        args = ATinsert(args, toATerm(*i));
+
+    ATermList env = ATempty;
+    for (StringPairs::const_iterator i = drv.env.begin();
+         i != drv.env.end(); ++i)
+        env = ATinsert(env,
+            makeEnvBinding(
+                toATerm(i->first),
+                toATerm(i->second)));
+
+    return makeDerive(
+        ATreverse(outputs),
+        ATreverse(inDrvs),
+        unparseStrings(drv.inputSrcs),
+        toATerm(drv.platform),
+        toATerm(drv.builder),
+        ATreverse(args),
+        ATreverse(env));
+}
+
+
+bool isDerivation(const string & fileName)
+{
+    return
+        fileName.size() >= drvExtension.size() &&
+        string(fileName, fileName.size() - drvExtension.size()) == drvExtension;
+}

src/libstore/derivations.hh

@@ -0,0 +1,67 @@
+#ifndef __DERIVATIONS_H
+#define __DERIVATIONS_H
+
+#include "aterm.hh"
+#include "store.hh"
+
+
+/* Extension of derivations in the Nix store. */
+const string drvExtension = ".drv";
+
+
+/* Abstract syntax of derivations. */
+
+struct DerivationOutput
+{
+    OutputEqClass eqClass;
+    string hashAlgo; /* hash used for expected hash computation */
+    string hash; /* expected hash, may be null */
+    DerivationOutput()
+    {
+    }
+    DerivationOutput(OutputEqClass eqClass, string hashAlgo, string hash)
+    {
+        this->eqClass = eqClass;
+        this->hashAlgo = hashAlgo;
+        this->hash = hash;
+    }
+};
+
+typedef map<string, DerivationOutput> DerivationOutputs;
+
+/* For inputs that are sub-derivations, we specify exactly which
+   output IDs we are interested in. */
+typedef map<Path, StringSet> DerivationInputs;
+
+typedef map<string, string> StringPairs;
+
+struct Derivation
+{
+    DerivationOutputs outputs; /* keyed on symbolic IDs */
+    DerivationInputs inputDrvs; /* inputs that are sub-derivations */
+    PathSet inputSrcs; /* inputs that are sources */
+    string platform;
+    Path builder;
+    Strings args;
+    StringPairs env;
+};
+
+
+/* Hash an aterm. */
+Hash hashTerm(ATerm t);
+
+/* Write a derivation to the Nix store, and return its path. */
+Path writeDerivation(const Derivation & drv, const string & name);
+
+/* Parse a derivation. */
+Derivation parseDerivation(ATerm t);
+
+/* Parse a derivation. */
+ATerm unparseDerivation(const Derivation & drv);
+
+/* Check whether a file name ends with the extensions for
+   derivations. */
+bool isDerivation(const string & fileName);
+
+
+#endif /* !__DERIVATIONS_H */

src/libstore/gc.cc

@@ -1,98 +1,468 @@
-#include "normalise.hh"
 #include "globals.hh"
+#include "gc.hh"
+#include "build.hh"
+#include "pathlocks.hh"
+#include "misc.hh"
 
+#include <boost/shared_ptr.hpp>
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
 #include <unistd.h>
 
 
-void followLivePaths(Path nePath, PathSet & live)
+static string gcLockName = "gc.lock";
+static string tempRootsDir = "temproots";
+static string gcRootsDir = "gcroots";
+
+
+/* Acquire the global GC lock.  This is used to prevent new Nix
+   processes from starting after the temporary root files have been
+   read.  To be precise: when they try to create a new temporary root
+   file, they will block until the garbage collector has finished /
+   yielded the GC lock. */
+static int openGCLock(LockType lockType)
 {
-    /* Just to be sure, canonicalise the path.  It is important to do
-       this here and in findDeadPath() to ensure that a live path is
-       not mistaken for a dead path due to some non-canonical
-       representation. */
-    nePath = canonPath(nePath);
+    Path fnGCLock = (format("%1%/%2%")
+        % nixStateDir % gcLockName).str();
+         
+    debug(format("acquiring global GC lock `%1%'") % fnGCLock);
     
-    if (live.find(nePath) != live.end()) return;
-    live.insert(nePath);
+    AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600);
+    if (fdGCLock == -1)
+        throw SysError(format("opening global GC lock `%1%'") % fnGCLock);
 
-    startNest(nest, lvlDebug, format("following `%1%'") % nePath);
-    assertStorePath(nePath);
+    lockFile(fdGCLock, lockType, true);
 
-    if (isValidPath(nePath)) {
+    /* !!! Restrict read permission on the GC root.  Otherwise any
+       process that can open the file for reading can DoS the
+       collector. */
+    
+    return fdGCLock.borrow();
+}
 
-        /* !!! should make sure that no substitutes are used */
-        StoreExpr ne = storeExprFromPath(nePath);
 
-        /* !!! painfully similar to requisitesWorker() */
-        if (ne.type == StoreExpr::neClosure)
-            for (ClosureElems::iterator i = ne.closure.elems.begin();
-                 i != ne.closure.elems.end(); ++i)
-            {
-                Path p = canonPath(i->first);
-                if (live.find(p) == live.end()) {
-                    debug(format("found live `%1%'") % p);
-                    assertStorePath(p);
-                    live.insert(p);
+void createSymlink(const Path & link, const Path & target, bool careful)
+{
+    /* Create directories up to `gcRoot'. */
+    createDirs(dirOf(link));
+
+    /* Remove the old symlink. */
+    if (pathExists(link)) {
+        if (careful && (!isLink(link) || !isInStore(readLink(link))))
+            throw Error(format("cannot create symlink `%1%'; already exists") % link);
+        unlink(link.c_str());
+    }
+
+    /* And create the new own. */
+    if (symlink(target.c_str(), link.c_str()) == -1)
+        throw SysError(format("symlinking `%1%' to `%2%'")
+            % link % target);
+}
+
+
+Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
+    bool indirect)
+{
+    Path storePath(canonPath(_storePath));
+    Path gcRoot(canonPath(_gcRoot));
+    assertStorePath(storePath);
+
+    /* Grab the global GC root.  This prevents the set of permanent
+       roots from increasing while a GC is in progress. */
+    AutoCloseFD fdGCLock = openGCLock(ltRead);
+
+    if (indirect) {
+        string hash = printHash32(hashString(htSHA1, gcRoot));
+        Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
+            % nixStateDir % gcRootsDir % hash).str());
+        
+        {
+            SwitchToOriginalUser sw;
+            createSymlink(gcRoot, storePath, true);
+        }
+        createSymlink(realRoot, gcRoot, false);
+    }
+
+    else {
+        Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+    
+        if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
+            throw Error(format(
+                "path `%1%' is not a valid garbage collector root; "
+                "it's not in the directory `%2%'")
+                % gcRoot % rootsDir);
+
+        createSymlink(gcRoot, storePath, false);
+    }
+
+    return gcRoot;
+}
+
+
+/* The file to which we write our temporary roots. */
+static Path fnTempRoots;
+static AutoCloseFD fdTempRoots;
+
+
+void addTempRoot(const Path & path)
+{
+    /* Create the temporary roots file for this process. */
+    if (fdTempRoots == -1) {
+
+        while (1) {
+            Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str();
+            createDirs(dir);
+            
+            fnTempRoots = (format("%1%/%2%")
+                % dir % getpid()).str();
+
+            AutoCloseFD fdGCLock = openGCLock(ltRead);
+            
+            fdTempRoots = open(fnTempRoots.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0600);
+            if (fdTempRoots == -1)
+                throw SysError(format("opening temporary roots file `%1%'") % fnTempRoots);
+
+            fdGCLock.close();
+            
+            debug(format("acquiring read lock on `%1%'") % fnTempRoots);
+            lockFile(fdTempRoots, ltRead, true);
+
+            /* Check whether the garbage collector didn't get in our
+               way. */
+            struct stat st;
+            if (fstat(fdTempRoots, &st) == -1)
+                throw SysError(format("statting `%1%'") % fnTempRoots);
+            if (st.st_size == 0) break;
+            
+            /* The garbage collector deleted this file before we could
+               get a lock.  (It won't delete the file after we get a
+               lock.)  Try again. */
+        }
+
+    }
+
+    /* Upgrade the lock to a write lock.  This will cause us to block
+       if the garbage collector is holding our lock. */
+    debug(format("acquiring write lock on `%1%'") % fnTempRoots);
+    lockFile(fdTempRoots, ltWrite, true);
+
+    string s = path + '\0';
+    writeFull(fdTempRoots, (const unsigned char *) s.c_str(), s.size());
+
+    /* Downgrade to a read lock. */
+    debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
+    lockFile(fdTempRoots, ltRead, true);
+}
+
+
+void removeTempRoots()
+{
+    if (fdTempRoots != -1) {
+        fdTempRoots.close();
+        unlink(fnTempRoots.c_str());
+    }
+}
+
+
+typedef shared_ptr<AutoCloseFD> FDPtr;
+typedef list<FDPtr> FDs;
+
+
+static void readTempRoots(PathSet & tempRoots, FDs & fds)
+{
+    /* Read the `temproots' directory for per-process temporary root
+       files. */
+    Strings tempRootFiles = readDirectory(
+        (format("%1%/%2%") % nixStateDir % tempRootsDir).str());
+
+    for (Strings::iterator i = tempRootFiles.begin();
+         i != tempRootFiles.end(); ++i)
+    {
+        Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
+
+        debug(format("reading temporary root file `%1%'") % path);
+            
+        FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
+        if (*fd == -1) {
+            /* It's okay if the file has disappeared. */
+            if (errno == ENOENT) continue;
+            throw SysError(format("opening temporary roots file `%1%'") % path);
+        }
+
+        /* Try to acquire a write lock without blocking.  This can
+           only succeed if the owning process has died.  In that case
+           we don't care about its temporary roots. */
+        if (lockFile(*fd, ltWrite, false)) {
+            printMsg(lvlError, format("removing stale temporary roots file `%1%'")
+                % path);
+            unlink(path.c_str());
+            writeFull(*fd, (const unsigned char *) "d", 1);
+            continue;
+        }
+
+        /* Acquire a read lock.  This will prevent the owning process
+           from upgrading to a write lock, therefore it will block in
+           addTempRoot(). */
+        debug(format("waiting for read lock on `%1%'") % path);
+        lockFile(*fd, ltRead, true);
+
+        /* Read the entire file. */
+        string contents = readFile(*fd);
+
+        /* Extract the roots. */
+        unsigned int pos = 0, end;
+
+        while ((end = contents.find((char) 0, pos)) != string::npos) {
+            Path root(contents, pos, end - pos);
+            debug(format("got temporary root `%1%'") % root);
+            assertStorePath(root);
+            tempRoots.insert(root);
+            pos = end + 1;
+        }
+
+        fds.push_back(fd); /* keep open */
+    }
+}
+
+
+static void findRoots(const Path & path, bool recurseSymlinks,
+    PathSet & roots)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1)
+        throw SysError(format("statting `%1%'") % path);
+
+    printMsg(lvlVomit, format("looking at `%1%'") % path);
+
+    if (S_ISDIR(st.st_mode)) {
+	Strings names = readDirectory(path);
+	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+            findRoots(path + "/" + *i, recurseSymlinks, roots);
+    }
+
+    else if (S_ISLNK(st.st_mode)) {
+        string target = readLink(path);
+        Path target2 = absPath(target, dirOf(path));
+
+        if (isInStore(target2)) {
+            debug(format("found root `%1%' in `%2%'")
+                % target2 % path);
+            Path target3 = toStorePath(target2);
+            if (isValidPath(target3)) 
+                roots.insert(target3);
+            else
+                printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'")
+                    % path % target3);
+        }
+
+        else if (recurseSymlinks) {
+            if (pathExists(target2))
+                findRoots(target2, false, roots);
+            else {
+                printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target2);
+                /* Note that we only delete when recursing, i.e., when
+                   we are still in the `gcroots' tree.  We never
+                   delete stuff outside that tree. */
+                unlink(path.c_str());
+            }
+        }
+    }
+}
+
+
+static void dfsVisit(const PathSet & paths, const Path & path,
+    PathSet & visited, Paths & sorted)
+{
+    if (visited.find(path) != visited.end()) return;
+    visited.insert(path);
+    
+    PathSet references;
+    if (isValidPath(path))
+        queryReferences(noTxn, path, references);
+    
+    for (PathSet::iterator i = references.begin();
+         i != references.end(); ++i)
+        /* Don't traverse into paths that don't exist.  That can
+           happen due to substitutes for non-existent paths. */
+        if (*i != path && paths.find(*i) != paths.end())
+            dfsVisit(paths, *i, visited, sorted);
+
+    sorted.push_front(path);
+}
+
+
+static Paths topoSort(const PathSet & paths)
+{
+    Paths sorted;
+    PathSet visited;
+    for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i)
+        dfsVisit(paths, *i, visited, sorted);
+    return sorted;
+}
+
+
+void collectGarbage(GCAction action, PathSet & result)
+{
+    result.clear();
+
+    bool gcKeepOutputs =
+        queryBoolSetting("gc-keep-outputs", false);
+    bool gcKeepDerivations =
+        queryBoolSetting("gc-keep-derivations", true);
+
+    /* Acquire the global GC root.  This prevents
+       a) New roots from being added.
+       b) Processes from creating new temporary root files. */
+    AutoCloseFD fdGCLock = openGCLock(ltWrite);
+
+    /* Find the roots.  Since we've grabbed the GC lock, the set of
+       permanent roots cannot increase now. */
+    Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+    PathSet roots;
+    findRoots(rootsDir, true, roots);
+
+    if (action == gcReturnRoots) {
+        result = roots;
+        return;
+    }
+
+    /* Determine the live paths which is just the closure of the
+       roots under the `references' relation. */
+    PathSet livePaths;
+    for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
+        computeFSClosure(canonPath(*i), livePaths);
+
+    if (gcKeepDerivations) {
+        for (PathSet::iterator i = livePaths.begin();
+             i != livePaths.end(); ++i)
+        {
+#if 0            
+            /* Note that the deriver need not be valid (e.g., if we
+               previously ran the collector with `gcKeepDerivations'
+               turned off). */
+            Path deriver = queryDeriver(noTxn, *i);
+            if (deriver != "" && isValidPath(deriver))
+                computeFSClosure(deriver, livePaths);
+#endif            
+        }
+    }
+
+    if (gcKeepOutputs) {
+        /* Hmz, identical to storePathRequisites in nix-store. */
+        for (PathSet::iterator i = livePaths.begin();
+             i != livePaths.end(); ++i)
+            if (isDerivation(*i)) {
+                Derivation drv = derivationFromPath(*i);
+                assert(0);
+#if 0                
+                for (DerivationOutputs::iterator j = drv.outputs.begin();
+                     j != drv.outputs.end(); ++j)
+                    if (isValidPath(j->second.path))
+                        computeFSClosure(j->second.path, livePaths);
+#endif
+            }
+    }
+
+    if (action == gcReturnLive) {
+        result = livePaths;
+        return;
+    }
+
+    /* Read the temporary roots.  This acquires read locks on all
+       per-process temporary root files.  So after this point no paths
+       can be added to the set of temporary roots. */
+    PathSet tempRoots;
+    FDs fds;
+    readTempRoots(tempRoots, fds);
+
+    /* Close the temporary roots.  Note that we *cannot* do this in
+       readTempRoots(), because there we may not have all locks yet,
+       meaning that an invalid path can become valid (and thus add to
+       the references graph) after we have added it to the closure
+       (and computeFSClosure() assumes that the presence of a path
+       means that it has already been closed). */
+    PathSet tempRootsClosed;
+    for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
+        if (isValidPath(*i))
+            computeFSClosure(*i, tempRootsClosed);
+        else
+            tempRootsClosed.insert(*i);
+
+    /* For testing - see tests/gc-concurrent.sh. */
+    if (getenv("NIX_DEBUG_GC_WAIT"))
+        sleep(2);
+    
+    /* After this point the set of roots or temporary roots cannot
+       increase, since we hold locks on everything.  So everything
+       that is not currently in in `livePaths' or `tempRootsClosed'
+       can be deleted. */
+    
+    /* Read the Nix store directory to find all currently existing
+       paths. */
+    Paths storePaths = readDirectory(nixStore);
+    PathSet storePaths2;
+    for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i)
+        storePaths2.insert(canonPath(nixStore + "/" + *i));
+
+    /* Topologically sort them under the `referers' relation.  That
+       is, a < b iff a is in referers(b).  This gives us the order in
+       which things can be deleted safely. */
+    /* !!! when we have multiple output paths per derivation, this
+       will not work anymore because we get cycles. */
+    storePaths = topoSort(storePaths2);
+
+    /* Try to delete store paths in the topologically sorted order. */
+    for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
+
+        debug(format("considering deletion of `%1%'") % *i);
+        
+        if (livePaths.find(*i) != livePaths.end()) {
+            debug(format("live path `%1%'") % *i);
+            continue;
+        }
+
+        if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
+            debug(format("temporary root `%1%'") % *i);
+            continue;
+        }
+
+        debug(format("dead path `%1%'") % *i);
+        result.insert(*i);
+
+        AutoCloseFD fdLock;
+
+        if (action == gcDeleteDead) {
+
+            /* Only delete a lock file if we can acquire a write lock
+               on it.  That means that it's either stale, or the
+               process that created it hasn't locked it yet.  In the
+               latter case the other process will detect that we
+               deleted the lock, and retry (see pathlocks.cc). */
+            if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
+
+                fdLock = open(i->c_str(), O_RDWR);
+                if (fdLock == -1) {
+                    if (errno == ENOENT) continue;
+                    throw SysError(format("opening lock file `%1%'") % *i);
+                }
+
+                if (!lockFile(fdLock, ltWrite, false)) {
+                    debug(format("skipping active lock `%1%'") % *i);
+                    continue;
                 }
             }
-    
-        else if (ne.type == StoreExpr::neDerivation)
-            for (PathSet::iterator i = ne.derivation.inputs.begin();
-                 i != ne.derivation.inputs.end(); ++i)
-                followLivePaths(*i, live);
 
-        else abort();
-        
-    }
+            printMsg(lvlInfo, format("deleting `%1%'") % *i);
+            
+            /* Okay, it's safe to delete. */
+            deleteFromStore(*i);
 
-    Path nfPath;
-    if (querySuccessor(nePath, nfPath))
-        followLivePaths(nfPath, live);
-}
-
-
-PathSet findLivePaths(const Paths & roots)
-{
-    PathSet live;
-
-    startNest(nest, lvlDebug, "finding live paths");
-
-    for (Paths::const_iterator i = roots.begin(); i != roots.end(); ++i)
-        followLivePaths(*i, live);
-
-    return live;
-}
-
-
-PathSet findDeadPaths(const PathSet & live, time_t minAge)
-{
-    PathSet dead;
-
-    startNest(nest, lvlDebug, "finding dead paths");
-
-    time_t now = time(0);
-
-    Strings storeNames = readDirectory(nixStore);
-
-    for (Strings::iterator i = storeNames.begin(); i != storeNames.end(); ++i) {
-        Path p = canonPath(nixStore + "/" + *i);
-
-        if (minAge > 0) {
-            struct stat st;
-            if (lstat(p.c_str(), &st) != 0)
-                throw SysError(format("obtaining information about `%1%'") % p);
-            if (st.st_atime + minAge >= now) continue;
+            if (fdLock != -1)
+                /* Write token to stale (deleted) lock file. */
+                writeFull(fdLock, (const unsigned char *) "d", 1);
         }
-        
-        if (live.find(p) == live.end()) {
-            debug(format("dead path `%1%'") % p);
-            dead.insert(p);
-        } else
-            debug(format("live path `%1%'") % p);
     }
-    
-    return dead;
 }