* Mark as stable.

`--isvalid' flag in nix-store.

Makefile.am

@@ -1,5 +1,6 @@
 SUBDIRS = externals src scripts corepkgs doc misc tests
-EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh svn-revision
+EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
+  svn-revision nix.conf.example
 
 include ./substitute.mk
 
@@ -12,6 +13,11 @@ relname:
 	echo -n $(distdir) > relname
 
 install-data-local: init-state
+	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
+	$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix
+	if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
+		$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
+	fi
 
 if INIT_STATE
 if SETUID_HACK
@@ -22,8 +28,10 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/db
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix/drvs
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles

NEWS

@@ -1,4 +1,178 @@
-Version 0.7
+Release 0.8.1 (April 13, 2005)
+
+This is a bug fix release.
+
+* Patch downloading was broken.
+
+* The garbage collector would not delete paths that had references
+  from invalid (but substitutable) paths.
+
+
+Release 0.8 (April 11, 2005)
+
+NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).  As a
+result, `nix-pull' manifests and channels built for Nix 0.7 and below
+will now work anymore.  However, the Nix expression language has not
+changed, so you can still build from source.  Also, existing user
+environments continue to work.  Nix 0.8 will automatically upgrade the
+database schema of previous installations when it is first run.
+
+If you get the error message
+
+  you have an old-style manifest `/nix/var/nix/manifests/[...]';
+  please delete it
+
+you should delete previously downloaded manifests:
+
+  $ rm /nix/var/nix/manifests/*
+
+If `nix-channel' gives the error message
+
+  manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST'
+  is too old (i.e., for Nix <= 0.7)
+
+then you should unsubscribe from the offending channel (`nix-channel
+--remove URL'; leave out `/MANIFEST'), and subscribe to the same URL,
+with `channels' replaced by `channels-v3' (e.g.,
+http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable).
+
+Nix 0.8 has the following improvements:
+
+* The cryptographic hashes used in store paths are now 160 bits long,
+  but encoded in base-32 so that they are still only 32 characters
+  long (e.g., /nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0).
+  (This is actually a 160 bit truncation of a SHA-256 hash.)
+
+* Big cleanups and simplifications of the basic store semantics.  The
+  notion of "closure store expressions" is gone (and so is the notion
+  of "successors"); the file system references of a store path are now
+  just stored in the database.
+
+  For instance, given any store path, you can query its closure:
+
+    $ nix-store -qR $(which firefox)
+    ... lots of paths ...
+
+  Also, Nix now remembers for each store path the derivation that
+  built it (the "deriver"):
+
+    $ nix-store -qR $(which firefox)
+    /nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv
+
+  So to see the build-time dependencies, you can do
+  
+    $ nix-store -qR $(nix-store -qd $(which firefox))
+
+  or, in a nicer format:
+
+    $ nix-store -q --tree $(nix-store -qd $(which firefox))
+
+  File system references are also stored in reverse.  For instance,
+  you can query all paths that directly or indirectly use a certain
+  Glibc:
+
+    $ nix-store -q --referers-closure \
+        /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+
+* The concept of fixed-output derivations has been formalised.
+  Previously, functions such as `fetchurl' in Nixpkgs used a hack
+  (namely, explicitly specifying a store path hash) to prevent changes
+  to, say, the URL of the file from propagating upwards through the
+  dependency graph, causing rebuilds of everything.  This can now be
+  done cleanly by specifying the `outputHash' and `outputHashAlgo'
+  attributes.  Nix itself checks that the content of the output has
+  the specified hash.  (This is important for maintaining certain
+  invariants necessary for future work on secure shared stores.)
+
+* One-click installation :-)  It is now possible to install any
+  top-level component in Nixpkgs directly, through the web - see,
+  e.g., http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/.  All you
+  have to do is associate `/nix/bin/nix-install-package' with the MIME
+  type `application/nix-package' (or the extension `.nixpkg'), and
+  clicking on a package link will cause it to be installed, with all
+  appropriate dependencies.  If you just want to install some specific
+  application, this is easier than subscribing to a channel.
+
+* `nix-store -r PATHS' now builds all the derivations PATHS in
+  parallel.  Previously it did them sequentially (though exploiting
+  possible parallelism between subderivations).  This is nice for
+  build farms.
+
+* `nix-channel' has new operations `--list' and `--remove'.
+
+* New ways of installing components into user environments:
+
+  - Copy from another user environment:
+
+      $ nix-env -i --from-profile .../other-profile firefox
+
+  - Install a store derivation directly (bypassing the Nix expression
+    language entirely):
+
+      $ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv
+
+    (This is used to implement `nix-install-package', which is
+    therefore immune to evolution in the Nix expression language.)
+
+  - Install an already built store path directly:
+
+      $ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1
+
+  - Install the result of a Nix expression specified as a command-line
+    argument:
+
+      $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'
+
+    The difference with the normal installation mode is that `-E' does
+    not use the `name' attributes of derivations.  Therefore, this can
+    be used to disambiguate multiple derivations with the same name.
+
+* A hash of the contents of a store path is now stored in the database
+  after a succesful build.  This allows you to check whether store
+  paths have been tampered with: `nix-store --verify --check-contents'.
+    
+* Implemented a concurrent garbage collector.  It is now always safe
+  to run the garbage collector, even if other Nix operations are
+  happening simultaneously.
+
+  However, there can still be GC races if you use `nix-instantiate'
+  and `nix-store -r' directly to build things.  To prevent races, use
+  the `--add-root' flag of those commands.
+
+* The garbage collector now finally deletes paths in the right order
+  (i.e., topologically sorted under the `references' relation), thus
+  making it safe to interrupt the collector without risking a store
+  that violates the closure invariant.
+
+* Likewise, the substitute mechanism now downloads files in the right
+  order, thus preserving the closure invariant at all times.
+
+* The result of `nix-build' is now registered as a root of the garbage
+  collector.  If the `./result' link is deleted, the GC root
+  disappears automatically.
+
+* The behaviour of the garbage collector can be changed globally by
+  setting options in `/nix/etc/nix/nix.conf'.
+
+  - `gc-keep-derivations' specifies whether deriver links should be
+    followed when searching for live paths.
+
+  - `gc-keep-outputs' specifies whether outputs of derivations should
+    be followed when searching for live paths.
+  
+  - `env-keep-derivations' specifies whether user environments should
+    store the paths of derivations when they are added (thus keeping
+    the derivations alive).
+
+* New `nix-env' query flags `--drv-path' and `--out-path'.
+
+* `fetchurl' allows SHA-1 and SHA-256 in addition to MD5.  Just
+  specify the attribute `sha1' or `sha256' instead of `md5'.
+
+* Manual updates.
+
+  
+Release 0.7 (January 12, 2005)
 
 * Binary patching.  When upgrading components using pre-built binaries
   (through nix-pull / nix-channel), Nix can automatically download and
@@ -20,7 +194,7 @@ Version 0.7
   dependencies are revealed.
 
 
-Version 0.6
+Release 0.6 (November 14, 2004)
 
 Major changes include the following:
 
@@ -86,6 +260,6 @@ Major changes include the following:
 * Many bug fixes.
 
 
-Version 0.5 and earlier
+Release 0.5 and earlier
 
 Please refer to the Subversion commit log messages.

README

@@ -1,5 +1,9 @@
-                             *** Nix ***
-
 For installation and usage instructions, please read the manual, which
 can be found in `docs/manual/manual.html', and additionally at the Nix
 website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
+
+
+Acknowledgments
+
+This product includes software developed by the OpenSSL Project for
+use in the OpenSSL Toolkit (http://www.OpenSSL.org/)

blacklisting/check-env.pl

@@ -0,0 +1,252 @@
+#! /usr/bin/perl -w -I /home/eelco/.nix-profile/lib/site_perl
+
+use strict;
+use XML::LibXML;
+#use XML::Simple;
+
+my $blacklistFN = shift @ARGV;
+die unless defined $blacklistFN;
+my $userEnv = shift @ARGV;
+die unless defined $userEnv;
+
+
+# Read the blacklist.
+my $parser = XML::LibXML->new();
+my $blacklist = $parser->parse_file($blacklistFN)->getDocumentElement;
+
+#print $blacklist->toString() , "\n";
+
+
+# Get all the elements of the user environment.
+my $userEnvElems = `nix-store --query --references '$userEnv'`;
+die "cannot query user environment elements" if $? != 0;
+my @userEnvElems = split ' ', $userEnvElems;
+
+
+my %storePathHashes;
+
+
+sub getElemNodes {
+    my $node = shift;
+    my @elems = ();
+    foreach my $node ($node->getChildNodes) {
+        push @elems, $node if $node->nodeType == XML_ELEMENT_NODE;
+    }
+    return @elems;
+}
+
+
+my %referencesCache;
+sub getReferences {
+    my $path = shift;
+    return $referencesCache{$path} if defined $referencesCache{$path};
+    
+    my $references = `nix-store --query --references '$path'`;
+    die "cannot query references" if $? != 0;
+    $referencesCache{$path} = [split ' ', $references];
+    
+    return $referencesCache{$path};
+}
+
+
+my %attrsCache;
+sub getAttr {
+    my $path = shift;
+    my $name = shift;
+    my $key = "$path/$name";
+    return $referencesCache{$key} if defined $referencesCache{$key};
+
+    my $value = `nix-store --query --binding '$name' '$path' 2> /dev/null`;
+    $value = "" if $? != 0; # !!!
+    chomp $value;
+    $referencesCache{$key} = $value;
+
+    return $value;
+}
+
+
+sub evalCondition;
+
+
+sub traverse {
+    my $done = shift;
+    my $set = shift;
+    my $path = shift;
+    my $stopCondition = shift;
+
+    return if defined $done->{$path};
+    $done->{$path} = 1;
+    $set->{$path} = 1;
+
+#    print "  in $path\n";
+
+    if (!evalCondition({$path => 1}, $stopCondition)) {
+#        print "  STOPPING in $path\n";
+        return;
+    }
+
+    # Get the requisites of the deriver.
+
+    foreach my $reference (@{getReferences $path}) {
+        traverse($done, $set, $reference, $stopCondition);
+    }
+}
+
+
+sub evalSet {
+    my $inSet = shift;
+    my $expr = shift;
+    my $name = $expr->getName;
+    
+    if ($name eq "traverse") {
+        my $stopCondition = (getElemNodes $expr)[0];
+        my $done = { };
+        my $set = { };
+        foreach my $path (keys %{$inSet}) {
+            traverse($done, $set, $path, $stopCondition);
+        }
+        return $set;
+    }
+
+    else {
+        die "unknown element `$name'";
+    }
+}
+
+
+# Function for evaluating conditions.
+sub evalCondition {
+    my $storePaths = shift;
+    my $condition = shift;
+    my $elemName = $condition->getName;
+    
+    if ($elemName eq "containsSource") {
+        my $hash = $condition->attributes->getNamedItem("hash")->getValue;
+        foreach my $path (keys %{$storePathHashes{$hash}}) {
+            return 1 if defined $storePaths->{$path};
+        }
+        return 0;
+    }
+
+    elsif ($elemName eq "hasName") { 
+        my $nameRE = $condition->attributes->getNamedItem("name")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            return 1 if $path =~ /$nameRE/;
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "hasAttr") { 
+        my $name = $condition->attributes->getNamedItem("name")->getValue;
+        my $valueRE = $condition->attributes->getNamedItem("value")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            if ($path =~ /\.drv$/) {
+                my $value = getAttr($path, $name);
+#                print "    $path $name $value\n";
+                return 1 if $value =~ /$valueRE/;
+            }
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "and") {
+        my $result = 1;
+        foreach my $node (getElemNodes $condition) {
+            $result &= evalCondition($storePaths, $node);
+        }
+        return $result;
+    }
+
+    elsif ($elemName eq "not") {
+        return !evalCondition($storePaths, (getElemNodes $condition)[0]);
+    }
+    
+    elsif ($elemName eq "within") {
+        my @elems = getElemNodes $condition;
+        my $set = evalSet($storePaths, $elems[0]);
+        return evalCondition($set, $elems[1]);
+    }
+
+    elsif ($elemName eq "true") {
+        return 1;
+    }
+
+    elsif ($elemName eq "false") {
+        return 0;
+    }
+
+    else {
+        die "unknown element `$elemName'";
+    }
+}
+
+
+sub evalOr {
+    my $storePaths = shift;
+    my $nodes = shift;
+
+    my $result = 0;
+    foreach my $node (@{$nodes}) {
+        $result |= evalCondition($storePaths, $node);
+    }
+    
+    return $result;
+}
+
+
+# Iterate over all elements, check them.
+foreach my $userEnvElem (@userEnvElems) {
+
+    # Get the deriver of this path.
+    my $deriver = `nix-store --query --deriver '$userEnvElem'`;
+    die "cannot query deriver" if $? != 0;
+    chomp $deriver;
+
+    if ($deriver eq "unknown-deriver") {
+#        print "  deriver unknown, cannot check sources\n";
+        next;
+    }
+
+    print "CHECKING $userEnvElem\n";
+
+
+    # Get the requisites of the deriver.
+#    my $requisites = `nix-store --query --requisites --include-outputs '$deriver'`;
+#    die "cannot query requisites" if $? != 0;
+#    my @requisites = split ' ', $requisites;
+
+
+    # Get the hashes of the requisites.
+#    my $hashes = `nix-store --query --hash @requisites`;
+#    die "cannot query hashes" if $? != 0;
+#    my @hashes = split ' ', $hashes;
+#    for (my $i = 0; $i < scalar @requisites; $i++) {
+#        die unless $i < scalar @hashes;
+#        my $hash = $hashes[$i];
+#        $storePathHashes{$hash} = {} unless defined $storePathHashes{$hash};
+#        my $r = $storePathHashes{$hash}; # !!! fix
+#        $$r{$requisites[$i]} = 1;
+#    }
+
+
+    # Evaluate each blacklist item.
+    foreach my $item ($blacklist->getChildrenByTagName("item")) {
+        my $itemId = $item->getAttributeNode("id")->getValue;
+#        print "  CHECKING FOR $itemId\n";
+
+        my $condition = ($item->getChildrenByTagName("condition"))[0];
+        die unless $condition;
+
+        # Evaluate the condition.
+        my @elems = getElemNodes $condition;
+        if (evalOr({$deriver => 1}, \@elems)) {
+            # Oops, condition triggered.
+            my $reason = ($item->getChildrenByTagName("reason"))[0]->getChildNodes->to_literal;
+            $reason =~ s/\s+/ /g;
+            $reason =~ s/^\s+//g;
+
+            print "    VULNERABLE TO `$itemId': $reason\n";
+        }
+    }
+}
+

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(nix, "0.7")
+AC_INIT(nix, "0.8.1")
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
 AM_INIT_AUTOMAKE
@@ -81,17 +81,19 @@ AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 NEED_PROG(perl, perl)
+NEED_PROG(tar, tar)
+                
+NEED_PROG(cat, cat)
+AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
+  [path of cat, mkdir, etc.]),
+  coreutils=$withval, coreutils=$(dirname $cat))
+AC_SUBST(coreutils)
 
 AC_ARG_WITH(docbook-catalog, AC_HELP_STRING([--with-docbook-catalog=PATH],
   [path of the DocBook XML DTD]),
   docbookcatalog=$withval, docbookcatalog=/docbook-dtd-missing)
 AC_SUBST(docbookcatalog)
 
-AC_ARG_WITH(docbook-ebnf-catalog, AC_HELP_STRING([--with-docbook-ebnf-catalog=PATH],
-  [path of the DocBook XML EBNF module DTD]),
-  docbookebnfcatalog=$withval, docbookcatalog=/docbook-ebnf-dtd-missing)
-AC_SUBST(docbookebnfcatalog)
-
 AC_ARG_WITH(docbook-xsl, AC_HELP_STRING([--with-docbook-xsl=PATH],
   [path of the DocBook XSL stylesheets]),
   docbookxsl=$withval, docbookxsl=/docbook-xsl-missing)
@@ -192,7 +194,6 @@ AC_CONFIG_FILES([Makefile
    src/bsdiff-4.2/Makefile
    scripts/Makefile
    corepkgs/Makefile
-   corepkgs/fetchurl/Makefile
    corepkgs/nar/Makefile
    corepkgs/buildenv/Makefile
    corepkgs/channels/Makefile

corepkgs/Makefile.am

@@ -1 +1 @@
-SUBDIRS = fetchurl nar buildenv channels
+SUBDIRS = nar buildenv channels

corepkgs/buildenv/builder.pl.in

@@ -25,6 +25,7 @@ sub createLinks {
         
 	if ($srcFile =~ /\/propagated-build-inputs$/ ||
             $srcFile =~ /\/nix-support$/ ||
+            $srcFile =~ /\/perllocal.pod$/ ||
             $srcFile =~ /\/log$/)
         {
             # Do nothing.
@@ -72,13 +73,27 @@ sub createLinks {
 
 my %done;
 
+sub addPkg;
 sub addPkg {
     my $pkgDir = shift;
 
     return if (defined $done{$pkgDir});
     $done{$pkgDir} = 1;
 
+    print "adding $pkgDir\n";
     createLinks("$pkgDir", "$out");
+
+    my $propagatedFN = "$pkgDir/nix-support/propagated-build-inputs";
+    if (-e $propagatedFN) {
+        open PROP, "<$propagatedFN" or die;
+        my $propagated = <PROP>;
+        close PROP;
+        my @propagated = split ' ', $propagated;
+        foreach my $p (@propagated) {
+            addPkg $p;
+        }
+    }
+        
 }
 
 
@@ -86,7 +101,6 @@ my @args = split ' ', $ENV{"derivations"};
 
 while (scalar @args > 0) {
     my $drvPath = shift @args;
-    print "adding $drvPath\n";
     addPkg($drvPath);
 }
 

corepkgs/channels/unpack.sh.in

@@ -1,9 +1,7 @@
 #! @shell@ -e
 
-export PATH=/bin:/usr/bin # !!! impure
-
-mkdir $out
-mkdir $out/tmp
+@coreutils@/mkdir $out
+@coreutils@/mkdir $out/tmp
 cd $out/tmp
 
 expr=$out/default.nix
@@ -12,8 +10,8 @@ echo '[' > $expr
 nr=0
 for i in $inputs; do
     echo "unpacking $i"
-    @bunzip2@ < $i | tar xvf -
-    mv * ../$nr # !!! hacky
+    @bunzip2@ < $i | @tar@ xvf -
+    @coreutils@/mv * ../$nr # !!! hacky
     echo "(import ./$nr)" >> $expr
     nr=$(($nr + 1))
 done
@@ -21,4 +19,4 @@ done
 echo ']' >> $expr
 
 cd ..
-rmdir tmp
+@coreutils@/rmdir tmp

corepkgs/fetchurl/Makefile.am

@@ -1,11 +0,0 @@
-all-local: builder.sh
-
-install-exec-local:
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-
-include ../../substitute.mk
-
-EXTRA_DIST = default.nix builder.sh.in

corepkgs/fetchurl/builder.sh.in

@@ -1,19 +0,0 @@
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-echo "downloading $url into $out"
-
-prefetch=@storedir@/nix-prefetch-url-$md5
-if test -f "$prefetch"; then
-    echo "using prefetched $prefetch";
-    mv $prefetch $out
-else
-    @curl@ --fail --location --max-redirs 20 "$url" > "$out"
-fi
-
-actual=$(@bindir@/nix-hash --flat $out)
-if test "$actual" != "$md5"; then
-    echo "hash is $actual, expected $md5"
-    exit 1
-fi

corepkgs/fetchurl/default.nix

@@ -1,8 +0,0 @@
-{system, url, md5}:
-
-derivation {
-  name = baseNameOf (toString url);
-  builder = ./builder.sh;
-  id = md5;
-  inherit system url md5;
-}

corepkgs/nar/Makefile.am

@@ -1,13 +1,11 @@
-all-local: nar.sh unnar.sh
+all-local: nar.sh
 
 install-exec-local:
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_DATA) nar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_PROGRAM) nar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_DATA) unnar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_PROGRAM) unnar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
 
 include ../../substitute.mk
 
-EXTRA_DIST = nar.nix nar.sh.in unnar.nix unnar.sh.in
+EXTRA_DIST = nar.nix nar.sh.in

corepkgs/nar/nar.nix

@@ -1,6 +1,5 @@
-{system, path}: derivation {
+{system, path, hashAlgo}: derivation {
   name = "nar";
   builder = ./nar.sh;
-  system = system;
-  path = path;
+  inherit system path hashAlgo;
 }

corepkgs/nar/nar.sh.in

@@ -1,19 +1,14 @@
 #! @shell@ -e
 
-# !!! impure; fix this
-export PATH=/bin:/usr/bin
-
 echo "packing $path into $out..."
-mkdir $out
-dst=$out/$(basename $path).nar.bz2
+@coreutils@/mkdir $out
+dst=$out/tmp.nar.bz2
 @bindir@/nix-store --dump "$path" > tmp
 
 @bzip2@ < tmp > $dst
 
-narHash=$(md5sum -b tmp | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $narHash > $out/nar-hash
+@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
 
-narbz2Hash=$(md5sum -b $dst | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $narbz2Hash > $out/narbz2-hash
+@bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
+
+@coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2

corepkgs/nar/unnar.nix

@@ -1,7 +0,0 @@
-{system, narFile, outPath}: derivation {
-  name = "unnar";
-  builder = ./unnar.sh;
-  system = system;
-  narFile = narFile;
-  outPath = outPath;
-}

corepkgs/nar/unnar.sh.in

@@ -1,4 +0,0 @@
-#! @shell@ -e
-
-echo "unpacking $narFile to $out..."
-@bunzip2@ < $narFile | @bindir@/nix-store --restore "$out"

doc/manual/Makefile.am

@@ -1,37 +1,41 @@
-ENV = SGML_CATALOG_FILES=$(docbookcatalog):$(docbookebnfcatalog)
+ENV = SGML_CATALOG_FILES=$(docbookcatalog)
 
 XMLLINT = $(ENV) $(xmllint) $(xmlflags) --catalogs
 XSLTPROC = $(ENV) $(xsltproc) $(xmlflags) --catalogs \
  --param section.autolabel 1 \
  --param section.label.includes.component.label 1 \
  --param html.stylesheet \'style.css\' \
- --param xref.with.number.and.title 0
+ --param xref.with.number.and.title 1 \
+ --param toc.section.depth 3
 
-man1_MANS = nix-env.1 nix-store.1 nix-instantiate.1 \
+man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
- nix-prefetch-url.1
+ nix-prefetch-url.1 nix-channel.1
 
 FIGURES = figures/user-environments.png
 
-SOURCES = manual.xml introduction.xml installation.xml \
+MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  package-management.xml writing-nix-expressions.xml \
  build-farm.xml \
  $(man1_MANS:.1=.xml) \
  troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
- quick-start.xml nix-lang-ref.xml style.css images
+ env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
+ conf-file.xml \
+ style.css images
 
-manual.is-valid: $(SOURCES) version.xml
-	$(XMLLINT) --noout --valid manual.xml
+manual.is-valid: $(MANUAL_SRCS) version.txt
+	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --valid -
 	touch $@
 
-version.xml:
-	echo -n $(VERSION) > version.xml
+version.txt:
+	echo -n $(VERSION) > version.txt
 
-man $(MANS): $(SOURCES) manual.is-valid
-	$(XSLTPROC) $(docbookxsl)/manpages/docbook.xsl manual.xml
+man $(MANS): $(MANUAL_SRCS) manual.is-valid
+	$(XSLTPROC) --nonet --xinclude $(docbookxsl)/manpages/docbook.xsl manual.xml
 
-manual.html: $(SOURCES) manual.is-valid images
-	$(XSLTPROC) --output manual.html $(docbookxsl)/html/docbook.xsl manual.xml
+manual.html: $(MANUAL_SRCS) manual.is-valid images
+	$(XSLTPROC) --nonet --xinclude --output manual.html \
+	  $(docbookxsl)/html/docbook.xsl manual.xml
 
 all-local: manual.html
 
@@ -50,8 +54,8 @@ images:
 	cp $(docbookxsl)/images/callouts/*.png images/callouts
 	chmod +w -R images
 
-KEEP = manual.html manual.is-valid version.xml $(MANS)
+KEEP = manual.html manual.is-valid version.txt $(MANS)
 
-EXTRA_DIST = $(SOURCES) $(FIGURES) $(KEEP)
+EXTRA_DIST = $(MANUAL_SRCS) $(FIGURES) $(KEEP)
 
 DISTCLEANFILES = $(KEEP)

doc/manual/bugs.xml

@@ -2,90 +2,25 @@
 
 <itemizedlist>
 
-    <listitem>
-      <para>
-        The man-pages generated from the DocBook documentation are ugly.
-      </para>
-    </listitem>
+<listitem><para>The man-pages generated from the DocBook documentation
+are ugly.</para></listitem>
 
-    <listitem>
-      <para>
-        Generations properly form a tree.  E.g., if after switching to
-        generation 39, we perform an installation action, a generation
-        43 is created which is a descendant of 39, not 42.  So a
-        rollback from 43 ought to go back to 39.  This is not
-        currently implemented; generations form a linear sequence.
-      </para>
-    </listitem>
+<listitem><para>Generations properly form a tree.  E.g., if after
+switching to generation 39, we perform an installation action, a
+generation 43 is created which is a descendant of 39, not 42.  So a
+rollback from 43 ought to go back to 39.  This is not currently
+implemented; generations form a linear sequence.</para></listitem>
 
-    <listitem>
-      <para>
-        Unify the concepts of successors and substitutes into a
-        general notion of <emphasis>equivalent expressions</emphasis>.
-        Expressions are equivalent if they have the same target paths
-        with the same identifiers.  However, even though they are
-        functionally equivalent, they may differ stronly with respect
-        to their <emphasis>performance characteristics</emphasis>.
-        For example, realising a closure expression is more efficient
-        that realising the derivation expression from which it was
-        produced.  On the other hand, distributing sources may be more
-        efficient (storage- or bandwidth-wise) than distributing
-        binaries.  So we need to be able to attach weigths or
-        priorities or performance annotations to expressions; Nix can
-        then choose the most efficient expression dependent on the
-        context.
-      </para>
-    </listitem>
-
-    <listitem>
-      <para>
-        <emphasis>Build management.</emphasis>  In principle it is already
-        possible to do build management using Nix (by writing builders that
-        perform appropriate build steps), but the Nix expression language is
-        not yet powerful enough to make this pleasant (?).  The language should
-        be extended with features from the <ulink
-          url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
-        Another interesting idea is to write a <command>make</command>
-        implementation that uses Nix as a back-end to support <ulink
-          url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink> 
-        build files.
-      </para>
-    </listitem>
-
-    <listitem>
-      <para>
-        There are race conditions between the garbage collector and
-        other Nix tools.  For instance, when we run
-        <command>nix-env</command> to build and install a derivation
-        and run the garbage collector at the same time, the garbage
-        collector may kick in exactly between the build and
-        installation steps, i.e., before the newly built derivation
-        has become reachable from a root of the garbage collector.
-      </para>
-
-      <para>
-        One solution would be for these programs to properly register
-        temporary roots for the collector.  Another would be to use
-        stop-the-world garbage collection: if any tool is running, the
-        garbage collector blocks, and vice versa.  These solutions do
-        not solve the situation where multiple tools are involved,
-        e.g.,
-
-        <screen>
-$ nix-store -r $(nix-instantiate foo.nix)</screen>
-
-        since even if <command>nix-instantiate</command> where to
-        register a temporary root, it would be released by the time
-        <command>nix-store</command> is started.  A solution would be
-        to write the intermediate value to a file that is used as a
-        root to the collector, e.g.,
-        
-        <screen>
-$ nix-instantiate foo.nix > /nix/var/nix/roots/bla
-$ nix-store -r $(cat /nix/var/nix/roots/bla)</screen>
-
-      </para>
-    </listitem>
+<listitem><para><emphasis>Build management.</emphasis> In principle it
+is already possible to do build management using Nix (by writing
+builders that perform appropriate build steps), but the Nix expression
+language is not yet powerful enough to make this pleasant (?).  The
+language should be extended with features from the <ulink
+url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
+Another interesting idea is to write a <command>make</command>
+implementation that uses Nix as a back-end to support <ulink
+url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink>
+build files.</para></listitem>
 
 <listitem><para>For security, <command>nix-push</command> manifests
 should be digitally signed, and <command>nix-pull</command> should
@@ -94,15 +29,18 @@ need to be signed, since the manifest contains cryptographic hashes of
 these files (and <filename>fetchurl.nix</filename> checks
 them).</para></listitem>
 
-<listitem><para>We should switch away from MD5, since it has been
-more-or-less cracked.  We don't currently depend very much on the
-collision-resistance of MD5, but we will once we start sharing build
-results between users.</para></listitem>
-
 <listitem><para>It would be useful to have an option in
 <command>nix-env --delete-generations</command> to remove non-current
 generations older than a certain age.</para></listitem>
 
+<listitem><para>There should be a flexible way to change the user
+environment builder.  Currently, you have to replace
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>,
+which is hard-coded into <command>nix-env</command>.  Also, the
+default builder should be more powerful.  For instance, there should
+be some way to specify priorities to resolve
+collisions.</para></listitem>
+
 </itemizedlist>
 
 </appendix>

doc/manual/build-farm.xml

@@ -65,7 +65,10 @@ will call whenever it wants to build a derivation.  The build hook
 will perform it in the usual way if possible, or it can accept it, in
 which case it is responsible for somehow getting the inputs of the
 build to another machine, doing the build there, and getting the
-results back.</para>
+results back.  The details of the build hook protocol are described in
+the documentation of the <link
+linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar>
+variable</link>.</para>
 
 <example id='ex-remote-systems'><title>Remote machine configuration:
 <filename>remote-systems.conf</filename></title>

doc/manual/conf-file.xml

@@ -0,0 +1,82 @@
+<sect1 id="sec-conf-file"><title>Nix configuration file</title>
+
+<para>A number of persistent settings of Nix are stored in the file
+<filename><replaceable>prefix</replaceable>/etc/nix/nix.conf</filename>.
+This file is a list of <literal><replaceable>name</replaceable> =
+<replaceable>value</replaceable></literal> pairs, one per line.
+Comments start with a <literal>#</literal> character.  An example
+configuration file is shown in <xref linkend="ex-nix-conf" />.</para>
+
+<example id='ex-nix-conf'><title>Nix configuration file</title>
+
+<programlisting>
+gc-keep-outputs = true       # Nice for developers
+gc-keep-derivations = true   # Idem
+env-keep-derivations = false
+</programlisting>
+</example>
+
+<para>The following variables are currently available: 
+
+<variablelist>
+
+  <varlistentry id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
+
+    <listitem><para>If <literal>true</literal>, the garbage collector
+    will keep the outputs of non-garbage derivations.  If
+    <literal>false</literal> (default), outputs will be deleted unless
+    they are GC roots themselves (or reachable from other roots).</para>
+ 
+    <para>In general, outputs must be registered as roots separately.
+    However, even if the output of a derivation is registered as a
+    root, the collector will still delete store paths that are used
+    only at build time (e.g., the C compiler, or source tarballs
+    downloaded from the network).  To prevent it from doing so, set
+    this option to <literal>true</literal>.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>true</literal> (default), the garbage
+    collector will keep the derivations from which non-garbage store
+    paths were built.  If <literal>false</literal>, they will be
+    deleted unless explicitly registered as a root (or reachable from
+    other roots).</para>
+
+    <para>Keeping derivation around is useful for querying and
+    traceability (e.g., it allows you to ask with what dependencies or
+    options a store path was built), so by default this option is on.
+    Turn it off to safe a bit of disk space (or a lot if
+    <literal>gc-keep-outputs</literal> is also turned on).</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><literal>env-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>false</literal> (default), derivations
+    are not stored in Nix user environments.  That is, the derivation
+    any build-time-only dependencies may be garbage-collected.</para>
+
+    <para>If <literal>true</literal>, when you add a Nix derivation to
+    a user environment, the path of the derivation is stored in the
+    user environment.  Thus, the derivation will not be
+    garbage-collected until the user environment generation is deleted
+    (<command>nix-env --delete-generations</command>).  To prevent
+    build-time-only dependencies from being collected, you should also
+    turn on <literal>gc-keep-outputs</literal>.</para>
+
+    <para>The difference between this option and
+    <literal>gc-keep-derivations</literal> is that this one is
+    “sticky”: it applies to any user environment created while this
+    option was enabled, while <literal>gc-keep-derivations</literal>
+    only applies at the moment the garbage collector is
+    run.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+</sect1>

doc/manual/env-common.xml

@@ -0,0 +1,274 @@
+<sect1 id="sec-common-env"><title>Common environment variables</title>
+
+<para>Most Nix commands interpret the following environment variables:</para>
+
+<variablelist>
+
+  
+<varlistentry><term><envar>NIX_ROOT</envar></term>
+
+  <listitem><para>If <envar>NIX_ROOT</envar> is set, the Nix command
+  will on startup perform a <function>chroot()</function> to the
+  specified directory.  This is useful in certain bootstrapping
+  situations (e.g., when installing a Nix installation onto a hard
+  disk from CD-ROM).</para></listitem>
+
+</varlistentry>
+
+  
+<varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term>
+
+  <listitem>
+
+  <para>Normally, the Nix store directory (typically
+  <filename>/nix/store</filename>) is not allowed to contain any
+  symlink components.  This is to prevent “impure” builds.  Builders
+  sometimes “canonicalise” paths by resolving all symlink components.
+  Thus, builds on different machines (with
+  <filename>/nix/store</filename> resolving to different locations)
+  could yield different results.  This is generally not a problem,
+  except when builds are deployed to machines where
+  <filename>/nix/store</filename> resolves differently.  If you are
+  sure that you’re not going to do that, you can set
+  <envar>NIX_IGNORE_SYMLINK_STORE</envar> to <envar>1</envar>.</para>
+
+  <para>Note that if you’re symlinking the Nix store so that you can
+  put it on another file system than the root file system, on Linux
+  you’re better off using <literal>bind</literal> mount points, e.g.,
+
+  <screen>
+$ mkdir /nix   
+$ mount -o bind /mnt/otherdisk/nix /nix</screen>
+
+  Consult the <citerefentry><refentrytitle>mount</refentrytitle>
+  <manvolnum>8</manvolnum></citerefentry> manual page for details.</para>
+
+  </listitem>
+
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STORE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix store (default
+  <filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DATA_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix static data
+  directory (default
+  <filename><replaceable>prefix</replaceable>/share</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix log directory
+  (default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STATE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix state directory
+  (default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DB_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix database (default
+  <filename><replaceable>$NIX_STATE_DIR</replaceable>/db</filename>, i.e.,
+  <filename><replaceable>prefix</replaceable>/var/nix/db</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_CONF_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix configuration
+  directory (default
+  <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_TYPE</envar></term>
+
+  <listitem><para>Equivalent to the <link
+  linkend="opt-log-type"><option>--log-type</option>
+  option</link>.</para></listitem>
+
+</varlistentry>
+  
+
+<varlistentry><term><envar>TMPDIR</envar></term>
+
+  <listitem><para>Use the specified directory to store temporary
+  files.  In particular, this includes temporary build directories;
+  these can take up substantial amounts of disk space.  The default is
+  <filename>/tmp</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term>
+
+  <listitem>
+
+  <para>Specifies the location of the <emphasis>build hook</emphasis>,
+  which is a program (typically some script) that Nix will call
+  whenever it wants to build a derivation.  This is used to implement
+  distributed builds (see <xref linkend="sec-distributed-builds"
+  />).  The protocol by which the calling Nix process and the build
+  hook communicate is as follows.</para>
+
+  <para>The build hook is called with the following command-line
+  arguments:
+
+  <orderedlist>
+
+    <listitem><para>A boolean value <literal>0</literal> or
+    <literal>1</literal> specifying whether Nix can locally execute
+    more builds, as per the <link
+    linkend="opt-max-jobs"><option>--max-jobs</option> option</link>.
+    The purpose of this argument is to allow the hook to not have to
+    maintain bookkeeping for the local machine.</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the local machine
+    (e.g., <literal>i686-linux</literal>).</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the derivation,
+    i.e., its <link linkend="attr-system"><varname>system</varname>
+    attribute</link>.</para></listitem>
+
+    <listitem><para>The store path of the derivation.</para></listitem>
+
+  </orderedlist>
+
+  </para>
+
+  <para>On the basis of this information, and whatever persistent
+  state the build hook keeps about other machines and their current
+  load, it has to decide what to do with the build.  It should print
+  out on file descriptor 3 one of the following responses (terminated
+  by a newline, <literal>"\n"</literal>):
+
+  <variablelist>
+
+    <varlistentry><term><literal>decline</literal></term>
+
+      <listitem><para>The build hook is not willing or able to perform
+      the build; the calling Nix process should do the build itself,
+      if possible.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>postpone</literal></term>
+
+      <listitem><para>The build hook cannot perform the build now, but
+      can do so in the future (e.g., because all available build slots
+      on remote machines are in use).  The calling Nix process should
+      postpone this build until at least one currently running build
+      has terminated.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>accept</literal></term>
+
+      <listitem><para>The build hook has accepted the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the build hook accepts the build, it is possible that it is
+  no longer necessary to do the build because some other process has
+  performed the build in the meantime.  To prevent races, the hook
+  must read from file descriptor 4 a single line that tells it whether
+  to continue:
+
+  <variablelist>
+
+    <varlistentry><term><literal>cancel</literal></term>
+
+      <listitem><para>The build has already been done, so the hook
+      should exit.</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><literal>okay</literal></term>
+
+      <listitem><para>The hook should proceed with the build.  At this
+      point, the calling Nix process has acquired locks on the output
+      path, so no other Nix process will perform the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the hook has been told to proceed, Nix will store in the
+  hook’s current directory a number of text files that contain
+  information about the derivation:
+
+  <variablelist>
+
+    <varlistentry><term><filename>inputs</filename></term>
+
+      <listitem><para>The set of store paths that are inputs to the
+      build process (one per line).  These have to be copied
+      <emphasis>to</emphasis> the remote machine (in addition to the
+      store derivation itself).</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><filename>outputs</filename></term>
+
+      <listitem><para>The set of store paths that are outputs of the
+      derivation (one per line).  These have to be copied
+      <emphasis>from</emphasis> the remote machine if the build
+      succeeds.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><filename>references</filename></term>
+
+      <listitem><para>The reference graph of the inputs, in the format
+      accepted by the command <command>nix-store
+      --register-validity</command>.  It is necessary to run this
+      command on the remote machine after copying the inputs to inform
+      Nix on the remote machine that the inputs are valid
+      paths.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>The hook should copy the inputs to the remote machine,
+  register the validity of the inputs, perform the remote build, and
+  copy the outputs back to the local machine.  An exit code other than
+  <literal>0</literal> indicates that the hook has failed.</para>
+
+  </listitem>
+
+
+</varlistentry>
+
+
+</variablelist>
+
+</sect1>

doc/manual/glossary.xml

@@ -0,0 +1,163 @@
+<appendix><title>Glossary</title>
+
+<glosslist>
+
+
+<glossentry id="gloss-derivation"><glossterm>derivation</glossterm>
+
+  <glossdef><para>A description of a build action.  The result of a
+  derivation is a store object.  Derivations are typically specified
+  in Nix expressions using the <link
+  linkend="ssec-derivation"><function>derivation</function>
+  primitive</link>.  These are translated into low-level
+  <emphasis>store derivations</emphasis> (implicitly by
+  <command>nix-env</command> and <command>nix-build</command>, or
+  explicitly by <command>nix-instantiate</command>).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store</glossterm>
+
+  <glossdef><para>The location in the file system where store objects
+  live.  Typically <filename>/nix/store</filename>.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store path</glossterm>
+
+  <glossdef><para>The location in the file system of a store object,
+  i.e., an immediate child of the Nix store
+  directory.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store object</glossterm>
+
+  <glossdef><para>A file that is an immediate child of the Nix store
+  directory.  These can be regular files, but also entire directory
+  trees.  Store objects can be sources (objects copied from outside of
+  the store), derivation outputs (objects produced by running a build
+  action), or derivations (files describing a build
+  action).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-substitute"><glossterm>substitute</glossterm>
+
+  <glossdef><para>A substitute is a command invocation stored in the
+  Nix database that describes how to build a store object, bypassing
+  normal the build mechanism (i.e., derivations).  Typically, the
+  substitute builds the store object by downloading a pre-built
+  version of the store object from some server.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>purity</glossterm>
+
+  <glossdef><para>The assumption that equal Nix derivations when run
+  always produce the same output.  This cannot be guaranteed in
+  general (e.g., a builder can rely on external inputs such as the
+  network or the system time) but the Nix model assumes
+  it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>Nix expression</glossterm>
+
+  <glossdef><para>A high-level description of software components and
+  compositions thereof.  Deploying software using Nix entails writing
+  Nix expressions for your components.  Nix expressions are translated
+  to derivations that are stored in the Nix store.  These derivations
+  can then be built.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-reference"><glossterm>reference</glossterm>
+
+  <glossdef><para>A store path <varname>P</varname> is said to have a
+  reference to a store path <varname>Q</varname> if the store object
+  at <varname>P</varname> contains the path <varname>Q</varname>
+  somewhere.  This implies than an execution involving
+  <varname>P</varname> potentially needs <varname>Q</varname> to be
+  present.  The <emphasis>references</emphasis> of a store path are
+  the set of store paths to which it has a reference.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-closure"><glossterm>closure</glossterm>
+
+  <glossdef><para>The closure of a store path is the set of store
+  paths that are directly or indirectly “reachable” from that store
+  path; that is, it’s the closure of the path under the <link
+  linkend="gloss-reference">references</link> relation.  For instance,
+  if the store object at path <varname>P</varname> contains a
+  reference to path <varname>Q</varname>, then <varname>Q</varname> is
+  in the closure of <varname>P</varname>.  For correct deployment it
+  is necessary to deploy whole closures, since otherwise at runtime
+  files could be missing.  The command <command>nix-store
+  -qR</command> prints out closures of store paths.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-output-path"><glossterm>output path</glossterm>
+
+  <glossdef><para>A store path produced by a derivation.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-deriver"><glossterm>deriver</glossterm>
+
+  <glossdef><para>The deriver of an <link
+  linkend="gloss-output-path">output path</link> is the store
+  derivation that built it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-validity"><glossterm>validity</glossterm>
+
+  <glossdef><para>A store path is considered
+  <emphasis>valid</emphasis> if it exists in the file system, is
+  listed in the Nix database as being valid, and if all paths in its
+  closure are also valid.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-user-env"><glossterm>user environment</glossterm>
+
+  <glossdef><para>An automatically generated store object that
+  consists of a set of symlinks to “active” applications, i.e., other
+  store paths.  These are generated automatically by <link
+  linkend="sec-nix-env"><command>nix-env</command></link>.  See <xref
+  linkend="sec-profiles" />.</para>
+
+  </glossdef>
+  
+</glossentry>
+
+
+<glossentry id="gloss-profile"><glossterm>profile</glossterm>
+
+  <glossdef><para>A symlink to the current <link
+  linkend="gloss-user-env">user environment</link> of a user, e.g.,
+  <filename>/nix/var/nix/profiles/default</filename>.</para></glossdef>
+
+</glossentry>
+
+
+
+</glosslist>
+
+
+</appendix>

doc/manual/installation.xml

@@ -5,8 +5,8 @@
 
 <para>The easiest way to obtain Nix is to download a <ulink
 url='http://www.cs.uu.nl/groups/ST/Trace/Nix'>source
-distribution</ulink>.  RPMs for Red Hat 9 are also available.  These
-distributions are generated automatically.</para>
+distribution</ulink>.  RPMs for Red Hat, SuSE, and Fedore Core are
+also available.</para>
 
 <para>Alternatively, the most recent sources of Nix can be obtained
 from its <ulink

doc/manual/introduction.xml

@@ -130,11 +130,7 @@ collection.  It also discusses some advanced topics, such as setting
 up a Nix-based build farm, and doing service deployment using
 Nix.</para>
 
-<warning><para>This manual is a work in progress.  It's quite likely
-to be incomplete, inconsistent with the current implementation, or
-simply wrong.</para></warning>
-
-<note><para>Some background information on Nix can be found in two
+<note><para>Some background information on Nix can be found in three
 papers.  The ICSE 2004 paper <ulink
 url='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'><citetitle>Imposing
 a Memory Management Discipline on Software
@@ -145,6 +141,10 @@ different versions and variants of packages.  The LISA 2004 paper
 url='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'><citetitle>Nix:
 A Safe and Policy-Free System for Software
 Deployment</citetitle></ulink> gives a more general discussion of Nix
-from a system-administration perspective.</para></note>
+from a system-administration perspective.  The CBSE 2005 paper <ulink
+url='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'><citetitle>Efficient
+Upgrading in a Purely Functional Component Deployment Model
+</citetitle></ulink> is about transparent patch deployment in
+Nix.</para></note>
 
 </chapter>

doc/manual/manual.xml

@@ -3,31 +3,14 @@
     PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
     "http://www.docbook.org/xml/4.3/docbook-xml-4.3.zip"
 [
-<!ENTITY introduction SYSTEM "introduction.xml">
-<!ENTITY quick-start SYSTEM "quick-start.xml">
-<!ENTITY installation SYSTEM "installation.xml">
-<!ENTITY package-management SYSTEM "package-management.xml">
-<!ENTITY writing-nix-expressions SYSTEM "writing-nix-expressions.xml">
-<!ENTITY build-farm SYSTEM "build-farm.xml">
-<!ENTITY opt-common SYSTEM "opt-common.xml">
-<!ENTITY opt-common-syn SYSTEM "opt-common-syn.xml">
-<!ENTITY nix-env SYSTEM "nix-env.xml">
-<!ENTITY nix-store SYSTEM "nix-store.xml">
-<!ENTITY nix-instantiate SYSTEM "nix-instantiate.xml">
-<!ENTITY nix-collect-garbage SYSTEM "nix-collect-garbage.xml">
-<!ENTITY nix-push SYSTEM "nix-push.xml">
-<!ENTITY nix-pull SYSTEM "nix-pull.xml">
-<!ENTITY nix-prefetch-url SYSTEM "nix-prefetch-url.xml">
-<!-- <!ENTITY nix-lang-ref SYSTEM "nix-lang-ref.xml"> -->
-<!ENTITY troubleshooting SYSTEM "troubleshooting.xml">
-<!ENTITY bugs SYSTEM "bugs.xml">
-<!ENTITY version SYSTEM "version.xml">
 ]>
 
 <book>
   <title>Nix User's Guide</title>
 
-  <subtitle>Draft (Version &version;)</subtitle>
+  <subtitle>Draft (Version <xi:include
+  xmlns:xi="http://www.w3.org/2001/XInclude"
+  href="version.txt" parse="text" />)</subtitle>
   
   <bookinfo>
     <author>
@@ -36,52 +19,65 @@
     </author>
     <copyright>
       <year>2004</year>
+      <year>2005</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
   </bookinfo>
-
-  &introduction;
-  &quick-start;
-  &installation;
-  &package-management;
-  &writing-nix-expressions;
-  &build-farm;
+  
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="introduction.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="quick-start.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="installation.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="package-management.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="writing-nix-expressions.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="build-farm.xml" />
 
   <appendix>
     <title>Command Reference</title>
-    <sect1>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="env-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="conf-file.xml" />
+    <sect1 id="sec-nix-env">
       <title>nix-env</title>
-      &nix-env;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-env.xml" />
+    </sect1>
+    <sect1 id="sec-nix-build">
+      <title>nix-build</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-build.xml" />
     </sect1>
     <sect1>
       <title>nix-store</title>
-      &nix-store;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-store.xml" />
     </sect1>
-    <sect1>
+    <sect1 id="sec-nix-instantiate">
       <title>nix-instantiate</title>
-      &nix-instantiate;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-instantiate.xml" />
     </sect1>
     <sect1>
       <title>nix-collect-garbage</title>
-      &nix-collect-garbage;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-collect-garbage.xml" />
+    </sect1>
+    <sect1 id="sec-nix-channel">
+      <title>nix-channel</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-channel.xml" />
     </sect1>
     <sect1>
       <title>nix-push</title>
-      &nix-push;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-push.xml" />
     </sect1>
     <sect1>
       <title>nix-pull</title>
-      &nix-pull;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-pull.xml" />
     </sect1>
     <sect1>
       <title>nix-prefetch-url</title>
-      &nix-prefetch-url;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-prefetch-url.xml" />
     </sect1>
   </appendix>
 
-<!--  &nix-lang-ref; -->
+  <!--  &nix-lang-ref; -->
 
-  &troubleshooting;
-  &bugs;
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="troubleshooting.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="bugs.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="glossary.xml" />
 
 </book>

doc/manual/nix-build.xml

@@ -0,0 +1,74 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-build</refname>
+  <refpurpose>build a Nix expression</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-build</command>
+    <arg><option>--add-drv-link</option></arg>
+    <arg><option>--no-link</option></arg>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>The <command>nix-build</command> command builds the derivations
+described by the Nix expressions in <replaceable>paths</replaceable>.
+If the build succeeds, it places a symlink to the result in the
+current directory.  The symlink is called <filename>result</filename>.
+If there are multiple Nix expressions, or the Nix expressions evaluate
+to multiple derivations, multiple sequentially numbered symlinks are
+created (<filename>result</filename>, <filename>result-2</filename>,
+and so on).</para>
+
+<note><para><command>nix-build</command> is essentially a wrapper
+around <link
+linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>
+(to translate a high-level Nix expression to a low-level store
+derivation) and <link
+linkend="rsec-nix-store-realise"><command>nix-store
+--realise</command></link> (to build the store
+derivation).</para></note>
+
+<warning><para>The result of the build is automatically registered as
+a root of the Nix garbage collector.  This root disappears
+automatically when the <filename>result</filename> symlink is deleted
+or renamed.  So don’t rename the symlink.</para></warning>
+
+</refsection>
+
+
+<refsection><title>Options</title>
+
+<variablelist>
+
+  <varlistentry><term><option>--add-drv-link</option></term>
+  
+    <listitem><para>Add a symlink in the current directory to the
+    store derivation produced by <command>nix-instantiate</command>.
+    The symlink is called <filename>derivation</filename> (which is
+    numbered in the case of multiple derivations).  The derivation is
+    a root of the garbage collector until the symlink is deleted or
+    renamed.</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--no-link</option></term>
+  
+    <listitem><para>Do not create a symlink to the output path.  Note
+    that as a result the output does not become a root of the garbage
+    collector, and so might be deleted by <command>nix-store
+    --gc</command>.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</refsection>
+
+
+</refentry>

doc/manual/nix-channel.xml

@@ -0,0 +1,83 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-channel</refname>
+  <refpurpose>manage Nix channels</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-channel</command>
+    <group choice='req'>
+      <arg choice='plain'><option>--add</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--remove</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--list</option></arg>
+      <arg choice='plain'><option>--update</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>A Nix channel is mechanism that allows you to automatically stay
+up-to-date with a set of pre-built Nix expressions.  A Nix channel is
+just a URL that points to a place that contains a set of Nix
+expressions, as well as a <command>nix-push</command> manifest.  See
+also <xref linkend="sec-channels" />.</para>
+
+<para>This command has the following operations:
+
+<variablelist>
+
+  <varlistentry><term><option>--add</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Adds <replaceable>url</replaceable> to the list of
+    subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--remove</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Removes <replaceable>url</replaceable> from the
+    list of subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--list</option></term>
+
+    <listitem><para>Prints the URLs of all subscribed channels on
+    standard output.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--update</option></term>
+
+    <listitem><para>Downloads the Nix expressions of all subscribed
+    channels, makes the conjunction of these the default for
+    <command>nix-env</command> operations (by calling <command>nix-env
+    -I</command>), and performs a <command>nix-pull</command> on the
+    manifests of all channels to make pre-built binaries
+    available.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+<para>Note that <option>--add</option> and <option>--remove</option>
+do not automatically perform an update.</para>
+
+<para>The list of subscribed channels is stored in
+<filename>~/.nix-channels</filename>.</para>
+
+<para>A channel consists of two elements: a bzipped Tar archive
+containing the Nix expressions, and a manifest created by
+<command>nix-push</command>.  These must be stored under
+<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal> and
+<literal><replaceable>url</replaceable>/MANIFEST</literal>,
+respectively.</para>
+
+</refsection>
+
+</refentry>

doc/manual/nix-collect-garbage.xml

@@ -1,87 +1,29 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-collect-garbage</refname>
-    <refpurpose>remove unreachable store paths</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-collect-garbage</refname>
+  <refpurpose>delete unreachable store paths</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-collect-garbage</command>
-      <group choice='opt'>
-        <arg choice='plain'><option>--print-live</option></arg>
-        <arg choice='plain'><option>--print-dead</option></arg>
-      </group>
-      <arg><option>--min-age</option> <replaceable>age</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-collect-garbage</command>
+    <group choice='opt'>
+      <arg choice='plain'><option>--print-roots</option></arg>
+      <arg choice='plain'><option>--print-live</option></arg>
+      <arg choice='plain'><option>--print-dead</option></arg>
+      <arg choice='plain'><option>--delete</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
+<refsection><title>Description</title>
 
-    <para>
-      The command <command>nix-collect-garbage</command> performs a
-      garbage collection on the Nix store: any paths in the Nix store
-      that are garbage (not reachable from a set of root store
-      expressions) are deleted.
-    </para>
+<para>The command <command>nix-collect-garbage</command> is an
+obsolete wrapper around <link
+linkend="rsec-nix-store-gc"><command>nix-store
+--gc</command></link>.</para>
 
-    <para>
-      The roots of the garbage collector are the store expressions
-      mentioned in the files in the directory
-      <filename><replaceable>prefix</replaceable>/var/nix/gcroots</filename>.
-      By default, the roots are all user environments in
-      <filename><replaceable>prefix</replaceable>/var/nix/profiles</filename>.
-      You can register other store expressions as roots by writing the
-      full path of the store expression to an arbitrary file in the
-      <filename>gcroots</filename> directory (or a subdirectory
-      thereof).
-    </para>
+</refsection>
 
-  </refsection>
-
-  <refsection>
-    <title>Options</title>
-
-    <variablelist>
-
-      <varlistentry>
-        <term><option>--print-live</option> / <option>--print-dead</option></term>
-        <listitem>
-          <para>
-            These options cause the set of live or dead paths to be
-            printed, respectively, rather than performing an actual
-            garbage collector.  They correspond exactly with the
-            sub-operations in <command>nix-store
-            <option>--gc</option></command>.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-      <varlistentry>
-        <term><option>--min-age</option> <replaceable>age</replaceable></term>
-        <listitem>
-          <para>
-            This option corresponds to the <option>--min-age</option>
-            option in <command>nix-store <option>--gc</option></command>.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-    </variablelist>
-
-  </refsection>
-
-  <refsection>
-    <title>Examples</title>
-
-    <para>
-      To delete all unreachable paths, just do:
-    
-      <screen>
-$ nix-collect-garbage</screen>
-
-    </para>
-
-  </refsection>
-    
 </refentry>

doc/manual/nix-instantiate.xml

@@ -1,90 +1,97 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-instantiate</refname>
-    <refpurpose>instantiate store expressions from Nix expressions</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-instantiate</refname>
+  <refpurpose>instantiate store derivations from Nix expressions</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-instantiate</command>
-      &opt-common-syn;
-      <group choice='opt'>
-        <arg choice='plain'><option>--parse-only</option></arg>
-        <arg choice='plain'><option>--eval-only</option></arg>
-      </group>
-      <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-instantiate</command>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
+    <arg><option>--indirect</option></arg>
+    <group choice='opt'>
+      <arg choice='plain'><option>--parse-only</option></arg>
+      <arg choice='plain'><option>--eval-only</option></arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-instantiate</command> generates
-      (low-level) store expressions from (high-level) Nix expressions.
-      It loads and evaluates the Nix expressions in each of
-      <replaceable>files</replaceable>.  Each top-level expression
-      should evaluate to a derivation, a list of derivations, or a set
-      of derivations.  The paths of the resulting store expressions
-      are printed on standard output.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is generally used for testing Nix expression before
-      they are used with <command>nix-env</command>.
-    </para>
+<para>The command <command>nix-instantiate</command> generates <link
+linkend="gloss-derivation">store derivations</link> from (high-level)
+Nix expressions.  It loads and evaluates the Nix expressions in each
+of <replaceable>files</replaceable>.  Each top-level expression should
+evaluate to a derivation, a list of derivations, or a set of
+derivations.  The paths of the resulting store derivations are printed
+on standard output.</para>
 
-  </refsection>
+<para>Most users and developers don’t need to use this command
+(<command>nix-env</command> and <command>nix-build</command> perform
+store derivation instantiation from Nix expressions automatically).
+It is most commonly used for implementing new deployment
+policies.</para>
 
-  <refsection>
-    <title>Options</title>
+<para>See also <xref linkend="sec-common-options" /> for a list of
+common options.</para>
 
-    <variablelist>
+</refsection>
 
-      &opt-common;
 
-      <varlistentry>
-        <term><option>--parse-only</option></term>
-        <listitem>
-          <para>
-            Just parse the input files, and print their abstract
-            syntax trees on standard output in ATerm format.
-          </para>
-        </listitem>
-      </varlistentry>
+<refsection><title>Options</title>
+
+<variablelist>
+
+  <varlistentry>
+    <term><option>--add-root</option> <replaceable>path</replaceable></term>
+    <term><option>--indirect</option></term>
+
+    <listitem><para>See the <link linkend="opt-add-root">corresponding
+    options</link> in <command>nix-store</command>.</para></listitem>
+
+  </varlistentry>
+
+    
+  <varlistentry><term><option>--parse-only</option></term>
+  
+    <listitem><para>Just parse the input files, and print their
+    abstract syntax trees on standard output in ATerm
+    format.</para></listitem>
+    
+  </varlistentry>
       
-      <varlistentry>
-        <term><option>--eval-only</option></term>
-        <listitem>
-          <para>
-            Just parse and evaluate the input files, and print the
-            resulting values on standard output.  No instantiation of
-            store expressions takes place.
-          </para>
-        </listitem>
-      </varlistentry>
+  <varlistentry><term><option>--eval-only</option></term>
+  
+    <listitem><para>Just parse and evaluate the input files, and print
+    the resulting values on standard output.  No instantiation of
+    store derivations takes place.</para></listitem>
+    
+  </varlistentry>
 
-    </variablelist>
+</variablelist>
 
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Examples</title>
 
-    <screen>
-$ nix-instantiate gcc.nix <lineannotation>(instantiate)</lineannotation>
-/nix/store/468abdcb93aa22bb721142615b97698b-d-gcc-3.3.2.store
+<refsection><title>Examples</title>
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(build)</lineannotation>
+<screen>
+$ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation>
+/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(print output path)</lineannotation>
-/nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
+$ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation>
+<replaceable>...</replaceable>
+/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation>
 
-$ ls -l /nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
-dr-xr-xr-x    2 eelco    users         360 2003-12-01 16:12 bin
-dr-xr-xr-x    3 eelco    users          72 2003-12-01 16:12 include
+$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
+dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
 ...</screen>
 
-  </refsection>
-    
+</refsection>
+
+
 </refentry>

doc/manual/nix-prefetch-url.xml

@@ -1,54 +1,69 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-prefetch-url</refname>
-    <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-prefetch-url</refname>
+  <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-prefetch-url</command>
-      <arg choice='plain'><replaceable>url</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-prefetch-url</command>
+    <arg choice='plain'><replaceable>url</replaceable></arg>
+    <arg><replaceable>hash</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-prefetch-url</command> downloads the
-      file referenced by the URL <replaceable>url</replaceable>,
-      prints its MD5 cryptographic hash code, and copies it into the
-      Nix store.  The file name in the store is
-      <filename><replaceable>hash</replaceable>-<replaceable>basename</replaceable></filename>,
-      where <replaceable>basename</replaceable> is everything
-      following the final slash in <replaceable>url</replaceable>.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is just a convenience to Nix expression writers.
-      Often a Nix expressions fetch some source distribution from the
-      network using the <literal>fetchurl</literal> expression
-      contained in <literal>nixpkgs</literal>.  However,
-      <literal>fetchurl</literal> requires an MD5 hash.  If you don't
-      know the hash, you would have to download the file first, and
-      then <literal>fetchurl</literal> would download it again when
-      you build your Nix expression.  Since
-      <literal>fetchurl</literal> uses the same name for the
-      downloaded file as <command>nix-prefetch-url</command>, the
-      redundant download can be avoided.
-    </para>
+<para>The command <command>nix-prefetch-url</command> downloads the
+file referenced by the URL <replaceable>url</replaceable>, prints its
+cryptographic hash, and copies it into the Nix store.  The file name
+in the store is
+<filename><replaceable>hash</replaceable>-<replaceable>baseName</replaceable></filename>,
+where <replaceable>baseName</replaceable> is everything following the
+final slash in <replaceable>url</replaceable>.</para>
 
-  </refsection>
+<para>This command is just a convenience for Nix expression writers.
+Often a Nix expression fetches some source distribution from the
+network using the <literal>fetchurl</literal> expression contained in
+Nixpkgs.  However, <literal>fetchurl</literal> requires a
+cryptographic hash.  If you don't know the hash, you would have to
+download the file first, and then <literal>fetchurl</literal> would
+download it again when you build your Nix expression.  Since
+<literal>fetchurl</literal> uses the same name for the downloaded file
+as <command>nix-prefetch-url</command>, the redundant download can be
+avoided.</para>
 
-  <refsection>
-    <title>Examples</title>
+<para>The environment variable <envar>NIX_HASH_ALGO</envar> specifies
+which hash algorithm to use.  It can be either <literal>md5</literal>,
+<literal>sha1</literal>, or <literal>sha256</literal>.  The default is
+<literal>md5</literal>.</para>
 
-    <screen>
+<para>If <replaceable>hash</replaceable> is specified, then a download
+is not performed if the Nix store already contains a file with the
+same hash and base name.  Otherwise, the file is downloaded, and an
+error if signaled if the actual hash of the file does not match the
+specified hash.</para>
+
+<para>This command prints the hash on standard output.  Additionally,
+if the environment variable <envar>PRINT_PATH</envar> is set, the path
+of the downloaded file in the Nix store is also printed.</para>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<screen>
 $ nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
-...
-file has hash 0bbd1df101bc0294d440471e50feca71
-...</screen>
+0bbd1df101bc0294d440471e50feca71
+
+$ PRINT_PATH=1 nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
+0bbd1df101bc0294d440471e50feca71
+/nix/store/wvyz8ifdn7wyz1p3pqyn0ra45ka2l492-make-3.80.tar.bz2</screen>
+
+</refsection>
 
-  </refsection>
     
 </refentry>

doc/manual/nix-push.xml

@@ -1,138 +1,116 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-push</refname>
-    <refpurpose>push store paths onto a network cache</refpurpose>
-  </refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-push</command>
-      <arg choice='plain'><replaceable>archives-put-url</replaceable></arg>
-      <arg choice='plain'><replaceable>archives-get-url</replaceable></arg>
-      <arg choice='plain'><replaceable>manifest-put-url</replaceable></arg>
-      <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refnamediv>
+  <refname>nix-push</refname>
+  <refpurpose>push store paths onto a network cache</refpurpose>
+</refnamediv>
 
-  <refsection>
-    <title>Description</title>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-push</command>
+    <group choice='req'>
+      <arg choice='req'>
+        <arg choice='plain'><replaceable>archivesPutURL</replaceable></arg>
+        <arg choice='plain'><replaceable>archivesGetURL</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestPutURL</replaceable></arg>
+      </arg>
+      <arg choice='req'>
+        <arg choice='plain'><option>--copy</option></arg>
+        <arg choice='plain'><replaceable>archivesDir</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestFile</replaceable></arg>
+      </arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-    <para>
-      The command <command>nix-push</command> builds a set of store
-      expressions (if necessary), and then packages and uploads all
-      store paths in the resulting closures to a server.  A network
-      cache thus populated can subsequently be used to speed up
-      software deployment on other machines using the
-      <command>nix-pull</command> command.
-    </para>
 
-    <para>
-      <command>nix-push</command> performs the following actions.
+<refsection><title>Description</title>
+
+<para>The command <command>nix-push</command> builds a set of store
+paths (if necessary), and then packages and uploads all store paths in
+the resulting closures to a server.  A network cache thus populated
+can subsequently be used to speed up software deployment on other
+machines using the <command>nix-pull</command> command.</para>
+
+<para><command>nix-push</command> performs the following actions.
       
-      <orderedlist>
+<orderedlist>
 
-        <listitem>
-          <para>
-            The store expressions stored in
-            <replaceable>paths</replaceable> are realised (using
-            <literal>nix-store --realise</literal>).
-          </para>
-        </listitem>
+  <listitem><para>Each path in <replaceable>paths</replaceable> is
+  realised (using <link
+  linkend='rsec-nix-store-realise'><literal>nix-store
+  --realise</literal></link>).</para></listitem>
 
-        <listitem>
-          <para>
-            All paths in the closure of the store expressions stored
-            in <replaceable>paths</replaceable> are determined (using
-            <literal>nix-store --query --requisites
-            --include-successors</literal>).  It should be noted that
-            since the <option>--include-successors</option> flag is
-            used, if you specify a derivation store expression, you
-            get a combined source/binary distribution.  If you only
-            want a binary distribution, you should specify the closure
-            store expression that result from realising these (see
-            below).
-          </para>
-        </listitem>
+  <listitem><para>All paths in the closure of the store expressions
+  stored in <replaceable>paths</replaceable> are determined (using
+  <literal>nix-store --query --requisites
+  --include-outputs</literal>).  It should be noted that since the
+  <option>--include-outputs</option> flag is used, you get a combined
+  source/binary distribution.</para></listitem>
 
-        <listitem>
-          <para>
-            All store paths determined in the previous step are
-            packaged and compressed into a <command>bzip</command>ped
-            NAR archive (extension <filename>.nar.bz2</filename>).
-          </para>
-        </listitem>
+  <listitem><para>All store paths determined in the previous step are
+  packaged and compressed into a <command>bzip</command>ped NAR
+  archive (extension <filename>.nar.bz2</filename>).</para></listitem>
 
-        <listitem>
-          <para>
-            A <emphasis>manifest</emphasis> is created that contains
-            information on the store paths, their eventual URLs in the
-            cache, and cryptographic hashes of the contents of the NAR
-            archives.
-          </para>
-        </listitem>
+  <listitem><para>A <emphasis>manifest</emphasis> is created that
+  contains information on the store paths, their eventual URLs in the
+  cache, and cryptographic hashes of the contents of the NAR
+  archives.</para></listitem>
 
-        <listitem>
-          <para>
-            Each store path is uploaded to the remote directory
-            specified by <replaceable>archives-put-url</replaceable>.
-            HTTP PUT requests are used to do this.  However, before a
-            file <varname>x</varname> is uploaded to
-            <literal><replaceable>archives-put-url</replaceable>/<varname>x</varname></literal>,
-            <command>nix-push</command> first determines whether this
-            upload is unnecessary by issuing a HTTP HEAD request on
-            <literal><replaceable>archives-get-url</replaceable>/<varname>x</varname></literal>.
-            This allows a cache to be shared between many partially
-            overlapping <command>nix-push</command> invocations.
-            (We use two URLs because the upload URL typically
-            refers to a CGI script, while the download URL just refers
-            to a file system directory on the server.)
-          </para>
-        </listitem>
+  <listitem><para>Each store path is uploaded to the remote directory
+  specified by <replaceable>archivesPutURL</replaceable>.  HTTP PUT
+  requests are used to do this.  However, before a file
+  <varname>x</varname> is uploaded to
+  <literal><replaceable>archivesPutURL</replaceable>/<varname>x</varname></literal>,
+  <command>nix-push</command> first determines whether this upload is
+  unnecessary by issuing a HTTP HEAD request on
+  <literal><replaceable>archivesGetURL</replaceable>/<varname>x</varname></literal>.
+  This allows a cache to be shared between many partially overlapping
+  <command>nix-push</command> invocations.  (We use two URLs because
+  the upload URL typically refers to a CGI script, while the download
+  URL just refers to a file system directory on the server.)</para></listitem>
 
-        <listitem>
-          <para>
-            The manifest is uploaded using an HTTP PUT request to
-            <replaceable>manifest-put-url</replaceable>.  The
-            corresponding URL to download the manifest can then be
-            used by <command>nix-pull</command>.
-          </para>
-        </listitem>
+  <listitem><para>The manifest is uploaded using an HTTP PUT request
+  to <replaceable>manifestPutURL</replaceable>.  The corresponding
+  URL to download the manifest can then be used by
+  <command>nix-pull</command>.</para></listitem>
         
-      </orderedlist>
-    </para>
+</orderedlist>
+
+</para>
+
+<para>TODO: <option>--copy</option></para>
             
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Examples</title>
 
-    <para>
-      To upload files there typically is some CGI script on the server
-      side.  This script should be be protected with a password.  The
-      following example uploads the store paths resulting from
-      building the Nix expressions in <filename>foo.nix</filename>,
-      passing appropriate authentication information:
+<refsection><title>Examples</title>
+
+<para>To upload files there typically is some CGI script on the server
+side.  This script should be be protected with a password.  The
+following example uploads the store paths resulting from building the
+Nix expressions in <filename>foo.nix</filename>, passing appropriate
+authentication information:
     
-      <screen>
+<screen>
 $ nix-push \
     http://foo@bar:server.domain/cgi-bin/upload.pl/cache \
     http://server.domain/cache \
     http://foo@bar:server.domain/cgi-bin/upload.pl/MANIFEST \
     $(nix-instantiate foo.nix)</screen>
 
-    This will push both sources and binaries (and any build-time
-    dependencies used in the build, such as compilers).
-    </para>
+This will push both sources and binaries (and any build-time
+dependencies used in the build, such as compilers).</para>
 
-    <para>
-      If we just want to push binaries, not sources and build-time
-      dependencies, we can do:
+<para>If we just want to push binaries, not sources and build-time
+dependencies, we can do:
       
-      <screen>
+<screen>
 $ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
     
-    </para>
+</para>
 
-  </refsection>
+</refsection>
     
 </refentry>

doc/manual/opt-common-syn.xml

@@ -1,3 +1,5 @@
+<nop>
+  
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'><option>--verbose</option></arg>
@@ -17,3 +19,6 @@
 <arg><option>-K</option></arg>
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
+<arg><option>--log-type</option> <replaceable>type</replaceable></arg>
+
+</nop>

doc/manual/opt-common.xml

@@ -1,184 +1,216 @@
-<varlistentry>
-  <term><option>--help</option></term>
+<sect1 id="sec-common-options"><title>Common options</title>
+
+<para>Most Nix commands accept the following command-line options:</para>
+
+<variablelist>
+
+<varlistentry><term><option>--help</option></term>
+  
+  <listitem><para>Prints out a summary of the command syntax and
+  exits.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--version</option></term>
+  
+  <listitem><para>Prints out the Nix version number on standard output
+  and exits.</para></listitem>
+</varlistentry>
+
+
+<varlistentry><term><option>--verbose</option></term>
+  <term><option>-v</option></term>
+
   <listitem>
-    <para>
-      Prints out a summary of the command syntax and exits.
+    
+  <para>Increases the level of verbosity of diagnostic messages
+  printed on standard error.  For each Nix operation, the information
+  printed on standard output is well-defined; any diagnostic
+  information is printed on standard error, never on standard
+  output.</para>
+
+  <para>This option may be specified repeatedly.  Currently, the
+  following verbosity levels exist:</para>
+
+  <variablelist>
+    
+    <varlistentry><term>0</term>
+    <listitem><para>“Errors only”: only print messages
+    explaining why the Nix invocation failed.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>1</term>
+    <listitem><para>“Informational”: print
+    <emphasis>useful</emphasis> messages about what Nix is doing.
+    This is the default.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>2</term>
+    <listitem><para>“Talkative”: print more informational
+    messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>3</term>
+    <listitem><para>“Chatty”: print even more
+    informational messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>4</term>
+    <listitem><para>“Debug”: print debug
+    information.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>5</term>
+    <listitem><para>“Vomit”: print vast amounts of debug
+    information.</para></listitem>
+    </varlistentry>
+    
+  </variablelist>
+
+  </listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--no-build-output</option></term>
+  <term><option>-Q</option></term>
+
+  <listitem><para>By default, output written by builders to standard
+  output and standard error is echoed to the Nix command's standard
+  error.  This option suppresses this behaviour.  Note that the
+  builder's standard output and error are always written to a log file
+  in
+  <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="opt-max-jobs"><term><option>--max-jobs</option></term>
+  <term><option>-j</option></term>
+
+  <listitem><para>Sets the maximum number of build jobs that Nix will
+  perform in parallel to the specified number.  The default is 1.  A
+  higher value is useful on SMP systems or to exploit I/O latency.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-going</option></term>
+  <term><option>-k</option></term>
+
+  <listitem><para>Keep going in case of failed builds, to the
+  greatest extent possible.  That is, if building an input of some
+  derivation fails, Nix will still build the other inputs, but not the
+  derivation itself.  Without this option, Nix stops if any build
+  fails (except for builds of substitutes), possibly killing builds in
+  progress (in case of parallel or distributed builds).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-failed</option></term>
+  <term><option>-K</option></term>
+
+  <listitem><para>Specifies that in case of a build failure, the
+  temporary directory (usually in <filename>/tmp</filename>) in which
+  the build takes place should not be deleted.  The path of the build
+  directory is printed as an informational message.
     </para>
   </listitem>
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--version</option></term>
+<varlistentry><term><option>--fallback</option></term>
+
   <listitem>
-    <para>
-      Prints out the Nix version number on standard output and exits.
-    </para>
+
+  <para>Whenever Nix attempts to build a derivation for which
+  substitutes are known for each output path, but realising the output
+  paths through the substitutes fails, fall back on building the
+  derivation.</para>
+
+  <para>The most common scenario in which this is useful is when we
+  have registered substitutes in order to perform binary distribution
+  from, say, a network repository.  If the repository is down, the
+  realisation of the derivation will fail.  When this option is
+  specified, Nix will build the derivation instead.  Thus,
+  installation from binaries falls back on nstallation from source.
+  This option is not the default since it is generally not desirable
+  for a transient failure in obtaining the substitutes to lead to a
+  full build from source (with the related consumption of
+  resources).</para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--verbose</option> / <option>-v</option></term>
-  <listitem>
-    <para>
-      Increases the level of verbosity of diagnostic messages printed
-      on standard error.  For each Nix operation, the information
-      printed on standard output is well-defined; any diagnostic
-      information is printed on standard error, never on standard
-      output.
-    </para>
+<varlistentry><term><option>--readonly-mode</option></term>
 
-    <para>
-      This option may be specified repeatedly.  Currently, the
-      following verbosity levels exist:
-    </para>
-
-    <variablelist>
-      <varlistentry>
-        <term>0</term>
-        <listitem>
-          <para>
-            <quote>Errors only</quote>: only print messages explaining
-            why the Nix invocation failed.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>1</term>
-        <listitem>
-          <para>
-            <quote>Informational</quote>: print
-            <emphasis>useful</emphasis> messages about what Nix is
-            doing.  This is the default.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>2</term>
-        <listitem>
-          <para>
-            <quote>Talkative</quote>: print more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>3</term>
-        <listitem>
-          <para>
-            <quote>Chatty</quote>: print even more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>4</term>
-        <listitem>
-          <para>
-            <quote>Debug</quote>: print debug information:
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>5</term>
-        <listitem>
-          <para>
-            <quote>Vomit</quote>: print vast amounts of debug
-            information.
-          </para>
-        </listitem>
-      </varlistentry>
-    </variablelist>
-
-  </listitem>
+  <listitem><para>When this option is used, no attempt is made to open
+  the Nix database.  Most Nix operations do need database access, so
+  those operations will fail.</para></listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--no-build-output</option> / <option>-Q</option></term>
+<varlistentry id="opt-log-type"><term><option>--log-type</option>
+<replaceable>type</replaceable></term>
+
   <listitem>
-    <para>
-      By default, output written by builders to standard output and
-      standard error is echoed to the Nix command's standard error.
-      This option suppresses this behaviour.  Note that the builder's
-      standard output and error are always written to a log file in
-      <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.
-    </para>
+
+  <para>This option determines how the output written to standard
+  error is formatted.  Nix’s diagnostic messages are typically
+  <emphasis>nested</emphasis>.  For instance, when tracing Nix
+  expression evaluation (<command>nix-env -vvvvv</command>, messages
+  from subexpressions are nested inside their parent expressions.  Nix
+  builder output is also often nested.  For instance, the Nix Packages
+  generic builder nests the various build tasks (unpack, configure,
+  compile, etc.), and the GNU Make in <literal>stdenv-linux</literal>
+  has been patched to provide nesting for recursive Make
+  invocations.</para>
+
+  <para><replaceable>type</replaceable> can be one of the
+  following:
+
+  <variablelist>
+
+    <varlistentry><term><literal>pretty</literal></term>
+
+      <listitem><para>Pretty-print the output, indicating different
+      nesting levels using spaces.  This is the
+      default.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>escapes</literal></term>
+
+      <listitem><para>Indicate nesting using escape codes that can be
+      interpreted by the <command>log2xml</command> tool in the Nix
+      source distribution.  The resulting XML file can be fed into the
+      <command>log2html.xsl</command> stylesheet to create an HTML
+      file that can be browsed interactively, using Javascript to
+      expand and collapse parts of the output.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>flat</literal></term>
+
+      <listitem><para>Remove all nesting.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>    
+  
+  </para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--max-jobs</option> / <option>-j</option></term>
-  <listitem>
-    <para>
-      Sets the maximum number of build jobs that Nix will perform in
-      parallel to the specified number.  The default is 1.  A higher
-      value is useful on SMP systems or to exploit I/O latency.
-    </para>
-  </listitem>
-</varlistentry>
+</variablelist>
 
 
-<varlistentry>
-  <term><option>--keep-going</option> / <option>-k</option></term>
-  <listitem>
-    <para>
-      Keep going in case of failed builds, to the greatest extent
-      possible.  That is, if building an input of some derivation
-      fails, Nix will still build the other inputs, but not the
-      derivation itself.  Without this option, Nix stops if any build
-      fails (except for builds of substitutes), possibly killing
-      builds in progress (in case of parallel or distributed builds).
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--keep-failed</option> / <option>-K</option></term>
-  <listitem>
-    <para>
-      Specifies that in case of a build failure, the temporary
-      directory (usually in <filename>/tmp</filename>) in which the
-      build takes place should not be deleted.  The path of the build
-      directory is printed as an informational message.
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--fallback</option></term>
-  <listitem>
-    <para>
-      Whenever Nix attempts to realise a derivation for which a
-      closure is already known, but this closure cannot be realised,
-      fall back on normalising the derivation.
-    </para>
-
-    <para>
-      The most common scenario in which this is useful is when we have
-      registered substitutes in order to perform binary distribution
-      from, say, a network repository.  If the repository is down, the
-      realisation of the derivation will fail.  When this option is
-      specified, Nix will build the derivation instead.  Thus,
-      binary installation falls back on a source installation.  This
-      option is not the default since it is generally not desirable
-      for a transient failure in obtaining the substitutes to lead to
-      a full build from source (with the related consumption of
-      resources).
-    </para>
-  </listitem>
-</varlistentry>
-
-
-<varlistentry>
-  <term><option>--readonly-mode</option></term>
-  <listitem>
-    <para>
-      When this option is used, no attempt is made to open the Nix
-      database.  Most Nix operations do need database access, so those
-      operations will fail.
-    </para>
-  </listitem>
-</varlistentry>
+</sect1>

doc/manual/package-management.xml

@@ -2,24 +2,24 @@
 
 <para>This chapter discusses how to do package management with Nix,
 i.e., how to obtain, install, upgrade, and erase components.  This is
-the <quote>user's</quote> perspective of the Nix system — people
+the “user’s” perspective of the Nix system — people
 who want to <emphasis>create</emphasis> components should consult
 <xref linkend='chap-writing-nix-expressions' />.</para>
 
 
 <sect1><title>Basic package management</title>
 
-<para>The main command for package management is
-<command>nix-env</command>.  You can use it to install, upgrade, and
-erase components, and to query what components are installed or are
-available for installation.</para>
+<para>The main command for package management is <link
+linkend="sec-nix-env"><command>nix-env</command></link>.  You can use
+it to install, upgrade, and erase components, and to query what
+components are installed or are available for installation.</para>
 
-<para>In Nix, different users can have different <quote>views</quote>
+<para>In Nix, different users can have different “views”
 on the set of installed applications.  That is, there might be lots of
 applications present on the system (possibly in many different
 versions), but users can have a specific selection of those active —
-where <quote>active</quote> just means that it appears in a directory
-in the user's <envar>PATH</envar>.  Such a view on the set of
+where “active” just means that it appears in a directory
+in the user’s <envar>PATH</envar>.  Such a view on the set of
 installed applications is called a <emphasis>user
 environment</emphasis>, which is just a directory tree consisting of
 symlinks to the files of the active applications.  </para>
@@ -31,11 +31,9 @@ Nix expressions called the Nix Package collection that contains
 components ranging from basic development stuff such as GCC and Glibc,
 to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
-expression based on it, or completely new ones.)  You can download the
-latest version from <ulink
-url='http://catamaran.labs.cs.uu.nl/dist/nix' />.  You probably want
-the latest unstable release; currently the stable releases tend to lag
-behind quite a bit.</para>
+expressions based on it, or completely new ones.)  You can download
+the latest version from <ulink
+url='http://catamaran.labs.cs.uu.nl/dist/nix' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available components in the release:
@@ -52,7 +50,7 @@ bzip2-1.0.2
 ...</screen>
 
 where <literal>nixpkgs-<replaceable>version</replaceable></literal> is
-where you've unpacked the release.</para>
+where you’ve unpacked the release.</para>
 
 <para>It is also possible to see the <emphasis>status</emphasis> of
 available components, i.e., whether they are installed into the user
@@ -72,7 +70,7 @@ component is installed in your current user environment.  The second
 (in which case installing it into your user environment would be a
 very quick operation).  The last one (<literal>S</literal>) indicates
 whether there is a so-called <emphasis>substitute</emphasis> for the
-component, which is Nix's mechanism for doing binary deployment.  It
+component, which is Nix’s mechanism for doing binary deployment.  It
 just means that Nix know that it can fetch a pre-built component from
 somewhere (typically a network server) instead of building it
 locally.</para>
@@ -98,7 +96,7 @@ available somewhere.  This is done using the
 <command>nix-pull</command> command, which must be supplied with a URL
 containing a <emphasis>manifest</emphasis> describing what binaries
 are available.  This URL should correspond to the Nix Packages release
-that you're using.  For instance, if you obtained a release from
+that you’re using.  For instance, if you obtained a release from
 <ulink
 url='http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />,
 then you should do:
@@ -111,7 +109,7 @@ downloading binaries from <systemitem
 class='fqdomainname'>catamaran.labs.cs.uu.nl</systemitem>, instead of
 building them from source.  This might still take a while since all
 dependencies must be downloaded, but on a reasonably fast connection
-such as an DSL line it's on the order of a few minutes.</para>
+such as an DSL line it’s on the order of a few minutes.</para>
 
 <para>Naturally, packages can also be uninstalled:
 
@@ -127,9 +125,9 @@ release of Nix Packages, you can do:
 $ nix-env -f nixpkgs-<replaceable>version</replaceable> -u subversion</screen>
 
 This will <emphasis>only</emphasis> upgrade Subversion if there is a
-<quote>newer</quote> version in the new set of Nix expressions, as
+“newer” version in the new set of Nix expressions, as
 defined by some pretty arbitrary rules regarding ordering of version
-numbers (which generally do what you'd expect of them).  To just
+numbers (which generally do what you’d expect of them).  To just
 unconditionally replace Subversion with whatever version is in the Nix
 expressions, use <parameter>-i</parameter> instead of
 <parameter>-u</parameter>; <parameter>-i</parameter> will remove
@@ -143,7 +141,7 @@ $ nix-env -f nixpkgs-<replaceable>version</replaceable> -u '*'</screen>
 
 </para>
 
-<para>Sometimes it's useful to be able to ask what
+<para>Sometimes it’s useful to be able to ask what
 <command>nix-env</command> would do, without actually doing it.  For
 instance, to find out what packages would be upgraded by
 <literal>nix-env -u '*'</literal>, you can do
@@ -175,30 +173,28 @@ set.</para></footnote></para>
 </sect1>
 
 
-<sect1><title>Profiles</title>
+<sect1 id="sec-profiles"><title>Profiles</title>
 
-<para>Profiles and user environments are Nix's mechanism for
+<para>Profiles and user environments are Nix’s mechanism for
 implementing the ability to allow differens users to have different
 configurations, and to do atomic upgrades and rollbacks.  To
-understand how they work, it's useful to know a bit about how Nix
+understand how they work, it’s useful to know a bit about how Nix
 works.  In Nix, components are stored in unique locations in the
 <emphasis>Nix store</emphasis> (typically,
 <filename>/nix/store</filename>).  For instance, a particular version
 of the Subversion component might be stored in a directory
-<filename>/nix/store/eeeeaf42e56b...-subversion-0.32.1/</filename>,
+<filename>/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/</filename>,
 while another version might be stored in
-<filename>/nix/store/58823d558a6a...-subversion-0.34/</filename>.  The
-long hexadecimal numbers prefixed to the directory names are
-cryptographic hashes<footnote><para>128 bit MD5 hashes, to be
-precise.</para></footnote> of <emphasis>all</emphasis> inputs involved
-in building the component — sources, dependencies, compiler flags, and
-so on.  So if two components differ in any way, they end up in
-different locations in the file system, so they don't interfere with
-each other.  <xref linkend='fig-user-environments'
-/><footnote><para>TODO: the figure isn't entirely up to date.  It
-should show multiple profiles and
-<filename>~/.nix-profile</filename>.</para></footnote> shows a part of
-a typical Nix store.</para>
+<filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>.
+The long strings prefixed to the directory names are cryptographic
+hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in
+a base-32 notation, to be precise.</para></footnote> of
+<emphasis>all</emphasis> inputs involved in building the component —
+sources, dependencies, compiler flags, and so on.  So if two
+components differ in any way, they end up in different locations in
+the file system, so they don’t interfere with each other.  <xref
+linkend='fig-user-environments' /> shows a part of a typical Nix
+store.</para>
 
 <figure id='fig-user-environments'><title>User environments</title>
   <mediaobject>
@@ -208,41 +204,42 @@ a typical Nix store.</para>
   </mediaobject>
 </figure>
 
-<para>Of course, you wouldn't want to type
+<para>Of course, you wouldn’t want to type
 
 <screen>
-$ /nix/store/eeeeaf42e56b...-subversion-0.32.1/bin/svn</screen>
+$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen>
 
 every time you want to run Subversion.  Of course we could set up the
 <envar>PATH</envar> environment variable to include the
 <filename>bin</filename> directory of every component we want to use,
 but this is not very convenient since changing <envar>PATH</envar>
-doesn't take effect for already existing processes.  The solution Nix
+doesn’t take effect for already existing processes.  The solution Nix
 uses is to create directory trees of symlinks to
 <emphasis>activated</emphasis> components.  These are called
 <emphasis>user environments</emphasis> and they are components
 themselves (though automatically generated by
 <command>nix-env</command>), so they too reside in the Nix store.  For
 instance, in <xref linkend='fig-user-environments' /> the user
-environment <filename>/nix/store/068150f63831...-user-env</filename>
-contains a symlink to just Subversion 0.32.1 (arrows in the figure
+environment <filename>/nix/store/5mq2jcn36ldl...-user-env</filename>
+contains a symlink to just Subversion 1.1.2 (arrows in the figure
 indicate symlinks).  This would be what we would obtain if we had done
 
 <screen>
 $ nix-env -i subversion</screen>
 
-on a set of Nix expressions that contained Subversion 0.32.1.</para>
+on a set of Nix expressions that contained Subversion 1.1.2.</para>
 
-<para>This doesn't in itself solve the problem, of course; you
-wouldn't want to type
-<filename>/nix/store/068150f63831...-user-env/bin/svn</filename>
-either.  Therefore there are symlinks outside of the store that point
+<para>This doesn’t in itself solve the problem, of course; you
+wouldn’t want to type
+<filename>/nix/store/0c1p5z4kda11...-user-env/bin/svn</filename>
+either.  That’s why there are symlinks outside of the store that point
 to the user environments in the store; for instance, the symlinks
-<filename>42</filename> and <filename>43</filename> in the example.
-These are called <emphasis>generations</emphasis> since every time you
-perform a <command>nix-env</command> operation, a new user environment
-is generated based on the current one.  For instance, generation 43
-was created from generation 42 when we did
+<filename>default-42-link</filename> and
+<filename>default-43-link</filename> in the example.  These are called
+<emphasis>generations</emphasis> since every time you perform a
+<command>nix-env</command> operation, a new user environment is
+generated based on the current one.  For instance, generation 43 was
+created from generation 42 when we did
 
 <screen>
 $ nix-env -i subversion mozilla</screen>
@@ -251,14 +248,14 @@ on a set of Nix expressions that contained Mozilla and a new version
 of Subversion.</para>
 
 <para>Generations are grouped together into
-<emphasis>profiles</emphasis> so that different users don't interfere
-with each other if they don't want to.  For example:
+<emphasis>profiles</emphasis> so that different users don’t interfere
+with each other if they don’t want to.  For example:
 
 <screen>
 $ ls -l /nix/var/nix/profiles/
 ...
-lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/068150f63831...-user-env
-lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/84c85f89ddbf...-user-env
+lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
+lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
 lrwxrwxrwx  1 eelco ... default -> default-43-link</screen>
 
 This shows a profile called <filename>default</filename>.  The file
@@ -268,7 +265,7 @@ operation, a new user environment and generation link are created
 based on the current one, and finally the <filename>default</filename>
 symlink is made to point at the new generation.  This last step is
 atomic on Unix, which explains how we can do atomic upgrades.  (Note
-that the building/installing of new components doesn't interfere in
+that the building/installing of new components doesn’t interfere in
 any way with old components, since they are stored in different
 locations in the Nix store.)</para>
 
@@ -293,13 +290,13 @@ can also see all available generations:
 $ nix-env --list-generations</screen></para>
 
 <para>Actually, there is another level of indirection not shown in the
-figure above.  You generally wouldn't have
+figure above.  You generally wouldn’t have
 <filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
 in your <envar>PATH</envar>.  Rather, there is a symlink
 <filename>~/.nix-profile</filename> that points to your current
 profile.  This means that you should put
 <filename>~/.nix-profile/bin</filename> in your <envar>PATH</envar>
-(and indeed, that's what the initialisation script
+(and indeed, that’s what the initialisation script
 <filename>/nix/etc/profile.d/nix.sh</filename> does).  This makes it
 easier to switch to a different profile.  You can do that using the
 command <command>nix-env --switch-profile</command>:
@@ -310,7 +307,7 @@ $ nix-env --switch-profile /nix/var/nix/profiles/my-profile
 $ nix-env --switch-profile /nix/var/nix/profiles/default</screen>
 
 These commands switch to the <filename>my-profile</filename> and
-default profile, respectively.  If the profile doesn't exist, it will
+default profile, respectively.  If the profile doesn’t exist, it will
 be created automatically.  You should be careful about storing a
 profile in another location than the <filename>profiles</filename>
 directory, since otherwise it might not be used as a root of the
@@ -337,7 +334,7 @@ This will <emphasis>not</emphasis> change the
 (<option>-u</option>) and uninstall (<option>-e</option>) never
 actually delete components from the system.  All they do (as shown
 above) is to create a new user environment that no longer contains
-symlinks to the <quote>deleted</quote> components.</para>
+symlinks to the “deleted” components.</para>
 
 <para>Of course, since disk space is not infinite, unused components
 should be removed at some point.  You can do this by running the Nix
@@ -346,7 +343,7 @@ not used (directly or indirectly) by any generation of any
 profile.</para>
 
 <para>Note however that as long as old generations reference a
-component, it will not be deleted.  After all, we wouldn't be able to
+component, it will not be deleted.  After all, we wouldn’t be able to
 do a rollback otherwise.  So in order for garbage collection to be
 effective, you should also delete (some) old generations.  Of course,
 this should only be done if you are certain that you will not need to
@@ -370,51 +367,61 @@ $ nix-env --delete-generations 10 11 14</screen>
 garbage collector as follows:
 
 <screen>
-$ nix-collect-garbage</screen>
+$ nix-store --gc</screen>
 
-You can alo first view what files would be deleted:
+If you are feeling uncertain, you can also first view what files would
+be deleted:
 
 <screen>
-$ nix-collect-garbage --print-dead</screen>
+$ nix-store --gc --print-dead</screen>
 
 Likewise, the option <option>--print-live</option> will show the paths
-that <emphasis>won't</emphasis> be deleted.</para>
+that <emphasis>won’t</emphasis> be deleted.</para>
 
-<sect2><title>Garbage collector roots</title>
 
-<para>TODO</para>
+<sect2 id="ssec-gc-roots"><title>Garbage collector roots</title>
 
-<para>The garbage collector uses as roots all store expressions
-mentioned in all files with extension <filename>.gcroot</filename> in
-the directory
-<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>,
-or in any file or directory symlinked to from that directory.  E.g.,
-by default,
-<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>
-contains a symlink to
-<filename><replaceable>prefix</replaceable>/var/nix/profiles/</filename>,
-so all generations of all profiles are also roots of the collector.</para>
+<para>The roots of the garbage collector are all store paths to which
+there are symlinks in the directory
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>.
+For instance, the following command makes the path
+<filename>/nix/store/d718ef...-foo</filename> a root of the collector:
+
+<screen>
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar</screen>
+	
+That is, after this command, the garbage collector will not remove
+<filename>/nix/store/d718ef...-foo</filename> or any of its
+dependencies.</para>
+
+<para>Subdirectories of
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>
+are also searched for symlinks.  Symlinks to non-store paths are
+followed and searched for roots, but symlinks to non-store paths
+<emphasis>inside</emphasis> the paths reached in that way are not
+followed to prevent infinite recursion.</para>
 
 </sect2>
 
 </sect1>
 
 
-<sect1><title>Channels</title>
+<sect1 id="sec-channels"><title>Channels</title>
 
-<para>If you want to stay up to date with a set of packages, it's not
+<para>If you want to stay up to date with a set of packages, it’s not
 very convenient to manually download the latest set of Nix expressions
 for those packages, use <command>nix-pull</command> to register
 pre-built binaries (if available), and upgrade using
-<command>nix-env</command>.  Fortunately, there's a better way:
+<command>nix-env</command>.  Fortunately, there’s a better way:
 <emphasis>Nix channels</emphasis>.</para>
 
 <para>A Nix channel is just a URL that points to a place that contains
-a set of Nix expressions and a manifest.  Using the command
-<command>nix-channel</command> you can automatically stay up to date
-with whatever is available at that URL.</para>
+a set of Nix expressions and a manifest.  Using the command <link
+linkend="sec-nix-channel"><command>nix-channel</command></link> you
+can automatically stay up to date with whatever is available at that
+URL.</para>
 
-<para>You can <quote>subscribe</quote> to a channel using
+<para>You can “subscribe” to a channel using
 <command>nix-channel --add</command>, e.g.,
 
 <screen>
@@ -427,7 +434,7 @@ of the Nix Packages collection.  (Instead of
 stability, but right now is just outdated.)  Subscribing really just
 means that the URL is added to the file
 <filename>~/.nix-channels</filename>.  Right now there is no command
-to <quote>unsubscribe</quote>; you should just edit that file manually
+to “unsubscribe”; you should just edit that file manually
 and delete the offending URL.</para>
 
 <para>To obtain the latest Nix expressions available in a channel, do
@@ -440,7 +447,7 @@ This downloads the Nix expressions in every channel (downloaded from
 and registers any available pre-built binaries in every channel
 (by <command>nix-pull</command>ing
 <literal><replaceable>url</replaceable>/MANIFEST</literal>).  It also
-makes the union of each channel's Nix expressions the default for
+makes the union of each channel’s Nix expressions the default for
 <command>nix-env</command> operations.  Consequently, you can then say
 
 <screen>

doc/manual/quick-start.xml

@@ -88,8 +88,8 @@ $ nix-channel --update
 $ nix-env -u '*'</screen>
 
 The latter command will upgrade each installed component for which
-there is a <quote>newer</quote> version (as determined by comparing
-the version numbers).</para></listitem>
+there is a “newer” version (as determined by comparing the version
+numbers).</para></listitem>
 
 <listitem><para>If you're unhappy with the result of a
 <command>nix-env</command> action (e.g., an upgraded component turned
@@ -106,12 +106,12 @@ actually delete them:
 
 <screen>
 $ nix-env --delete-generations old
-$ nix-collect-garbage</screen>
+$ nix-store --gc</screen>
 
-The first command deletes old <quote>generations</quote> of your
-profile (making rollbacks impossible, but also making the components
-in those old generations available for garbage collection), while the
-second command actually deletes them.</para></listitem>
+The first command deletes old “generations” of your profile (making
+rollbacks impossible, but also making the components in those old
+generations available for garbage collection), while the second
+command actually deletes them.</para></listitem>
 
 </orderedlist>
 

doc/manual/style.css

@@ -102,6 +102,8 @@ pre.screen
 
 .note,.warning
 {
+    margin-top: 1em;
+    margin-bottom: 1em;
     border: 1px solid #6185a0;
     padding: 0px 1em;
     background: #fffff5;
@@ -154,7 +156,7 @@ a:hover { background: #ffffcd; }
                                Special elements:
  ***************************************************************************/
 
-tt
+tt, code
 {
     color: #400000;
 }
@@ -229,4 +231,4 @@ div.epigraph
 table.productionset table.productionset
 {
     font-family: monospace;
-}
+}

doc/manual/troubleshooting.xml

@@ -1,14 +1,71 @@
-<appendix>
-  <title>Troubleshooting</title>
+<appendix><title>Troubleshooting</title>
+
+<para>This section provides solutions for some common problems.</para>
+
+
+<sect1><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
+
+<para>Symptom: Nix operations (in particular the
+<command>nix-store</command> operations <option>--gc</option>,
+<option>--verify</option>, and <option>--clear-substitutes</option> —
+the latter being called by <command>nix-channel --update</command>)
+failing:
+
+<screen>
+$ nix-store --verify
+error: Db::del: Cannot allocate memory</screen>
+
+</para>
+
+<para>Possible solution: make sure that no Nix processes are running,
+then do:
+
+<screen>
+$ cd /nix/var/nix/db
+$ rm __db.00*</screen>
+
+</para>
+
+</sect1>
+
+
+<sect1><title>Collisions in <command>nix-env</command></title>
+
+<para>Symptom: when installing or upgrading, you get an error message such as
+
+<screen>
+$ nix-env -i docbook-xml
+...
+adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
+collission between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
+  and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
+  at /nix/store/...-builder.pl line 62.</screen>
+
+</para>
+
+<para>The cause is that two installed packages in the user environment
+have overlapping filenames (e.g.,
+<filename>xml/dtd/docbook/calstblx.dtd</filename>.  This usually
+happens when you accidentally try to install two versions of the same
+package.  For instance, in the example above, the Nix Packages
+collection contains two versions of <literal>docbook-xml</literal>, so
+<command>nix-env -i</command> will try to install both.  The default
+user environment builder has no way to way to resolve such conflicts,
+so it just gives up.</para>
+
+<para>Solution: remove one of the offending packages from the user
+environment (if already installed) using <command>nix-env
+-u</command>, or specify exactly which version should be installed
+(e.g., <literal>nix-env -i docbook-xml-4.2</literal>).</para>
+
+<para>Alternatively, you can modify the user environment builder
+script (in
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>)
+to implement some conflict resolution policy.  E.g., the script could
+be modified to rename conflicting file names, or to pick one over the
+other.</para>
+
+</sect1>
+
 
-  <para>
-    (Nothing.)
-  </para>
-	
 </appendix>
-
-<!--
-local variables:
-sgml-parent-document: ("book.xml" "appendix")
-end:
--->

doc/manual/writing-nix-expressions.xml

@@ -383,7 +383,7 @@ some fragments of
     that can be built by Nix (since when we fill in the arguments of
     the function, what we get is its body, which is the call to
     <varname>stdenv.mkDerivation</varname> in <xref
-    linkend='ex-hello-nix ' />).</para>
+    linkend='ex-hello-nix' />).</para>
 
   </callout>
 
@@ -447,7 +447,8 @@ following:
 (import pkgs/system/i686-linux.nix).hello</programlisting>
 
 Call it <filename>test.nix</filename>.  You can then build it without
-installing it using the command <command>nix-build</command>:
+installing it using the command <link
+linkend="sec-nix-build"><command>nix-build</command></link>:
 
 <screen>
 $ nix-build ./test.nix
@@ -817,7 +818,7 @@ set.</para>
 </simplesect>
 
 
-<simplesect><title>Functions</title>
+<simplesect id="ss-functions"><title>Functions</title>
 
 <para>Functions have the following form:
 
@@ -1103,7 +1104,7 @@ weakest binding).</para>
 </simplesect>
 
 
-<simplesect><title>Derivations</title>
+<simplesect id="ssec-derivation"><title>Derivations</title>
 
 <para>The most important built-in function is
 <function>derivation</function>, which is used to describe a
@@ -1112,7 +1113,7 @@ set, the attributes of which specify the inputs of the build.</para>
 
 <itemizedlist>
 
-  <listitem><para>There must be an attribute named
+  <listitem id="attr-system"><para>There must be an attribute named
   <varname>system</varname> whose value must be a string specifying a
   Nix platform identifier, such as <literal>"i686-linux"</literal> or
   <literal>"powerpc-darwin"</literal><footnote><para>To figure out

nix.conf.example

@@ -0,0 +1,49 @@
+### Option `gc-keep-outputs'
+#
+# If `true', the garbage collector will keep the outputs of
+# non-garbage derivations.  If `false' (default), outputs will be
+# deleted unless they are GC roots themselves (or reachable from other
+# roots).
+# 
+# In general, outputs must be registered as roots separately.
+# However, even if the output of a derivation is registered as a root,
+# the collector will still delete store paths that are used only at
+# build time (e.g., the C compiler, or source tarballs downloaded from
+# the network).  To prevent it from doing so, set this option to
+# `true'.
+gc-keep-outputs = false
+
+
+### Option `gc-keep-derivations'
+#
+# If `true' (default), the garbage collector will keep the derivations
+# from which non-garbage store paths were built.  If `false', they
+# will be deleted unless explicitly registered as a root (or reachable
+# from other roots).
+#
+# Keeping derivation around is useful for querying and traceability
+# (e.g., it allows you to ask with what dependencies or options a
+# store path was built), so by default this option is on.  Turn it off
+# to safe a bit of disk space (or a lot if `gc-keep-outputs' is also
+# turned on).
+gc-keep-derivations = true
+
+
+### Option `env-keep-derivations'
+#
+# If `false' (default), derivations are not stored in Nix user
+# environments.  That is, the derivation any build-time-only
+# dependencies may be garbage-collected.
+#
+# If `true', when you add a Nix derivation to a user environment, the
+# path of the derivation is stored in the user environment.  Thus, the
+# derivation will not be garbage-collected until the user environment
+# generation is deleted (`nix-env --delete-generations').  To prevent
+# build-time-only dependencies from being collected, you should also
+# turn on `gc-keep-outputs'.
+#
+# The difference between this option and `gc-keep-derivations' is that
+# this one is `sticky': it applies to any user environment created
+# while this option was enabled, while `gc-keep-derivations' only
+# applies at the moment the garbage collector is run.
+env-keep-derivations = false

scripts/Makefile.am

@@ -2,7 +2,7 @@ bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build
 
-noinst_SCRIPTS = nix-profile.sh
+noinst_SCRIPTS = nix-profile.sh generate-patches.pl
 
 nix-pull nix-push: readmanifest.pm download-using-manifests.pl
 
@@ -22,4 +22,5 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-channel.in \
   readmanifest.pm.in \
   nix-build.in \
-  download-using-manifests.pl.in
+  download-using-manifests.pl.in \
+  generate-patches.pl.in

scripts/copying-collector.pl

@@ -0,0 +1,111 @@
+#! /usr/bin/perl -w
+
+use strict;
+
+my @paths = `nix-store -qR /home/eelco/.nix-profile/bin/firefox`;
+
+my %copyMap;
+my %rewriteMap;
+
+
+my $counter = 0;
+
+foreach my $path (@paths) {
+    chomp $path;
+
+    $path =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$path'";
+    my $hash = $2;
+
+#    my $newHash = "deadbeef" . (sprintf "%024d", $counter++);
+    my $newHash = "deadbeef" . substr($hash, 0, 24);
+    my $newPath = "/home/eelco/chroot/$1/$newHash-$3";
+
+    die unless length $newHash == length $hash;
+
+    $copyMap{$path} = $newPath;
+    $rewriteMap{$hash} = $newHash;
+}
+
+
+my %rewriteMap2;
+
+
+sub rewrite;
+sub rewrite {
+    my $src = shift;
+    my $dst = shift;
+
+    if (-l $dst) {
+
+        my $target = readlink $dst or die;
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+            $target =~ s/$srcHash/$dstHash/g;
+        }
+
+        unlink $dst or die;
+
+        symlink $target, $dst;
+        
+    }
+
+    elsif (-f $dst) {
+
+        print "$dst\n";
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+
+            my @stats = lstat $dst or die;
+            
+            system "sed s/$srcHash/$dstHash/g < '$dst' > '$dst.tmp'";
+            die if $? != 0;
+            rename "$dst.tmp", $dst or die;
+
+            chmod $stats[2], $dst or die;
+        }
+
+    }
+
+    elsif (-d $dst) {
+
+        chmod 0755, $dst;
+        
+        opendir(DIR, "$dst") or die "cannot open `$dst': $!";
+        my @files = readdir DIR;
+        closedir DIR;
+        
+        foreach my $file (@files) {
+            next if $file eq "." || $file eq "..";
+            rewrite "$src/$file", "$dst/$file";
+        }
+    }
+}
+
+
+foreach my $src (keys %copyMap) {
+    my $dst = $copyMap{$src};
+    print "$src -> $dst\n";
+
+    if (!-e $dst) {
+        system "cp -prd $src $dst";
+        die if $? != 0;
+
+        my @refs = `nix-store -q --references $src`;
+
+        %rewriteMap2 = ();
+        foreach my $ref (@refs) {
+            chomp $ref;
+
+            $ref =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$ref'";
+            my $hash = $2;
+
+            $rewriteMap2{$hash} = $rewriteMap{$hash};
+        }
+
+        rewrite $src, $dst;
+    }
+}

scripts/download-using-manifests.pl.in

@@ -13,7 +13,7 @@ open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
 die unless scalar @ARGV == 1;
 my $targetPath = $ARGV[0];
 
-my $date = `date`;
+my $date = `date` or die;
 chomp $date;
 print LOGFILE "$$ get $targetPath $date\n";
 
@@ -27,7 +27,10 @@ my %successors;
 
 for my $manifest (glob "$manifestDir/*.nixmanifest") {
 #    print STDERR "reading $manifest\n";
-    readManifest $manifest, \%narFiles, \%patches, \%successors;
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
+        exit 1;
+    }
 }
 
 
@@ -73,10 +76,19 @@ addToQueue $targetPath;
 
 sub isValidPath {
     my $p = shift;
-    system "nix-store --isvalid '$p' 2> /dev/null";
+    system "@bindir@/nix-store --check-validity '$p' 2> /dev/null";
     return $? == 0;
 }
 
+sub parseHash {
+    my $hash = shift;
+    if ($hash =~ /^(.+):(.+)$/) {
+        return ($1, $2);
+    } else {
+        return ("md5", $hash);
+    }
+}
+
 while ($queueFront < scalar @queue) {
     my $u = $queue[$queueFront++];
 #    print "$u\n";
@@ -96,10 +108,13 @@ while ($queueFront < scalar @queue) {
         foreach my $patch (@{$patchList}) {
             if (isValidPath($patch->{basePath})) {
                 # !!! this should be cached
-                my $hash = `nix-hash "$patch->{basePath}"`;
+                my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
+                my $format = "--base32";
+                $format = "" if $baseHashAlgo eq "md5";
+                my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
                 chomp $hash;
 #                print "  MY HASH is $hash\n";
-                if ($hash ne $patch->{baseHash}) {
+                if ($hash ne $baseHash) {
                     print LOGFILE "$$ rejecting $patch->{basePath}\n";
                     next;
                 }
@@ -174,13 +189,15 @@ my $maxStep = scalar @path;
 
 sub downloadFile {
     my $url = shift;
-    my $hash = shift;
+    my ($hashAlgo, $hash) = parseHash(shift);
     $ENV{"PRINT_PATH"} = 1;
     $ENV{"QUIET"} = 1;
-    my ($hash2, $path) = `nix-prefetch-url '$url' '$hash'`;
+    $ENV{"NIX_HASH_ALGO"} = $hashAlgo;
+    my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
+    die "download of `$url' failed" unless $? == 0;
     chomp $hash2;
     chomp $path;
-    die "hash mismatch" if $hash ne $hash2;
+    die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
     return $path;
 }
 
@@ -210,7 +227,7 @@ while (scalar @path > 0) {
         # Turn the base path into a NAR archive, to which we can
         # actually apply the patch.
         print "  packing base path...\n";
-        system "nix-store --dump $patch->{basePath} > /tmp/nar";
+        system "@bindir@/nix-store --dump $patch->{basePath} > /tmp/nar";
         die "cannot dump `$patch->{basePath}'" if ($? != 0);
 
         # Apply the patch.
@@ -220,7 +237,7 @@ while (scalar @path > 0) {
 
         # Unpack the resulting NAR archive into the target path.
         print "  unpacking patched archive...\n";
-        system "nix-store --restore $v < /tmp/nar2";
+        system "@bindir@/nix-store --restore $v < /tmp/nar2";
         die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
     }
 
@@ -236,7 +253,7 @@ while (scalar @path > 0) {
 
         # Unpack the archive into the target path.
         print "  unpacking archive...\n";
-        system "bunzip2 < '$narFilePath' | nix-store --restore '$v'";
+        system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
         die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
     }
 }

scripts/gc-releases.pl

@@ -0,0 +1,75 @@
+#! /usr/bin/perl -w
+
+use strict;
+use readmanifest;
+
+
+# Read the archive directories.
+my @archives = ();
+my %archives;
+
+sub readDir {
+    my $dir = shift;
+    opendir(DIR, "$dir") or die "cannot open `$dir': $!";
+    my @as = readdir DIR;
+    foreach my $archive (@as) {
+        push @archives, $archive;
+        $archives{$archive} = "$dir/$archive";
+    }
+    closedir DIR;
+}
+
+readDir "/mnt/scratchy/eelco/public_html/nix-cache";
+readDir "/mnt/scratchy/eelco/public_html/patches";
+
+print STDERR scalar @archives, "\n";
+
+
+# Read the manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+foreach my $manifest (@ARGV) {
+    print STDERR "loading $manifest\n";
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors, 1) < 3) {
+#        die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n";
+    }
+}
+
+
+# Find the live archives.
+my %usedFiles;
+
+foreach my $narFile (keys %narFiles) {
+    foreach my $file (@{$narFiles{$narFile}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+foreach my $patch (keys %patches) {
+    foreach my $file (@{$patches{$patch}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+
+# Print out the dead archives.
+foreach my $archive (@archives) {
+    next if $archive eq "." || $archive eq "..";
+    if (!defined $usedFiles{$archive}) {
+        print $archives{$archive}, "\n";
+    }
+}

scripts/generate-patches.pl.in

@@ -1,4 +1,4 @@
-#! /usr/bin/perl -w -I/home/eelco/nix/scripts
+#! @perl@ -w -I@libexecdir@/nix
 
 use strict;
 use POSIX qw(tmpnam);
@@ -6,6 +6,8 @@ use readmanifest;
 
 die unless scalar @ARGV == 5;
 
+my $hashAlgo = "sha256";
+
 my $cacheDir = $ARGV[0];
 my $patchesDir = $ARGV[1];
 my $patchesURL = $ARGV[2];
@@ -45,6 +47,7 @@ sub findOutputPaths {
 
         # Ignore store expressions.
         next if ($p =~ /\.store$/);
+        next if ($p =~ /\.drv$/);
         
         # Ignore builders (too much ambiguity -- they're all called
         # `builder.sh').
@@ -69,7 +72,7 @@ my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
 
 sub getNameVersion {
     my $p = shift;
-    $p =~ /\/[0-9a-f]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
+    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
     my $name = $1;
     my $version = $2;
     $name =~ s/^-//;
@@ -125,6 +128,64 @@ sub containsPatch {
 }
 
 
+# Compute the "weighted" number of uses of a path in the build graph.
+sub computeUses {
+    my $narFiles = shift;
+    my $path = shift;
+
+    # Find the deriver of $path.
+    return 1 unless defined $$narFiles{$path};
+    my $deriver = @{$$narFiles{$path}}[0]->{deriver};
+    return 1 unless defined $deriver && $deriver ne "";
+
+#    print "  DERIVER $deriver\n";
+
+    # Optimisation: build the referers graph from the references
+    # graph.
+    my %referers;
+    foreach my $q (keys %{$narFiles}) {
+        my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
+        foreach my $r (@refs) {
+            $referers{$r} = [] unless defined $referers{$r};
+            push @{$referers{$r}}, $q;
+        }
+    }
+
+    # Determine the shortest path from $deriver to all other reachable
+    # paths in the `referers' graph.
+
+    my %dist;
+    $dist{$deriver} = 0;
+
+    my @queue = ($deriver);
+    my $pos = 0;
+    
+    while ($pos < scalar @queue) {
+        my $p = $queue[$pos];
+        $pos++;
+
+        foreach my $q (@{$referers{$p}}) {
+            if (!defined $dist{$q}) {
+                $dist{$q} = $dist{$p} + 1;
+#                print "    $q $dist{$q}\n";
+                push @queue, $q;
+            }
+        }
+    }
+
+    my $wuse = 1.0;
+    foreach my $user (keys %dist) {
+        next if $user eq $deriver;
+#        print "    $user $dist{$user}\n";
+        $wuse += 1.0 / 2.0**$dist{$user};
+    }
+
+#    print "    XXX $path $wuse\n";
+    
+    return $wuse;
+}
+
+
 # For each output path in the destination, see if we need to / can
 # create a patch.
 
@@ -135,9 +196,9 @@ foreach my $p (keys %dstOutPaths) {
     # If exactly the same path already exists in the source, skip it.
     next if defined $srcOutPaths{$p};
     
-#    print "  $p\n";
+    print "  $p\n";
 
-    # If not, then we should find the path in the source that is
+    # If not, then we should find the paths in the source that are
     # `most' likely to be present on a system that wants to install
     # this path.
 
@@ -152,6 +213,37 @@ foreach my $p (keys %dstOutPaths) {
     foreach my $q (keys %srcOutPaths) {
         (my $name2, my $version2) = getNameVersion $q;
         if ($name eq $name2) {
+
+            # If the sizes differ too much, then skip.  This
+            # disambiguates between, e.g., a real component and a
+            # wrapper component (cf. Firefox in Nixpkgs).
+            my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
+            my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
+            my $ratio = $srcSize / $dstSize;
+            $ratio = 1 / $ratio if $ratio < 1;
+#            print "  SIZE $srcSize $dstSize $ratio $q\n";
+
+            if ($ratio >= 3) {
+                print "  SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
+                next;
+            }
+
+            # If the numbers of weighted uses differ too much, then
+            # skip.  This disambiguates between, e.g., the bootstrap
+            # GCC and the final GCC in Nixpkgs.
+            my $srcUses = computeUses \%srcNarFiles, $q;
+            my $dstUses = computeUses \%dstNarFiles, $p;
+            $ratio = $srcUses / $dstUses;
+            $ratio = 1 / $ratio if $ratio < 1;
+            print "  USE $srcUses $dstUses $ratio $q\n";
+            
+            if ($ratio >= 2) {
+                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
+                next;
+            }
+
+            # If there are multiple matching names, include the ones
+            # with the closest version numbers.
             my $dist = versionDiff $version, $version2;
             if ($dist > $minDist) {
                 $minDist = $dist;
@@ -186,22 +278,22 @@ foreach my $p (keys %dstOutPaths) {
         my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
         my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
         
-        system("bunzip2 < $srcNarBz2 > $tmpdir/A") == 0
+        system("@bunzip2@ < $srcNarBz2 > $tmpdir/A") == 0
             or die "cannot unpack $srcNarBz2";
 
-        system("bunzip2 < $dstNarBz2 > $tmpdir/B") == 0
+        system("@bunzip2@ < $dstNarBz2 > $tmpdir/B") == 0
             or die "cannot unpack $dstNarBz2";
 
-        system("bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
+        system("@libexecdir@/bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
             or die "cannot compute binary diff";
 
-        my $baseHash = `nix-hash --flat $tmpdir/A` or die;
+        my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/A` or die;
         chomp $baseHash;
 
-        my $narHash = `nix-hash --flat $tmpdir/B` or die;
+        my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/B` or die;
         chomp $narHash;
 
-        my $narDiffHash = `nix-hash --flat $tmpdir/DIFF` or die;
+        my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/DIFF` or die;
         chomp $narDiffHash;
 
         my $narDiffSize = (stat "$tmpdir/DIFF")[7];
@@ -213,7 +305,7 @@ foreach my $p (keys %dstOutPaths) {
         }
     
         my $finalName =
-            "$narDiffHash-$name-$closestVersion-to-$version.nar-bsdiff";
+            "$narDiffHash.nar-bsdiff";
 
         print "    size $narDiffSize; full size $dstNarBz2Size\n";
         
@@ -233,11 +325,10 @@ foreach my $p (keys %dstOutPaths) {
         
         # Add the patch to the manifest.
         addPatch \%dstPatches, $p,
-            { url => "$patchesURL/$finalName", hash => $narDiffHash
-            , size => $narDiffSize
-            , basePath => $closest, baseHash => $baseHash
-            , narHash => $narHash, patchType => "nar-bsdiff"
-            };
+            { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
+            , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
+            , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
+            }, 0;
     }
 }
 

scripts/nix-build.in

@@ -8,34 +8,52 @@ if test -z "$nixExpr"; then
 fi
 
 extraArgs=
-noLink=
+addDrvLink=0
+addOutLink=1
 
+
+trap 'rm -f ./.nix-build-tmp-*' EXIT
+
+
+# Process the arguments.
 for i in "$@"; do
     case "$i" in
-        --no-link)
-            noLink=1
+    
+        --add-drv-link)
+            addDrvLink=1
             ;;
+            
+        --no-link)
+            addOutLink=0
+            ;;
+            
         -*)
             extraArgs="$extraArgs $i"
             ;;
+            
         *)
-            storeExprs=$(nix-instantiate "$i")
+            # Instantiate the Nix expression.
+            prefix=
+            if test "$addDrvLink" = 0; then prefix=.nix-build-tmp-; fi
+            storeExprs=$(@bindir@/nix-instantiate \
+                --add-root ./${prefix}derivation --indirect \
+                "$i")
+                
             for j in $storeExprs; do
-                echo "store expression is $j" >&2
+                echo "store expression is $(readlink "$j")" >&2
             done
-            outPaths=$(nix-store -qnfv $extraArgs $storeExprs)
+
+            # Build the resulting store derivation.
+            prefix=
+            if test "$addOutLink" = 0; then prefix=.nix-build-tmp-; fi
+            outPaths=$(@bindir@/nix-store \
+                --add-root ./${prefix}result --indirect \
+                -rv $extraArgs $storeExprs)
+                
             for j in $outPaths; do
-                echo "$j"
-                if test -z "$noLink"; then
-                    if test -L result; then
-                        rm result
-                    elif test -e result; then
-                        echo "cannot remove \`result' (not a symlink)"
-                        exit 1
-                    fi
-                    ln -s "$j" result
-                fi
+                echo "$(readlink "$j")"
             done
+            
             ;;
     esac
 done

scripts/nix-channel.in

@@ -48,6 +48,19 @@ sub addChannel {
 }
 
 
+# Remove a channel from the file $channelsList;
+sub removeChannel {
+    my $url = shift;
+    my @left = ();
+    readChannels;
+    foreach my $url2 (@channels) {
+        push @left, $url2 if $url ne $url2;
+    }
+    @channels = @left;
+    writeChannels;
+}
+
+
 # Fetch Nix expressions and pull cache manifests from the subscribed
 # channels.
 sub update {
@@ -68,69 +81,94 @@ sub update {
     # expressions.
 
     my $nixExpr = "[";
-    
     foreach my $url (@channels) {
         my $fullURL = "$url/nixexprs.tar.bz2";
-        my $hash = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`
-            or die "cannot fetch `$fullURL'";
-        chomp $hash;
-        # !!! escaping
-        $nixExpr .= "((import @datadir@/nix/corepkgs/fetchurl) " .
-            "{url = $fullURL; md5 = \"$hash\"; system = \"@system@\";}) "
+        $ENV{"PRINT_PATH"} = 1;
+        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
+        die "cannot fetch `$fullURL'" if $? != 0;
+        chomp $path;
+        $nixExpr .= $path . " ";
     }
-
     $nixExpr .= "]";
 
     $nixExpr =
         "(import @datadir@/nix/corepkgs/channels/unpack.nix) " .
         "{inputs = $nixExpr; system = \"@system@\";}";
 
-    # Instantiate the Nix expression.
-    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
-        or die "cannot instantiate Nix expression";
-    chomp $storeExpr;
-
-    # Register the store expression as a root of the garbage
-    # collector.
+    # Figure out a name for the GC root.
     my $userName = getpwuid($<);
     die "who ARE you? go away" unless defined $userName;
 
-    my $rootFile = "$rootsDir/$userName.gcroot";
-    my $tmpRootFile = "$rootsDir/$userName-tmp.gcroot";
+    my $rootFile = "$rootsDir/$userName";
     
-    open ROOT, ">$tmpRootFile" or die "cannot create `$tmpRootFile': $!";
-    print ROOT "$storeExpr";
-    close ROOT;
+    # Instantiate the Nix expression.
+    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate --add-root '$rootFile'.tmp -`
+        or die "cannot instantiate Nix expression";
+    chomp $storeExpr;
 
-    # Realise the store expression.
-    my $outPath = `nix-store -qnf '$storeExpr'`
+    # Build the resulting derivation.
+    my $outPath = `nix-store --add-root '$rootFile' -r '$storeExpr'`
         or die "cannot realise store expression";
     chomp $outPath;
 
+    unlink "$rootFile.tmp";
+
     # Make it the default Nix expression for `nix-env'.
     system "@bindir@/nix-env --import '$outPath'";
     die "cannot pull set default Nix expression to `$outPath'" if ($? != 0);
-
-    rename $tmpRootFile, $rootFile or die "cannot rename `$tmpRootFile' to `$rootFile': $!";
 }
 
 
+sub usageError {
+    print STDERR <<EOF;
+Usage:
+  nix-channel --add URL
+  nix-channel --remove URL
+  nix-channel --list
+  nix-channel --update
+EOF
+    exit 1;
+}
+
+
+usageError if scalar @ARGV == 0;
+
+
 while (scalar @ARGV) {
     my $arg = shift @ARGV;
 
     if ($arg eq "--add") {
-        die "syntax: nix-channel --add URL" if (scalar @ARGV != 1);
+        usageError if scalar @ARGV != 1;
         addChannel (shift @ARGV);
         last;
     }
 
-    elsif ($arg eq "--update") {
-        die "syntax: nix-channel --update" if (scalar @ARGV != 0);
-        update;
+    if ($arg eq "--remove") {
+        usageError if scalar @ARGV != 1;
+        removeChannel (shift @ARGV);
         last;
     }
 
+    if ($arg eq "--list") {
+        usageError if scalar @ARGV != 0;
+        readChannels;
+        foreach my $url (@channels) {
+            print "$url\n";
+        }
+        last;
+    }
+
+    elsif ($arg eq "--update") {
+        usageError if scalar @ARGV != 0;
+        update;
+        last;
+    }
+    
+    elsif ($arg eq "--help") {
+        usageError;
+    }
+
     else {
-        die "unknown argument `$arg'";
+        die "unknown argument `$arg'; try `--help'";
     }
 }

scripts/nix-collect-garbage.in

@@ -1,90 +1,2 @@
-#! @perl@ -w
-
-use strict;
-use IPC::Open2;
-
-my $rootsDir = "@localstatedir@/nix/gcroots";
-my $storeDir = "@storedir@";
-
-my %alive;
-
-my $gcOper = "--delete";
-my $minAge = 0;
-
-my @roots = ();
-
-
-# Parse the command line.
-for (my $i = 0; $i < scalar @ARGV; $i++) {
-    my $arg = $ARGV[$i];
-    if ($arg eq "--delete" || $arg eq "--print-live" || $arg eq "--print-dead") {
-        $gcOper = $arg;
-    }
-    elsif ($arg eq "--min-age") {
-        $i++;
-        $minAge = undef;
-        $minAge = $ARGV[$i];
-        die "invalid minimum age" unless defined $minAge && $minAge =~ /^\d*$/;
-    }
-    else { die "unknown argument `$arg'" };
-}
-
-
-# Read all GC roots from the given file.
-sub readRoots {
-    my $fileName = shift;
-    open ROOT, "<$fileName" or die "cannot open `$fileName': $!";
-    while (<ROOT>) {
-        chomp;
-        foreach my $root (split ' ') {
-            die "bad root `$root' in file `$fileName'"
-                unless $root =~ /^\S+$/;
-            push @roots, $root;
-        }
-    }
-    close ROOT;
-}
-
-
-# Recursively finds all *.gcroot files in the given directory.
-sub findRoots;
-sub findRoots {
-    my $followSymlinks = shift;
-    my $dir = shift;
-
-    opendir(DIR, $dir) or die "cannot open directory `$dir': $!";
-    my @names = readdir DIR or die "cannot read directory `$dir': $!";
-    closedir DIR;
-
-    foreach my $name (@names) {
-        next if $name eq "." || $name eq "..";
-        $name = $dir . "/" . $name;
-        if ($name =~ /.gcroot$/ && -f $name) {
-            readRoots $name;
-        }
-        elsif (-d $name) {
-            if ($followSymlinks || !-l $name) {
-                findRoots 0, $name;
-            }
-        }
-    }
-    
-}
-
-
-# Find GC roots, starting at $rootsDir.
-findRoots 1, $rootsDir;
-
-
-# Run the collector with the roots we found.
-my $pid = open2(">&1", \*WRITE, "@bindir@/nix-store --gc $gcOper --min-age $minAge")
-    or die "cannot run `nix-store --gc'";
-
-foreach my $root (@roots) {
-    print WRITE "$root\n";
-}
-
-close WRITE;
-
-waitpid $pid, 0;
-$? == 0 or die "`nix-store --gc' failed";
+#! @shell@ -e
+exec @bindir@/nix-store --gc "$@"

scripts/nix-install-package.in

@@ -3,35 +3,49 @@
 use strict;
 use POSIX qw(tmpnam);
 
-my $pkgfile = $ARGV[0];
-die unless defined $pkgfile;
+my $pkgFile = $ARGV[0];
+die unless defined $pkgFile;
 
-my $tmpdir;
-do { $tmpdir = tmpnam(); }
-until mkdir $tmpdir, 0777;
 
-# !!! remove tmpdir on exit
+# Re-execute in a terminal, if necessary, so that if we're executed
+# from a web browser, the user gets to see us.
+if (!defined $ENV{"NIX_HAVE_TERMINAL"}) {
+    $ENV{"NIX_HAVE_TERMINAL"} = "1";
+    $ENV{"LD_LIBRARY_PATH"} = "";
+    exec("xterm", "-e", "@shell@", "-c", "@bindir@/nix-install-package '$pkgFile' || read");
+    die "cannot execute `xterm'";
+}
 
-print "Unpacking $pkgfile in $tmpdir...\n";
-system "bunzip2 < $pkgfile | (cd $tmpdir && tar xf -)";
-die if $?;
 
-print "This package contains the following derivations:\n";
-system "@bindir@/nix-env -qasf $tmpdir/default.nix";
-die if $?;
+# Read and parse the package file.
+open PKGFILE, "<$pkgFile" or die "cannot open `$pkgFile': $!";
+my $contents = <PKGFILE>;
+close PKGFILE;
 
-print "Do you wish to install these (Y/N)? ";
+$contents =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ or die "invalid package contents";
+my $version = $1;
+my $manifestURL = $2;
+my $drvName = $3;
+my $system = $4;
+my $drvPath = $5;
+my $outPath = $6;
+
+die "invalid package version `$version'" unless $version eq "NIXPKG1";
+
+
+# Ask confirmation.
+print "Do you want to install `$drvName' (Y/N)? ";
 my $reply = <STDIN>;
 chomp $reply;
-exit if (!($reply eq "y"));
+exit if $reply ne "y" && $reply ne "Y";
 
-print "Pulling caches...\n";
-system "@bindir@/nix-pull `cat $tmpdir/caches`";
-die if $?;
+print "\nPulling manifests...\n";
+system "@bindir@/nix-pull '$manifestURL'";
+die if $? != 0;
 
-print "Installing package...\n";
-system "@bindir@/nix-env -if $tmpdir/default.nix '*'";
-die if $?;
+print "\nInstalling package...\n";
+system "@bindir@/nix-env -i '$outPath'";
+die if $? != 0;
 
-print "Installation succeeded! Press Enter to continue.\n";
+print "\nInstallation succeeded! Press Enter to continue.\n";
 <STDIN>;

scripts/nix-prefetch-url.in

@@ -1,58 +1,67 @@
 #! @shell@ -e
 
 url=$1
-hash=$2
+expHash=$2
+
+hashType=$NIX_HASH_ALGO
+if test -z "$hashType"; then
+    hashType=md5
+fi
+
+hashFormat=
+if test "$hashType" != "md5"; then
+    hashFormat=--base32
+fi
 
 if test -z "$url"; then
-    echo "syntax: nix-prefetch-url URL" >&2
+    echo "syntax: nix-prefetch-url URL [EXPECTED-HASH]" >&2
     exit 1
 fi
 
-# Determine the hash, unless it was given.
-if test -z "$hash"; then
+name=$(basename "$url")
+if test -z "$name"; then echo "invalid url"; exit 1; fi
 
-    # !!! race
-    tmpPath1=@storedir@/nix-prefetch-url-$$
 
-    # Test whether we have write permission in the store.  If not,
-    # fetch to /tmp and don't copy to the store.  This is a hack to
-    # make this script at least work somewhat in setuid installations.
-    if ! touch $tmpPath1 2> /dev/null; then
-        echo "(cannot write to the store, result won't be cached)" >&2
-        dummyMode=1
-        tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
+# If the hash was given, a file with that hash may already be in the
+# store.
+if test -n "$expHash"; then
+    finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$expHash" "$name")
+    if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
+        finalPath=
     fi
+    hash=$expHash
+fi
 
-    # Perform the checkout.
-    @curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
+
+# If we don't know the hash or a file with that hash doesn't exist,
+# download the file and add it to the store.
+if test -z "$finalPath"; then
+
+    tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
+    tmpFile=$tmpPath/$name
+    mkdir $tmpPath
+
+    # Perform the download.
+    @curl@ --fail --location --max-redirs 20 "$url" > $tmpFile
 
     # Compute the hash.
-    hash=$(@bindir@/nix-hash --flat $tmpPath1)
+    hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
     if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
 
-    # Rename it so that the fetchurl builder can find it.
-    if test "$dummyMode" != 1; then
-        tmpPath2=@storedir@/nix-prefetch-url-$hash
-        test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
+    # Add the downloaded file to the Nix store.
+    finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+
+    if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+
+    if test -n "$expHash" -a "$expHash" != "$hash"; then
+        echo "hash mismatch for URL \`$url'"
+        exit 1
     fi
-
 fi
 
-# Create a Nix expression that does a fetchurl.
-storeExpr=$( \
-    echo "(import @datadir@/nix/corepkgs/fetchurl) \
-        {url = $url; md5 = \"$hash\"; system = \"@system@\";}" \
-    | @bindir@/nix-instantiate -)
 
-# Realise it.
-finalPath=$(@bindir@/nix-store -qnB --force-realise $storeExpr)
-    
 if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
 
-if test -n "$tmpPath1" -o -n "$tmpPath2"; then
-    rm -rf $tmpPath1 $tmpPath2 || true
-fi
-
 echo $hash
 
 if test -n "$PRINT_PATH"; then

scripts/nix-pull.in

@@ -13,6 +13,19 @@ my $manifest = "$tmpdir/manifest";
 
 END { unlink $manifest; rmdir $tmpdir; }
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
+$libexecDir = "@libexecdir@" unless defined $libexecDir;
+
+my $stateDir = $ENV{"NIX_STATE_DIR"};
+$stateDir = "@localstatedir@/nix" unless defined $stateDir;
+
+
+# Prevent access problems in shared-stored installations.
+umask 0022;
+
 
 # Obtain URLs either from the command line or from a configuration file.
 my %narFiles;
@@ -29,20 +42,22 @@ sub processURL {
            "'$url' > '$manifest'") == 0
            or die "curl failed: $?";
 
-    readManifest $manifest, \%narFiles, \%patches, \%successors;
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
+    }
 
     my $baseName = "unnamed";
     if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
         $baseName = $1;
     }
 
-    my $hash = `@bindir@/nix-hash --flat '$manifest'`
+    my $hash = `$binDir/nix-hash --flat '$manifest'`
         or die "cannot hash `$manifest'";
     chomp $hash;
     
-    my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest";
+    my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
     
-    system("mv '$manifest' '$finalPath'") == 0
+    system("mv -f '$manifest' '$finalPath'") == 0
         or die "cannot move `$manifest' to `$finalPath";
 }
 
@@ -59,7 +74,7 @@ print "$size store paths in manifest\n";
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
 
-my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --substitute")
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --register-substitutes")
     or die "cannot run nix-store";
 
 close READ;
@@ -68,8 +83,15 @@ foreach my $storePath (keys %narFiles) {
     my $narFileList = $narFiles{$storePath};
     foreach my $narFile (@{$narFileList}) {
         print WRITE "$storePath\n";
-        print WRITE "@libexecdir@/nix/download-using-manifests.pl\n";
+        print WRITE "$narFile->{deriver}\n";
+        print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
         print WRITE "0\n";
+        my @references = split " ", $narFile->{references};
+        my $count = scalar @references;
+        print WRITE "$count\n";
+        foreach my $reference (@references) {
+            print WRITE "$reference\n";
+        }
     }
 }
 
@@ -77,16 +99,3 @@ close WRITE;
 
 waitpid $pid, 0;
 $? == 0 or die "nix-store failed";
-
-
-# Register all successors.
-print STDERR "registering successors...\n";
-my @sucs = %successors;
-while (scalar @sucs > 0) {
-    my $n = scalar @sucs;
-    if ($n > 256) { $n = 256 };
-    my @sucs2 = @sucs[0..$n - 1];
-    @sucs = @sucs[$n..scalar @sucs - 1];
-    system "@bindir@/nix-store --successor @sucs2";
-    if ($?) { die "`nix-store --successor' failed"; }
-}

scripts/nix-push.in

@@ -1,9 +1,12 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
 
 use strict;
+use IPC::Open2;
 use POSIX qw(tmpnam);
 use readmanifest;
 
+my $hashAlgo = "sha256";
+
 my $tmpdir;
 do { $tmpdir = tmpnam(); }
 until mkdir $tmpdir, 0777;
@@ -17,32 +20,62 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $dataDir = $ENV{"NIX_DATA_DIR"};
+$dataDir = "@datadir@" unless defined $dataDir;
+
 
 # Parse the command line.
-my $archives_put_url = shift @ARGV;
-my $archives_get_url = shift @ARGV;
-my $manifest_put_url = shift @ARGV;
+my $localCopy;
+my $localArchivesDir;
+my $localManifestFile;
+
+my $archivesPutURL;
+my $archivesGetURL;
+my $manifestPutURL;
+
+if ($ARGV[0] eq "--copy") {
+    die "syntax: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 1;
+    shift @ARGV;
+    $localArchivesDir = shift @ARGV;
+    $localManifestFile = shift @ARGV;
+}
+else {
+    die "syntax: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL " .
+        "MANIFEST_PUT_URL PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 0;
+    $archivesPutURL = shift @ARGV;
+    $archivesGetURL = shift @ARGV;
+    $manifestPutURL = shift @ARGV;
+}
 
 
-# From the given store expressions, determine the requisite store
-# paths.
+# From the given store paths, determine the set of requisite store
+# paths, i.e, the paths required to realise them.
 my %storePaths;
 
-foreach my $storeexpr (@ARGV) {
-    die unless $storeexpr =~ /^\//;
+foreach my $path (@ARGV) {
+    die unless $path =~ /^\//;
 
     # Get all paths referenced by the normalisation of the given 
     # Nix expression.
-    system "@bindir@/nix-store --realise $storeexpr > /dev/null";
-    die if ($?);
-
-    open PATHS, "@bindir@/nix-store --query --requisites --include-successors $storeexpr 2> /dev/null |" or die;
-    while (<PATHS>) {
+    my $pid = open2(\*READ, \*WRITE,
+        "$binDir/nix-store --query --requisites --force-realise " .
+        "--include-outputs '$path'") or die;
+    close WRITE;
+    
+    while (<READ>) {
         chomp;
         die "bad: $_" unless /^\//;
         $storePaths{$_} = "";
     }
-    close PATHS;
+    close READ;
+    
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
 my @storePaths = keys %storePaths;
@@ -58,9 +91,8 @@ foreach my $storePath (@storePaths) {
 
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
-        "((import @datadir@/nix/corepkgs/nar/nar.nix) " .
-        # !!! $storePath should be represented as a closure
-        "{path = \"$storePath\"; system = \"@system@\";}) ";
+        "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
+        "{path = \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
     
     print NIX $nixexpr;
 }
@@ -70,39 +102,46 @@ close NIX;
 
 
 # Instantiate store expressions from the Nix expression.
-my @storeexprs;
+my @storeExprs;
 print STDERR "instantiating store expressions...\n";
-open STOREEXPRS, "@bindir@/nix-instantiate $nixfile |" or die "cannot run nix-instantiate";
-while (<STOREEXPRS>) {
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-instantiate $nixfile")
+    or die "cannot run nix-instantiate";
+close WRITE;
+while (<READ>) {
     chomp;
     die unless /^\//;
-    push @storeexprs, $_;
+    push @storeExprs, $_;
 }
-close STOREEXPRS;
+close READ;
+
+waitpid $pid, 0;
+$? == 0 or die "nix-instantiate failed";
 
 
 # Realise the store expressions.
 print STDERR "creating archives...\n";
 
-my @narpaths;
+my @narPaths;
 
-my @tmp = @storeexprs;
+my @tmp = @storeExprs;
 while (scalar @tmp > 0) {
     my $n = scalar @tmp;
     if ($n > 256) { $n = 256 };
     my @tmp2 = @tmp[0..$n - 1];
     @tmp = @tmp[$n..scalar @tmp - 1];
 
-    system "@bindir@/nix-store --realise @tmp2 > /dev/null";
-    if ($?) { die "`nix-store --realise' failed"; }
-
-    open NARPATHS, "@bindir@/nix-store --query --list @tmp2 |" or die "cannot run nix";
-    while (<NARPATHS>) {
+    my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --realise @tmp2")
+        or die "cannot run nix-store";
+    close WRITE;
+    while (<READ>) {
         chomp;
         die unless (/^\//);
-        push @narpaths, "$_";
+        push @narPaths, "$_";
     }
-    close NARPATHS;
+    close READ;
+
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
 
@@ -111,83 +150,110 @@ print STDERR "creating manifest...\n";
 
 my %narFiles;
 my %patches;
-my %successors;
 
-my @nararchives;
+my @narArchives;
 for (my $n = 0; $n < scalar @storePaths; $n++) {
     my $storePath = $storePaths[$n];
-    my $nardir = $narpaths[$n];
+    my $narDir = $narPaths[$n];
     
     $storePath =~ /\/([^\/]*)$/;
     my $basename = $1;
     defined $basename or die;
 
-    my $narname = "$basename.nar.bz2";
-
-    my $narfile = "$nardir/$narname";
-    (-f $narfile) or die "narfile for $storePath not found";
-    push @nararchives, $narfile;
-
-    open MD5, "$nardir/narbz2-hash" or die "cannot open narbz2-hash";
-    my $narbz2Hash = <MD5>;
+    open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
+    my $narbz2Hash = <HASH>;
     chomp $narbz2Hash;
-    $narbz2Hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
-    close MD5;
+    $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
 
-    open MD5, "$nardir/nar-hash" or die "cannot open nar-hash";
-    my $narHash = <MD5>;
+    open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
+    my $narHash = <HASH>;
     chomp $narHash;
-    $narHash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
-    close MD5;
+    $narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
     
-    my $narbz2Size = (stat $narfile)[7];
+    my $narName = "$narbz2Hash.nar.bz2";
 
+    my $narFile = "$narDir/$narName";
+    (-f $narFile) or die "narfile for $storePath not found";
+    push @narArchives, $narFile;
+
+    my $narbz2Size = (stat $narFile)[7];
+
+    my $references = `$binDir/nix-store --query --references '$storePath'`;
+    die "cannot query references for `$storePath'" if $? != 0;
+    $references = join(" ", split(" ", $references));
+
+    my $deriver = `$binDir/nix-store --query --deriver '$storePath'`;
+    die "cannot query deriver for `$storePath'" if $? != 0;
+    chomp $deriver;
+    $deriver = "" if $deriver eq "unknown-deriver";
+
+    my $url;
+    if ($localCopy) {
+        $url = "file://$localArchivesDir/$narName";
+    } else {
+        $url = "$archivesGetURL/$narName";
+    }
     $narFiles{$storePath} = [
-        { url => $archives_get_url/$narname
-        , hash => $narbz2Hash
+        { url => $url
+        , hash => "$hashAlgo:$narbz2Hash"
         , size => $narbz2Size
-        , narHash => $narHash
+        , narHash => "$hashAlgo:$narHash"
+        , references => $references
+        , deriver => $deriver
         }
     ];
-                            
-    if ($storePath =~ /\.store$/) {
-        open PREDS, "@bindir@/nix-store --query --predecessors $storePath |" or die "cannot run nix";
-        while (<PREDS>) {
-            chomp;
-            die unless (/^\//);
-            my $pred = $_;
-            # Only include predecessors that are themselves being
-            # pushed.
-            if (defined $storePaths{$pred}) {
-                $successors{$pred} = $storePath;
-            }
-        }
-        close PREDS;
-    }
-
 }
 
-writeManifest $manifest, \%narFiles, \%patches, \%successors;
+writeManifest $manifest, \%narFiles, \%patches;
+
+
+sub copyFile {
+    my $src = shift;
+    my $dst = shift;
+    system("cp '$src' '$dst.tmp'") == 0 or die "cannot copy file";
+    rename("$dst.tmp", "$dst") or die "cannot rename file";
+}
 
 
 # Upload the archives.
 print STDERR "uploading archives...\n";
-foreach my $nararchive (@nararchives) {
 
-    $nararchive =~ /\/([^\/]*)$/;
+sub archiveExists {
+    my $name = shift;
+    print STDERR "  HEAD on $archivesGetURL/$name\n";
+    return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
+}
+
+foreach my $narArchive (@narArchives) {
+
+    $narArchive =~ /\/([^\/]*)$/;
     my $basename = $1;
 
-    if (system("$curl --head $archives_get_url/$basename > /dev/null") != 0) {
-        print STDERR "  $nararchive\n";
-        system("$curl --show-error --upload-file " .
-               "'$nararchive' '$archives_put_url/$basename' > /dev/null") == 0 or
-            die "curl failed on $nararchive: $?";
+    if ($localCopy) {
+        if (! -f "$localArchivesDir/$basename") {
+            print STDERR "  $narArchive\n";
+            copyFile $narArchive, "$localArchivesDir/$basename";
+        }
+    }
+    else {
+        if (!archiveExists("$basename")) {
+            print STDERR "  $narArchive\n";
+            system("$curl --show-error --upload-file " .
+                   "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
+                   die "curl failed on $narArchive: $?";
+        }
     }
 }
 
 
 # Upload the manifest.
 print STDERR "uploading manifest...\n";
-system("$curl  --show-error --upload-file " .
-       "'$manifest' '$manifest_put_url' > /dev/null") == 0 or
-    die "curl failed on $manifest: $?";
+if ($localCopy) {
+    copyFile $manifest, $localManifestFile;
+} else {
+    system("$curl  --show-error --upload-file " .
+           "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
+           die "curl failed on $manifest: $?";
+}

scripts/readmanifest.pm.in

@@ -5,6 +5,7 @@ sub addPatch {
     my $patches = shift;
     my $storePath = shift;
     my $patch = shift;
+    my $allowConflicts = shift;
 
     $$patches{$storePath} = []
         unless defined $$patches{$storePath};
@@ -18,7 +19,8 @@ sub addPatch {
                 $found = 1 if ($patch2->{basePath} eq $patch->{basePath});
             } else {
                 die "conflicting hashes for URL $patch->{url}, " .
-                    "namely $patch2->{hash} and $patch->{hash}";
+                    "namely $patch2->{hash} and $patch->{hash}"
+                    unless $allowConflicts;
             }
         }
     }
@@ -34,12 +36,17 @@ sub readManifest {
     my $narFiles = shift;
     my $patches = shift;
     my $successors = shift;
+    my $allowConflicts = shift;
+    $allowConflicts = 0 unless defined $allowConflicts;
 
-    open MANIFEST, "<$manifest";
+    open MANIFEST, "<$manifest"
+        or die "cannot open `$manifest': $!";
 
     my $inside = 0;
     my $type;
 
+    my $manifestVersion = 2;
+
     my $storePath;
     my $url;
     my $hash;
@@ -49,6 +56,9 @@ sub readManifest {
     my $baseHash;
     my $patchType;
     my $narHash;
+    my $references;
+    my $deriver;
+    my $hashAlgo;
 
     while (<MANIFEST>) {
         chomp;
@@ -70,6 +80,9 @@ sub readManifest {
                 undef $basePath;
                 undef $baseHash;
                 undef $patchType;
+                $references = "";
+                $deriver = "";
+                $hashAlgo = "md5";
 	    }
 
         } else {
@@ -91,14 +104,16 @@ sub readManifest {
                                 $found = 1;
                             } else {
                                 die "conflicting hashes for URL $url, " .
-                                    "namely $narFile->{hash} and $hash";
+                                    "namely $narFile->{hash} and $hash"
+                                    unless $allowConflicts;
                             }
                         }
                     }
                     if (!$found) {
                         push @{$narFileList},
                             { url => $url, hash => $hash, size => $size
-                            , narHash => $narHash
+                            , narHash => $narHash, references => $references
+                            , deriver => $deriver, hashAlgo => $hashAlgo
                             };
                     }
                 
@@ -113,7 +128,8 @@ sub readManifest {
                         { url => $url, hash => $hash, size => $size
                         , basePath => $basePath, baseHash => $baseHash
                         , narHash => $narHash, patchType => $patchType
-                        };
+                        , hashAlgo => $hashAlgo
+                        }, $allowConflicts;
                 }
 
             }
@@ -127,15 +143,20 @@ sub readManifest {
             elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
             elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
             elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+            elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
+            elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
+            elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
 
             # Compatibility;
             elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
-            elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = $1; }
+            elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; }
 
         }
     }
 
     close MANIFEST;
+
+    return $manifestVersion;
 }
 
 
@@ -144,24 +165,26 @@ sub writeManifest
     my $manifest = shift;
     my $narFiles = shift;
     my $patches = shift;
-    my $successors = shift;
 
     open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
 
+    print MANIFEST "version {\n";
+    print MANIFEST "  ManifestVersion: 3\n";
+    print MANIFEST "}\n";
+
     foreach my $storePath (keys %{$narFiles}) {
         my $narFileList = $$narFiles{$storePath};
         foreach my $narFile (@{$narFileList}) {
             print MANIFEST "{\n";
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $narFile->{url}\n";
-            print MANIFEST "  MD5: $narFile->{hash}\n";
+            print MANIFEST "  Hash: $narFile->{hash}\n";
             print MANIFEST "  NarHash: $narFile->{narHash}\n";
             print MANIFEST "  Size: $narFile->{size}\n";
-            foreach my $p (keys %{$successors}) { # !!! quadratic
-                if ($$successors{$p} eq $storePath) {
-                    print MANIFEST "  SuccOf: $p\n";
-                }
-            }
+            print MANIFEST "  References: $narFile->{references}\n"
+                if defined $narFile->{references} && $narFile->{references} ne "";
+            print MANIFEST "  Deriver: $narFile->{deriver}\n"
+                if defined $narFile->{deriver} && $narFile->{deriver} ne "";
             print MANIFEST "}\n";
         }
     }
@@ -172,7 +195,7 @@ sub writeManifest
             print MANIFEST "patch {\n";
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $patch->{url}\n";
-            print MANIFEST "  MD5: $patch->{hash}\n";
+            print MANIFEST "  Hash: $patch->{hash}\n";
             print MANIFEST "  NarHash: $patch->{narHash}\n";
             print MANIFEST "  Size: $patch->{size}\n";
             print MANIFEST "  BasePath: $patch->{basePath}\n";

src/aterm-helper.pl

@@ -56,6 +56,7 @@ while (<STDIN>) {
         my $unpack = "";
         my $n = 1;
         foreach my $type (@types) {
+            my $realType = $type;
             $args .= ", ";
             if ($type eq "string") {
 #                $args .= "(ATerm) ATmakeAppl0(ATmakeAFun((char *) e$n, 0, ATtrue))";
@@ -83,6 +84,9 @@ while (<STDIN>) {
                 $unpack .= "    e$n = (ATermList) ATgetArgument(e, $m);\n";
             } elsif ($type eq "ATermBlob") {
                 $unpack .= "    e$n = (ATermBlob) ATgetArgument(e, $m);\n";
+            } elsif ($realType eq "string") {
+                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
+                $unpack .= "    if (ATgetType(e$n) != AT_APPL) return false;\n";
             } else {
                 $unpack .= "    e$n = ATgetArgument(e, $m);\n";
             }
@@ -96,12 +100,18 @@ while (<STDIN>) {
         print IMPL "AFun sym$funname = 0;\n";
         
         print HEADER "static inline $result make$funname($formals) {\n";
-        print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
+        if ($arity <= 6) {
+            print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
+        } else {
+            $args =~ s/^,//;
+            print HEADER "    ATerm array[$arity] = {$args};\n";
+            print HEADER "    return (ATerm) ATmakeApplArray(sym$funname, array);\n";
+        }
         print HEADER "}\n\n";
 
         print HEADER "#ifdef __cplusplus\n";
         print HEADER "static inline bool match$funname(ATerm e$formals2) {\n";
-        print HEADER "    if (ATgetType(e) != AT_APPL || ATgetAFun(e) != sym$funname) return false;\n";
+        print HEADER "    if (ATgetType(e) != AT_APPL || (AFun) ATgetAFun(e) != sym$funname) return false;\n";
         print HEADER "$unpack";
         print HEADER "    return true;\n";
         print HEADER "}\n";

src/libexpr/Makefile.am

@@ -29,7 +29,7 @@ lexer-tab.c lexer-tab.h: lexer.l
 nixexpr-ast.cc nixexpr-ast.hh: ../aterm-helper.pl nixexpr-ast.def
 	$(perl) ../aterm-helper.pl nixexpr-ast.hh nixexpr-ast.cc < nixexpr-ast.def
 
-nixexpr.hh: nixexpr-ast.hh
+nixexpr.cc nixexpr.hh: nixexpr-ast.hh
 
 
 CLEANFILES =

src/libexpr/nixexpr.cc

@@ -1,5 +1,5 @@
 #include "nixexpr.hh"
-#include "storeexpr.hh"
+#include "derivations.hh"
 
 
 #include "nixexpr-ast.hh"

src/libexpr/nixexpr.hh

@@ -89,11 +89,5 @@ void checkVarDefs(const ATermMap & def, Expr e);
 /* Create an expression representing a boolean. */
 Expr makeBool(bool b);
 
-/* Create an expression representing a string. */
-Expr makeString(const string & s);
-
-/* Create an expression representing a path. */
-Expr makePath(const Path & path);
-
 
 #endif /* !__NIXEXPR_H */

src/libexpr/parser.cc

@@ -75,6 +75,65 @@ int yyparse(yyscan_t scanner, ParseData * data);
 }
 
 
+static void checkAttrs(ATermMap & names, ATermList bnds)
+{
+    for (ATermIterator i(bnds); i; ++i) {
+        ATerm name;
+        Expr e;
+        ATerm pos;
+        if (!matchBind(*i, name, e, pos)) abort(); /* can't happen */
+        if (names.get(name))
+            throw Error(format("duplicate attribute `%1%' at %2%")
+                % aterm2String(name) % showPos(pos));
+        names.set(name, name);
+    }
+}
+
+
+static void checkAttrSets(ATerm e)
+{
+    ATermList formals;
+    ATerm body, pos;
+    if (matchFunction(e, formals, body, pos)) {
+        ATermMap names;
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm name;
+            Expr deflt;
+            if (!matchNoDefFormal(*i, name) &&
+                !matchDefFormal(*i, name, deflt))
+                abort();
+            if (names.get(name))
+                throw Error(format("duplicate formal function argument `%1%' at %2%")
+                    % aterm2String(name) % showPos(pos));
+            names.set(name, name);
+        }
+    }
+
+    ATermList bnds;
+    if (matchAttrs(e, bnds)) {
+        ATermMap names;
+        checkAttrs(names, bnds);
+    }
+    
+    ATermList rbnds, nrbnds;
+    if (matchRec(e, rbnds, nrbnds)) {
+        ATermMap names;
+        checkAttrs(names, rbnds);
+        checkAttrs(names, nrbnds);
+    }
+    
+    if (ATgetType(e) == AT_APPL) {
+        int arity = ATgetArity(ATgetAFun(e));
+        for (int i = 0; i < arity; ++i)
+            checkAttrSets(ATgetArgument(e, i));
+    }
+
+    else if (ATgetType(e) == AT_LIST)
+        for (ATermIterator i((ATermList) e); i; ++i)
+            checkAttrSets(*i);
+}
+
+
 static Expr parse(EvalState & state,
     const char * text, const Path & path,
     const Path & basePath)
@@ -96,6 +155,8 @@ static Expr parse(EvalState & state,
     } catch (Error & e) {
         throw Error(format("%1%, in `%2%'") % e.msg() % path);
     }
+    
+    checkAttrSets(data.result);
 
     return data.result;
 }

src/libexpr/primops.cc

@@ -1,4 +1,4 @@
-#include "normalise.hh"
+#include "build.hh"
 #include "eval.hh"
 #include "globals.hh"
 #include "nixexpr-ast.hh"
@@ -16,72 +16,62 @@ static Expr primImport(EvalState & state, const ATermVector & args)
 }
 
 
-static PathSet storeExprRootsCached(EvalState & state, const Path & nePath)
-{
-    DrvRoots::iterator i = state.drvRoots.find(nePath);
-    if (i != state.drvRoots.end())
-        return i->second;
-    else {
-        PathSet paths = storeExprRoots(nePath);
-        state.drvRoots[nePath] = paths;
-        return paths;
-    }
-}
+/* Returns the hash of a derivation modulo fixed-output
+   subderivations.  A fixed-output derivation is a derivation with one
+   output (`out') for which an expected hash and hash algorithm are
+   specified (using the `outputHash' and `outputHashAlgo'
+   attributes).  We don't want changes to such derivations to
+   propagate upwards through the dependency graph, changing output
+   paths everywhere.
 
+   For instance, if we change the url in a call to the `fetchurl'
+   function, we do not want to rebuild everything depending on it
+   (after all, (the hash of) the file being downloaded is unchanged).
+   So the *output paths* should not change.  On the other hand, the
+   *derivation store expression paths* should change to reflect the
+   new dependency graph.
 
-static Hash hashDerivation(EvalState & state, StoreExpr ne)
+   That's what this function does: it returns a hash which is just the
+   of the derivation ATerm, except that any input store expression
+   paths have been replaced by the result of a recursive call to this
+   function, and that for fixed-output derivations we return
+   (basically) its outputHash. */
+static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 {
-    if (ne.type == StoreExpr::neDerivation) {
-	PathSet inputs2;
-        for (PathSet::iterator i = ne.derivation.inputs.begin();
-             i != ne.derivation.inputs.end(); i++)
+    /* Return a fixed hash for fixed-output derivations. */
+    if (drv.outputs.size() == 1) {
+        DerivationOutputs::const_iterator i = drv.outputs.begin();
+        if (i->first == "out" &&
+            i->second.hash != "")
         {
-            DrvHashes::iterator j = state.drvHashes.find(*i);
-            if (j == state.drvHashes.end())
-                throw Error(format("don't know expression `%1%'") % (string) *i);
-            inputs2.insert(j->second);
+            return hashString(htSHA256, "fixed:out:"
+                + i->second.hashAlgo + ":"
+                + i->second.hash + ":"
+                + i->second.path);
         }
-	ne.derivation.inputs = inputs2;
     }
-    return hashTerm(unparseStoreExpr(ne));
+
+    /* For other derivations, replace the inputs paths with recursive
+       calls to this function.*/
+    DerivationInputs inputs2;
+    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+    {
+        Hash h = state.drvHashes[i->first];
+        if (h.type == htUnknown) {
+            Derivation drv2 = derivationFromPath(i->first);
+            h = hashDerivationModulo(state, drv2);
+            state.drvHashes[i->first] = h;
+        }
+        inputs2[printHash(h)] = i->second;
+    }
+    drv.inputDrvs = inputs2;
+    
+    return hashTerm(unparseDerivation(drv));
 }
 
 
-static Path copyAtom(EvalState & state, const Path & srcPath)
-{
-    /* !!! should be cached */
-    Path dstPath(addToStore(srcPath));
-
-    ClosureElem elem;
-    StoreExpr ne;
-    ne.type = StoreExpr::neClosure;
-    ne.closure.roots.insert(dstPath);
-    ne.closure.elems[dstPath] = elem;
-
-    Hash drvHash = hashDerivation(state, ne);
-    Path drvPath = writeTerm(unparseStoreExpr(ne), "");
-    state.drvHashes[drvPath] = drvHash;
-
-    state.drvRoots[drvPath] = ne.closure.roots;
-
-    printMsg(lvlChatty, format("copied `%1%' -> closure `%2%'")
-        % srcPath % drvPath);
-    return drvPath;
-}
-
-
-static string addInput(EvalState & state, 
-    Path & nePath, StoreExpr & ne)
-{
-    PathSet paths = storeExprRootsCached(state, nePath);
-    if (paths.size() != 1) abort();
-    Path path = *(paths.begin());
-    ne.derivation.inputs.insert(nePath);
-    return path;
-}
-
-
-static void processBinding(EvalState & state, Expr e, StoreExpr & ne,
+static void processBinding(EvalState & state, Expr e, Derivation & drv,
     Strings & ss)
 {
     e = evalExpr(state, e);
@@ -104,37 +94,61 @@ static void processBinding(EvalState & state, Expr e, StoreExpr & ne,
 
     else if (matchAttrs(e, es)) {
         Expr a = queryAttr(e, "type");
+        
         if (a && evalString(state, a) == "derivation") {
             a = queryAttr(e, "drvPath");
             if (!a) throw Error("derivation name missing");
             Path drvPath = evalPath(state, a);
 
-            a = queryAttr(e, "drvHash");
-            if (!a) throw Error("derivation hash missing");
-            Hash drvHash = parseHash(evalString(state, a));
+            a = queryAttr(e, "outPath");
+            if (!a) throw Error("output path missing");
+            /* !!! supports only single output path */
+            Path outPath = evalPath(state, a);
+
+            drv.inputDrvs[drvPath] = singleton<StringSet>("out");
+            ss.push_back(outPath);
+        }
+
+        else if (a && evalString(state, a) == "storePath") {
 
             a = queryAttr(e, "outPath");
             if (!a) throw Error("output path missing");
-            PathSet drvRoots;
-            drvRoots.insert(evalPath(state, a));
-            
-            state.drvHashes[drvPath] = drvHash;
-            state.drvRoots[drvPath] = drvRoots;
+            /* !!! supports only single output path */
+            Path outPath = evalPath(state, a);
 
-            ss.push_back(addInput(state, drvPath, ne));
-        } else
-            throw Error("invalid derivation attribute");
+            drv.inputSrcs.insert(outPath);
+            ss.push_back(outPath);
+        }
+
+        else throw Error("invalid derivation attribute");
     }
 
     else if (matchPath(e, s)) {
-        Path drvPath = copyAtom(state, aterm2String(s));
-        ss.push_back(addInput(state, drvPath, ne));
+        Path srcPath(canonPath(aterm2String(s)));
+
+        if (isStorePath(srcPath)) {
+            printMsg(lvlChatty, format("using store path `%1%' as source")
+                % srcPath);
+            drv.inputSrcs.insert(srcPath);
+            ss.push_back(srcPath);
+        }
+
+        else {
+            if (isDerivation(srcPath))
+                throw Error(format("file names are not allowed to end in `%1%'")
+                    % drvExtension);
+            Path dstPath(addToStore(srcPath));
+            printMsg(lvlChatty, format("copied source `%1%' -> `%2%'")
+                % srcPath % dstPath);
+            drv.inputSrcs.insert(dstPath);
+            ss.push_back(dstPath);
+        }
     }
     
     else if (matchList(e, es)) {
         for (ATermIterator i(es); i; ++i) {
             startNest(nest, lvlVomit, format("processing list element"));
-	    processBinding(state, evalExpr(state, *i), ne, ss);
+	    processBinding(state, evalExpr(state, *i), drv, ss);
         }
     }
 
@@ -142,7 +156,7 @@ static void processBinding(EvalState & state, Expr e, StoreExpr & ne,
 
     else if (matchSubPath(e, e1, e2)) {
         Strings ss2;
-        processBinding(state, evalExpr(state, e1), ne, ss2);
+        processBinding(state, evalExpr(state, e1), drv, ss2);
         if (ss2.size() != 1)
             throw Error("left-hand side of `~' operator cannot be a list");
         e2 = evalExpr(state, e2);
@@ -184,12 +198,13 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
     queryAllAttrs(args, attrs, true);
 
     /* Build the derivation expression by processing the attributes. */
-    StoreExpr ne;
-    ne.type = StoreExpr::neDerivation;
+    Derivation drv;
 
     string drvName;
-    Hash outHash;
-    bool outHashGiven = false;
+    
+    string outputHash;
+    string outputHashAlgo;
+    bool outputHashRecursive = false;
 
     for (ATermIterator i(attrs.keys()); i; ++i) {
         string key = aterm2String(*i);
@@ -201,7 +216,7 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
 
         Strings ss;
         try {
-            processBinding(state, value, ne, ss);
+            processBinding(state, value, drv, ss);
         } catch (Error & e) {
             throw Error(format("while processing the derivation attribute `%1%' at %2%:\n%3%")
                 % key % showPos(pos) % e.msg());
@@ -211,68 +226,98 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
            command-line arguments to the builder. */
         if (key == "args") {
             for (Strings::iterator i = ss.begin(); i != ss.end(); ++i)
-                ne.derivation.args.push_back(*i);
+                drv.args.push_back(*i);
         }
 
         /* All other attributes are passed to the builder through the
            environment. */
         else {
             string s = concatStrings(ss);
-            ne.derivation.env[key] = s;
-            if (key == "builder") ne.derivation.builder = s;
-            else if (key == "system") ne.derivation.platform = s;
+            drv.env[key] = s;
+            if (key == "builder") drv.builder = s;
+            else if (key == "system") drv.platform = s;
             else if (key == "name") drvName = s;
-            else if (key == "id") { 
-                outHash = parseHash(s);
-                outHashGiven = true;
+            else if (key == "outputHash") outputHash = s;
+            else if (key == "outputHashAlgo") outputHashAlgo = s;
+            else if (key == "outputHashMode") {
+                if (s == "recursive") outputHashRecursive = true; 
+                else if (s == "flat") outputHashRecursive = false;
+                else throw Error(format("invalid value `%1%' for `outputHashMode' attribute") % s);
             }
         }
     }
     
     /* Do we have all required attributes? */
-    if (ne.derivation.builder == "")
+    if (drv.builder == "")
         throw Error("required attribute `builder' missing");
-    if (ne.derivation.platform == "")
+    if (drv.platform == "")
         throw Error("required attribute `system' missing");
     if (drvName == "")
         throw Error("required attribute `name' missing");
 
+    /* If an output hash was given, check it. */
+    if (outputHash == "")
+        outputHashAlgo = "";
+    else {
+        HashType ht = parseHashType(outputHashAlgo);
+        if (ht == htUnknown)
+            throw Error(format("unknown hash algorithm `%1%'") % outputHashAlgo);
+        Hash h;
+        if (outputHash.size() == Hash(ht).hashSize * 2)
+            /* hexadecimal representation */
+            h = parseHash(ht, outputHash);
+        else
+            /* base-32 representation */
+            h = parseHash32(ht, outputHash);
+        string s = outputHash;
+        outputHash = printHash(h);
+        if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
+    }
+
     /* Check the derivation name.  It shouldn't contain whitespace,
        but we are conservative here: we check whether only
        alphanumerics and some other characters appear. */
-    string validChars = "+-._?=";
-    for (string::iterator i = drvName.begin(); i != drvName.end(); ++i)
-        if (!((*i >= 'A' && *i <= 'Z') ||
-              (*i >= 'a' && *i <= 'z') ||
-              (*i >= '0' && *i <= '9') ||
-              validChars.find(*i) != string::npos))
-        {
-            throw Error(format("invalid character `%1%' in derivation name `%2%'")
-                % *i % drvName);
-        }
+    checkStoreName(drvName);
+    if (isDerivation(drvName))
+        throw Error(format("derivation names are not allowed to end in `%1%'")
+            % drvExtension);
+
+    /* !!! the name should not end in the derivation extension (.drv).
+       Likewise for sources. */
+
+    /* Construct the "masked" derivation store expression, which is
+       the final one except that in the list of outputs, the output
+       paths are empty, and the corresponding environment variables
+       have an empty value.  This ensures that changes in the set of
+       output names do get reflected in the hash. */
+    drv.env["out"] = "";
+    drv.outputs["out"] =
+        DerivationOutput("", outputHashAlgo, outputHash);
         
-    /* Determine the output path by hashing the Nix expression with no
-       outputs to produce a unique but deterministic path name for
-       this derivation. */
-    if (!outHashGiven) outHash = hashDerivation(state, ne);
-    Path outPath = canonPath(nixStore + "/" + 
-        ((string) outHash).c_str() + "-" + drvName);
-    ne.derivation.env["out"] = outPath;
-    ne.derivation.outputs.insert(outPath);
+    /* Use the masked derivation expression to compute the output
+       path. */
+    Path outPath = makeStorePath("output:out",
+        hashDerivationModulo(state, drv), drvName);
+
+    /* Construct the final derivation store expression. */
+    drv.env["out"] = outPath;
+    drv.outputs["out"] =
+        DerivationOutput(outPath, outputHashAlgo, outputHash);
 
     /* Write the resulting term into the Nix store directory. */
-    Hash drvHash = outHashGiven
-        ? hashString((string) outHash + outPath)
-        : hashDerivation(state, ne);
-    Path drvPath = writeTerm(unparseStoreExpr(ne), "-d-" + drvName);
+    Path drvPath = writeDerivation(drv, drvName);
 
     printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'")
         % drvName % drvPath);
 
+    /* Optimisation, but required in read-only mode! because in that
+       case we don't actually write store expressions, so we can't
+       read them later. */
+    state.drvHashes[drvPath] = hashDerivationModulo(state, drv);
+
+    /* !!! assumes a single output */
     attrs.set("outPath", makeAttrRHS(makePath(toATerm(outPath)), makeNoPos()));
     attrs.set("drvPath", makeAttrRHS(makePath(toATerm(drvPath)), makeNoPos()));
-    attrs.set("drvHash",
-        makeAttrRHS(makeStr(toATerm((string) drvHash)), makeNoPos()));
     attrs.set("type", makeAttrRHS(makeStr(toATerm("derivation")), makeNoPos()));
 
     return makeAttrs(attrs);
@@ -312,21 +357,21 @@ static Expr primFalse(EvalState & state, const ATermVector & args)
 
 
 /* Return the null value. */
-Expr primNull(EvalState & state, const ATermVector & args)
+static Expr primNull(EvalState & state, const ATermVector & args)
 {
     return makeNull();
 }
 
 
 /* Determine whether the argument is the null value. */
-Expr primIsNull(EvalState & state, const ATermVector & args)
+static Expr primIsNull(EvalState & state, const ATermVector & args)
 {
     return makeBool(matchNull(evalExpr(state, args[0])));
 }
 
 
 /* Apply a function to every element of a list. */
-Expr primMap(EvalState & state, const ATermVector & args)
+static Expr primMap(EvalState & state, const ATermVector & args)
 {
     Expr fun = evalExpr(state, args[0]);
     Expr list = evalExpr(state, args[1]);
@@ -343,11 +388,22 @@ Expr primMap(EvalState & state, const ATermVector & args)
 }
 
 
+/* Return a string constant representing the current platform.  Note!
+   that differs between platforms, so Nix expressions using
+   `__currentSystem' can evaluate to different values on different
+   platforms. */
+static Expr primCurrentSystem(EvalState & state, const ATermVector & args)
+{
+    return makeStr(toATerm(thisSystem));
+}
+
+
 void EvalState::addPrimOps()
 {
     addPrimOp("true", 0, primTrue);
     addPrimOp("false", 0, primFalse);
     addPrimOp("null", 0, primNull);
+    addPrimOp("__currentSystem", 0, primCurrentSystem);
 
     addPrimOp("import", 1, primImport);
     addPrimOp("derivation", 1, primDerivation);

src/libmain/Makefile.am

@@ -7,5 +7,6 @@ AM_CXXFLAGS = \
  -DNIX_DATA_DIR=\"$(datadir)\" \
  -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \
  -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \
+ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_VERSION=\"$(VERSION)\" \
  -I.. ${aterm_include} -I../libutil -I../libstore

src/libmain/shared.cc

@@ -13,6 +13,7 @@ extern "C" {
 }
 
 #include "globals.hh"
+#include "gc.hh"
 #include "shared.hh"
 
 #include "config.h"
@@ -30,6 +31,28 @@ void sigintHandler(int signo)
 }
 
 
+Path makeRootName(const Path & gcRoot, int & counter)
+{
+    counter++;
+    if (counter == 1)
+        return gcRoot;
+    else
+        return (format("%1%-%2%") % gcRoot % counter).str();
+}
+
+
+void printGCWarning()
+{
+    static bool warned = false;
+    if (!warned) {
+        printMsg(lvlInfo,
+            "warning: you did not specify `--add-root'; "
+            "the result might be removed by the garbage collector");
+        warned = true;
+    }
+}
+
+
 void setLogType(string lt)
 {
     if (lt == "pretty") logType = ltPretty;
@@ -55,7 +78,16 @@ void checkStoreNotSymlink(Path path)
 }
 
 
-void initStoreExprHelpers();
+struct RemoveTempRoots 
+{
+    ~RemoveTempRoots()
+    {
+        removeTempRoots();
+    }
+};
+
+
+void initDerivationsHelpers();
 
 
 /* Initialize and reorder arguments, then call the actual argument
@@ -69,11 +101,12 @@ static void initAndRun(int argc, char * * argv)
     }
     
     /* Setup Nix paths. */
-    nixStore = getEnv("NIX_STORE_DIR", canonPath(NIX_STORE_DIR));
-    nixDataDir = getEnv("NIX_DATA_DIR", canonPath(NIX_DATA_DIR));
-    nixLogDir = getEnv("NIX_LOG_DIR", canonPath(NIX_LOG_DIR));
-    nixStateDir = getEnv("NIX_STATE_DIR", canonPath(NIX_STATE_DIR));
+    nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
+    nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
+    nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
+    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
     nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
+    nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
 
     /* Check that the store directory and its parent are not
        symlinks. */
@@ -105,7 +138,7 @@ static void initAndRun(int argc, char * * argv)
     if (lt != "") setLogType(lt);
 
     /* ATerm stuff.  !!! find a better place to put this */
-    initStoreExprHelpers();
+    initDerivationsHelpers();
     
     /* Put the arguments in a vector. */
     Strings args, remaining;
@@ -171,6 +204,10 @@ static void initAndRun(int argc, char * * argv)
         else remaining.push_back(arg);
     }
 
+    /* Automatically clean up the temporary roots file when we
+       exit. */
+    RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
+
     run(remaining);
 }
 

src/libmain/shared.hh

@@ -17,6 +17,10 @@ void run(Strings args);
 /* Should print a help message to stdout and return. */
 void printHelp();
 
+/* Ugh.  No better place to put this. */
+Path makeRootName(const Path & gcRoot, int & counter);
+void printGCWarning();
+
 
 extern string programId;
 

src/libstore/Makefile.am

@@ -1,18 +1,18 @@
 noinst_LIBRARIES = libstore.a
 
 libstore_a_SOURCES = \
- store.cc store.hh storeexpr.cc storeexpr.hh \
- normalise.cc misc.cc normalise.hh \
+ store.cc store.hh derivations.cc derivations.hh \
+ build.cc misc.cc build.hh \
  globals.cc globals.hh db.cc db.hh \
  references.cc references.hh pathlocks.cc pathlocks.hh \
- gc.cc gc.hh storeexpr-ast.hh
+ gc.cc gc.hh derivations-ast.hh
 
-EXTRA_DIST = storeexpr-ast.def storeexpr-ast.cc
+EXTRA_DIST = derivations-ast.def derivations-ast.cc
 
 AM_CXXFLAGS = -Wall \
  -I.. ${bdb_include} ${aterm_include} -I../libutil
 
-storeexpr-ast.cc storeexpr-ast.hh: ../aterm-helper.pl storeexpr-ast.def
-	$(perl) ../aterm-helper.pl storeexpr-ast.hh storeexpr-ast.cc < storeexpr-ast.def
+derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
+	$(perl) ../aterm-helper.pl derivations-ast.hh derivations-ast.cc < derivations-ast.def
 
-storeexpr.cc: storeexpr-ast.hh
+derivations.cc store.cc: derivations-ast.hh

src/libstore/build.hh

@@ -0,0 +1,36 @@
+#ifndef __BUILD_H
+#define __BUILD_H
+
+#include "derivations.hh"
+
+/* Ensure that the output paths of the derivation are valid.  If they
+   are already valid, this is a no-op.  Otherwise, validity can
+   be reached in two ways.  First, if the output paths have
+   substitutes, then those can be used.  Second, the output paths can
+   be created by running the builder, after recursively building any
+   sub-derivations. */
+void buildDerivations(const PathSet & drvPaths);
+
+/* Ensure that a path is valid.  If it is not currently valid, it may
+   be made valid by running a substitute (if defined for the path). */
+void ensurePath(const Path & storePath);
+
+/* Read a derivation, after ensuring its existence through
+   ensurePath(). */
+Derivation derivationFromPath(const Path & drvPath);
+
+/* Place in `paths' the set of all store paths in the file system
+   closure of `storePath'; that is, all paths than can be directly or
+   indirectly reached from it.  `paths' is not cleared.  If
+   `flipDirection' is true, the set of paths that can reach
+   `storePath' is returned; that is, the closures under the `referers'
+   relation instead of the `references' relation is returned. */
+void computeFSClosure(const Path & storePath,
+    PathSet & paths, bool flipDirection = false);
+
+/* Return the path corresponding to the output identifier `id' in the
+   given derivation. */
+Path findOutput(const Derivation & drv, string id);
+    
+
+#endif /* !__BUILD_H */

src/libstore/db.cc

@@ -33,11 +33,9 @@ Transaction::Transaction()
 
 
 Transaction::Transaction(Database & db)
+    : txn(0)
 {
-    db.requireEnv();
-    try {
-        db.env->txn_begin(0, &txn, 0);
-    } catch (DbException e) { rethrow(e); }
+    begin(db);
 }
 
 
@@ -47,6 +45,16 @@ Transaction::~Transaction()
 }
 
 
+void Transaction::begin(Database & db)
+{
+    assert(txn == 0);
+    db.requireEnv();
+    try {
+        db.env->txn_begin(0, &txn, 0);
+    } catch (DbException e) { rethrow(e); }
+}
+
+
 void Transaction::commit()
 {
     if (!txn) throw Error("commit called on null transaction");
@@ -82,7 +90,7 @@ void Transaction::moveTo(Transaction & t)
 void Database::requireEnv()
 {
     checkInterrupt();
-    if (!env)throw Error("database environment is not open "
+    if (!env) throw Error("database environment is not open "
         "(maybe you don't have sufficient permission?)");
 }
 
@@ -186,9 +194,9 @@ void Database::open(const string & path)
            number, then run db_recover on the database to remove the
            existing DB environment (since changes only take effect on
            new environments). */
-        env->set_lk_max_locks(4000);
-        env->set_lk_max_lockers(4000);
-        env->set_lk_max_objects(4000);
+        env->set_lk_max_locks(10000);
+        env->set_lk_max_lockers(10000);
+        env->set_lk_max_objects(10000);
         env->set_lk_detect(DB_LOCK_DEFAULT);
 
         /* Dangerous, probably, but from the docs it *seems* that BDB

src/libstore/db.hh

@@ -27,6 +27,7 @@ public:
     Transaction(Database & _db);
     ~Transaction();
 
+    void begin(Database & db);
     void abort();
     void commit();
 

src/libstore/derivations-ast.def

@@ -0,0 +1,10 @@
+init initDerivationsHelpers
+
+Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
+
+| string string | ATerm | EnvBinding |
+| string ATermList | ATerm | DerivationInput |
+| string string string string | ATerm | DerivationOutput |
+
+Closure | ATermList ATermList | ATerm | OldClosure |
+| string ATermList | ATerm | OldClosureElem |

src/libstore/derivations.cc

@@ -0,0 +1,169 @@
+#include "derivations.hh"
+#include "globals.hh"
+#include "store.hh"
+
+#include "derivations-ast.hh"
+#include "derivations-ast.cc"
+
+
+Hash hashTerm(ATerm t)
+{
+    return hashString(htSHA256, atPrint(t));
+}
+
+
+Path writeDerivation(const Derivation & drv, const string & name)
+{
+    PathSet references;
+    references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
+    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+        references.insert(i->first);
+    /* Note that the outputs of a derivation are *not* references
+       (that can be missing (of course) and should not necessarily be
+       held during a garbage collection). */
+    return addTextToStore(name + drvExtension,
+        atPrint(unparseDerivation(drv)), references);
+}
+
+
+static void checkPath(const string & s)
+{
+    if (s.size() == 0 || s[0] != '/')
+        throw Error(format("bad path `%1%' in derivation") % s);
+}
+    
+
+static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
+{
+    for (ATermIterator i(paths); i; ++i) {
+        if (ATgetType(*i) != AT_APPL)
+            throw badTerm("not a path", *i);
+        string s = aterm2String(*i);
+        if (arePaths) checkPath(s);
+        out.insert(s);
+    }
+}
+
+
+void throwBadDrv(ATerm t)
+{
+    throw badTerm("not a valid derivation", t);
+}
+
+
+Derivation parseDerivation(ATerm t)
+{
+    Derivation drv;
+    ATermList outs, inDrvs, inSrcs, args, bnds;
+    ATerm builder, platform;
+
+    if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds))
+        throwBadDrv(t);
+
+    for (ATermIterator i(outs); i; ++i) {
+        ATerm id, path, hashAlgo, hash;
+        if (!matchDerivationOutput(*i, id, path, hashAlgo, hash))
+            throwBadDrv(t);
+        DerivationOutput out;
+        out.path = aterm2String(path);
+        checkPath(out.path);
+        out.hashAlgo = aterm2String(hashAlgo);
+        out.hash = aterm2String(hash);
+        drv.outputs[aterm2String(id)] = out;
+    }
+
+    for (ATermIterator i(inDrvs); i; ++i) {
+        ATerm drvPath;
+        ATermList ids;
+        if (!matchDerivationInput(*i, drvPath, ids))
+            throwBadDrv(t);
+        Path drvPath2 = aterm2String(drvPath);
+        checkPath(drvPath2);
+        StringSet ids2;
+        parseStrings(ids, ids2, false);
+        drv.inputDrvs[drvPath2] = ids2;
+    }
+    
+    parseStrings(inSrcs, drv.inputSrcs, true);
+
+    drv.builder = aterm2String(builder);
+    drv.platform = aterm2String(platform);
+    
+    for (ATermIterator i(args); i; ++i) {
+        if (ATgetType(*i) != AT_APPL)
+            throw badTerm("string expected", *i);
+        drv.args.push_back(aterm2String(*i));
+    }
+
+    for (ATermIterator i(bnds); i; ++i) {
+        ATerm s1, s2;
+        if (!matchEnvBinding(*i, s1, s2))
+            throw badTerm("tuple of strings expected", *i);
+        drv.env[aterm2String(s1)] = aterm2String(s2);
+    }
+
+    return drv;
+}
+
+
+static ATermList unparseStrings(const StringSet & paths)
+{
+    ATermList l = ATempty;
+    for (PathSet::const_iterator i = paths.begin();
+         i != paths.end(); ++i)
+        l = ATinsert(l, toATerm(*i));
+    return ATreverse(l);
+}
+
+
+ATerm unparseDerivation(const Derivation & drv)
+{
+    ATermList outputs = ATempty;
+    for (DerivationOutputs::const_iterator i = drv.outputs.begin();
+         i != drv.outputs.end(); ++i)
+        outputs = ATinsert(outputs,
+            makeDerivationOutput(
+                toATerm(i->first),
+                toATerm(i->second.path),
+                toATerm(i->second.hashAlgo),
+                toATerm(i->second.hash)));
+
+    ATermList inDrvs = ATempty;
+    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
+         i != drv.inputDrvs.end(); ++i)
+        inDrvs = ATinsert(inDrvs,
+            makeDerivationInput(
+                toATerm(i->first),
+                unparseStrings(i->second)));
+    
+    ATermList args = ATempty;
+    for (Strings::const_iterator i = drv.args.begin();
+         i != drv.args.end(); ++i)
+        args = ATinsert(args, toATerm(*i));
+
+    ATermList env = ATempty;
+    for (StringPairs::const_iterator i = drv.env.begin();
+         i != drv.env.end(); ++i)
+        env = ATinsert(env,
+            makeEnvBinding(
+                toATerm(i->first),
+                toATerm(i->second)));
+
+    return makeDerive(
+        ATreverse(outputs),
+        ATreverse(inDrvs),
+        unparseStrings(drv.inputSrcs),
+        toATerm(drv.platform),
+        toATerm(drv.builder),
+        ATreverse(args),
+        ATreverse(env));
+}
+
+
+bool isDerivation(const string & fileName)
+{
+    return
+        fileName.size() >= drvExtension.size() &&
+        string(fileName, fileName.size() - drvExtension.size()) == drvExtension;
+}

src/libstore/derivations.hh

@@ -0,0 +1,67 @@
+#ifndef __DERIVATIONS_H
+#define __DERIVATIONS_H
+
+#include "aterm.hh"
+#include "store.hh"
+
+
+/* Extension of derivations in the Nix store. */
+const string drvExtension = ".drv";
+
+
+/* Abstract syntax of derivations. */
+
+struct DerivationOutput
+{
+    Path path;
+    string hashAlgo; /* hash used for expected hash computation */
+    string hash; /* expected hash, may be null */
+    DerivationOutput()
+    {
+    }
+    DerivationOutput(Path path, string hashAlgo, string hash)
+    {
+        this->path = path;
+        this->hashAlgo = hashAlgo;
+        this->hash = hash;
+    }
+};
+
+typedef map<string, DerivationOutput> DerivationOutputs;
+
+/* For inputs that are sub-derivations, we specify exactly which
+   output IDs we are interested in. */
+typedef map<Path, StringSet> DerivationInputs;
+
+typedef map<string, string> StringPairs;
+
+struct Derivation
+{
+    DerivationOutputs outputs; /* keyed on symbolic IDs */
+    DerivationInputs inputDrvs; /* inputs that are sub-derivations */
+    PathSet inputSrcs; /* inputs that are sources */
+    string platform;
+    Path builder;
+    Strings args;
+    StringPairs env;
+};
+
+
+/* Hash an aterm. */
+Hash hashTerm(ATerm t);
+
+/* Write a derivation to the Nix store, and return its path. */
+Path writeDerivation(const Derivation & drv, const string & name);
+
+/* Parse a derivation. */
+Derivation parseDerivation(ATerm t);
+
+/* Parse a derivation. */
+ATerm unparseDerivation(const Derivation & drv);
+
+/* Check whether a file name ends with the extensions for
+   derivations. */
+bool isDerivation(const string & fileName);
+
+
+#endif /* !__DERIVATIONS_H */

src/libstore/gc.cc

@@ -1,98 +1,457 @@
-#include "normalise.hh"
 #include "globals.hh"
+#include "gc.hh"
+#include "build.hh"
+#include "pathlocks.hh"
 
+#include <boost/shared_ptr.hpp>
 
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <errno.h>
+#include <fcntl.h>
 #include <unistd.h>
 
 
-void followLivePaths(Path nePath, PathSet & live)
+static string gcLockName = "gc.lock";
+static string tempRootsDir = "temproots";
+static string gcRootsDir = "gcroots";
+
+
+/* Acquire the global GC lock.  This is used to prevent new Nix
+   processes from starting after the temporary root files have been
+   read.  To be precise: when they try to create a new temporary root
+   file, they will block until the garbage collector has finished /
+   yielded the GC lock. */
+static int openGCLock(LockType lockType)
 {
-    /* Just to be sure, canonicalise the path.  It is important to do
-       this here and in findDeadPath() to ensure that a live path is
-       not mistaken for a dead path due to some non-canonical
-       representation. */
-    nePath = canonPath(nePath);
+    Path fnGCLock = (format("%1%/%2%")
+        % nixStateDir % gcLockName).str();
+         
+    debug(format("acquiring global GC lock `%1%'") % fnGCLock);
     
-    if (live.find(nePath) != live.end()) return;
-    live.insert(nePath);
+    AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600);
+    if (fdGCLock == -1)
+        throw SysError(format("opening global GC lock `%1%'") % fnGCLock);
 
-    startNest(nest, lvlDebug, format("following `%1%'") % nePath);
-    assertStorePath(nePath);
+    lockFile(fdGCLock, lockType, true);
 
-    if (isValidPath(nePath)) {
+    /* !!! Restrict read permission on the GC root.  Otherwise any
+       process that can open the file for reading can DoS the
+       collector. */
+    
+    return fdGCLock.borrow();
+}
 
-        /* !!! should make sure that no substitutes are used */
-        StoreExpr ne = storeExprFromPath(nePath);
 
-        /* !!! painfully similar to requisitesWorker() */
-        if (ne.type == StoreExpr::neClosure)
-            for (ClosureElems::iterator i = ne.closure.elems.begin();
-                 i != ne.closure.elems.end(); ++i)
-            {
-                Path p = canonPath(i->first);
-                if (live.find(p) == live.end()) {
-                    debug(format("found live `%1%'") % p);
-                    assertStorePath(p);
-                    live.insert(p);
+void createSymlink(const Path & link, const Path & target, bool careful)
+{
+    /* Create directories up to `gcRoot'. */
+    createDirs(dirOf(link));
+
+    /* Remove the old symlink. */
+    if (pathExists(link)) {
+        if (careful && (!isLink(link) || !isInStore(readLink(link))))
+            throw Error(format("cannot create symlink `%1%'; already exists") % link);
+        unlink(link.c_str());
+    }
+
+    /* And create the new own. */
+    if (symlink(target.c_str(), link.c_str()) == -1)
+        throw SysError(format("symlinking `%1%' to `%2%'")
+            % link % target);
+}
+
+
+Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
+    bool indirect)
+{
+    Path storePath(canonPath(_storePath));
+    Path gcRoot(canonPath(_gcRoot));
+    assertStorePath(storePath);
+
+    /* Grab the global GC root.  This prevents the set of permanent
+       roots from increasing while a GC is in progress. */
+    AutoCloseFD fdGCLock = openGCLock(ltRead);
+
+    if (indirect) {
+        string hash = printHash32(hashString(htSHA1, gcRoot));
+        Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
+            % nixStateDir % gcRootsDir % hash).str());
+        
+        {
+            SwitchToOriginalUser sw;
+            createSymlink(gcRoot, storePath, true);
+        }
+        createSymlink(realRoot, gcRoot, false);
+    }
+
+    else {
+        Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+    
+        if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
+            throw Error(format(
+                "path `%1%' is not a valid garbage collector root; "
+                "it's not in the directory `%2%'")
+                % gcRoot % rootsDir);
+
+        createSymlink(gcRoot, storePath, false);
+    }
+
+    return gcRoot;
+}
+
+
+/* The file to which we write our temporary roots. */
+static Path fnTempRoots;
+static AutoCloseFD fdTempRoots;
+
+
+void addTempRoot(const Path & path)
+{
+    /* Create the temporary roots file for this process. */
+    if (fdTempRoots == -1) {
+
+        while (1) {
+            Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str();
+            createDirs(dir);
+            
+            fnTempRoots = (format("%1%/%2%")
+                % dir % getpid()).str();
+
+            AutoCloseFD fdGCLock = openGCLock(ltRead);
+            
+            fdTempRoots = open(fnTempRoots.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0600);
+            if (fdTempRoots == -1)
+                throw SysError(format("opening temporary roots file `%1%'") % fnTempRoots);
+
+            fdGCLock.close();
+            
+            debug(format("acquiring read lock on `%1%'") % fnTempRoots);
+            lockFile(fdTempRoots, ltRead, true);
+
+            /* Check whether the garbage collector didn't get in our
+               way. */
+            struct stat st;
+            if (fstat(fdTempRoots, &st) == -1)
+                throw SysError(format("statting `%1%'") % fnTempRoots);
+            if (st.st_size == 0) break;
+            
+            /* The garbage collector deleted this file before we could
+               get a lock.  (It won't delete the file after we get a
+               lock.)  Try again. */
+        }
+
+    }
+
+    /* Upgrade the lock to a write lock.  This will cause us to block
+       if the garbage collector is holding our lock. */
+    debug(format("acquiring write lock on `%1%'") % fnTempRoots);
+    lockFile(fdTempRoots, ltWrite, true);
+
+    string s = path + '\0';
+    writeFull(fdTempRoots, (const unsigned char *) s.c_str(), s.size());
+
+    /* Downgrade to a read lock. */
+    debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
+    lockFile(fdTempRoots, ltRead, true);
+}
+
+
+void removeTempRoots()
+{
+    if (fdTempRoots != -1) {
+        fdTempRoots.close();
+        unlink(fnTempRoots.c_str());
+    }
+}
+
+
+typedef shared_ptr<AutoCloseFD> FDPtr;
+typedef list<FDPtr> FDs;
+
+
+static void readTempRoots(PathSet & tempRoots, FDs & fds)
+{
+    /* Read the `temproots' directory for per-process temporary root
+       files. */
+    Strings tempRootFiles = readDirectory(
+        (format("%1%/%2%") % nixStateDir % tempRootsDir).str());
+
+    for (Strings::iterator i = tempRootFiles.begin();
+         i != tempRootFiles.end(); ++i)
+    {
+        Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
+
+        debug(format("reading temporary root file `%1%'") % path);
+            
+        FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
+        if (*fd == -1) {
+            /* It's okay if the file has disappeared. */
+            if (errno == ENOENT) continue;
+            throw SysError(format("opening temporary roots file `%1%'") % path);
+        }
+
+        /* Try to acquire a write lock without blocking.  This can
+           only succeed if the owning process has died.  In that case
+           we don't care about its temporary roots. */
+        if (lockFile(*fd, ltWrite, false)) {
+            printMsg(lvlError, format("removing stale temporary roots file `%1%'")
+                % path);
+            unlink(path.c_str());
+            writeFull(*fd, (const unsigned char *) "d", 1);
+            continue;
+        }
+
+        /* Acquire a read lock.  This will prevent the owning process
+           from upgrading to a write lock, therefore it will block in
+           addTempRoot(). */
+        debug(format("waiting for read lock on `%1%'") % path);
+        lockFile(*fd, ltRead, true);
+
+        /* Read the entire file. */
+        string contents = readFile(*fd);
+
+        /* Extract the roots. */
+        unsigned int pos = 0, end;
+
+        while ((end = contents.find((char) 0, pos)) != string::npos) {
+            Path root(contents, pos, end - pos);
+            debug(format("got temporary root `%1%'") % root);
+            assertStorePath(root);
+            tempRoots.insert(root);
+            pos = end + 1;
+        }
+
+        fds.push_back(fd); /* keep open */
+    }
+}
+
+
+static void findRoots(const Path & path, bool recurseSymlinks,
+    PathSet & roots)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1)
+        throw SysError(format("statting `%1%'") % path);
+
+    printMsg(lvlVomit, format("looking at `%1%'") % path);
+
+    if (S_ISDIR(st.st_mode)) {
+	Strings names = readDirectory(path);
+	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+            findRoots(path + "/" + *i, recurseSymlinks, roots);
+    }
+
+    else if (S_ISLNK(st.st_mode)) {
+        string target = readLink(path);
+        Path target2 = absPath(target, dirOf(path));
+
+        if (isInStore(target2)) {
+            debug(format("found root `%1%' in `%2%'")
+                % target2 % path);
+            roots.insert(toStorePath(target2));
+        }
+
+        else if (recurseSymlinks) {
+            if (pathExists(target2))
+                findRoots(target2, false, roots);
+            else {
+                printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target2);
+                /* Note that we only delete when recursing, i.e., when
+                   we are still in the `gcroots' tree.  We never
+                   delete stuff outside that tree. */
+                unlink(path.c_str());
+            }
+        }
+    }
+}
+
+
+static void dfsVisit(const PathSet & paths, const Path & path,
+    PathSet & visited, Paths & sorted)
+{
+    if (visited.find(path) != visited.end()) return;
+    visited.insert(path);
+    
+    PathSet references;
+    if (isValidPath(path))
+        queryReferences(noTxn, path, references);
+    
+    for (PathSet::iterator i = references.begin();
+         i != references.end(); ++i)
+        /* Don't traverse into paths that don't exist.  That can
+           happen due to substitutes for non-existent paths. */
+        if (*i != path && paths.find(*i) != paths.end())
+            dfsVisit(paths, *i, visited, sorted);
+
+    sorted.push_front(path);
+}
+
+
+static Paths topoSort(const PathSet & paths)
+{
+    Paths sorted;
+    PathSet visited;
+    for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i)
+        dfsVisit(paths, *i, visited, sorted);
+    return sorted;
+}
+
+
+void collectGarbage(GCAction action, PathSet & result)
+{
+    result.clear();
+
+    bool gcKeepOutputs =
+        queryBoolSetting("gc-keep-outputs", false);
+    bool gcKeepDerivations =
+        queryBoolSetting("gc-keep-derivations", true);
+
+    /* Acquire the global GC root.  This prevents
+       a) New roots from being added.
+       b) Processes from creating new temporary root files. */
+    AutoCloseFD fdGCLock = openGCLock(ltWrite);
+
+    /* Find the roots.  Since we've grabbed the GC lock, the set of
+       permanent roots cannot increase now. */
+    Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+    PathSet roots;
+    findRoots(rootsDir, true, roots);
+
+    if (action == gcReturnRoots) {
+        result = roots;
+        return;
+    }
+
+    /* Determine the live paths which is just the closure of the
+       roots under the `references' relation. */
+    PathSet livePaths;
+    for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
+        computeFSClosure(canonPath(*i), livePaths);
+
+    if (gcKeepDerivations) {
+        for (PathSet::iterator i = livePaths.begin();
+             i != livePaths.end(); ++i)
+        {
+            /* Note that the deriver need not be valid (e.g., if we
+               previously ran the collector with `gcKeepDerivations'
+               turned off). */
+            Path deriver = queryDeriver(noTxn, *i);
+            if (deriver != "" && isValidPath(deriver))
+                computeFSClosure(deriver, livePaths);
+        }
+    }
+
+    if (gcKeepOutputs) {
+        /* Hmz, identical to storePathRequisites in nix-store. */
+        for (PathSet::iterator i = livePaths.begin();
+             i != livePaths.end(); ++i)
+            if (isDerivation(*i)) {
+                Derivation drv = derivationFromPath(*i);
+                for (DerivationOutputs::iterator j = drv.outputs.begin();
+                     j != drv.outputs.end(); ++j)
+                    if (isValidPath(j->second.path))
+                        computeFSClosure(j->second.path, livePaths);
+            }
+    }
+
+    if (action == gcReturnLive) {
+        result = livePaths;
+        return;
+    }
+
+    /* Read the temporary roots.  This acquires read locks on all
+       per-process temporary root files.  So after this point no paths
+       can be added to the set of temporary roots. */
+    PathSet tempRoots;
+    FDs fds;
+    readTempRoots(tempRoots, fds);
+
+    /* Close the temporary roots.  Note that we *cannot* do this in
+       readTempRoots(), because there we may not have all locks yet,
+       meaning that an invalid path can become valid (and thus add to
+       the references graph) after we have added it to the closure
+       (and computeFSClosure() assumes that the presence of a path
+       means that it has already been closed). */
+    PathSet tempRootsClosed;
+    for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
+        if (isValidPath(*i))
+            computeFSClosure(*i, tempRootsClosed);
+        else
+            tempRootsClosed.insert(*i);
+
+    /* For testing - see tests/gc-concurrent.sh. */
+    if (getenv("NIX_DEBUG_GC_WAIT"))
+        sleep(2);
+    
+    /* After this point the set of roots or temporary roots cannot
+       increase, since we hold locks on everything.  So everything
+       that is not currently in in `livePaths' or `tempRootsClosed'
+       can be deleted. */
+    
+    /* Read the Nix store directory to find all currently existing
+       paths. */
+    Paths storePaths = readDirectory(nixStore);
+    PathSet storePaths2;
+    for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i)
+        storePaths2.insert(canonPath(nixStore + "/" + *i));
+
+    /* Topologically sort them under the `referers' relation.  That
+       is, a < b iff a is in referers(b).  This gives us the order in
+       which things can be deleted safely. */
+    /* !!! when we have multiple output paths per derivation, this
+       will not work anymore because we get cycles. */
+    storePaths = topoSort(storePaths2);
+
+    /* Try to delete store paths in the topologically sorted order. */
+    for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
+
+        debug(format("considering deletion of `%1%'") % *i);
+        
+        if (livePaths.find(*i) != livePaths.end()) {
+            debug(format("live path `%1%'") % *i);
+            continue;
+        }
+
+        if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
+            debug(format("temporary root `%1%'") % *i);
+            continue;
+        }
+
+        debug(format("dead path `%1%'") % *i);
+        result.insert(*i);
+
+        AutoCloseFD fdLock;
+
+        if (action == gcDeleteDead) {
+
+            /* Only delete a lock file if we can acquire a write lock
+               on it.  That means that it's either stale, or the
+               process that created it hasn't locked it yet.  In the
+               latter case the other process will detect that we
+               deleted the lock, and retry (see pathlocks.cc). */
+            if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
+
+                fdLock = open(i->c_str(), O_RDWR);
+                if (fdLock == -1) {
+                    if (errno == ENOENT) continue;
+                    throw SysError(format("opening lock file `%1%'") % *i);
+                }
+
+                if (!lockFile(fdLock, ltWrite, false)) {
+                    debug(format("skipping active lock `%1%'") % *i);
+                    continue;
                 }
             }
-    
-        else if (ne.type == StoreExpr::neDerivation)
-            for (PathSet::iterator i = ne.derivation.inputs.begin();
-                 i != ne.derivation.inputs.end(); ++i)
-                followLivePaths(*i, live);
 
-        else abort();
-        
-    }
+            printMsg(lvlInfo, format("deleting `%1%'") % *i);
+            
+            /* Okay, it's safe to delete. */
+            deleteFromStore(*i);
 
-    Path nfPath;
-    if (querySuccessor(nePath, nfPath))
-        followLivePaths(nfPath, live);
-}
-
-
-PathSet findLivePaths(const Paths & roots)
-{
-    PathSet live;
-
-    startNest(nest, lvlDebug, "finding live paths");
-
-    for (Paths::const_iterator i = roots.begin(); i != roots.end(); ++i)
-        followLivePaths(*i, live);
-
-    return live;
-}
-
-
-PathSet findDeadPaths(const PathSet & live, time_t minAge)
-{
-    PathSet dead;
-
-    startNest(nest, lvlDebug, "finding dead paths");
-
-    time_t now = time(0);
-
-    Strings storeNames = readDirectory(nixStore);
-
-    for (Strings::iterator i = storeNames.begin(); i != storeNames.end(); ++i) {
-        Path p = canonPath(nixStore + "/" + *i);
-
-        if (minAge > 0) {
-            struct stat st;
-            if (lstat(p.c_str(), &st) != 0)
-                throw SysError(format("obtaining information about `%1%'") % p);
-            if (st.st_atime + minAge >= now) continue;
+            if (fdLock != -1)
+                /* Write token to stale (deleted) lock file. */
+                writeFull(fdLock, (const unsigned char *) "d", 1);
         }
-        
-        if (live.find(p) == live.end()) {
-            debug(format("dead path `%1%'") % p);
-            dead.insert(p);
-        } else
-            debug(format("live path `%1%'") % p);
     }
-    
-    return dead;
 }

src/libstore/gc.hh

@@ -1,26 +1,40 @@
 #ifndef __GC_H
 #define __GC_H
 
-#include "storeexpr.hh"
+#include "util.hh"
 
 
-/* Determine the set of "live" store paths, given a set of root store
-   expressions.  The live store paths are those that are reachable
-   from the roots.  The roots are reachable by definition.  Any path
-   mentioned in a reachable store expression is also reachable.  If a
-   derivation store expression is reachable, then its successor (if it
-   exists) if also reachable.  It is not an error for store
-   expressions not to exist (since this can happen on derivation store
-   expressions, for instance, due to the substitute mechanism), but
-   successor links are followed even for non-existant derivations. */
-PathSet findLivePaths(const Paths & roots);
+/* Garbage collector operation. */
+typedef enum {
+    gcReturnRoots,
+    gcReturnLive,
+    gcReturnDead,
+    gcDeleteDead,
+} GCAction;
 
-/* Given a set of "live" store paths, determine the set of "dead"
-   store paths (which are simply all store paths that are not in the
-   live set).  The value `minAge' specifies the minimum age in seconds
-   for an unreachable file to be considered dead (0 meaning that any
-   unreachable file is dead). */
-PathSet findDeadPaths(const PathSet & live, time_t minAge);
+/* If `action' is set to `gcReturnRoots', find and return the set of
+   roots for the garbage collector.  These are the store paths
+   symlinked to in the `gcroots' directory.  If `action' is
+   `gcReturnLive', return the set of paths reachable from (i.e. in the
+   closure of) the roots.  If `action' is `gcReturnDead', return the
+   set of paths not reachable from the roots.  If `action' is
+   `gcDeleteDead', actually delete the latter set. */
+void collectGarbage(GCAction action, PathSet & result);
+
+/* Register a temporary GC root.  This root will automatically
+   disappear when this process exits.  WARNING: this function should
+   not be called inside a BDB transaction, otherwise we can
+   deadlock. */
+void addTempRoot(const Path & path);
+
+/* Remove the temporary roots file for this process.  Any temporary
+   root becomes garbage after this point unless it has been registered
+   as a (permanent) root. */
+void removeTempRoots();
+
+/* Register a permanent GC root. */
+Path addPermRoot(const Path & storePath, const Path & gcRoot,
+    bool indirect);
 
 
 #endif /* !__GC_H */

src/libstore/globals.cc

@@ -1,10 +1,14 @@
 #include "globals.hh"
 
+#include <map>
+
+
 string nixStore = "/UNINIT";
 string nixDataDir = "/UNINIT";
 string nixLogDir = "/UNINIT";
 string nixStateDir = "/UNINIT";
 string nixDBPath = "/UNINIT";
+string nixConfDir = "/UNINIT";
 
 bool keepFailed = false;
 
@@ -17,3 +21,59 @@ Verbosity buildVerbosity = lvlInfo;
 unsigned int maxBuildJobs = 1;
 
 bool readOnlyMode = false;
+
+
+static bool settingsRead = false;
+
+static map<string, string> settings;
+
+
+static void readSettings()
+{
+    Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
+    if (!pathExists(settingsFile)) return;
+    string contents = readFile(settingsFile);
+
+    unsigned int pos = 0;
+
+    while (pos < contents.size()) {
+        string line;
+        while (pos < contents.size() && contents[pos] != '\n')
+            line += contents[pos++];
+        pos++;
+
+        unsigned int hash = line.find('#');
+        if (hash != string::npos)
+            line = string(line, 0, hash);
+
+        if (line.find_first_not_of(" ") == string::npos) continue;
+
+        istringstream is(line);
+        string name, sep, value;
+        is >> name >> sep >> value;
+        if (sep != "=" || !is)
+            throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile);
+        
+        settings[name] = value;
+    };
+    
+    settingsRead = true;
+}
+
+
+string querySetting(const string & name, const string & def)
+{
+    if (!settingsRead) readSettings();
+    map<string, string>::iterator i = settings.find(name);
+    return i == settings.end() ? def : i->second;
+}
+
+
+bool queryBoolSetting(const string & name, bool def)
+{
+    string value = querySetting(name, def ? "true" : "false");
+    if (value == "true") return true;
+    else if (value == "false") return false;
+    else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'")
+        % name % value);
+}

src/libstore/globals.hh

@@ -23,6 +23,11 @@ extern string nixStateDir;
 /* nixDBPath is the path name of our Berkeley DB environment. */
 extern string nixDBPath;
 
+/* nixConfDir is the directory where configuration files are
+   stored. */
+extern string nixConfDir;
+
+
 
 /* Misc. global flags. */
 
@@ -48,4 +53,9 @@ extern unsigned int maxBuildJobs;
 extern bool readOnlyMode;
 
 
+string querySetting(const string & name, const string & def);
+
+bool queryBoolSetting(const string & name, bool def);
+
+
 #endif /* !__GLOBALS_H */

src/libstore/misc.cc

@@ -1,72 +1,38 @@
-#include "normalise.hh"
+#include "build.hh"
 
 
-StoreExpr storeExprFromPath(const Path & path)
+Derivation derivationFromPath(const Path & drvPath)
 {
-    assertStorePath(path);
-    ensurePath(path);
-    ATerm t = ATreadFromNamedFile(path.c_str());
-    if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
-    return parseStoreExpr(t);
+    assertStorePath(drvPath);
+    ensurePath(drvPath);
+    ATerm t = ATreadFromNamedFile(drvPath.c_str());
+    if (!t) throw Error(format("cannot read aterm from `%1%'") % drvPath);
+    return parseDerivation(t);
 }
 
 
-PathSet storeExprRoots(const Path & nePath)
+void computeFSClosure(const Path & storePath,
+    PathSet & paths, bool flipDirection)
 {
-    PathSet paths;
+    if (paths.find(storePath) != paths.end()) return;
+    paths.insert(storePath);
 
-    StoreExpr ne = storeExprFromPath(nePath);
+    PathSet references;
+    if (flipDirection)
+        queryReferers(noTxn, storePath, references);
+    else
+        queryReferences(noTxn, storePath, references);
 
-    if (ne.type == StoreExpr::neClosure)
-        paths.insert(ne.closure.roots.begin(), ne.closure.roots.end());
-    else if (ne.type == StoreExpr::neDerivation)
-        paths.insert(ne.derivation.outputs.begin(),
-            ne.derivation.outputs.end());
-    else abort();
-
-    return paths;
+    for (PathSet::iterator i = references.begin();
+         i != references.end(); ++i)
+        computeFSClosure(*i, paths, flipDirection);
 }
 
 
-static void requisitesWorker(const Path & nePath,
-    bool includeExprs, bool includeSuccessors,
-    PathSet & paths, PathSet & doneSet)
+ Path findOutput(const Derivation & drv, string id)
 {
-    checkInterrupt();
-    
-    if (doneSet.find(nePath) != doneSet.end()) return;
-    doneSet.insert(nePath);
-
-    StoreExpr ne = storeExprFromPath(nePath);
-
-    if (ne.type == StoreExpr::neClosure)
-        for (ClosureElems::iterator i = ne.closure.elems.begin();
-             i != ne.closure.elems.end(); ++i)
-            paths.insert(i->first);
-    
-    else if (ne.type == StoreExpr::neDerivation)
-        for (PathSet::iterator i = ne.derivation.inputs.begin();
-             i != ne.derivation.inputs.end(); ++i)
-            requisitesWorker(*i,
-                includeExprs, includeSuccessors, paths, doneSet);
-
-    else abort();
-
-    if (includeExprs) paths.insert(nePath);
-
-    Path nfPath;
-    if (includeSuccessors && querySuccessor(nePath, nfPath))
-        requisitesWorker(nfPath, includeExprs, includeSuccessors,
-            paths, doneSet);
-}
-
-
-PathSet storeExprRequisites(const Path & nePath,
-    bool includeExprs, bool includeSuccessors)
-{
-    PathSet paths;
-    PathSet doneSet;
-    requisitesWorker(nePath, includeExprs, includeSuccessors,
-        paths, doneSet);
-    return paths;
+    for (DerivationOutputs::const_iterator i = drv.outputs.begin();
+         i != drv.outputs.end(); ++i)
+        if (i->first == id) return i->second.path;
+    throw Error(format("derivation has no output `%1%'") % id);
 }

src/libstore/normalise.hh

@@ -1,44 +0,0 @@
-#ifndef __NORMALISE_H
-#define __NORMALISE_H
-
-#include "storeexpr.hh"
-
-
-/* Normalise a store expression.  That is, if the expression is a
-   derivation, a path containing an equivalent closure expression is
-   returned.  This requires that the derivation is performed, unless a
-   successor is known. */
-Path normaliseStoreExpr(const Path & nePath);
-
-/* Realise a store expression.  If the expression is a derivation, it
-   is first normalised into a closure.  The closure is then realised
-   in the file system (i.e., it is ensured that each path in the
-   closure exists in the file system, if necessary by using the
-   substitute mechanism).  Returns the normal form of the expression
-   (i.e., its closure expression). */
-Path realiseStoreExpr(const Path & nePath);
-
-/* Ensure that a path exists, possibly by instantiating it by
-   realising a substitute. */
-void ensurePath(const Path & path);
-
-/* Read a store expression, after ensuring its existence through
-   ensurePath(). */
-StoreExpr storeExprFromPath(const Path & path);
-
-/* Get the list of root (output) paths of the given store
-   expression. */
-PathSet storeExprRoots(const Path & nePath);
-
-/* Get the list of paths that are required to realise the given store
-   expression.  For a derive expression, this is the union of
-   requisites of the inputs; for a closure expression, it is the path
-   of each element in the closure.  If `includeExprs' is true, include
-   the paths of the store expressions themselves.  If
-   `includeSuccessors' is true, include the requisites of
-   successors. */
-PathSet storeExprRequisites(const Path & nePath,
-    bool includeExprs, bool includeSuccessors);
-
-
-#endif /* !__NORMALISE_H */

src/libstore/pathlocks.cc

@@ -61,7 +61,7 @@ PathLocks::PathLocks(const PathSet & paths)
 void PathLocks::lockPaths(const PathSet & _paths)
 {
     /* May be called only once! */
-    assert(this->paths.empty());
+    assert(fds.empty());
     
     /* Note that `fds' is built incrementally so that the destructor
        will only release those locks that we have already acquired. */
@@ -83,20 +83,38 @@ void PathLocks::lockPaths(const PathSet & _paths)
             debug(format("already holding lock on `%1%'") % lockPath);
             continue;
         }
+
+        AutoCloseFD fd;
         
-        /* Open/create the lock file. */
-        int fd = open(lockPath.c_str(), O_WRONLY | O_CREAT, 0666);
-        if (fd == -1)
-            throw SysError(format("opening lock file `%1%'") % lockPath);
+        while (1) {
+        
+            /* Open/create the lock file. */
+            fd = open(lockPath.c_str(), O_WRONLY | O_CREAT, 0666);
+            if (fd == -1)
+                throw SysError(format("opening lock file `%1%'") % lockPath);
 
-        fds.push_back(fd);
-        this->paths.push_back(lockPath);
+            /* Acquire an exclusive lock. */
+            lockFile(fd, ltWrite, true);
 
-        /* Acquire an exclusive lock. */
-        lockFile(fd, ltWrite, true);
+            debug(format("lock acquired on `%1%'") % lockPath);
 
-        debug(format("lock acquired on `%1%'") % lockPath);
+            /* Check that the lock file hasn't become stale (i.e.,
+               hasn't been unlinked). */
+            struct stat st;
+            if (fstat(fd, &st) == -1)
+                throw SysError(format("statting lock file `%1%'") % lockPath);
+            if (st.st_size != 0)
+                /* This lock file has been unlinked, so we're holding
+                   a lock on a deleted file.  This means that other
+                   processes may create and acquire a lock on
+                   `lockPath', and proceed.  So we must retry. */
+                debug(format("open lock file `%1%' has become stale") % lockPath);
+            else
+                break;
+        }
 
+        /* Use borrow so that the descriptor isn't closed. */
+        fds.push_back(FDPair(fd.borrow(), lockPath));
         lockedPaths.insert(lockPath);
     }
 }
@@ -104,19 +122,21 @@ void PathLocks::lockPaths(const PathSet & _paths)
 
 PathLocks::~PathLocks()
 {
-    for (list<int>::iterator i = fds.begin(); i != fds.end(); i++)
-        if (close(*i) != 0) throw SysError("closing fd");
-
-    for (Paths::iterator i = paths.begin(); i != paths.end(); i++) {
-        checkInterrupt();
+    for (list<FDPair>::iterator i = fds.begin(); i != fds.end(); i++) {
         if (deletePaths) {
-            /* This is not safe in general! */
-            unlink(i->c_str());
+            /* Write a (meaningless) token to the file to indicate to
+               other processes waiting on this lock that the lock is
+               stale (deleted). */
+            unlink(i->second.c_str());
+            writeFull(i->first, (const unsigned char *) "d", 1);
             /* Note that the result of unlink() is ignored; removing
                the lock file is an optimisation, not a necessity. */
         }
-        lockedPaths.erase(*i);
-        debug(format("lock released on `%1%'") % *i);
+        lockedPaths.erase(i->second);
+        if (close(i->first) == -1)
+            printMsg(lvlError,
+                format("error (ignored): cannot close lock file on `%1%'") % i->second);
+        debug(format("lock released on `%1%'") % i->second);
     }
 }
 

src/libstore/pathlocks.hh

@@ -12,8 +12,8 @@ bool lockFile(int fd, LockType lockType, bool wait);
 class PathLocks 
 {
 private:
-    list<int> fds;
-    Paths paths;
+    typedef pair<int, Path> FDPair;
+    list<FDPair> fds;
     bool deletePaths;
 
 public:

src/libstore/references.cc

@@ -34,6 +34,8 @@ void checkPath(const string & path,
 {
     checkInterrupt();
     
+    debug(format("checking `%1%'") % path);
+
     struct stat st;
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting attributes of path `%1%'") % path);
@@ -48,8 +50,6 @@ void checkPath(const string & path,
 
     else if (S_ISREG(st.st_mode)) {
         
-        debug(format("checking `%1%'") % path);
-
         AutoCloseFD fd = open(path.c_str(), O_RDONLY);
         if (fd == -1) throw SysError(format("opening file `%1%'") % path);
 
@@ -81,8 +81,12 @@ Strings filterReferences(const string & path, const Strings & paths)
     for (Strings::const_iterator i = paths.begin();
          i != paths.end(); i++)
     {
-        string s = string(baseNameOf(*i), 0, 32);
-        parseHash(s);
+        string baseName = baseNameOf(*i);
+        unsigned int pos = baseName.find('-');
+        if (pos == string::npos)
+            throw Error(format("bad reference `%1%'") % *i);
+        string s = string(baseName, 0, pos);
+        // parseHash(htSHA256, s);
         ids.push_back(s);
         backMap[s] = *i;
     }

src/libstore/store.hh

@@ -9,11 +9,17 @@
 using namespace std;
 
 
+const int nixSchemaVersion = 2;
+
+
 /* A substitute is a program invocation that constructs some store
    path (typically by fetching it from somewhere, e.g., from the
    network). */
 struct Substitute
-{
+{       
+    /* The derivation that built this store path (empty if none). */
+    Path deriver;
+    
     /* Program to be executed to create the store path.  Must be in
        the output path of `storeExpr'. */
     Path program;
@@ -40,60 +46,115 @@ void createStoreTransaction(Transaction & txn);
 /* Copy a path recursively. */
 void copyPath(const Path & src, const Path & dst);
 
-/* Register a successor.  This function accepts a transaction handle
-   so that it can be enclosed in an atomic operation with calls to
-   registerValidPath().  This must be atomic, since if we register a
-   successor for a derivation without registering the paths built in
-   the derivation, we have a successor with dangling pointers, and if
-   we do it in reverse order, we can get an obstructed build (since to
-   rebuild the successor, the outputs paths must not exist). */
-void registerSuccessor(const Transaction & txn,
-    const Path & srcPath, const Path & sucPath);
-
-/* Remove a successor mapping. */
-void unregisterSuccessor(const Path & srcPath);
-
-/* Return the predecessors of the Nix expression stored at the given
-   path. */
-bool querySuccessor(const Path & srcPath, Path & sucPath);
-
-/* Return the predecessors of the Nix expression stored at the given
-   path. */
-Paths queryPredecessors(const Path & sucPath);
-
 /* Register a substitute. */
-typedef list<pair<Path, Substitute> > SubstitutePairs;
-void registerSubstitutes(const Transaction & txn,
-    const SubstitutePairs & subPairs);
+void registerSubstitute(const Transaction & txn,
+    const Path & srcPath, const Substitute & sub);
 
-/* Return the substitutes expression for the given path. */
-Substitutes querySubstitutes(const Path & srcPath);
+/* Return the substitutes for the given path. */
+Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath);
 
 /* Deregister all substitutes. */
 void clearSubstitutes();
 
-/* Register the validity of a path. */
-void registerValidPath(const Transaction & txn, const Path & path);
+/* Register the validity of a path, i.e., that `path' exists, that the
+   paths referenced by it exists, and in the case of an output path of
+   a derivation, that it has been produced by a succesful execution of
+   the derivation (or something equivalent).  Also register the hash
+   of the file system contents of the path.  The hash must be a
+   SHA-256 hash. */
+void registerValidPath(const Transaction & txn,
+    const Path & path, const Hash & hash, const PathSet & references,
+    const Path & deriver);
+
+struct ValidPathInfo 
+{
+    Path path;
+    Path deriver;
+    Hash hash;
+    PathSet references;
+};
+
+typedef list<ValidPathInfo> ValidPathInfos;
+
+void registerValidPaths(const Transaction & txn,
+    const ValidPathInfos & infos);
 
 /* Throw an exception if `path' is not directly in the Nix store. */
 void assertStorePath(const Path & path);
 
+bool isInStore(const Path & path);
+bool isStorePath(const Path & path);
+
+void checkStoreName(const string & name);
+
+/* Chop off the parts after the top-level store name, e.g.,
+   /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
+Path toStorePath(const Path & path);
+
+/* "Fix", or canonicalise, the meta-data of the files in a store path
+   after it has been built.  In particular:
+   - the last modification date on each file is set to 0 (i.e.,
+     00:00:00 1/1/1970 UTC)
+   - the permissions are set of 444 or 555 (i.e., read-only with or
+     without execute permission; setuid bits etc. are cleared)
+   - the owner and group are set to the Nix user and group, if we're
+     in a setuid Nix installation. */
+void canonicalisePathMetaData(const Path & path);
+
 /* Checks whether a path is valid. */ 
+bool isValidPathTxn(const Transaction & txn, const Path & path);
 bool isValidPath(const Path & path);
 
+/* Queries the hash of a valid path. */ 
+Hash queryPathHash(const Path & path);
+
+/* Sets the set of outgoing FS references for a store path.  Use with
+   care! */
+void setReferences(const Transaction & txn, const Path & storePath,
+    const PathSet & references);
+
+/* Queries the set of outgoing FS references for a store path.  The
+   result is not cleared. */
+void queryReferences(const Transaction & txn,
+    const Path & storePath, PathSet & references);
+
+/* Queries the set of incoming FS references for a store path.  The
+   result is not cleared. */
+void queryReferers(const Transaction & txn,
+    const Path & storePath, PathSet & referers);
+
+/* Sets the deriver of a store path.  Use with care! */
+void setDeriver(const Transaction & txn, const Path & storePath,
+    const Path & deriver);
+
+/* Query the deriver of a store path.  Return the empty string if no
+   deriver has been set. */
+Path queryDeriver(const Transaction & txn, const Path & storePath);
+
+/* Constructs a unique store path name. */
+Path makeStorePath(const string & type,
+    const Hash & hash, const string & suffix);
+    
 /* Copy the contents of a path to the store and register the validity
    the resulting path.  The resulting path is returned. */
 Path addToStore(const Path & srcPath);
 
-/* Like addToStore, but the path of the output is given, and the
-   contents written to the output path is a regular file containing
-   the given string. */
-void addTextToStore(const Path & dstPath, const string & s);
+/* Like addToStore(), but for pre-adding the outputs of fixed-output
+   derivations. */
+Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath);
+
+Path makeFixedOutputPath(bool recursive,
+    string hashAlgo, Hash hash, string name);
+
+/* Like addToStore, but the contents written to the output path is a
+   regular file containing the given string. */
+Path addTextToStore(const string & suffix, const string & s,
+    const PathSet & references);
 
 /* Delete a value from the nixStore directory. */
 void deleteFromStore(const Path & path);
 
-void verifyStore();
+void verifyStore(bool checkContents);
 
 
 #endif /* !__STORE_H */

src/libstore/storeexpr-ast.def

@@ -1,7 +0,0 @@
-init initStoreExprHelpers
-
-Closure | ATermList ATermList | ATerm |
-Derive | ATermList ATermList string string ATermList ATermList | ATerm |
-
-| string string | ATerm | EnvBinding |
-| string ATermList | ATerm | ClosureElem |

src/libstore/storeexpr.cc

@@ -1,198 +0,0 @@
-#include "storeexpr.hh"
-#include "globals.hh"
-#include "store.hh"
-
-#include "storeexpr-ast.hh"
-#include "storeexpr-ast.cc"
-
-
-Hash hashTerm(ATerm t)
-{
-    return hashString(atPrint(t));
-}
-
-
-Path writeTerm(ATerm t, const string & suffix)
-{
-    /* The id of a term is its hash. */
-    Hash h = hashTerm(t);
-
-    Path path = canonPath(nixStore + "/" + 
-        (string) h + suffix + ".store");
-
-    if (!readOnlyMode && !isValidPath(path)) {
-        char * s = ATwriteToString(t);
-        if (!s) throw Error(format("cannot write aterm to `%1%'") % path);
-        addTextToStore(path, string(s));
-    }
-    
-    return path;
-}
-
-
-static void parsePaths(ATermList paths, PathSet & out)
-{
-    for (ATermIterator i(paths); i; ++i) {
-        if (ATgetType(*i) != AT_APPL)
-            throw badTerm("not a path", *i);
-        string s = aterm2String(*i);
-        if (s.size() == 0 || s[0] != '/')
-            throw badTerm("not a path", *i);
-        out.insert(s);
-    }
-}
-
-
-static void checkClosure(const Closure & closure)
-{
-    if (closure.elems.size() == 0)
-        throw Error("empty closure");
-
-    PathSet decl;
-    for (ClosureElems::const_iterator i = closure.elems.begin();
-         i != closure.elems.end(); i++)
-        decl.insert(i->first);
-    
-    for (PathSet::const_iterator i = closure.roots.begin();
-         i != closure.roots.end(); i++)
-        if (decl.find(*i) == decl.end())
-            throw Error(format("undefined root path `%1%'") % *i);
-    
-    for (ClosureElems::const_iterator i = closure.elems.begin();
-         i != closure.elems.end(); i++)
-        for (PathSet::const_iterator j = i->second.refs.begin();
-             j != i->second.refs.end(); j++)
-            if (decl.find(*j) == decl.end())
-                throw Error(
-		    format("undefined path `%1%' referenced by `%2%'")
-		    % *j % i->first);
-}
-
-
-/* Parse a closure. */
-static bool parseClosure(ATerm t, Closure & closure)
-{
-    ATermList roots, elems;
-
-    if (!matchClosure(t, roots, elems))
-        return false;
-
-    parsePaths(roots, closure.roots);
-
-    for (ATermIterator i(elems); i; ++i) {
-        ATerm path;
-        ATermList refs;
-        if (!matchClosureElem(*i, path, refs))
-            throw badTerm("not a closure element", *i);
-        ClosureElem elem;
-        parsePaths(refs, elem.refs);
-        closure.elems[aterm2String(path)] = elem;
-    }
-
-    checkClosure(closure);
-    return true;
-}
-
-
-static bool parseDerivation(ATerm t, Derivation & derivation)
-{
-    ATermList outs, ins, args, bnds;
-    ATerm builder, platform;
-
-    if (!matchDerive(t, outs, ins, platform, builder, args, bnds))
-        return false;
-
-    parsePaths(outs, derivation.outputs);
-    parsePaths(ins, derivation.inputs);
-
-    derivation.builder = aterm2String(builder);
-    derivation.platform = aterm2String(platform);
-    
-    for (ATermIterator i(args); i; ++i) {
-        if (ATgetType(*i) != AT_APPL)
-            throw badTerm("string expected", *i);
-        derivation.args.push_back(aterm2String(*i));
-    }
-
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm s1, s2;
-        if (!matchEnvBinding(*i, s1, s2))
-            throw badTerm("tuple of strings expected", *i);
-        derivation.env[aterm2String(s1)] = aterm2String(s2);
-    }
-
-    return true;
-}
-
-
-StoreExpr parseStoreExpr(ATerm t)
-{
-    StoreExpr ne;
-    if (parseClosure(t, ne.closure))
-        ne.type = StoreExpr::neClosure;
-    else if (parseDerivation(t, ne.derivation))
-        ne.type = StoreExpr::neDerivation;
-    else throw badTerm("not a store expression", t);
-    return ne;
-}
-
-
-static ATermList unparsePaths(const PathSet & paths)
-{
-    ATermList l = ATempty;
-    for (PathSet::const_iterator i = paths.begin();
-         i != paths.end(); i++)
-        l = ATinsert(l, toATerm(*i));
-    return ATreverse(l);
-}
-
-
-static ATerm unparseClosure(const Closure & closure)
-{
-    ATermList roots = unparsePaths(closure.roots);
-    
-    ATermList elems = ATempty;
-    for (ClosureElems::const_iterator i = closure.elems.begin();
-         i != closure.elems.end(); i++)
-        elems = ATinsert(elems,
-            makeClosureElem(
-                toATerm(i->first),
-                unparsePaths(i->second.refs)));
-
-    return makeClosure(roots, elems);
-}
-
-
-static ATerm unparseDerivation(const Derivation & derivation)
-{
-    ATermList args = ATempty;
-    for (Strings::const_iterator i = derivation.args.begin();
-         i != derivation.args.end(); i++)
-        args = ATinsert(args, toATerm(*i));
-
-    ATermList env = ATempty;
-    for (StringPairs::const_iterator i = derivation.env.begin();
-         i != derivation.env.end(); i++)
-        env = ATinsert(env,
-            makeEnvBinding(
-                toATerm(i->first),
-                toATerm(i->second)));
-
-    return makeDerive(
-        unparsePaths(derivation.outputs),
-        unparsePaths(derivation.inputs),
-        toATerm(derivation.platform),
-        toATerm(derivation.builder),
-        ATreverse(args),
-        ATreverse(env));
-}
-
-
-ATerm unparseStoreExpr(const StoreExpr & ne)
-{
-    if (ne.type == StoreExpr::neClosure)
-        return unparseClosure(ne.closure);
-    else if (ne.type == StoreExpr::neDerivation)
-        return unparseDerivation(ne.derivation);
-    else abort();
-}

src/libstore/storeexpr.hh

@@ -1,56 +0,0 @@
-#ifndef __STOREEXPR_H
-#define __STOREEXPR_H
-
-#include "aterm.hh"
-#include "store.hh"
-
-
-/* Abstract syntax of store expressions. */
-
-struct ClosureElem
-{
-    PathSet refs;
-};
-
-typedef map<Path, ClosureElem> ClosureElems;
-
-struct Closure
-{
-    PathSet roots;
-    ClosureElems elems;
-};
-
-typedef map<string, string> StringPairs;
-
-struct Derivation
-{
-    PathSet outputs;
-    PathSet inputs; /* Store expressions, not actual inputs */
-    string platform;
-    Path builder;
-    Strings args;
-    StringPairs env;
-};
-
-struct StoreExpr
-{
-    enum { neClosure, neDerivation } type;
-    Closure closure;
-    Derivation derivation;
-};
-
-
-/* Hash an aterm. */
-Hash hashTerm(ATerm t);
-
-/* Write an aterm to the Nix store directory, and return its path. */
-Path writeTerm(ATerm t, const string & suffix);
-
-/* Parse a store expression. */
-StoreExpr parseStoreExpr(ATerm t);
-
-/* Parse a store expression. */
-ATerm unparseStoreExpr(const StoreExpr & ne);
-
-
-#endif /* !__STOREEXPR_H */

src/libutil/Makefile.am

@@ -1,6 +1,7 @@
 noinst_LIBRARIES = libutil.a
 
 libutil_a_SOURCES = util.cc util.hh hash.cc hash.hh \
- archive.cc archive.hh md5.c md5.h aterm.cc aterm.hh
+ archive.cc archive.hh aterm.cc aterm.hh \
+ md5.c md5.h sha1.c sha1.h sha256.c sha256.h md32_common.h
 
 AM_CXXFLAGS = -Wall -I.. ${aterm_include}

src/libutil/archive.hh

@@ -42,6 +42,7 @@
 
 struct DumpSink 
 {
+    virtual ~DumpSink() { }
     virtual void operator () (const unsigned char * data, unsigned int len) = 0;
 };
 
@@ -50,6 +51,8 @@ void dumpPath(const Path & path, DumpSink & sink);
 
 struct RestoreSource
 {
+    virtual ~RestoreSource() { }
+    
     /* The callee should store exactly *len bytes in the buffer
        pointed to by data.  It should block if that much data is not
        yet available, or throw an error if it is not going to be

src/libutil/hash.cc

@@ -2,20 +2,42 @@
 
 extern "C" {
 #include "md5.h"
+#include "sha1.h"
+#include "sha256.h"
 }
 
 #include "hash.hh"
 #include "archive.hh"
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+
 
 Hash::Hash()
 {
-    memset(hash, 0, sizeof(hash));
+    type = htUnknown;
+    hashSize = 0;
+    memset(hash, 0, maxHashSize);
+}
+
+
+Hash::Hash(HashType type)
+{
+    this->type = type;
+    if (type == htMD5) hashSize = md5HashSize;
+    else if (type == htSHA1) hashSize = sha1HashSize;
+    else if (type == htSHA256) hashSize = sha256HashSize;
+    else throw Error("unknown hash type");
+    assert(hashSize <= maxHashSize);
+    memset(hash, 0, maxHashSize);
 }
 
 
 bool Hash::operator == (const Hash & h2) const
 {
+    if (hashSize != h2.hashSize) return false;
     for (unsigned int i = 0; i < hashSize; i++)
         if (hash[i] != h2.hash[i]) return false;
     return true;
@@ -38,24 +60,24 @@ bool Hash::operator < (const Hash & h) const
 }
 
 
-Hash::operator string() const
+string printHash(const Hash & hash)
 {
     ostringstream str;
-    for (unsigned int i = 0; i < hashSize; i++) {
+    for (unsigned int i = 0; i < hash.hashSize; i++) {
         str.fill('0');
         str.width(2);
-        str << hex << (int) hash[i];
+        str << hex << (int) hash.hash[i];
     }
     return str.str();
 }
 
     
-Hash parseHash(const string & s)
+Hash parseHash(HashType ht, const string & s)
 {
-    Hash hash;
-    if (s.length() != Hash::hashSize * 2)
+    Hash hash(ht);
+    if (s.length() != hash.hashSize * 2)
         throw Error(format("invalid hash `%1%'") % s);
-    for (unsigned int i = 0; i < Hash::hashSize; i++) {
+    for (unsigned int i = 0; i < hash.hashSize; i++) {
         string s2(s, i * 2, 2);
         if (!isxdigit(s2[0]) || !isxdigit(s2[1])) 
             throw Error(format("invalid hash `%1%'") % s);
@@ -68,6 +90,97 @@ Hash parseHash(const string & s)
 }
 
 
+static unsigned char divMod(unsigned char * bytes, unsigned char y)
+{
+    unsigned int borrow = 0;
+
+    int pos = Hash::maxHashSize - 1;
+    while (pos >= 0 && !bytes[pos]) --pos;
+
+    for ( ; pos >= 0; --pos) {
+        unsigned int s = bytes[pos] + (borrow << 8);
+        unsigned int d = s / y;
+        borrow = s % y;
+        bytes[pos] = d;
+    }
+
+    return borrow;
+}
+
+
+// omitted: E O U T
+char chars[] = "0123456789abcdfghijklmnpqrsvwxyz";
+
+
+string printHash32(const Hash & hash)
+{
+    Hash hash2(hash);
+    unsigned int len = (hash.hashSize * 8 - 1) / 5 + 1;
+    
+    string s(len, '0');
+
+    int pos = len - 1;
+    while (pos >= 0) {
+        unsigned char digit = divMod(hash2.hash, 32);
+        s[pos--] = chars[digit];
+    }
+
+    for (unsigned int i = 0; i < hash2.maxHashSize; ++i)
+        assert(hash2.hash[i] == 0);
+
+    return s;
+}
+
+
+static bool mul(unsigned char * bytes, unsigned char y, int maxSize)
+{
+    unsigned char carry = 0;
+
+    for (int pos = 0; pos < maxSize; ++pos) {
+        unsigned int m = bytes[pos] * y + carry;
+        bytes[pos] = m & 0xff;
+        carry = m >> 8;
+    }
+
+    return carry;
+}
+
+
+static bool add(unsigned char * bytes, unsigned char y, int maxSize)
+{
+    unsigned char carry = y;
+
+    for (int pos = 0; pos < maxSize; ++pos) {
+        unsigned int m = bytes[pos] + carry;
+        bytes[pos] = m & 0xff;
+        carry = m >> 8;
+        if (carry == 0) break;
+    }
+
+    return carry;
+}
+
+
+Hash parseHash32(HashType ht, const string & s)
+{
+    Hash hash(ht);
+
+    for (unsigned int i = 0; i < s.length(); ++i) {
+        char c = s[i];
+        unsigned char digit;
+        for (digit = 0; digit < sizeof(chars); ++digit) /* !!! slow */
+            if (chars[digit] == c) break;
+        if (digit >= 32)
+            throw Error(format("invalid base-32 hash `%1%'") % s);
+        if (mul(hash.hash, 32, hash.hashSize) ||
+            add(hash.hash, digit, hash.hashSize))
+            throw Error(format("base-32 hash `%1%' is too large") % s);
+    }
+
+    return hash;
+}
+
+
 bool isHash(const string & s)
 {
     if (s.length() != 32) return false;
@@ -81,44 +194,113 @@ bool isHash(const string & s)
 }
 
 
-Hash hashString(const string & s)
+struct Ctx
 {
-    Hash hash;
-    md5_buffer(s.c_str(), s.length(), hash.hash);
+    md5_ctx md5;
+    sha_ctx sha1;
+    SHA256_CTX sha256;
+};
+
+
+static void start(HashType ht, Ctx & ctx)
+{
+    if (ht == htMD5) md5_init_ctx(&ctx.md5);
+    else if (ht == htSHA1) sha_init(&ctx.sha1);
+    else if (ht == htSHA256) SHA256_Init(&ctx.sha256);
+}
+
+
+static void update(HashType ht, Ctx & ctx,
+    const unsigned char * bytes, unsigned int len)
+{
+    if (ht == htMD5) md5_process_bytes(bytes, len, &ctx.md5);
+    else if (ht == htSHA1) sha_update(&ctx.sha1, bytes, len);
+    else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len);
+}
+
+
+static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
+{
+    if (ht == htMD5) md5_finish_ctx(&ctx.md5, hash);
+    else if (ht == htSHA1) {
+        sha_final(&ctx.sha1);
+        sha_digest(&ctx.sha1, hash);
+    }
+    else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256);
+}
+
+
+Hash hashString(HashType ht, const string & s)
+{
+    Ctx ctx;
+    Hash hash(ht);
+    start(ht, ctx);
+    update(ht, ctx, (const unsigned char *) s.c_str(), s.length());
+    finish(ht, ctx, hash.hash);
     return hash;
 }
 
 
-Hash hashFile(const Path & path)
+Hash hashFile(HashType ht, const Path & path)
 {
-    Hash hash;
-    FILE * file = fopen(path.c_str(), "rb");
-    if (!file)
-        throw SysError(format("file `%1%' does not exist") % path);
-    int err = md5_stream(file, hash.hash);
-    fclose(file);
-    if (err) throw SysError(format("cannot hash file `%1%'") % path);
+    Ctx ctx;
+    Hash hash(ht);
+    start(ht, ctx);
+
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY);
+    if (fd == -1) throw SysError(format("opening file `%1%'") % path);
+
+    unsigned char buf[8192];
+    ssize_t n;
+    while ((n = read(fd, buf, sizeof(buf)))) {
+        checkInterrupt();
+        if (n == -1) throw SysError(format("reading file `%1%'") % path);
+        update(ht, ctx, buf, n);
+    }
+    
+    finish(ht, ctx, hash.hash);
     return hash;
 }
 
 
 struct HashSink : DumpSink
 {
-    struct md5_ctx ctx;
+    HashType ht;
+    Ctx ctx;
     virtual void operator ()
         (const unsigned char * data, unsigned int len)
     {
-        md5_process_bytes(data, len, &ctx);
+        update(ht, ctx, data, len);
     }
 };
 
 
-Hash hashPath(const Path & path)
+Hash hashPath(HashType ht, const Path & path)
 {
-    Hash hash;
     HashSink sink;
-    md5_init_ctx(&sink.ctx);
+    sink.ht = ht;
+    Hash hash(ht);
+    start(ht, sink.ctx);
     dumpPath(path, sink);
-    md5_finish_ctx(&sink.ctx, hash.hash);
+    finish(ht, sink.ctx, hash.hash);
     return hash;
 }
+
+
+Hash compressHash(const Hash & hash, unsigned int newSize)
+{
+    Hash h;
+    h.hashSize = newSize;
+    for (unsigned int i = 0; i < hash.hashSize; ++i)
+        h.hash[i % newSize] ^= hash.hash[i];
+    return h;
+}
+
+
+HashType parseHashType(const string & s)
+{
+    if (s == "md5") return htMD5;
+    else if (s == "sha1") return htSHA1;
+    else if (s == "sha256") return htSHA256;
+    else return htUnknown;
+}

src/libutil/hash.hh

@@ -8,14 +8,28 @@
 using namespace std;
 
 
+typedef enum { htUnknown, htMD5, htSHA1, htSHA256 } HashType;
+
+
+const int md5HashSize = 16;
+const int sha1HashSize = 20;
+const int sha256HashSize = 32;
+
+
 struct Hash
 {
-    static const unsigned int hashSize = 16;
-    unsigned char hash[hashSize];
+    static const unsigned int maxHashSize = 32;
+    unsigned int hashSize;
+    unsigned char hash[maxHashSize];
 
-    /* Create a zeroed hash object. */
+    HashType type;
+
+    /* Create an unusable hash object. */
     Hash();
 
+    /* Create a zero-filled hash object. */
+    Hash(HashType type);
+
     /* Check whether two hash are equal. */
     bool operator == (const Hash & h2) const;
 
@@ -25,27 +39,40 @@ struct Hash
     /* For sorting. */
     bool operator < (const Hash & h) const;
 
-    /* Convert a hash code into a hexadecimal representation. */
-    operator string() const;
 };
 
 
+/* Convert a hash to a hexadecimal representation. */
+string printHash(const Hash & hash);
+
 /* Parse a hexadecimal representation of a hash code. */
-Hash parseHash(const string & s);
+Hash parseHash(HashType ht, const string & s);
+
+/* Convert a hash to a base-32 representation. */
+string printHash32(const Hash & hash);
+
+/* Parse a base-32 representation of a hash code. */
+Hash parseHash32(HashType ht, const string & s);
 
 /* Verify that the given string is a valid hash code. */
 bool isHash(const string & s);
 
 /* Compute the hash of the given string. */
-Hash hashString(const string & s);
+Hash hashString(HashType ht, const string & s);
 
 /* Compute the hash of the given file. */
-Hash hashFile(const Path & path);
+Hash hashFile(HashType ht, const Path & path);
 
 /* Compute the hash of the given path.  The hash is defined as
-   md5(dump(path)).
-*/
-Hash hashPath(const Path & path);
+   md5(dump(path)). */
+Hash hashPath(HashType ht, const Path & path);
+
+/* Compress a hash to the specified number of bytes by cyclically
+   XORing bytes together. */
+Hash compressHash(const Hash & hash, unsigned int newSize);
+
+/* Parse a string representing a hash type. */
+HashType parseHashType(const string & s);
 
 
 #endif /* !__HASH_H */

src/libutil/md32_common.h

@@ -0,0 +1,620 @@
+/* crypto/md32_common.h */
+/* ====================================================================
+ * Copyright (c) 1999-2002 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    licensing@OpenSSL.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This product includes cryptographic software written by Eric Young
+ * (eay@cryptsoft.com).  This product includes software written by Tim
+ * Hudson (tjh@cryptsoft.com).
+ *
+ */
+
+/*
+ * This is a generic 32 bit "collector" for message digest algorithms.
+ * Whenever needed it collects input character stream into chunks of
+ * 32 bit values and invokes a block function that performs actual hash
+ * calculations.
+ *
+ * Porting guide.
+ *
+ * Obligatory macros:
+ *
+ * DATA_ORDER_IS_BIG_ENDIAN or DATA_ORDER_IS_LITTLE_ENDIAN
+ *	this macro defines byte order of input stream.
+ * HASH_CBLOCK
+ *	size of a unit chunk HASH_BLOCK operates on.
+ * HASH_LONG
+ *	has to be at lest 32 bit wide, if it's wider, then
+ *	HASH_LONG_LOG2 *has to* be defined along
+ * HASH_CTX
+ *	context structure that at least contains following
+ *	members:
+ *		typedef struct {
+ *			...
+ *			HASH_LONG	Nl,Nh;
+ *			HASH_LONG	data[HASH_LBLOCK];
+ *			unsigned int	num;
+ *			...
+ *			} HASH_CTX;
+ * HASH_UPDATE
+ *	name of "Update" function, implemented here.
+ * HASH_TRANSFORM
+ *	name of "Transform" function, implemented here.
+ * HASH_FINAL
+ *	name of "Final" function, implemented here.
+ * HASH_BLOCK_HOST_ORDER
+ *	name of "block" function treating *aligned* input message
+ *	in host byte order, implemented externally.
+ * HASH_BLOCK_DATA_ORDER
+ *	name of "block" function treating *unaligned* input message
+ *	in original (data) byte order, implemented externally (it
+ *	actually is optional if data and host are of the same
+ *	"endianess").
+ * HASH_MAKE_STRING
+ *	macro convering context variables to an ASCII hash string.
+ *
+ * Optional macros:
+ *
+ * B_ENDIAN or L_ENDIAN
+ *	defines host byte-order.
+ * HASH_LONG_LOG2
+ *	defaults to 2 if not states otherwise.
+ * HASH_LBLOCK
+ *	assumed to be HASH_CBLOCK/4 if not stated otherwise.
+ * HASH_BLOCK_DATA_ORDER_ALIGNED
+ *	alternative "block" function capable of treating
+ *	aligned input message in original (data) order,
+ *	implemented externally.
+ *
+ * MD5 example:
+ *
+ *	#define DATA_ORDER_IS_LITTLE_ENDIAN
+ *
+ *	#define HASH_LONG		MD5_LONG
+ *	#define HASH_LONG_LOG2		MD5_LONG_LOG2
+ *	#define HASH_CTX		MD5_CTX
+ *	#define HASH_CBLOCK		MD5_CBLOCK
+ *	#define HASH_LBLOCK		MD5_LBLOCK
+ *	#define HASH_UPDATE		MD5_Update
+ *	#define HASH_TRANSFORM		MD5_Transform
+ *	#define HASH_FINAL		MD5_Final
+ *	#define HASH_BLOCK_HOST_ORDER	md5_block_host_order
+ *	#define HASH_BLOCK_DATA_ORDER	md5_block_data_order
+ *
+ *					<appro@fy.chalmers.se>
+ */
+
+#if !defined(DATA_ORDER_IS_BIG_ENDIAN) && !defined(DATA_ORDER_IS_LITTLE_ENDIAN)
+#error "DATA_ORDER must be defined!"
+#endif
+
+#ifndef HASH_CBLOCK
+#error "HASH_CBLOCK must be defined!"
+#endif
+#ifndef HASH_LONG
+#error "HASH_LONG must be defined!"
+#endif
+#ifndef HASH_CTX
+#error "HASH_CTX must be defined!"
+#endif
+
+#ifndef HASH_UPDATE
+#error "HASH_UPDATE must be defined!"
+#endif
+#ifndef HASH_TRANSFORM
+#error "HASH_TRANSFORM must be defined!"
+#endif
+#ifndef HASH_FINAL
+#error "HASH_FINAL must be defined!"
+#endif
+
+#ifndef HASH_BLOCK_HOST_ORDER
+#error "HASH_BLOCK_HOST_ORDER must be defined!"
+#endif
+
+#if 0
+/*
+ * Moved below as it's required only if HASH_BLOCK_DATA_ORDER_ALIGNED
+ * isn't defined.
+ */
+#ifndef HASH_BLOCK_DATA_ORDER
+#error "HASH_BLOCK_DATA_ORDER must be defined!"
+#endif
+#endif
+
+#ifndef HASH_LBLOCK
+#define HASH_LBLOCK	(HASH_CBLOCK/4)
+#endif
+
+#ifndef HASH_LONG_LOG2
+#define HASH_LONG_LOG2	2
+#endif
+
+/*
+ * Engage compiler specific rotate intrinsic function if available.
+ */
+#undef ROTATE
+#ifndef PEDANTIC
+# if defined(_MSC_VER) || defined(__ICC)
+#  define ROTATE(a,n)	_lrotl(a,n)
+# elif defined(__MWERKS__)
+#  if defined(__POWERPC__)
+#   define ROTATE(a,n)	__rlwinm(a,n,0,31)
+#  elif defined(__MC68K__)
+    /* Motorola specific tweak. <appro@fy.chalmers.se> */
+#   define ROTATE(a,n)	( n<24 ? __rol(a,n) : __ror(a,32-n) )
+#  else
+#   define ROTATE(a,n)	__rol(a,n)
+#  endif
+# elif defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
+  /*
+   * Some GNU C inline assembler templates. Note that these are
+   * rotates by *constant* number of bits! But that's exactly
+   * what we need here...
+   * 					<appro@fy.chalmers.se>
+   */
+#  if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__)
+#   define ROTATE(a,n)	({ register unsigned int ret;	\
+				asm (			\
+				"roll %1,%0"		\
+				: "=r"(ret)		\
+				: "I"(n), "0"(a)	\
+				: "cc");		\
+			   ret;				\
+			})
+#  elif defined(__powerpc) || defined(__ppc__) || defined(__powerpc64__)
+#   define ROTATE(a,n)	({ register unsigned int ret;	\
+				asm (			\
+				"rlwinm %0,%1,%2,0,31"	\
+				: "=r"(ret)		\
+				: "r"(a), "I"(n));	\
+			   ret;				\
+			})
+#  endif
+# endif
+#endif /* PEDANTIC */
+
+#if HASH_LONG_LOG2==2	/* Engage only if sizeof(HASH_LONG)== 4 */
+/* A nice byte order reversal from Wei Dai <weidai@eskimo.com> */
+#ifdef ROTATE
+/* 5 instructions with rotate instruction, else 9 */
+#define REVERSE_FETCH32(a,l)	(					\
+		l=*(const HASH_LONG *)(a),				\
+		((ROTATE(l,8)&0x00FF00FF)|(ROTATE((l&0x00FF00FF),24)))	\
+				)
+#else
+/* 6 instructions with rotate instruction, else 8 */
+#define REVERSE_FETCH32(a,l)	(				\
+		l=*(const HASH_LONG *)(a),			\
+		l=(((l>>8)&0x00FF00FF)|((l&0x00FF00FF)<<8)),	\
+		ROTATE(l,16)					\
+				)
+/*
+ * Originally the middle line started with l=(((l&0xFF00FF00)>>8)|...
+ * It's rewritten as above for two reasons:
+ *	- RISCs aren't good at long constants and have to explicitely
+ *	  compose 'em with several (well, usually 2) instructions in a
+ *	  register before performing the actual operation and (as you
+ *	  already realized:-) having same constant should inspire the
+ *	  compiler to permanently allocate the only register for it;
+ *	- most modern CPUs have two ALUs, but usually only one has
+ *	  circuitry for shifts:-( this minor tweak inspires compiler
+ *	  to schedule shift instructions in a better way...
+ *
+ *				<appro@fy.chalmers.se>
+ */
+#endif
+#endif
+
+#ifndef ROTATE
+#define ROTATE(a,n)     (((a)<<(n))|(((a)&0xffffffff)>>(32-(n))))
+#endif
+
+/*
+ * Make some obvious choices. E.g., HASH_BLOCK_DATA_ORDER_ALIGNED
+ * and HASH_BLOCK_HOST_ORDER ought to be the same if input data
+ * and host are of the same "endianess". It's possible to mask
+ * this with blank #define HASH_BLOCK_DATA_ORDER though...
+ *
+ *				<appro@fy.chalmers.se>
+ */
+#if defined(B_ENDIAN)
+#  if defined(DATA_ORDER_IS_BIG_ENDIAN)
+#    if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2
+#      define HASH_BLOCK_DATA_ORDER_ALIGNED	HASH_BLOCK_HOST_ORDER
+#    endif
+#  endif
+#elif defined(L_ENDIAN)
+#  if defined(DATA_ORDER_IS_LITTLE_ENDIAN)
+#    if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED) && HASH_LONG_LOG2==2
+#      define HASH_BLOCK_DATA_ORDER_ALIGNED	HASH_BLOCK_HOST_ORDER
+#    endif
+#  endif
+#endif
+
+#if !defined(HASH_BLOCK_DATA_ORDER_ALIGNED)
+#ifndef HASH_BLOCK_DATA_ORDER
+#error "HASH_BLOCK_DATA_ORDER must be defined!"
+#endif
+#endif
+
+#if defined(DATA_ORDER_IS_BIG_ENDIAN)
+
+#ifndef PEDANTIC
+# if defined(__GNUC__) && __GNUC__>=2 && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_NO_INLINE_ASM)
+#  if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__)
+    /*
+     * This gives ~30-40% performance improvement in SHA-256 compiled
+     * with gcc [on P4]. Well, first macro to be frank. We can pull
+     * this trick on x86* platforms only, because these CPUs can fetch
+     * unaligned data without raising an exception.
+     */
+#   define HOST_c2l(c,l)	({ unsigned int r=*((const unsigned int *)(c));	\
+				   asm ("bswapl %0":"=r"(r):"0"(r));	\
+				   (c)+=4; (l)=r;			})
+#   define HOST_l2c(l,c)	({ unsigned int r=(l);			\
+				   asm ("bswapl %0":"=r"(r):"0"(r));	\
+				   *((unsigned int *)(c))=r; (c)+=4; r;	})
+#  endif
+# endif
+#endif
+
+#ifndef HOST_c2l
+#define HOST_c2l(c,l)	(l =(((unsigned long)(*((c)++)))<<24),		\
+			 l|=(((unsigned long)(*((c)++)))<<16),		\
+			 l|=(((unsigned long)(*((c)++)))<< 8),		\
+			 l|=(((unsigned long)(*((c)++)))    ),		\
+			 l)
+#endif
+#define HOST_p_c2l(c,l,n)	{					\
+			switch (n) {					\
+			case 0: l =((unsigned long)(*((c)++)))<<24;	\
+			case 1: l|=((unsigned long)(*((c)++)))<<16;	\
+			case 2: l|=((unsigned long)(*((c)++)))<< 8;	\
+			case 3: l|=((unsigned long)(*((c)++)));		\
+				} }
+#define HOST_p_c2l_p(c,l,sc,len) {					\
+			switch (sc) {					\
+			case 0: l =((unsigned long)(*((c)++)))<<24;	\
+				if (--len == 0) break;			\
+			case 1: l|=((unsigned long)(*((c)++)))<<16;	\
+				if (--len == 0) break;			\
+			case 2: l|=((unsigned long)(*((c)++)))<< 8;	\
+				} }
+/* NOTE the pointer is not incremented at the end of this */
+#define HOST_c2l_p(c,l,n)	{					\
+			l=0; (c)+=n;					\
+			switch (n) {					\
+			case 3: l =((unsigned long)(*(--(c))))<< 8;	\
+			case 2: l|=((unsigned long)(*(--(c))))<<16;	\
+			case 1: l|=((unsigned long)(*(--(c))))<<24;	\
+				} }
+#ifndef HOST_l2c
+#define HOST_l2c(l,c)	(*((c)++)=(unsigned char)(((l)>>24)&0xff),	\
+			 *((c)++)=(unsigned char)(((l)>>16)&0xff),	\
+			 *((c)++)=(unsigned char)(((l)>> 8)&0xff),	\
+			 *((c)++)=(unsigned char)(((l)    )&0xff),	\
+			 l)
+#endif
+
+#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN)
+
+#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__)
+  /* See comment in DATA_ORDER_IS_BIG_ENDIAN section. */
+# define HOST_c2l(c,l)	((l)=*((const unsigned int *)(c)), (c)+=4, l)
+# define HOST_l2c(l,c)	(*((unsigned int *)(c))=(l), (c)+=4, l)
+#endif
+
+#ifndef HOST_c2l
+#define HOST_c2l(c,l)	(l =(((unsigned long)(*((c)++)))    ),		\
+			 l|=(((unsigned long)(*((c)++)))<< 8),		\
+			 l|=(((unsigned long)(*((c)++)))<<16),		\
+			 l|=(((unsigned long)(*((c)++)))<<24),		\
+			 l)
+#endif
+#define HOST_p_c2l(c,l,n)	{					\
+			switch (n) {					\
+			case 0: l =((unsigned long)(*((c)++)));		\
+			case 1: l|=((unsigned long)(*((c)++)))<< 8;	\
+			case 2: l|=((unsigned long)(*((c)++)))<<16;	\
+			case 3: l|=((unsigned long)(*((c)++)))<<24;	\
+				} }
+#define HOST_p_c2l_p(c,l,sc,len) {					\
+			switch (sc) {					\
+			case 0: l =((unsigned long)(*((c)++)));		\
+				if (--len == 0) break;			\
+			case 1: l|=((unsigned long)(*((c)++)))<< 8;	\
+				if (--len == 0) break;			\
+			case 2: l|=((unsigned long)(*((c)++)))<<16;	\
+				} }
+/* NOTE the pointer is not incremented at the end of this */
+#define HOST_c2l_p(c,l,n)	{					\
+			l=0; (c)+=n;					\
+			switch (n) {					\
+			case 3: l =((unsigned long)(*(--(c))))<<16;	\
+			case 2: l|=((unsigned long)(*(--(c))))<< 8;	\
+			case 1: l|=((unsigned long)(*(--(c))));		\
+				} }
+#ifndef HOST_l2c
+#define HOST_l2c(l,c)	(*((c)++)=(unsigned char)(((l)    )&0xff),	\
+			 *((c)++)=(unsigned char)(((l)>> 8)&0xff),	\
+			 *((c)++)=(unsigned char)(((l)>>16)&0xff),	\
+			 *((c)++)=(unsigned char)(((l)>>24)&0xff),	\
+			 l)
+#endif
+
+#endif
+
+/*
+ * Time for some action:-)
+ */
+
+int HASH_UPDATE (HASH_CTX *c, const void *data_, size_t len)
+	{
+	const unsigned char *data=data_;
+	register HASH_LONG * p;
+	register HASH_LONG l;
+	size_t sw,sc,ew,ec;
+
+	if (len==0) return 1;
+
+	l=(c->Nl+(((HASH_LONG)len)<<3))&0xffffffffUL;
+	/* 95-05-24 eay Fixed a bug with the overflow handling, thanks to
+	 * Wei Dai <weidai@eskimo.com> for pointing it out. */
+	if (l < c->Nl) /* overflow */
+		c->Nh++;
+	c->Nh+=(len>>29);	/* might cause compiler warning on 16-bit */
+	c->Nl=l;
+
+	if (c->num != 0)
+		{
+		p=c->data;
+		sw=c->num>>2;
+		sc=c->num&0x03;
+
+		if ((c->num+len) >= HASH_CBLOCK)
+			{
+			l=p[sw]; HOST_p_c2l(data,l,sc); p[sw++]=l;
+			for (; sw<HASH_LBLOCK; sw++)
+				{
+				HOST_c2l(data,l); p[sw]=l;
+				}
+			HASH_BLOCK_HOST_ORDER (c,p,1);
+			len-=(HASH_CBLOCK-c->num);
+			c->num=0;
+			/* drop through and do the rest */
+			}
+		else
+			{
+			c->num+=(unsigned int)len;
+			if ((sc+len) < 4) /* ugly, add char's to a word */
+				{
+				l=p[sw]; HOST_p_c2l_p(data,l,sc,len); p[sw]=l;
+				}
+			else
+				{
+				ew=(c->num>>2);
+				ec=(c->num&0x03);
+				if (sc)
+					l=p[sw];
+				HOST_p_c2l(data,l,sc);
+				p[sw++]=l;
+				for (; sw < ew; sw++)
+					{
+					HOST_c2l(data,l); p[sw]=l;
+					}
+				if (ec)
+					{
+					HOST_c2l_p(data,l,ec); p[sw]=l;
+					}
+				}
+			return 1;
+			}
+		}
+
+	sw=len/HASH_CBLOCK;
+	if (sw > 0)
+		{
+#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED)
+		/*
+		 * Note that HASH_BLOCK_DATA_ORDER_ALIGNED gets defined
+		 * only if sizeof(HASH_LONG)==4.
+		 */
+		if ((((size_t)data)%4) == 0)
+			{
+			/* data is properly aligned so that we can cast it: */
+			HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,sw);
+			sw*=HASH_CBLOCK;
+			data+=sw;
+			len-=sw;
+			}
+		else
+#if !defined(HASH_BLOCK_DATA_ORDER)
+			while (sw--)
+				{
+				memcpy (p=c->data,data,HASH_CBLOCK);
+				HASH_BLOCK_DATA_ORDER_ALIGNED(c,p,1);
+				data+=HASH_CBLOCK;
+				len-=HASH_CBLOCK;
+				}
+#endif
+#endif
+#if defined(HASH_BLOCK_DATA_ORDER)
+			{
+			HASH_BLOCK_DATA_ORDER(c,data,sw);
+			sw*=HASH_CBLOCK;
+			data+=sw;
+			len-=sw;
+			}
+#endif
+		}
+
+	if (len!=0)
+		{
+		p = c->data;
+		c->num = len;
+		ew=len>>2;	/* words to copy */
+		ec=len&0x03;
+		for (; ew; ew--,p++)
+			{
+			HOST_c2l(data,l); *p=l;
+			}
+		HOST_c2l_p(data,l,ec);
+		*p=l;
+		}
+	return 1;
+	}
+
+
+void HASH_TRANSFORM (HASH_CTX *c, const unsigned char *data)
+	{
+#if defined(HASH_BLOCK_DATA_ORDER_ALIGNED)
+	if ((((size_t)data)%4) == 0)
+		/* data is properly aligned so that we can cast it: */
+		HASH_BLOCK_DATA_ORDER_ALIGNED (c,(const HASH_LONG *)data,1);
+	else
+#if !defined(HASH_BLOCK_DATA_ORDER)
+		{
+		memcpy (c->data,data,HASH_CBLOCK);
+		HASH_BLOCK_DATA_ORDER_ALIGNED (c,c->data,1);
+		}
+#endif
+#endif
+#if defined(HASH_BLOCK_DATA_ORDER)
+	HASH_BLOCK_DATA_ORDER (c,data,1);
+#endif
+	}
+
+
+int HASH_FINAL (unsigned char *md, HASH_CTX *c)
+	{
+	register HASH_LONG *p;
+	register unsigned long l;
+	register int i,j;
+	static const unsigned char end[4]={0x80,0x00,0x00,0x00};
+	const unsigned char *cp=end;
+
+	/* c->num should definitly have room for at least one more byte. */
+	p=c->data;
+	i=c->num>>2;
+	j=c->num&0x03;
+
+#if 0
+	/* purify often complains about the following line as an
+	 * Uninitialized Memory Read.  While this can be true, the
+	 * following p_c2l macro will reset l when that case is true.
+	 * This is because j&0x03 contains the number of 'valid' bytes
+	 * already in p[i].  If and only if j&0x03 == 0, the UMR will
+	 * occur but this is also the only time p_c2l will do
+	 * l= *(cp++) instead of l|= *(cp++)
+	 * Many thanks to Alex Tang <altitude@cic.net> for pickup this
+	 * 'potential bug' */
+#ifdef PURIFY
+	if (j==0) p[i]=0; /* Yeah, but that's not the way to fix it:-) */
+#endif
+	l=p[i];
+#else
+	l = (j==0) ? 0 : p[i];
+#endif
+	HOST_p_c2l(cp,l,j); p[i++]=l; /* i is the next 'undefined word' */
+
+	if (i>(HASH_LBLOCK-2)) /* save room for Nl and Nh */
+		{
+		if (i<HASH_LBLOCK) p[i]=0;
+		HASH_BLOCK_HOST_ORDER (c,p,1);
+		i=0;
+		}
+	for (; i<(HASH_LBLOCK-2); i++)
+		p[i]=0;
+
+#if   defined(DATA_ORDER_IS_BIG_ENDIAN)
+	p[HASH_LBLOCK-2]=c->Nh;
+	p[HASH_LBLOCK-1]=c->Nl;
+#elif defined(DATA_ORDER_IS_LITTLE_ENDIAN)
+	p[HASH_LBLOCK-2]=c->Nl;
+	p[HASH_LBLOCK-1]=c->Nh;
+#endif
+	HASH_BLOCK_HOST_ORDER (c,p,1);
+
+#ifndef HASH_MAKE_STRING
+#error "HASH_MAKE_STRING must be defined!"
+#else
+	HASH_MAKE_STRING(c,md);
+#endif
+
+	c->num=0;
+	/* clear stuff, HASH_BLOCK may be leaving some stuff on the stack
+	 * but I'm not worried :-)
+	OPENSSL_cleanse((void *)c,sizeof(HASH_CTX));
+	 */
+	return 1;
+	}
+
+#ifndef MD32_REG_T
+#define MD32_REG_T long
+/*
+ * This comment was originaly written for MD5, which is why it
+ * discusses A-D. But it basically applies to all 32-bit digests,
+ * which is why it was moved to common header file.
+ *
+ * In case you wonder why A-D are declared as long and not
+ * as MD5_LONG. Doing so results in slight performance
+ * boost on LP64 architectures. The catch is we don't
+ * really care if 32 MSBs of a 64-bit register get polluted
+ * with eventual overflows as we *save* only 32 LSBs in
+ * *either* case. Now declaring 'em long excuses the compiler
+ * from keeping 32 MSBs zeroed resulting in 13% performance
+ * improvement under SPARC Solaris7/64 and 5% under AlphaLinux.
+ * Well, to be honest it should say that this *prevents* 
+ * performance degradation.
+ *				<appro@fy.chalmers.se>
+ * Apparently there're LP64 compilers that generate better
+ * code if A-D are declared int. Most notably GCC-x86_64
+ * generates better code.
+ *				<appro@fy.chalmers.se>
+ */
+#endif

src/libutil/md5.c

@@ -20,10 +20,6 @@
 
 /* Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995.  */
 
-#ifdef HAVE_CONFIG_H
-# include <config.h>
-#endif
-
 #include <sys/types.h>
 
 #include <stdlib.h>
@@ -123,85 +119,6 @@ md5_finish_ctx (ctx, resbuf)
   return md5_read_ctx (ctx, resbuf);
 }
 
-/* Compute MD5 message digest for bytes read from STREAM.  The
-   resulting message digest number will be written into the 16 bytes
-   beginning at RESBLOCK.  */
-int
-md5_stream (stream, resblock)
-     FILE *stream;
-     void *resblock;
-{
-  /* Important: BLOCKSIZE must be a multiple of 64.  */
-#define BLOCKSIZE 4096
-  struct md5_ctx ctx;
-  char buffer[BLOCKSIZE + 72];
-  size_t sum;
-
-  /* Initialize the computation context.  */
-  md5_init_ctx (&ctx);
-
-  /* Iterate over full file contents.  */
-  while (1)
-    {
-      /* We read the file in blocks of BLOCKSIZE bytes.  One call of the
-	 computation function processes the whole buffer so that with the
-	 next round of the loop another block can be read.  */
-      size_t n;
-      sum = 0;
-
-      /* Read block.  Take care for partial reads.  */
-      do
-	{
-	  n = fread (buffer + sum, 1, BLOCKSIZE - sum, stream);
-
-	  sum += n;
-	}
-      while (sum < BLOCKSIZE && n != 0);
-      if (n == 0 && ferror (stream))
-        return 1;
-
-      /* If end of file is reached, end the loop.  */
-      if (n == 0)
-	break;
-
-      /* Process buffer with BLOCKSIZE bytes.  Note that
-			BLOCKSIZE % 64 == 0
-       */
-      md5_process_block (buffer, BLOCKSIZE, &ctx);
-    }
-
-  /* Add the last bytes if necessary.  */
-  if (sum > 0)
-    md5_process_bytes (buffer, sum, &ctx);
-
-  /* Construct result in desired memory.  */
-  md5_finish_ctx (&ctx, resblock);
-  return 0;
-}
-
-/* Compute MD5 message digest for LEN bytes beginning at BUFFER.  The
-   result is always in little endian byte order, so that a byte-wise
-   output yields to the wanted ASCII representation of the message
-   digest.  */
-void *
-md5_buffer (buffer, len, resblock)
-     const char *buffer;
-     size_t len;
-     void *resblock;
-{
-  struct md5_ctx ctx;
-
-  /* Initialize the computation context.  */
-  md5_init_ctx (&ctx);
-
-  /* Process whole buffer but last len % 64 bytes.  */
-  md5_process_bytes (buffer, len, &ctx);
-
-  /* Put result in desired memory area.  */
-  return md5_finish_ctx (&ctx, resblock);
-}
-
-
 void
 md5_process_bytes (buffer, len, ctx)
      const void *buffer;

src/libutil/md5.h

@@ -21,65 +21,9 @@
 #ifndef _MD5_H
 #define _MD5_H 1
 
-#include <stdio.h>
-
-#if defined HAVE_LIMITS_H || _LIBC
-# include <limits.h>
-#endif
-
-/* The following contortions are an attempt to use the C preprocessor
-   to determine an unsigned integral type that is 32 bits wide.  An
-   alternative approach is to use autoconf's AC_CHECK_SIZEOF macro, but
-   doing that would require that the configure script compile and *run*
-   the resulting executable.  Locally running cross-compiled executables
-   is usually not possible.  */
-
-#ifdef _LIBC
-# include <stdint.h>
+#include <stdint.h>
 typedef uint32_t md5_uint32;
 typedef uintptr_t md5_uintptr;
-#else
-# if defined __STDC__ && __STDC__
-#  define UINT_MAX_32_BITS 4294967295U
-# else
-#  define UINT_MAX_32_BITS 0xFFFFFFFF
-# endif
-
-/* If UINT_MAX isn't defined, assume it's a 32-bit type.
-   This should be valid for all systems GNU cares about because
-   that doesn't include 16-bit systems, and only modern systems
-   (that certainly have <limits.h>) have 64+-bit integral types.  */
-
-# ifndef UINT_MAX
-#  define UINT_MAX UINT_MAX_32_BITS
-# endif
-
-# if UINT_MAX == UINT_MAX_32_BITS
-   typedef unsigned int md5_uint32;
-# else
-#  if USHRT_MAX == UINT_MAX_32_BITS
-    typedef unsigned short md5_uint32;
-#  else
-#   if ULONG_MAX == UINT_MAX_32_BITS
-     typedef unsigned long md5_uint32;
-#   else
-     /* The following line is intended to evoke an error.
-        Using #error is not portable enough.  */
-     "Cannot determine unsigned 32-bit data type."
-#   endif
-#  endif
-# endif
-/* We have to make a guess about the integer type equivalent in size
-   to pointers which should always be correct.  */
-typedef unsigned long int md5_uintptr;
-#endif
-
-#undef __P
-#if defined (__STDC__) && __STDC__
-# define __P(x) x
-#else
-# define __P(x) ()
-#endif
 
 /* Structure to save state of computation between the single steps.  */
 struct md5_ctx
@@ -136,16 +80,4 @@ extern void *md5_finish_ctx __P ((struct md5_ctx *ctx, void *resbuf));
 extern void *md5_read_ctx __P ((const struct md5_ctx *ctx, void *resbuf));
 
 
-/* Compute MD5 message digest for bytes read from STREAM.  The
-   resulting message digest number will be written into the 16 bytes
-   beginning at RESBLOCK.  */
-extern int md5_stream __P ((FILE *stream, void *resblock));
-
-/* Compute MD5 message digest for LEN bytes beginning at BUFFER.  The
-   result is always in little endian byte order, so that a byte-wise
-   output yields to the wanted ASCII representation of the message
-   digest.  */
-extern void *md5_buffer __P ((const char *buffer, size_t len,
-				void *resblock));
-    
 #endif /* md5.h */

src/libutil/sha1.c

@@ -0,0 +1,368 @@
+/* $Id$ */
+
+/* sha.c - Implementation of the Secure Hash Algorithm
+ *
+ * Copyright (C) 1995, A.M. Kuchling
+ *
+ * Distribute and use freely; there are no restrictions on further 
+ * dissemination and usage except those imposed by the laws of your 
+ * country of residence.
+ *
+ * Adapted to pike and some cleanup by Niels Möller.
+ */
+
+/* $Id$ */
+
+/* SHA: NIST's Secure Hash Algorithm */
+
+/* Based on SHA code originally posted to sci.crypt by Peter Gutmann
+   in message <30ajo5$oe8@ccu2.auckland.ac.nz>.
+   Modified to test for endianness on creation of SHA objects by AMK.
+   Also, the original specification of SHA was found to have a weakness
+   by NSA/NIST.  This code implements the fixed version of SHA.
+*/
+
+/* Here's the first paragraph of Peter Gutmann's posting:
+   
+The following is my SHA (FIPS 180) code updated to allow use of the "fixed"
+SHA, thanks to Jim Gillogly and an anonymous contributor for the information on
+what's changed in the new version.  The fix is a simple change which involves
+adding a single rotate in the initial expansion function.  It is unknown
+whether this is an optimal solution to the problem which was discovered in the
+SHA or whether it's simply a bandaid which fixes the problem with a minimum of
+effort (for example the reengineering of a great many Capstone chips).
+*/
+
+#include "sha1.h"
+
+#include <string.h>
+
+void sha_copy(struct sha_ctx *dest, struct sha_ctx *src)
+{
+  unsigned int i;
+
+  dest->count_l=src->count_l;
+  dest->count_h=src->count_h;
+  for(i=0; i<SHA_DIGESTLEN; i++)
+    dest->digest[i]=src->digest[i];
+  for(i=0; i < src->index; i++)
+    dest->block[i] = src->block[i];
+  dest->index = src->index;
+}
+
+
+/* The SHA f()-functions.  The f1 and f3 functions can be optimized to
+   save one boolean operation each - thanks to Rich Schroeppel,
+   rcs@cs.arizona.edu for discovering this */
+
+/*#define f1(x,y,z) ( ( x & y ) | ( ~x & z ) )          // Rounds  0-19 */
+#define f1(x,y,z)   ( z ^ ( x & ( y ^ z ) ) )           /* Rounds  0-19 */
+#define f2(x,y,z)   ( x ^ y ^ z )                       /* Rounds 20-39 */
+/*#define f3(x,y,z) ( ( x & y ) | ( x & z ) | ( y & z ) )   // Rounds 40-59 */
+#define f3(x,y,z)   ( ( x & y ) | ( z & ( x | y ) ) )   /* Rounds 40-59 */
+#define f4(x,y,z)   ( x ^ y ^ z )                       /* Rounds 60-79 */
+
+/* The SHA Mysterious Constants */
+
+#define K1  0x5A827999L                                 /* Rounds  0-19 */
+#define K2  0x6ED9EBA1L                                 /* Rounds 20-39 */
+#define K3  0x8F1BBCDCL                                 /* Rounds 40-59 */
+#define K4  0xCA62C1D6L                                 /* Rounds 60-79 */
+
+/* SHA initial values */
+
+#define h0init  0x67452301L
+#define h1init  0xEFCDAB89L
+#define h2init  0x98BADCFEL
+#define h3init  0x10325476L
+#define h4init  0xC3D2E1F0L
+
+/* 32-bit rotate left - kludged with shifts */
+
+#define ROTL(n,X)  ( ( (X) << (n) ) | ( (X) >> ( 32 - (n) ) ) )
+
+/* The initial expanding function.  The hash function is defined over an
+   80-word expanded input array W, where the first 16 are copies of the input
+   data, and the remaining 64 are defined by
+
+        W[ i ] = W[ i - 16 ] ^ W[ i - 14 ] ^ W[ i - 8 ] ^ W[ i - 3 ]
+
+   This implementation generates these values on the fly in a circular
+   buffer - thanks to Colin Plumb, colin@nyx10.cs.du.edu for this
+   optimization.
+
+   The updated SHA changes the expanding function by adding a rotate of 1
+   bit.  Thanks to Jim Gillogly, jim@rand.org, and an anonymous contributor
+   for this information */
+
+#define expand(W,i) ( W[ i & 15 ] = \
+		      ROTL( 1, ( W[ i & 15 ] ^ W[ (i - 14) & 15 ] ^ \
+				 W[ (i - 8) & 15 ] ^ W[ (i - 3) & 15 ] ) ) )
+
+
+/* The prototype SHA sub-round.  The fundamental sub-round is:
+
+        a' = e + ROTL( 5, a ) + f( b, c, d ) + k + data;
+        b' = a;
+        c' = ROTL( 30, b );
+        d' = c;
+        e' = d;
+
+   but this is implemented by unrolling the loop 5 times and renaming the
+   variables ( e, a, b, c, d ) = ( a', b', c', d', e' ) each iteration.
+   This code is then replicated 20 times for each of the 4 functions, using
+   the next 20 values from the W[] array each time */
+
+#define subRound(a, b, c, d, e, f, k, data) \
+    ( e += ROTL( 5, a ) + f( b, c, d ) + k + data, b = ROTL( 30, b ) )
+
+/* Initialize the SHA values */
+
+void sha_init(struct sha_ctx *ctx)
+{
+  /* Set the h-vars to their initial values */
+  ctx->digest[ 0 ] = h0init;
+  ctx->digest[ 1 ] = h1init;
+  ctx->digest[ 2 ] = h2init;
+  ctx->digest[ 3 ] = h3init;
+  ctx->digest[ 4 ] = h4init;
+
+  /* Initialize bit count */
+  ctx->count_l = ctx->count_h = 0;
+  
+  /* Initialize buffer */
+  ctx->index = 0;
+}
+
+/* Perform the SHA transformation.  Note that this code, like MD5, seems to
+   break some optimizing compilers due to the complexity of the expressions
+   and the size of the basic block.  It may be necessary to split it into
+   sections, e.g. based on the four subrounds
+
+   Note that this function destroys the data area */
+
+static void sha_transform(struct sha_ctx *ctx, uint32_t *data )
+{
+  uint32_t A, B, C, D, E;     /* Local vars */
+
+  /* Set up first buffer and local data buffer */
+  A = ctx->digest[0];
+  B = ctx->digest[1];
+  C = ctx->digest[2];
+  D = ctx->digest[3];
+  E = ctx->digest[4];
+
+  /* Heavy mangling, in 4 sub-rounds of 20 interations each. */
+  subRound( A, B, C, D, E, f1, K1, data[ 0] );
+  subRound( E, A, B, C, D, f1, K1, data[ 1] );
+  subRound( D, E, A, B, C, f1, K1, data[ 2] );
+  subRound( C, D, E, A, B, f1, K1, data[ 3] );
+  subRound( B, C, D, E, A, f1, K1, data[ 4] );
+  subRound( A, B, C, D, E, f1, K1, data[ 5] );
+  subRound( E, A, B, C, D, f1, K1, data[ 6] );
+  subRound( D, E, A, B, C, f1, K1, data[ 7] );
+  subRound( C, D, E, A, B, f1, K1, data[ 8] );
+  subRound( B, C, D, E, A, f1, K1, data[ 9] );
+  subRound( A, B, C, D, E, f1, K1, data[10] );
+  subRound( E, A, B, C, D, f1, K1, data[11] );
+  subRound( D, E, A, B, C, f1, K1, data[12] );
+  subRound( C, D, E, A, B, f1, K1, data[13] );
+  subRound( B, C, D, E, A, f1, K1, data[14] );
+  subRound( A, B, C, D, E, f1, K1, data[15] );
+  subRound( E, A, B, C, D, f1, K1, expand( data, 16 ) );
+  subRound( D, E, A, B, C, f1, K1, expand( data, 17 ) );
+  subRound( C, D, E, A, B, f1, K1, expand( data, 18 ) );
+  subRound( B, C, D, E, A, f1, K1, expand( data, 19 ) );
+
+  subRound( A, B, C, D, E, f2, K2, expand( data, 20 ) );
+  subRound( E, A, B, C, D, f2, K2, expand( data, 21 ) );
+  subRound( D, E, A, B, C, f2, K2, expand( data, 22 ) );
+  subRound( C, D, E, A, B, f2, K2, expand( data, 23 ) );
+  subRound( B, C, D, E, A, f2, K2, expand( data, 24 ) );
+  subRound( A, B, C, D, E, f2, K2, expand( data, 25 ) );
+  subRound( E, A, B, C, D, f2, K2, expand( data, 26 ) );
+  subRound( D, E, A, B, C, f2, K2, expand( data, 27 ) );
+  subRound( C, D, E, A, B, f2, K2, expand( data, 28 ) );
+  subRound( B, C, D, E, A, f2, K2, expand( data, 29 ) );
+  subRound( A, B, C, D, E, f2, K2, expand( data, 30 ) );
+  subRound( E, A, B, C, D, f2, K2, expand( data, 31 ) );
+  subRound( D, E, A, B, C, f2, K2, expand( data, 32 ) );
+  subRound( C, D, E, A, B, f2, K2, expand( data, 33 ) );
+  subRound( B, C, D, E, A, f2, K2, expand( data, 34 ) );
+  subRound( A, B, C, D, E, f2, K2, expand( data, 35 ) );
+  subRound( E, A, B, C, D, f2, K2, expand( data, 36 ) );
+  subRound( D, E, A, B, C, f2, K2, expand( data, 37 ) );
+  subRound( C, D, E, A, B, f2, K2, expand( data, 38 ) );
+  subRound( B, C, D, E, A, f2, K2, expand( data, 39 ) );
+
+  subRound( A, B, C, D, E, f3, K3, expand( data, 40 ) );
+  subRound( E, A, B, C, D, f3, K3, expand( data, 41 ) );
+  subRound( D, E, A, B, C, f3, K3, expand( data, 42 ) );
+  subRound( C, D, E, A, B, f3, K3, expand( data, 43 ) );
+  subRound( B, C, D, E, A, f3, K3, expand( data, 44 ) );
+  subRound( A, B, C, D, E, f3, K3, expand( data, 45 ) );
+  subRound( E, A, B, C, D, f3, K3, expand( data, 46 ) );
+  subRound( D, E, A, B, C, f3, K3, expand( data, 47 ) );
+  subRound( C, D, E, A, B, f3, K3, expand( data, 48 ) );
+  subRound( B, C, D, E, A, f3, K3, expand( data, 49 ) );
+  subRound( A, B, C, D, E, f3, K3, expand( data, 50 ) );
+  subRound( E, A, B, C, D, f3, K3, expand( data, 51 ) );
+  subRound( D, E, A, B, C, f3, K3, expand( data, 52 ) );
+  subRound( C, D, E, A, B, f3, K3, expand( data, 53 ) );
+  subRound( B, C, D, E, A, f3, K3, expand( data, 54 ) );
+  subRound( A, B, C, D, E, f3, K3, expand( data, 55 ) );
+  subRound( E, A, B, C, D, f3, K3, expand( data, 56 ) );
+  subRound( D, E, A, B, C, f3, K3, expand( data, 57 ) );
+  subRound( C, D, E, A, B, f3, K3, expand( data, 58 ) );
+  subRound( B, C, D, E, A, f3, K3, expand( data, 59 ) );
+
+  subRound( A, B, C, D, E, f4, K4, expand( data, 60 ) );
+  subRound( E, A, B, C, D, f4, K4, expand( data, 61 ) );
+  subRound( D, E, A, B, C, f4, K4, expand( data, 62 ) );
+  subRound( C, D, E, A, B, f4, K4, expand( data, 63 ) );
+  subRound( B, C, D, E, A, f4, K4, expand( data, 64 ) );
+  subRound( A, B, C, D, E, f4, K4, expand( data, 65 ) );
+  subRound( E, A, B, C, D, f4, K4, expand( data, 66 ) );
+  subRound( D, E, A, B, C, f4, K4, expand( data, 67 ) );
+  subRound( C, D, E, A, B, f4, K4, expand( data, 68 ) );
+  subRound( B, C, D, E, A, f4, K4, expand( data, 69 ) );
+  subRound( A, B, C, D, E, f4, K4, expand( data, 70 ) );
+  subRound( E, A, B, C, D, f4, K4, expand( data, 71 ) );
+  subRound( D, E, A, B, C, f4, K4, expand( data, 72 ) );
+  subRound( C, D, E, A, B, f4, K4, expand( data, 73 ) );
+  subRound( B, C, D, E, A, f4, K4, expand( data, 74 ) );
+  subRound( A, B, C, D, E, f4, K4, expand( data, 75 ) );
+  subRound( E, A, B, C, D, f4, K4, expand( data, 76 ) );
+  subRound( D, E, A, B, C, f4, K4, expand( data, 77 ) );
+  subRound( C, D, E, A, B, f4, K4, expand( data, 78 ) );
+  subRound( B, C, D, E, A, f4, K4, expand( data, 79 ) );
+
+  /* Build message digest */
+  ctx->digest[0] += A;
+  ctx->digest[1] += B;
+  ctx->digest[2] += C;
+  ctx->digest[3] += D;
+  ctx->digest[4] += E;
+}
+
+#if 1
+
+#ifndef EXTRACT_UCHAR
+#define EXTRACT_UCHAR(p)  (*(unsigned char *)(p))
+#endif
+
+#define STRING2INT(s) ((((((EXTRACT_UCHAR(s) << 8)    \
+			 | EXTRACT_UCHAR(s+1)) << 8)  \
+			 | EXTRACT_UCHAR(s+2)) << 8)  \
+			 | EXTRACT_UCHAR(s+3))
+#else
+uint32_t STRING2INT(unsigned char *s)
+{
+  uint32_t r;
+  unsigned int i;
+  
+  for (i = 0, r = 0; i < 4; i++, s++)
+    r = (r << 8) | *s;
+  return r;
+}
+#endif
+
+static void sha_block(struct sha_ctx *ctx, const unsigned char *block)
+{
+  uint32_t data[SHA_DATALEN];
+  unsigned int i;
+  
+  /* Update block count */
+  if (!++ctx->count_l)
+    ++ctx->count_h;
+
+  /* Endian independent conversion */
+  for (i = 0; i<SHA_DATALEN; i++, block += 4)
+    data[i] = STRING2INT(block);
+
+  sha_transform(ctx, data);
+}
+
+void sha_update(struct sha_ctx *ctx, const unsigned char *buffer, uint32_t len)
+{
+  if (ctx->index)
+    { /* Try to fill partial block */
+      unsigned left = SHA_DATASIZE - ctx->index;
+      if (len < left)
+	{
+	  memcpy(ctx->block + ctx->index, buffer, len);
+	  ctx->index += len;
+	  return; /* Finished */
+	}
+      else
+	{
+	  memcpy(ctx->block + ctx->index, buffer, left);
+	  sha_block(ctx, ctx->block);
+	  buffer += left;
+	  len -= left;
+	}
+    }
+  while (len >= SHA_DATASIZE)
+    {
+      sha_block(ctx, buffer);
+      buffer += SHA_DATASIZE;
+      len -= SHA_DATASIZE;
+    }
+  if ((ctx->index = len))     /* This assignment is intended */
+    /* Buffer leftovers */
+    memcpy(ctx->block, buffer, len);
+}
+	  
+/* Final wrapup - pad to SHA_DATASIZE-byte boundary with the bit pattern
+   1 0* (64-bit count of bits processed, MSB-first) */
+
+void sha_final(struct sha_ctx *ctx)
+{
+  uint32_t data[SHA_DATALEN];
+  unsigned int i;
+  unsigned int words;
+  
+  i = ctx->index;
+  /* Set the first char of padding to 0x80.  This is safe since there is
+     always at least one byte free */
+  ctx->block[i++] = 0x80;
+
+  /* Fill rest of word */
+  for( ; i & 3; i++)
+    ctx->block[i] = 0;
+
+  /* i is now a multiple of the word size 4 */
+  words = i >> 2;
+  for (i = 0; i < words; i++)
+    data[i] = STRING2INT(ctx->block + 4*i);
+  
+  if (words > (SHA_DATALEN-2))
+    { /* No room for length in this block. Process it and
+       * pad with another one */
+      for (i = words ; i < SHA_DATALEN; i++)
+	data[i] = 0;
+      sha_transform(ctx, data);
+      for (i = 0; i < (SHA_DATALEN-2); i++)
+	data[i] = 0;
+    }
+  else
+    for (i = words ; i < SHA_DATALEN - 2; i++)
+      data[i] = 0;
+  /* Theres 512 = 2^9 bits in one block */
+  data[SHA_DATALEN-2] = (ctx->count_h << 9) | (ctx->count_l >> 23);
+  data[SHA_DATALEN-1] = (ctx->count_l << 9) | (ctx->index << 3);
+  sha_transform(ctx, data);
+}
+
+void sha_digest(struct sha_ctx *ctx, unsigned char *s)
+{
+  unsigned int i;
+
+  for (i = 0; i < SHA_DIGESTLEN; i++)
+    {
+      *s++ =         ctx->digest[i] >> 24;
+      *s++ = 0xff & (ctx->digest[i] >> 16);
+      *s++ = 0xff & (ctx->digest[i] >> 8);
+      *s++ = 0xff &  ctx->digest[i];
+    }
+}

src/libutil/sha1.h

@@ -0,0 +1,28 @@
+#ifndef _SHA_H
+#define _SHA_H
+
+#include <stdint.h>
+
+/* The SHA block size and message digest sizes, in bytes */
+
+#define SHA_DATASIZE    64
+#define SHA_DATALEN     16
+#define SHA_DIGESTSIZE  20
+#define SHA_DIGESTLEN    5
+/* The structure for storing SHA info */
+
+struct sha_ctx {
+  uint32_t digest[SHA_DIGESTLEN];  /* Message digest */
+  uint32_t count_l, count_h;       /* 64-bit block count */
+  uint8_t block[SHA_DATASIZE];     /* SHA data buffer */
+  unsigned int index;            /* index into buffer */
+};
+
+void sha_init(struct sha_ctx *ctx);
+void sha_update(struct sha_ctx *ctx, const unsigned char *buffer, uint32_t len);
+void sha_final(struct sha_ctx *ctx);
+void sha_digest(struct sha_ctx *ctx, unsigned char *s);
+void sha_copy(struct sha_ctx *dest, struct sha_ctx *src);
+
+
+#endif /* !_SHA_H */

src/libutil/sha256.c

@@ -0,0 +1,238 @@
+/* crypto/sha/sha256.c */
+/* ====================================================================
+ * Copyright (c) 2004 The OpenSSL Project.  All rights reserved
+ * according to the OpenSSL license [found in ./md32_common.h].
+ * ====================================================================
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "sha256.h"
+
+int SHA224_Init (SHA256_CTX *c)
+	{
+	c->h[0]=0xc1059ed8UL;	c->h[1]=0x367cd507UL;
+	c->h[2]=0x3070dd17UL;	c->h[3]=0xf70e5939UL;
+	c->h[4]=0xffc00b31UL;	c->h[5]=0x68581511UL;
+	c->h[6]=0x64f98fa7UL;	c->h[7]=0xbefa4fa4UL;
+	c->Nl=0;	c->Nh=0;
+	c->num=0;	c->md_len=SHA224_DIGEST_LENGTH;
+	return 1;
+	}
+
+int SHA256_Init (SHA256_CTX *c)
+	{
+	c->h[0]=0x6a09e667UL;	c->h[1]=0xbb67ae85UL;
+	c->h[2]=0x3c6ef372UL;	c->h[3]=0xa54ff53aUL;
+	c->h[4]=0x510e527fUL;	c->h[5]=0x9b05688cUL;
+	c->h[6]=0x1f83d9abUL;	c->h[7]=0x5be0cd19UL;
+	c->Nl=0;	c->Nh=0;
+	c->num=0;	c->md_len=SHA256_DIGEST_LENGTH;
+	return 1;
+	}
+
+unsigned char *SHA224(const unsigned char *d, size_t n, unsigned char *md)
+	{
+	SHA256_CTX c;
+	static unsigned char m[SHA224_DIGEST_LENGTH];
+
+	if (md == NULL) md=m;
+	SHA224_Init(&c);
+	SHA256_Update(&c,d,n);
+	SHA256_Final(md,&c);
+	return(md);
+	}
+
+unsigned char *SHA256(const unsigned char *d, size_t n, unsigned char *md)
+	{
+	SHA256_CTX c;
+	static unsigned char m[SHA256_DIGEST_LENGTH];
+
+	if (md == NULL) md=m;
+	SHA256_Init(&c);
+	SHA256_Update(&c,d,n);
+	SHA256_Final(md,&c);
+	return(md);
+	}
+
+int SHA224_Update(SHA256_CTX *c, const void *data, size_t len)
+{   return SHA256_Update (c,data,len);   }
+int SHA224_Final (unsigned char *md, SHA256_CTX *c)
+{   return SHA256_Final (md,c);   }
+
+#define	DATA_ORDER_IS_BIG_ENDIAN
+
+#define	HASH_LONG		uint32_t
+#define	HASH_LONG_LOG2		2
+#define	HASH_CTX		SHA256_CTX
+#define	HASH_CBLOCK		SHA_CBLOCK
+#define	HASH_LBLOCK		SHA_LBLOCK
+/*
+ * Note that FIPS180-2 discusses "Truncation of the Hash Function Output."
+ * default: case below covers for it. It's not clear however if it's
+ * permitted to truncate to amount of bytes not divisible by 4. I bet not,
+ * but if it is, then default: case shall be extended. For reference.
+ * Idea behind separate cases for pre-defined lenghts is to let the
+ * compiler decide if it's appropriate to unroll small loops.
+ */
+#define	HASH_MAKE_STRING(c,s)	do {	\
+	unsigned long ll;		\
+	unsigned int  n;		\
+	switch ((c)->md_len)		\
+	{   case SHA224_DIGEST_LENGTH:	\
+		for (n=0;n<SHA224_DIGEST_LENGTH/4;n++)	\
+		{   ll=(c)->h[n]; HOST_l2c(ll,(s));   }	\
+		break;			\
+	    case SHA256_DIGEST_LENGTH:	\
+		for (n=0;n<SHA256_DIGEST_LENGTH/4;n++)	\
+		{   ll=(c)->h[n]; HOST_l2c(ll,(s));   }	\
+		break;			\
+	    default:			\
+		if ((c)->md_len > SHA256_DIGEST_LENGTH)	\
+		    return 0;				\
+		for (n=0;n<(c)->md_len/4;n++)		\
+		{   ll=(c)->h[n]; HOST_l2c(ll,(s));   }	\
+		break;			\
+	}				\
+	} while (0)
+
+#define	HASH_UPDATE		SHA256_Update
+#define	HASH_TRANSFORM		SHA256_Transform
+#define	HASH_FINAL		SHA256_Final
+#define	HASH_BLOCK_HOST_ORDER	sha256_block_host_order
+#define	HASH_BLOCK_DATA_ORDER	sha256_block_data_order
+void sha256_block_host_order (SHA256_CTX *ctx, const void *in, size_t num);
+void sha256_block_data_order (SHA256_CTX *ctx, const void *in, size_t num);
+
+#include "md32_common.h"
+
+static const uint32_t K256[64] = {
+	0x428a2f98UL,0x71374491UL,0xb5c0fbcfUL,0xe9b5dba5UL,
+	0x3956c25bUL,0x59f111f1UL,0x923f82a4UL,0xab1c5ed5UL,
+	0xd807aa98UL,0x12835b01UL,0x243185beUL,0x550c7dc3UL,
+	0x72be5d74UL,0x80deb1feUL,0x9bdc06a7UL,0xc19bf174UL,
+	0xe49b69c1UL,0xefbe4786UL,0x0fc19dc6UL,0x240ca1ccUL,
+	0x2de92c6fUL,0x4a7484aaUL,0x5cb0a9dcUL,0x76f988daUL,
+	0x983e5152UL,0xa831c66dUL,0xb00327c8UL,0xbf597fc7UL,
+	0xc6e00bf3UL,0xd5a79147UL,0x06ca6351UL,0x14292967UL,
+	0x27b70a85UL,0x2e1b2138UL,0x4d2c6dfcUL,0x53380d13UL,
+	0x650a7354UL,0x766a0abbUL,0x81c2c92eUL,0x92722c85UL,
+	0xa2bfe8a1UL,0xa81a664bUL,0xc24b8b70UL,0xc76c51a3UL,
+	0xd192e819UL,0xd6990624UL,0xf40e3585UL,0x106aa070UL,
+	0x19a4c116UL,0x1e376c08UL,0x2748774cUL,0x34b0bcb5UL,
+	0x391c0cb3UL,0x4ed8aa4aUL,0x5b9cca4fUL,0x682e6ff3UL,
+	0x748f82eeUL,0x78a5636fUL,0x84c87814UL,0x8cc70208UL,
+	0x90befffaUL,0xa4506cebUL,0xbef9a3f7UL,0xc67178f2UL };
+
+/*
+ * FIPS specification refers to right rotations, while our ROTATE macro
+ * is left one. This is why you might notice that rotation coefficients
+ * differ from those observed in FIPS document by 32-N...
+ */
+#define Sigma0(x)	(ROTATE((x),30) ^ ROTATE((x),19) ^ ROTATE((x),10))
+#define Sigma1(x)	(ROTATE((x),26) ^ ROTATE((x),21) ^ ROTATE((x),7))
+#define sigma0(x)	(ROTATE((x),25) ^ ROTATE((x),14) ^ ((x)>>3))
+#define sigma1(x)	(ROTATE((x),15) ^ ROTATE((x),13) ^ ((x)>>10))
+
+#define Ch(x,y,z)	(((x) & (y)) ^ ((~(x)) & (z)))
+#define Maj(x,y,z)	(((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
+
+#define	ROUND_00_15(i,a,b,c,d,e,f,g,h)		do {	\
+	T1 += h + Sigma1(e) + Ch(e,f,g) + K256[i];	\
+	h = Sigma0(a) + Maj(a,b,c);			\
+	d += T1;	h += T1;		} while (0)
+
+#define	ROUND_16_63(i,a,b,c,d,e,f,g,h,X)	do {	\
+	s0 = X[(i+1)&0x0f];	s0 = sigma0(s0);	\
+	s1 = X[(i+14)&0x0f];	s1 = sigma1(s1);	\
+	T1 = X[(i)&0x0f] += s0 + s1 + X[(i+9)&0x0f];	\
+	ROUND_00_15(i,a,b,c,d,e,f,g,h);		} while (0)
+
+static void sha256_block (SHA256_CTX *ctx, const void *in, size_t num, int host)
+	{
+	uint32_t a,b,c,d,e,f,g,h,s0,s1,T1;
+	uint32_t	X[16];
+	int i;
+	const unsigned char *data=in;
+
+			while (num--) {
+
+	a = ctx->h[0];	b = ctx->h[1];	c = ctx->h[2];	d = ctx->h[3];
+	e = ctx->h[4];	f = ctx->h[5];	g = ctx->h[6];	h = ctx->h[7];
+
+	if (host)
+		{
+		const uint32_t *W=(const uint32_t *)data;
+
+		T1 = X[0] = W[0];	ROUND_00_15(0,a,b,c,d,e,f,g,h);
+		T1 = X[1] = W[1];	ROUND_00_15(1,h,a,b,c,d,e,f,g);
+		T1 = X[2] = W[2];	ROUND_00_15(2,g,h,a,b,c,d,e,f);
+		T1 = X[3] = W[3];	ROUND_00_15(3,f,g,h,a,b,c,d,e);
+		T1 = X[4] = W[4];	ROUND_00_15(4,e,f,g,h,a,b,c,d);
+		T1 = X[5] = W[5];	ROUND_00_15(5,d,e,f,g,h,a,b,c);
+		T1 = X[6] = W[6];	ROUND_00_15(6,c,d,e,f,g,h,a,b);
+		T1 = X[7] = W[7];	ROUND_00_15(7,b,c,d,e,f,g,h,a);
+		T1 = X[8] = W[8];	ROUND_00_15(8,a,b,c,d,e,f,g,h);
+		T1 = X[9] = W[9];	ROUND_00_15(9,h,a,b,c,d,e,f,g);
+		T1 = X[10] = W[10];	ROUND_00_15(10,g,h,a,b,c,d,e,f);
+		T1 = X[11] = W[11];	ROUND_00_15(11,f,g,h,a,b,c,d,e);
+		T1 = X[12] = W[12];	ROUND_00_15(12,e,f,g,h,a,b,c,d);
+		T1 = X[13] = W[13];	ROUND_00_15(13,d,e,f,g,h,a,b,c);
+		T1 = X[14] = W[14];	ROUND_00_15(14,c,d,e,f,g,h,a,b);
+		T1 = X[15] = W[15];	ROUND_00_15(15,b,c,d,e,f,g,h,a);
+
+		data += SHA256_CBLOCK;
+		}
+	else
+		{
+		uint32_t l;
+
+		HOST_c2l(data,l); T1 = X[0] = l;  ROUND_00_15(0,a,b,c,d,e,f,g,h);
+		HOST_c2l(data,l); T1 = X[1] = l;  ROUND_00_15(1,h,a,b,c,d,e,f,g);
+		HOST_c2l(data,l); T1 = X[2] = l;  ROUND_00_15(2,g,h,a,b,c,d,e,f);
+		HOST_c2l(data,l); T1 = X[3] = l;  ROUND_00_15(3,f,g,h,a,b,c,d,e);
+		HOST_c2l(data,l); T1 = X[4] = l;  ROUND_00_15(4,e,f,g,h,a,b,c,d);
+		HOST_c2l(data,l); T1 = X[5] = l;  ROUND_00_15(5,d,e,f,g,h,a,b,c);
+		HOST_c2l(data,l); T1 = X[6] = l;  ROUND_00_15(6,c,d,e,f,g,h,a,b);
+		HOST_c2l(data,l); T1 = X[7] = l;  ROUND_00_15(7,b,c,d,e,f,g,h,a);
+		HOST_c2l(data,l); T1 = X[8] = l;  ROUND_00_15(8,a,b,c,d,e,f,g,h);
+		HOST_c2l(data,l); T1 = X[9] = l;  ROUND_00_15(9,h,a,b,c,d,e,f,g);
+		HOST_c2l(data,l); T1 = X[10] = l; ROUND_00_15(10,g,h,a,b,c,d,e,f);
+		HOST_c2l(data,l); T1 = X[11] = l; ROUND_00_15(11,f,g,h,a,b,c,d,e);
+		HOST_c2l(data,l); T1 = X[12] = l; ROUND_00_15(12,e,f,g,h,a,b,c,d);
+		HOST_c2l(data,l); T1 = X[13] = l; ROUND_00_15(13,d,e,f,g,h,a,b,c);
+		HOST_c2l(data,l); T1 = X[14] = l; ROUND_00_15(14,c,d,e,f,g,h,a,b);
+		HOST_c2l(data,l); T1 = X[15] = l; ROUND_00_15(15,b,c,d,e,f,g,h,a);
+		}
+
+	for (i=16;i<64;i+=8)
+		{
+		ROUND_16_63(i+0,a,b,c,d,e,f,g,h,X);
+		ROUND_16_63(i+1,h,a,b,c,d,e,f,g,X);
+		ROUND_16_63(i+2,g,h,a,b,c,d,e,f,X);
+		ROUND_16_63(i+3,f,g,h,a,b,c,d,e,X);
+		ROUND_16_63(i+4,e,f,g,h,a,b,c,d,X);
+		ROUND_16_63(i+5,d,e,f,g,h,a,b,c,X);
+		ROUND_16_63(i+6,c,d,e,f,g,h,a,b,X);
+		ROUND_16_63(i+7,b,c,d,e,f,g,h,a,X);
+		}
+
+	ctx->h[0] += a;	ctx->h[1] += b;	ctx->h[2] += c;	ctx->h[3] += d;
+	ctx->h[4] += e;	ctx->h[5] += f;	ctx->h[6] += g;	ctx->h[7] += h;
+
+			}
+	}
+
+/*
+ * Idea is to trade couple of cycles for some space. On IA-32 we save
+ * about 4K in "big footprint" case. In "small footprint" case any gain
+ * is appreciated:-)
+ */
+void HASH_BLOCK_HOST_ORDER (SHA256_CTX *ctx, const void *in, size_t num)
+{   sha256_block (ctx,in,num,1);   }
+
+void HASH_BLOCK_DATA_ORDER (SHA256_CTX *ctx, const void *in, size_t num)
+{   sha256_block (ctx,in,num,0);   }
+
+

src/libutil/sha256.h

@@ -0,0 +1,35 @@
+#ifndef _SHA256_H
+#define _SHA256_H 1
+
+#include <stdint.h>
+
+#define SHA_LBLOCK	16
+#define SHA_CBLOCK	(SHA_LBLOCK*4)	/* SHA treats input data as a
+					 * contiguous array of 32 bit
+					 * wide big-endian values. */
+
+#define SHA256_CBLOCK	(SHA_LBLOCK*4)	/* SHA-256 treats input data as a
+					 * contiguous array of 32 bit
+					 * wide big-endian values. */
+#define SHA224_DIGEST_LENGTH	28
+#define SHA256_DIGEST_LENGTH	32
+
+typedef struct SHA256state_st
+	{
+	uint32_t h[8];
+	uint32_t Nl,Nh;
+	uint32_t data[SHA_LBLOCK];
+	unsigned int num,md_len;
+	} SHA256_CTX;
+
+int SHA224_Init(SHA256_CTX *c);
+int SHA224_Update(SHA256_CTX *c, const void *data, size_t len);
+int SHA224_Final(unsigned char *md, SHA256_CTX *c);
+unsigned char *SHA224(const unsigned char *d, size_t n,unsigned char *md);
+int SHA256_Init(SHA256_CTX *c);
+int SHA256_Update(SHA256_CTX *c, const void *data, size_t len);
+int SHA256_Final(unsigned char *md, SHA256_CTX *c);
+unsigned char *SHA256(const unsigned char *d, size_t n,unsigned char *md);
+void SHA256_Transform(SHA256_CTX *c, const unsigned char *data);
+
+#endif

src/libutil/util.cc

@@ -135,6 +135,15 @@ Path readLink(const Path & path)
 }
 
 
+bool isLink(const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+        throw SysError(format("getting status of `%1%'") % path);
+    return S_ISLNK(st.st_mode);
+}
+
+
 Strings readDirectory(const Path & path)
 {
     Strings names;
@@ -155,6 +164,36 @@ Strings readDirectory(const Path & path)
 }
 
 
+string readFile(int fd)
+{
+    struct stat st;
+    if (fstat(fd, &st) == -1)
+        throw SysError("statting file");
+    unsigned char buf[st.st_size]; /* !!! stack space */
+    readFull(fd, buf, st.st_size);
+
+    return string((char *) buf, st.st_size);
+}
+
+
+string readFile(const Path & path)
+{
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY);
+    if (fd == -1)
+        throw SysError(format("opening file `%1%'") % path);
+    return readFile(fd);
+}
+
+
+void writeFile(const Path & path, const string & s)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
+    if (fd == -1)
+        throw SysError(format("opening file `%1%'") % path);
+    writeFull(fd, (unsigned char *) s.c_str(), s.size());
+}
+
+
 static void _deletePath(const Path & path)
 {
     checkInterrupt();
@@ -232,10 +271,20 @@ Path createTempDir()
 }
 
 
+void createDirs(const Path & path)
+{
+    if (path == "") return;
+    createDirs(dirOf(path));
+    if (!pathExists(path))
+        if (mkdir(path.c_str(), 0777) == -1)
+            throw SysError(format("creating directory `%1%'") % path);
+}
+
+
 void writeStringToFile(const Path & path, const string & s)
 {
-    AutoCloseFD fd = open(path.c_str(),
-        O_CREAT | O_EXCL | O_WRONLY, 0666);
+    AutoCloseFD fd(open(path.c_str(),
+        O_CREAT | O_EXCL | O_WRONLY, 0666));
     if (fd == -1)
         throw SysError(format("creating file `%1%'") % path);
     writeFull(fd, (unsigned char *) s.c_str(), s.size());
@@ -375,6 +424,12 @@ AutoCloseFD::AutoCloseFD(int fd)
 }
 
 
+AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd)
+{
+    abort();
+}
+
+
 AutoCloseFD::~AutoCloseFD()
 {
     try {
@@ -392,7 +447,7 @@ void AutoCloseFD::operator =(int fd)
 }
 
 
-AutoCloseFD::operator int()
+AutoCloseFD::operator int() const
 {
     return fd;
 }
@@ -415,6 +470,15 @@ bool AutoCloseFD::isOpen()
 }
 
 
+/* Pass responsibility for closing this fd to the caller. */
+int AutoCloseFD::borrow()
+{
+    int oldFD = fd;
+    fd = -1;
+    return oldFD;
+}
+
+
 void Pipe::create()
 {
     int fds[2];
@@ -587,6 +651,8 @@ Strings unpackStrings(const string & s)
         len |= ((unsigned char) *i++) << 8;
         len |= ((unsigned char) *i++) << 16;
         len |= ((unsigned char) *i++) << 24;
+
+        if (len == 0xffffffff) return strings; /* explicit end-of-list */
         
         if (i + len > s.end())
             throw Error(format("short db entry: `%1%'") % s);

src/libutil/util.hh

@@ -84,10 +84,19 @@ bool pathExists(const Path & path);
    in any way canonicalised. */
 Path readLink(const Path & path);
 
+bool isLink(const Path & path);
+
 /* Read the contents of a directory.  The entries `.' and `..' are
    removed. */
 Strings readDirectory(const Path & path);
 
+/* Read the contents of a file into a string. */
+string readFile(int fd);
+string readFile(const Path & path);
+
+/* Write a string to a file. */
+void writeFile(const Path & path, const string & s);
+
 /* Delete a path; i.e., in the case of a directory, it is deleted
    recursively.  Don't use this at home, kids. */
 void deletePath(const Path & path);
@@ -98,11 +107,22 @@ void makePathReadOnly(const Path & path);
 /* Create a temporary directory. */
 Path createTempDir();
 
+/* Create a directory and all its parents, if necessary. */
+void createDirs(const Path & path);
+
 /* Create a file and write the given text to it.  The file is written
    in binary mode (i.e., no end-of-line conversions).  The path should
    not already exist. */
 void writeStringToFile(const Path & path, const string & s);
 
+template<class T, class A>
+T singleton(const A & a)
+{
+    T t;
+    t.insert(a);
+    return t;
+}
+
 
 /* Messages. */
 
@@ -179,11 +199,13 @@ class AutoCloseFD
 public:
     AutoCloseFD();
     AutoCloseFD(int fd);
+    AutoCloseFD(const AutoCloseFD & fd);
     ~AutoCloseFD();
     void operator =(int fd);
-    operator int();
+    operator int() const;
     void close();
     bool isOpen();
+    int borrow();
 };