* Sync with thesis: do not include store location in the hash

computation (in the intensional model).

AUTHORS

@@ -0,0 +1,8 @@
+The following people contributed to Nix, in alphabetical order:
+
+Martin Bravenboer
+Eelco Dolstra
+Niels Janssen
+Armijn Hemel
+Rob Vermaas
+Eelco Visser

Makefile.am

@@ -1,5 +1,6 @@
-SUBDIRS = externals src scripts corepkgs doc
-EXTRA_DIST = substitute.mk nix.spec nix.spec.in
+SUBDIRS = externals src scripts corepkgs doc misc tests
+EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
+  svn-revision nix.conf.example
 
 include ./substitute.mk
 
@@ -12,19 +13,35 @@ relname:
 	echo -n $(distdir) > relname
 
 install-data-local: init-state
+	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
+	$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix
+	if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
+		$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
+	fi
 
 if INIT_STATE
+if SETUID_HACK
+INIT_FLAGS = -g @NIX_GROUP@ -o @NIX_USER@
+GROUP_WRITABLE = -m 775
+endif
 init-state:
-	$(INSTALL) -d $(DESTDIR)$(localstatedir)/nix
-	$(INSTALL) -d $(DESTDIR)$(localstatedir)/nix/db
-	$(INSTALL) -d $(DESTDIR)$(localstatedir)/log/nix
-	$(INSTALL) -d $(DESTDIR)$(localstatedir)/nix/profiles
-	$(INSTALL) -d $(DESTDIR)$(localstatedir)/nix/gcroots
-	$(INSTALL) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/db
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix/drvs
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
+	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
+	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
-	$(INSTALL) -d $(DESTDIR)$(prefix)/store
+	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(prefix)/store
+	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 #	$(bindir)/nix-store --init
 else
 init-state:
 endif
+
+svn-revision:
+	svnversion . > svn-revision

NEWS

@@ -0,0 +1,261 @@
+Version 0.9
+
+* Unpacking of patch sequences is much faster now by not doing
+  redundant unpacking and repacking of intermediate paths.
+
+
+Version 0.8 (April 11, 2005)
+
+NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).  As a
+result, `nix-pull' manifests and channels built for Nix 0.7 and below
+will now work anymore.  However, the Nix expression language has not
+changed, so you can still build from source.  Also, existing user
+environments continue to work.  Nix 0.8 will automatically upgrade the
+database schema of previous installations when it is first run.
+
+If you get the error message
+
+  you have an old-style manifest `/nix/var/nix/manifests/[...]';
+  please delete it
+
+you should delete previously downloaded manifests:
+
+  $ rm /nix/var/nix/manifests/*
+
+If `nix-channel' gives the error message
+
+  manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST'
+  is too old (i.e., for Nix <= 0.7)
+
+then you should unsubscribe from the offending channel (`nix-channel
+--remove URL'; leave out `/MANIFEST'), and subscribe to the same URL,
+with `channels' replaced by `channels-v3' (e.g.,
+http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable).
+
+Nix 0.8 has the following improvements:
+
+* The cryptographic hashes used in store paths are now 160 bits long,
+  but encoded in base-32 so that they are still only 32 characters
+  long (e.g., /nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0).
+  (This is actually a 160 bit truncation of a SHA-256 hash.)
+
+* Big cleanups and simplifications of the basic store semantics.  The
+  notion of "closure store expressions" is gone (and so is the notion
+  of "successors"); the file system references of a store path are now
+  just stored in the database.
+
+  For instance, given any store path, you can query its closure:
+
+    $ nix-store -qR $(which firefox)
+    ... lots of paths ...
+
+  Also, Nix now remembers for each store path the derivation that
+  built it (the "deriver"):
+
+    $ nix-store -qR $(which firefox)
+    /nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv
+
+  So to see the build-time dependencies, you can do
+  
+    $ nix-store -qR $(nix-store -qd $(which firefox))
+
+  or, in a nicer format:
+
+    $ nix-store -q --tree $(nix-store -qd $(which firefox))
+
+  File system references are also stored in reverse.  For instance,
+  you can query all paths that directly or indirectly use a certain
+  Glibc:
+
+    $ nix-store -q --referers-closure \
+        /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+
+* The concept of fixed-output derivations has been formalised.
+  Previously, functions such as `fetchurl' in Nixpkgs used a hack
+  (namely, explicitly specifying a store path hash) to prevent changes
+  to, say, the URL of the file from propagating upwards through the
+  dependency graph, causing rebuilds of everything.  This can now be
+  done cleanly by specifying the `outputHash' and `outputHashAlgo'
+  attributes.  Nix itself checks that the content of the output has
+  the specified hash.  (This is important for maintaining certain
+  invariants necessary for future work on secure shared stores.)
+
+* One-click installation :-)  It is now possible to install any
+  top-level component in Nixpkgs directly, through the web - see,
+  e.g., http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/.  All you
+  have to do is associate `/nix/bin/nix-install-package' with the MIME
+  type `application/nix-package' (or the extension `.nixpkg'), and
+  clicking on a package link will cause it to be installed, with all
+  appropriate dependencies.  If you just want to install some specific
+  application, this is easier than subscribing to a channel.
+
+* `nix-store -r PATHS' now builds all the derivations PATHS in
+  parallel.  Previously it did them sequentially (though exploiting
+  possible parallelism between subderivations).  This is nice for
+  build farms.
+
+* `nix-channel' has new operations `--list' and `--remove'.
+
+* New ways of installing components into user environments:
+
+  - Copy from another user environment:
+
+      $ nix-env -i --from-profile .../other-profile firefox
+
+  - Install a store derivation directly (bypassing the Nix expression
+    language entirely):
+
+      $ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv
+
+    (This is used to implement `nix-install-package', which is
+    therefore immune to evolution in the Nix expression language.)
+
+  - Install an already built store path directly:
+
+      $ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1
+
+  - Install the result of a Nix expression specified as a command-line
+    argument:
+
+      $ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'
+
+    The difference with the normal installation mode is that `-E' does
+    not use the `name' attributes of derivations.  Therefore, this can
+    be used to disambiguate multiple derivations with the same name.
+
+* A hash of the contents of a store path is now stored in the database
+  after a succesful build.  This allows you to check whether store
+  paths have been tampered with: `nix-store --verify --check-contents'.
+    
+* Implemented a concurrent garbage collector.  It is now always safe
+  to run the garbage collector, even if other Nix operations are
+  happening simultaneously.
+
+  However, there can still be GC races if you use `nix-instantiate'
+  and `nix-store -r' directly to build things.  To prevent races, use
+  the `--add-root' flag of those commands.
+
+* The garbage collector now finally deletes paths in the right order
+  (i.e., topologically sorted under the `references' relation), thus
+  making it safe to interrupt the collector without risking a store
+  that violates the closure invariant.
+
+* Likewise, the substitute mechanism now downloads files in the right
+  order, thus preserving the closure invariant at all times.
+
+* The result of `nix-build' is now registered as a root of the garbage
+  collector.  If the `./result' link is deleted, the GC root
+  disappears automatically.
+
+* The behaviour of the garbage collector can be changed globally by
+  setting options in `/nix/etc/nix/nix.conf'.
+
+  - `gc-keep-derivations' specifies whether deriver links should be
+    followed when searching for live paths.
+
+  - `gc-keep-outputs' specifies whether outputs of derivations should
+    be followed when searching for live paths.
+  
+  - `env-keep-derivations' specifies whether user environments should
+    store the paths of derivations when they are added (thus keeping
+    the derivations alive).
+
+* New `nix-env' query flags `--drv-path' and `--out-path'.
+
+* `fetchurl' allows SHA-1 and SHA-256 in addition to MD5.  Just
+  specify the attribute `sha1' or `sha256' instead of `md5'.
+
+* Manual updates.
+
+  
+Version 0.7 (January 12, 2005)
+
+* Binary patching.  When upgrading components using pre-built binaries
+  (through nix-pull / nix-channel), Nix can automatically download and
+  apply binary patches to already installed components instead of full
+  downloads.  Patching is "smart": if there is a *sequence* of patches
+  to an installed component, Nix will use it.  Patches are currently
+  generated automatically between Nixpkgs (pre-)releases.
+
+* Simplifications to the substitute mechanism.
+
+* Nix-pull now stores downloaded manifests in /nix/var/nix/manifests.
+
+* Metadata on files in the Nix store is canonicalised after builds:
+  the last-modified timestamp is set to 0 (00:00:00 1/1/1970), the
+  mode is set to 0444 or 0555 (readable and possibly executable by
+  all; setuid/setgid bits are dropped), and the group is set to the
+  default.  This ensures that the result of a build and an
+  installation through a substitute is the same; and that timestamp
+  dependencies are revealed.
+
+
+Version 0.6 (November 14, 2004)
+
+Major changes include the following:
+
+* Rewrite of the normalisation engine.
+
+  * Multiple builds can now be performed in parallel (option `-j').
+
+  * Distributed builds.  Nix can now call a shell script to forward
+    builds to Nix installations on remote machines, which may or may
+    not be of the same platform type.
+
+  * Option `--fallback' allows recovery from broken substitutes.
+
+  * Option `--keep-going' causes building of other (unaffected)
+    derivations to continue if one failed.
+    
+* Improvements to the garbage collector (i.e., it should actually work
+  now).
+
+* Setuid Nix installations allow a Nix store to be shared among
+  multiple users.
+
+* Substitute registration is much faster now.
+
+* A utility `nix-build' to build a Nix expression and create a symlink
+  to the result int the current directory; useful for testing Nix
+  derivations.
+
+* Manual updates.
+
+* `nix-env' changes:
+
+  * Derivations for other platforms are filtered out (which can be
+    overriden using `--system-filter').
+
+  * `--install' by default now uninstall previous derivations with the
+    same name.
+
+  * `--upgrade' allows upgrading to a specific version.
+
+  * New operation `--delete-generations' to remove profile
+    generations (necessary for effective garbage collection).
+
+  * Nicer output (sorted, columnised).
+
+* More sensible verbosity levels all around (builder output is now
+  shown always, unless `-Q' is given).
+
+* Nix expression language changes:
+
+  * New language construct: `with E1; E2' brings all attributes
+    defined in the attribute set E1 in scope in E2.
+
+  * Added a `map' function.
+
+  * Various new operators (e.g., string concatenation).
+
+* Expression evaluation is much faster.
+
+* An Emacs mode for editing Nix expressions (with syntax highlighting
+  and indentation) has been added.
+
+* Many bug fixes.
+
+
+Version 0.5 and earlier
+
+Please refer to the Subversion commit log messages.

README

@@ -1,5 +1,9 @@
-                             *** Nix ***
-
 For installation and usage instructions, please read the manual, which
 can be found in `docs/manual/manual.html', and additionally at the Nix
 website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
+
+
+Acknowledgments
+
+This product includes software developed by the OpenSSL Project for
+use in the OpenSSL Toolkit (http://www.OpenSSL.org/)

blacklisting/check-env.pl

@@ -0,0 +1,252 @@
+#! /usr/bin/perl -w -I /home/eelco/.nix-profile/lib/site_perl
+
+use strict;
+use XML::LibXML;
+#use XML::Simple;
+
+my $blacklistFN = shift @ARGV;
+die unless defined $blacklistFN;
+my $userEnv = shift @ARGV;
+die unless defined $userEnv;
+
+
+# Read the blacklist.
+my $parser = XML::LibXML->new();
+my $blacklist = $parser->parse_file($blacklistFN)->getDocumentElement;
+
+#print $blacklist->toString() , "\n";
+
+
+# Get all the elements of the user environment.
+my $userEnvElems = `nix-store --query --references '$userEnv'`;
+die "cannot query user environment elements" if $? != 0;
+my @userEnvElems = split ' ', $userEnvElems;
+
+
+my %storePathHashes;
+
+
+sub getElemNodes {
+    my $node = shift;
+    my @elems = ();
+    foreach my $node ($node->getChildNodes) {
+        push @elems, $node if $node->nodeType == XML_ELEMENT_NODE;
+    }
+    return @elems;
+}
+
+
+my %referencesCache;
+sub getReferences {
+    my $path = shift;
+    return $referencesCache{$path} if defined $referencesCache{$path};
+    
+    my $references = `nix-store --query --references '$path'`;
+    die "cannot query references" if $? != 0;
+    $referencesCache{$path} = [split ' ', $references];
+    
+    return $referencesCache{$path};
+}
+
+
+my %attrsCache;
+sub getAttr {
+    my $path = shift;
+    my $name = shift;
+    my $key = "$path/$name";
+    return $referencesCache{$key} if defined $referencesCache{$key};
+
+    my $value = `nix-store --query --binding '$name' '$path' 2> /dev/null`;
+    $value = "" if $? != 0; # !!!
+    chomp $value;
+    $referencesCache{$key} = $value;
+
+    return $value;
+}
+
+
+sub evalCondition;
+
+
+sub traverse {
+    my $done = shift;
+    my $set = shift;
+    my $path = shift;
+    my $stopCondition = shift;
+
+    return if defined $done->{$path};
+    $done->{$path} = 1;
+    $set->{$path} = 1;
+
+#    print "  in $path\n";
+
+    if (!evalCondition({$path => 1}, $stopCondition)) {
+#        print "  STOPPING in $path\n";
+        return;
+    }
+
+    # Get the requisites of the deriver.
+
+    foreach my $reference (@{getReferences $path}) {
+        traverse($done, $set, $reference, $stopCondition);
+    }
+}
+
+
+sub evalSet {
+    my $inSet = shift;
+    my $expr = shift;
+    my $name = $expr->getName;
+    
+    if ($name eq "traverse") {
+        my $stopCondition = (getElemNodes $expr)[0];
+        my $done = { };
+        my $set = { };
+        foreach my $path (keys %{$inSet}) {
+            traverse($done, $set, $path, $stopCondition);
+        }
+        return $set;
+    }
+
+    else {
+        die "unknown element `$name'";
+    }
+}
+
+
+# Function for evaluating conditions.
+sub evalCondition {
+    my $storePaths = shift;
+    my $condition = shift;
+    my $elemName = $condition->getName;
+    
+    if ($elemName eq "containsSource") {
+        my $hash = $condition->attributes->getNamedItem("hash")->getValue;
+        foreach my $path (keys %{$storePathHashes{$hash}}) {
+            return 1 if defined $storePaths->{$path};
+        }
+        return 0;
+    }
+
+    elsif ($elemName eq "hasName") { 
+        my $nameRE = $condition->attributes->getNamedItem("name")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            return 1 if $path =~ /$nameRE/;
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "hasAttr") { 
+        my $name = $condition->attributes->getNamedItem("name")->getValue;
+        my $valueRE = $condition->attributes->getNamedItem("value")->getValue;
+        foreach my $path (keys %{$storePaths}) {
+            if ($path =~ /\.drv$/) {
+                my $value = getAttr($path, $name);
+#                print "    $path $name $value\n";
+                return 1 if $value =~ /$valueRE/;
+            }
+        }
+        return 0;
+    }
+    
+    elsif ($elemName eq "and") {
+        my $result = 1;
+        foreach my $node (getElemNodes $condition) {
+            $result &= evalCondition($storePaths, $node);
+        }
+        return $result;
+    }
+
+    elsif ($elemName eq "not") {
+        return !evalCondition($storePaths, (getElemNodes $condition)[0]);
+    }
+    
+    elsif ($elemName eq "within") {
+        my @elems = getElemNodes $condition;
+        my $set = evalSet($storePaths, $elems[0]);
+        return evalCondition($set, $elems[1]);
+    }
+
+    elsif ($elemName eq "true") {
+        return 1;
+    }
+
+    elsif ($elemName eq "false") {
+        return 0;
+    }
+
+    else {
+        die "unknown element `$elemName'";
+    }
+}
+
+
+sub evalOr {
+    my $storePaths = shift;
+    my $nodes = shift;
+
+    my $result = 0;
+    foreach my $node (@{$nodes}) {
+        $result |= evalCondition($storePaths, $node);
+    }
+    
+    return $result;
+}
+
+
+# Iterate over all elements, check them.
+foreach my $userEnvElem (@userEnvElems) {
+
+    # Get the deriver of this path.
+    my $deriver = `nix-store --query --deriver '$userEnvElem'`;
+    die "cannot query deriver" if $? != 0;
+    chomp $deriver;
+
+    if ($deriver eq "unknown-deriver") {
+#        print "  deriver unknown, cannot check sources\n";
+        next;
+    }
+
+    print "CHECKING $userEnvElem\n";
+
+
+    # Get the requisites of the deriver.
+#    my $requisites = `nix-store --query --requisites --include-outputs '$deriver'`;
+#    die "cannot query requisites" if $? != 0;
+#    my @requisites = split ' ', $requisites;
+
+
+    # Get the hashes of the requisites.
+#    my $hashes = `nix-store --query --hash @requisites`;
+#    die "cannot query hashes" if $? != 0;
+#    my @hashes = split ' ', $hashes;
+#    for (my $i = 0; $i < scalar @requisites; $i++) {
+#        die unless $i < scalar @hashes;
+#        my $hash = $hashes[$i];
+#        $storePathHashes{$hash} = {} unless defined $storePathHashes{$hash};
+#        my $r = $storePathHashes{$hash}; # !!! fix
+#        $$r{$requisites[$i]} = 1;
+#    }
+
+
+    # Evaluate each blacklist item.
+    foreach my $item ($blacklist->getChildrenByTagName("item")) {
+        my $itemId = $item->getAttributeNode("id")->getValue;
+#        print "  CHECKING FOR $itemId\n";
+
+        my $condition = ($item->getChildrenByTagName("condition"))[0];
+        die unless $condition;
+
+        # Evaluate the condition.
+        my @elems = getElemNodes $condition;
+        if (evalOr({$deriver => 1}, \@elems)) {
+            # Oops, condition triggered.
+            my $reason = ($item->getChildrenByTagName("reason"))[0]->getChildNodes->to_literal;
+            $reason =~ s/\s+/ /g;
+            $reason =~ s/^\s+//g;
+
+            print "    VULNERABLE TO `$itemId': $reason\n";
+        }
+    }
+}
+

bootstrap.sh

@@ -0,0 +1,5 @@
+#! /bin/sh -e
+aclocal
+autoheader
+automake --add-missing --copy
+autoconf

configure.ac

@@ -1,7 +1,7 @@
-AC_INIT(nix, "0.5")
+AC_INIT(nix, "0.9")
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
-AM_INIT_AUTOMAKE
+AM_INIT_AUTOMAKE([dist-bzip2])
 
 # Change to `1' to produce a `stable' release (i.e., the `preREVISION'
 # suffix is not added).
@@ -22,9 +22,24 @@ AC_CANONICAL_HOST
 
 # Construct a Nix system name (like "i686-linux").
 AC_MSG_CHECKING([for the canonical Nix system name])
-machine_name=`uname -m`
-sys_name=`uname -s | tr [A-Z] [a-z]`
-system="${machine_name}-${sys_name}"
+cpu_name=$(uname -p | tr 'A-Z ' 'a-z_')
+machine_name=$(uname -m | tr 'A-Z ' 'a-z_')
+
+case $machine_name in
+    i*86)
+        machine_name=i686
+        ;;
+    *)
+        if test "$cpu_name" != "unknown"; then
+            machine_name=$cpu_name
+        fi
+        ;;
+esac
+
+sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
+AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM],
+  [platform identifier (e.g., `i686-linux')]),
+  system=$withval, system="${machine_name}-${sys_name}")
 AC_MSG_RESULT($system)
 AC_SUBST(system)
 AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
@@ -53,7 +68,7 @@ AC_DEFUN([NEED_PROG],
 [
 AC_PATH_PROG($1, $2)
 if test -z "$$1"; then
-    AC_MSG_ERROR([$1 is required])
+    AC_MSG_ERROR([$2 is required])
 fi
 ])
 
@@ -66,17 +81,19 @@ AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 NEED_PROG(perl, perl)
+NEED_PROG(tar, tar)
+                
+NEED_PROG(cat, cat)
+AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
+  [path of cat, mkdir, etc.]),
+  coreutils=$withval, coreutils=$(dirname $cat))
+AC_SUBST(coreutils)
 
 AC_ARG_WITH(docbook-catalog, AC_HELP_STRING([--with-docbook-catalog=PATH],
   [path of the DocBook XML DTD]),
   docbookcatalog=$withval, docbookcatalog=/docbook-dtd-missing)
 AC_SUBST(docbookcatalog)
 
-AC_ARG_WITH(docbook-ebnf-catalog, AC_HELP_STRING([--with-docbook-ebnf-catalog=PATH],
-  [path of the DocBook XML EBNF module DTD]),
-  docbookebnfcatalog=$withval, docbookcatalog=/docbook-ebnf-dtd-missing)
-AC_SUBST(docbookebnfcatalog)
-
 AC_ARG_WITH(docbook-xsl, AC_HELP_STRING([--with-docbook-xsl=PATH],
   [path of the DocBook XSL stylesheets]),
   docbookxsl=$withval, docbookxsl=/docbook-xsl-missing)
@@ -100,7 +117,7 @@ if test -z "$bdb"; then
   bdb_lib='-L${top_builddir}/externals/inst-bdb/lib -ldb_cxx'
   bdb_include='-I${top_builddir}/externals/inst-bdb/include'
 else
-  bdb_lib="-L$bdb/lib -Wl,-rpath,$bdb/lib -ldb_cxx"
+  bdb_lib="-L$bdb/lib -ldb_cxx"
   bdb_include="-I$bdb/include"
 fi
 AC_SUBST(bdb_lib)
@@ -113,12 +130,15 @@ AM_CONDITIONAL(HAVE_ATERM, test -n "$aterm")
 if test -z "$aterm"; then
   aterm_lib='-L${top_builddir}/externals/inst-aterm/lib -lATerm'
   aterm_include='-I${top_builddir}/externals/inst-aterm/include'
+  aterm_bin='${top_builddir}/externals/inst-aterm/bin'
 else
-  aterm_lib="-L$aterm/lib -Wl,-rpath,$aterm/lib -lATerm"
+  aterm_lib="-L$aterm/lib -lATerm"
   aterm_include="-I$aterm/include"
+  aterm_bin="$aterm/bin"
 fi
 AC_SUBST(aterm_lib)
 AC_SUBST(aterm_include)
+AC_SUBST(aterm_bin)
 
 AC_CHECK_LIB(pthread, pthread_mutex_init)
 
@@ -127,6 +147,34 @@ AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
   init_state=$enableval, init_state=yes)
 AM_CONDITIONAL(INIT_STATE, test "$init_state" = "yes")
 
+
+AC_ARG_ENABLE(setuid, AC_HELP_STRING([--enable-setuid],
+  [install Nix setuid]),
+  setuid_hack=$enableval, setuid_hack=no)
+AM_CONDITIONAL(SETUID_HACK, test "$setuid_hack" = "yes")
+if test "$setuid_hack" = "yes"; then
+    AC_DEFINE(SETUID_HACK, 1, [whether to install Nix setuid])
+fi
+
+AC_CHECK_FUNC(setresuid, [HAVE_SETRESUID=1], [HAVE_SETRESUID=])
+AM_CONDITIONAL(HAVE_SETRESUID, test "$HAVE_SETRESUID" = "1")
+if test "$HAVE_SETRESUID" = "1"; then
+    AC_DEFINE(HAVE_SETRESUID, 1, [whether we have setresuid()])
+fi
+
+AC_ARG_WITH(nix-user, AC_HELP_STRING([--with-nix-user=USER],
+  [user for Nix setuid binaries]),
+  NIX_USER=$withval, NIX_USER=nix)
+AC_SUBST(NIX_USER)
+AC_DEFINE_UNQUOTED(NIX_USER, ["$NIX_USER"], [Nix user])
+                                                      
+AC_ARG_WITH(nix-group, AC_HELP_STRING([--with-nix-group=USER],
+  [group for Nix setuid binaries]),
+  NIX_GROUP=$withval, NIX_GROUP=nix)
+AC_SUBST(NIX_GROUP)
+AC_DEFINE_UNQUOTED(NIX_GROUP, ["$NIX_GROUP"], [Nix group])
+
+                                                                                                            
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    externals/Makefile
@@ -143,13 +191,16 @@ AC_CONFIG_FILES([Makefile
    src/nix-instantiate/Makefile
    src/nix-env/Makefile
    src/log2xml/Makefile
+   src/bsdiff-4.2/Makefile
    scripts/Makefile
    corepkgs/Makefile
-   corepkgs/fetchurl/Makefile
    corepkgs/nar/Makefile
    corepkgs/buildenv/Makefile
    corepkgs/channels/Makefile
    doc/Makefile
    doc/manual/Makefile
+   misc/Makefile
+   misc/emacs/Makefile
+   tests/Makefile
   ])
 AC_OUTPUT

corepkgs/Makefile.am

@@ -1 +1 @@
-SUBDIRS = fetchurl nar buildenv channels
+SUBDIRS = nar buildenv channels

corepkgs/buildenv/builder.pl.in

@@ -24,9 +24,11 @@ sub createLinks {
         my $dstFile = "$dstDir/$baseName";
         
 	if ($srcFile =~ /\/propagated-build-inputs$/ ||
-            $srcFile =~ /\/nix-support$/)
+            $srcFile =~ /\/nix-support$/ ||
+            $srcFile =~ /\/perllocal.pod$/ ||
+            $srcFile =~ /\/log$/)
         {
-            # Do noting.
+            # Do nothing.
 	}
 
         elsif (-d $srcFile) {
@@ -71,13 +73,27 @@ sub createLinks {
 
 my %done;
 
+sub addPkg;
 sub addPkg {
     my $pkgDir = shift;
 
     return if (defined $done{$pkgDir});
     $done{$pkgDir} = 1;
 
+    print "adding $pkgDir\n";
     createLinks("$pkgDir", "$out");
+
+    my $propagatedFN = "$pkgDir/nix-support/propagated-build-inputs";
+    if (-e $propagatedFN) {
+        open PROP, "<$propagatedFN" or die;
+        my $propagated = <PROP>;
+        close PROP;
+        my @propagated = split ' ', $propagated;
+        foreach my $p (@propagated) {
+            addPkg $p;
+        }
+    }
+        
 }
 
 
@@ -85,7 +101,6 @@ my @args = split ' ', $ENV{"derivations"};
 
 while (scalar @args > 0) {
     my $drvPath = shift @args;
-    print "adding $drvPath\n";
     addPkg($drvPath);
 }
 

corepkgs/channels/unpack.sh.in

@@ -1,9 +1,7 @@
 #! @shell@ -e
 
-export PATH=/bin:/usr/bin # !!! impure
-
-mkdir $out
-mkdir $out/tmp
+@coreutils@/mkdir $out
+@coreutils@/mkdir $out/tmp
 cd $out/tmp
 
 expr=$out/default.nix
@@ -12,8 +10,8 @@ echo '[' > $expr
 nr=0
 for i in $inputs; do
     echo "unpacking $i"
-    @bunzip2@ < $i | tar xvf -
-    mv * ../$nr # !!! hacky
+    @bunzip2@ < $i | @tar@ xf -
+    @coreutils@/mv * ../$nr # !!! hacky
     echo "(import ./$nr)" >> $expr
     nr=$(($nr + 1))
 done
@@ -21,4 +19,4 @@ done
 echo ']' >> $expr
 
 cd ..
-rmdir tmp
+@coreutils@/rmdir tmp

corepkgs/fetchurl/Makefile.am

@@ -1,11 +0,0 @@
-all-local: builder.sh
-
-install-exec-local:
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-	$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
-
-include ../../substitute.mk
-
-EXTRA_DIST = default.nix builder.sh.in

corepkgs/fetchurl/builder.sh.in

@@ -1,19 +0,0 @@
-#! @shell@ -e
-
-export PATH=/bin:/usr/bin
-
-echo "downloading $url into $out"
-
-prefetch=@storedir@/nix-prefetch-url-$md5
-if test -f "$prefetch"; then
-    echo "using prefetched $prefetch";
-    mv $prefetch $out
-else
-    @curl@ --fail --location --max-redirs 20 "$url" > "$out"
-fi
-
-actual=$(@bindir@/nix-hash --flat $out)
-if test "$actual" != "$md5"; then
-    echo "hash is $actual, expected $md5"
-    exit 1
-fi

corepkgs/fetchurl/default.nix

@@ -1,8 +0,0 @@
-{system, url, md5}:
-
-derivation {
-  name = baseNameOf (toString url);
-  builder = ./builder.sh;
-  id = md5;
-  inherit system url md5;
-}

corepkgs/nar/Makefile.am

@@ -1,13 +1,11 @@
-all-local: nar.sh unnar.sh
+all-local: nar.sh
 
 install-exec-local:
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
 	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_DATA) nar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
 	$(INSTALL_PROGRAM) nar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_DATA) unnar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
-	$(INSTALL_PROGRAM) unnar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
 
 include ../../substitute.mk
 
-EXTRA_DIST = nar.nix nar.sh.in unnar.nix unnar.sh.in
+EXTRA_DIST = nar.nix nar.sh.in

corepkgs/nar/nar.nix

@@ -1,6 +1,5 @@
-{system, path}: derivation {
+{system, path, hashAlgo}: derivation {
   name = "nar";
   builder = ./nar.sh;
-  system = system;
-  path = path;
+  inherit system path hashAlgo;
 }

corepkgs/nar/nar.sh.in

@@ -1,13 +1,14 @@
 #! @shell@ -e
 
-# !!! impure; fix this
-export PATH=/bin:/usr/bin
-
 echo "packing $path into $out..."
-mkdir $out
-dst=$out/$(basename $path).nar.bz2
-@bindir@/nix-store --dump "$path" | @bzip2@ > $dst
+@coreutils@/mkdir $out
+dst=$out/tmp.nar.bz2
+@bindir@/nix-store --dump "$path" > tmp
 
-md5=$(md5sum -b $dst | cut -c1-32)
-if test $? != 0; then exit 1; fi
-echo $md5 > $out/md5
+@bzip2@ < tmp > $dst
+
+@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
+
+@bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
+
+@coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2

corepkgs/nar/unnar.nix

@@ -1,7 +0,0 @@
-{system, narFile, outPath}: derivation {
-  name = "unnar";
-  builder = ./unnar.sh;
-  system = system;
-  narFile = narFile;
-  outPath = outPath;
-}

corepkgs/nar/unnar.sh.in

@@ -1,4 +0,0 @@
-#! @shell@ -e
-
-echo "unpacking $narFile to $out..."
-@bunzip2@ < $narFile | @bindir@/nix-store --restore "$out"

doc/manual/Makefile.am

@@ -1,32 +1,41 @@
-ENV = SGML_CATALOG_FILES=$(docbookcatalog):$(docbookebnfcatalog)
+ENV = SGML_CATALOG_FILES=$(docbookcatalog)
 
 XMLLINT = $(ENV) $(xmllint) $(xmlflags) --catalogs
 XSLTPROC = $(ENV) $(xsltproc) $(xmlflags) --catalogs \
  --param section.autolabel 1 \
  --param section.label.includes.component.label 1 \
- --param html.stylesheet \'style.css\'
+ --param html.stylesheet \'style.css\' \
+ --param xref.with.number.and.title 1 \
+ --param toc.section.depth 3
 
-man1_MANS = nix-env.1 nix-store.1 nix-instantiate.1 \
+man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
- nix-prefetch-url.1
+ nix-prefetch-url.1 nix-channel.1
 
-SOURCES = manual.xml introduction.xml installation.xml overview.xml \
+FIGURES = figures/user-environments.png
+
+MANUAL_SRCS = manual.xml introduction.xml installation.xml \
+ package-management.xml writing-nix-expressions.xml \
+ build-farm.xml \
  $(man1_MANS:.1=.xml) \
  troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
- quick-start.xml nix-lang-ref.xml style.css images
+ env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
+ conf-file.xml \
+ style.css images
 
-manual.is-valid: $(SOURCES) version.xml
-	$(XMLLINT) --noout --valid manual.xml
+manual.is-valid: $(MANUAL_SRCS) version.txt
+	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --valid -
 	touch $@
 
-version.xml:
-	echo -n $(VERSION) > version.xml
+version.txt:
+	echo -n $(VERSION) > version.txt
 
-man $(MANS): $(SOURCES) manual.is-valid
-	$(XSLTPROC) $(docbookxsl)/manpages/docbook.xsl manual.xml
+man $(MANS): $(MANUAL_SRCS) manual.is-valid
+	$(XSLTPROC) --nonet --xinclude $(docbookxsl)/manpages/docbook.xsl manual.xml
 
-manual.html: $(SOURCES) manual.is-valid images
-	$(XSLTPROC) --output manual.html $(docbookxsl)/html/docbook.xsl manual.xml
+manual.html: $(MANUAL_SRCS) manual.is-valid images
+	$(XSLTPROC) --nonet --xinclude --output manual.html \
+	  $(docbookxsl)/html/docbook.xsl manual.xml
 
 all-local: manual.html
 
@@ -35,6 +44,8 @@ install-data-local: manual.html
 	$(INSTALL_DATA) manual.html $(DESTDIR)$(datadir)/nix/manual
 	$(INSTALL_DATA) style.css $(DESTDIR)$(datadir)/nix/manual
 	cp -r images $(DESTDIR)$(datadir)/nix/manual/images
+	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/manual/figures
+	$(INSTALL_DATA) $(FIGURES) $(DESTDIR)$(datadir)/nix/manual/figures
 
 images:
 	mkdir images
@@ -43,8 +54,8 @@ images:
 	cp $(docbookxsl)/images/callouts/*.png images/callouts
 	chmod +w -R images
 
-KEEP = manual.html manual.is-valid version.xml $(MANS)
+KEEP = manual.html manual.is-valid version.txt $(MANS)
 
-EXTRA_DIST = $(SOURCES) $(KEEP)
+EXTRA_DIST = $(MANUAL_SRCS) $(FIGURES) $(KEEP)
 
 DISTCLEANFILES = $(KEEP)

doc/manual/bugs.xml

@@ -1,115 +1,46 @@
-<appendix>
-  <title>Bugs / To-Do</title>
+<appendix><title>Bugs / To-Do</title>
 
-  <itemizedlist>
+<itemizedlist>
 
-    <listitem>
-      <para>
-        The man-pages generated from the DocBook documentation are ugly.
-      </para>
-    </listitem>
+<listitem><para>The man-pages generated from the DocBook documentation
+are ugly.</para></listitem>
 
-    <listitem>
-      <para>
-        Generations properly form a tree.  E.g., if after switching to
-        generation 39, we perform an installation action, a generation
-        43 is created which is a descendant of 39, not 42.  So a
-        rollback from 43 ought to go back to 39.  This is not
-        currently implemented; generations form a linear sequence.
-      </para>
-    </listitem>
+<listitem><para>Generations properly form a tree.  E.g., if after
+switching to generation 39, we perform an installation action, a
+generation 43 is created which is a descendant of 39, not 42.  So a
+rollback from 43 ought to go back to 39.  This is not currently
+implemented; generations form a linear sequence.</para></listitem>
 
-    <listitem>
-      <para>
-        Unify the concepts of successors and substitutes into a
-        general notion of <emphasis>equivalent expressions</emphasis>.
-        Expressions are equivalent if they have the same target paths
-        with the same identifiers.  However, even though they are
-        functionally equivalent, they may differ stronly with respect
-        to their <emphasis>performance characteristics</emphasis>.
-        For example, realising a closure expression is more efficient
-        that realising the derivation expression from which it was
-        produced.  On the other hand, distributing sources may be more
-        efficient (storage- or bandwidth-wise) than distributing
-        binaries.  So we need to be able to attach weigths or
-        priorities or performance annotations to expressions; Nix can
-        then choose the most efficient expression dependent on the
-        context.
-      </para>
-    </listitem>
+<listitem><para><emphasis>Build management.</emphasis> In principle it
+is already possible to do build management using Nix (by writing
+builders that perform appropriate build steps), but the Nix expression
+language is not yet powerful enough to make this pleasant (?).  The
+language should be extended with features from the <ulink
+url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
+Another interesting idea is to write a <command>make</command>
+implementation that uses Nix as a back-end to support <ulink
+url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink>
+build files.</para></listitem>
 
-    <listitem>
-      <para>
-        <emphasis>Build management.</emphasis>  In principle it is already
-        possible to do build management using Nix (by writing builders that
-        perform appropriate build steps), but the Nix expression language is
-        not yet powerful enough to make this pleasant (?).  The language should
-        be extended with features from the <ulink
-          url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
-        Another interesting idea is to write a <command>make</command>
-        implementation that uses Nix as a back-end to support <ulink
-          url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink> 
-        build files.
-      </para>
-    </listitem>
+<listitem><para>For security, <command>nix-push</command> manifests
+should be digitally signed, and <command>nix-pull</command> should
+verify the signatures.  The actual NAR archives in the cache do not
+need to be signed, since the manifest contains cryptographic hashes of
+these files (and <filename>fetchurl.nix</filename> checks
+them).</para></listitem>
 
-    <listitem>
-      <para>
-        The current garbage collector is a hack.  It should be
-        integrated into <command>nix-store</command>.  It should
-        delete derivations in an order determined by topologically
-        sorting derivations under the points-to relation.  This
-        ensures that no store paths ever exist that point to
-        non-existant store paths.
-      </para>
-    </listitem>
+<listitem><para>It would be useful to have an option in
+<command>nix-env --delete-generations</command> to remove non-current
+generations older than a certain age.</para></listitem>
 
-    <listitem>
-      <para>
-        There are race conditions between the garbage collector and
-        other Nix tools.  For instance, when we run
-        <command>nix-env</command> to build and install a derivation
-        and run the garbage collector at the same time, the garbage
-        collector may kick in exactly between the build and
-        installation steps, i.e., before the newly built derivation
-        has become reachable from a root of the garbage collector.
-      </para>
+<listitem><para>There should be a flexible way to change the user
+environment builder.  Currently, you have to replace
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>,
+which is hard-coded into <command>nix-env</command>.  Also, the
+default builder should be more powerful.  For instance, there should
+be some way to specify priorities to resolve
+collisions.</para></listitem>
 
-      <para>
-        One solution would be for these programs to properly register
-        temporary roots for the collector.  Another would be to use
-        stop-the-world garbage collection: if any tool is running, the
-        garbage collector blocks, and vice versa.  These solutions do
-        not solve the situation where multiple tools are involved,
-        e.g.,
-
-        <screen>
-$ nix-store -r $(nix-instantiate foo.nix)</screen>
-
-        since even if <command>nix-instantiate</command> where to
-        register a temporary root, it would be released by the time
-        <command>nix-store</command> is started.  A solution would be
-        to write the intermediate value to a file that is used as a
-        root to the collector, e.g.,
-        
-        <screen>
-$ nix-instantiate foo.nix > /nix/var/nix/roots/bla
-$ nix-store -r $(cat /nix/var/nix/roots/bla)</screen>
-
-      </para>
-    </listitem>
-
-    <listitem>
-      <para>
-        For security, <command>nix-push</command> manifests should be
-        digitally signed, and <command>nix-pull</command> should
-        verify the signatures.  The actual NAR archives in the cache
-        do not need to be signed, since the manifest contains
-        cryptographic hashes of these files (and
-        <filename>fetchurl.nix</filename> checks them).
-      </para>
-    </listitem>
-
-  </itemizedlist>
+</itemizedlist>
 
 </appendix>

doc/manual/build-farm.xml

@@ -0,0 +1,132 @@
+<chapter id='chap-build-farm'><title>Setting up a Build Farm</title>
+
+<para>This chapter provides some sketchy information on how to set up
+a Nix-based build farm.  Nix is particularly suited as a basis for a
+build farm, since:
+
+<itemizedlist>
+
+  <listitem><para>Nix supports distributed builds: a local Nix
+  installation can forward Nix builds to other machines over the
+  network.  This allows multiple builds to be performed in parallel
+  (thus improving performance), but more in importantly, it allows Nix
+  to perform multi-platform builds in a semi-transparent way.  For
+  instance, if you perform a build for a
+  <literal>powerpc-darwin</literal> on an
+  <literal>i686-linux</literal> machine, Nix can automatically forward
+  the build to a <literal>powerpc-darwin</literal> machine, if
+  available.</para></listitem>
+
+  <listitem><para>The Nix expression language is ideal for describing
+  build jobs, plus all their dependencies.  For instance, if your
+  package has some dependency, you don't have to manually install it
+  on all the machines in the build farm; they will be built
+  automatically.</para></listitem>
+
+  <listitem><para>Proper release management requires that builds (if
+  deployed) are traceable: it should be possible to figure out from
+  exactly what sources they were built, in what configuration, etc.;
+  and it should be possible to reproduce the build, if necessary.  Nix
+  makes this possible since Nix's hashing scheme uniquely identifies
+  builds, and Nix expressions are self-contained.</para></listitem>
+
+  <listitem><para>Nix will only rebuild things that have actually
+  changed.  For instance, if the sources of a component haven't
+  changed between runs of the build farm, the component won't be
+  rebuild (unless it was garbage-collected).  Also, dependencies
+  typically don't change very often, so they only need to be built
+  once.</para></listitem>
+
+  <listitem><para>The results of a Nix build farm can be made
+  available through a channel, so successful builds can be deployed to
+  users immediately.</para></listitem>
+
+</itemizedlist>
+
+</para>
+
+
+<section><title>Overview</title>
+
+<para>TODO</para>
+
+<para>The sources of the Nix build farm are at <ulink
+url='https://svn.cs.uu.nl:12443/repos/trace/release/trunk' />.</para>
+
+</section>
+
+
+<section id='sec-distributed-builds'><title>Setting up distributed builds</title>
+
+<para>You can enable distributed builds by setting the environment
+variable <envar>NIX_BUILD_HOOK</envar> to point to a program that Nix
+will call whenever it wants to build a derivation.  The build hook
+(typically a shell or Perl script) can decline the build, in which Nix
+will perform it in the usual way if possible, or it can accept it, in
+which case it is responsible for somehow getting the inputs of the
+build to another machine, doing the build there, and getting the
+results back.  The details of the build hook protocol are described in
+the documentation of the <link
+linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar>
+variable</link>.</para>
+
+<example id='ex-remote-systems'><title>Remote machine configuration:
+<filename>remote-systems.conf</filename></title>
+<programlisting>
+nix@mcflurry.labs.cs.uu.nl  powerpc-darwin  /home/nix/.ssh/id_quarterpounder_auto  2
+nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto        1
+</programlisting>
+</example>
+
+<para>An example build hook can be found in the Nix build farm
+sources: <ulink
+url='https://svn.cs.uu.nl:12443/repos/trace/release/trunk/common/distributed/build-remote.pl'
+/>.  It should be suitable for most purposes, with maybe some minor
+adjustments.  It uses <command>ssh</command> and
+<command>rsync</command> to copy the build inputs and outputs and
+perform the remote build.  You should define a list of available build
+machines and set the environment variable
+<envar>REMOTE_SYSTEMS</envar> to point to it.  An example
+configuration is shown in <xref linkend='ex-remote-systems' />.  Each
+line in the file specifies a machine, with the following bits of
+information:
+
+<orderedlist>
+  
+  <listitem><para>The name of the remote machine, with optionally the
+  user under which the remote build should be performed.  This is
+  actually passed as an argument to <command>ssh</command>, so it can
+  be an alias defined in your
+  <filename>~/.ssh/config</filename>.</para></listitem>
+
+  <listitem><para>The Nix platform type identifier, such as
+  <literal>powerpc-darwin</literal>.</para></listitem>
+
+  <listitem><para>The SSH private key to be used to log in to the
+  remote machine.  Since builds should be non-interactive, this key
+  should not have a passphrase!</para></listitem>
+
+  <listitem><para>The maximum <quote>load</quote> of the remote
+  machine.  This is just the maximum number of jobs that
+  <filename>build-remote.pl</filename> will execute in parallel on the
+  machine.  Typically this should be equal to the number of
+  CPUs.</para></listitem>
+
+</orderedlist>
+
+You should also set up the environment variable
+<envar>CURRENT_LOAD</envar> to point at a file that
+<filename>build-remote.pl</filename> uses to remember how many jobs it
+is currently executing remotely.  It doesn't look at the actual load
+on the remote machine, so if you have multiple instances of Nix
+running, they should use the same <envar>CURRENT_LOAD</envar>
+file<footnote><para>Although there are probably some race conditions
+in the script right now.</para></footnote>.  Maybe in the future
+<filename>build-remote.pl</filename> will look at the actual remote
+load.  The load file should exist, so you should just create it as an
+empty file initially.</para>
+  
+</section>
+
+
+</chapter>

doc/manual/conf-file.xml

@@ -0,0 +1,82 @@
+<sect1 id="sec-conf-file"><title>Nix configuration file</title>
+
+<para>A number of persistent settings of Nix are stored in the file
+<filename><replaceable>prefix</replaceable>/etc/nix/nix.conf</filename>.
+This file is a list of <literal><replaceable>name</replaceable> =
+<replaceable>value</replaceable></literal> pairs, one per line.
+Comments start with a <literal>#</literal> character.  An example
+configuration file is shown in <xref linkend="ex-nix-conf" />.</para>
+
+<example id='ex-nix-conf'><title>Nix configuration file</title>
+
+<programlisting>
+gc-keep-outputs = true       # Nice for developers
+gc-keep-derivations = true   # Idem
+env-keep-derivations = false
+</programlisting>
+</example>
+
+<para>The following variables are currently available: 
+
+<variablelist>
+
+  <varlistentry id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
+
+    <listitem><para>If <literal>true</literal>, the garbage collector
+    will keep the outputs of non-garbage derivations.  If
+    <literal>false</literal> (default), outputs will be deleted unless
+    they are GC roots themselves (or reachable from other roots).</para>
+ 
+    <para>In general, outputs must be registered as roots separately.
+    However, even if the output of a derivation is registered as a
+    root, the collector will still delete store paths that are used
+    only at build time (e.g., the C compiler, or source tarballs
+    downloaded from the network).  To prevent it from doing so, set
+    this option to <literal>true</literal>.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>true</literal> (default), the garbage
+    collector will keep the derivations from which non-garbage store
+    paths were built.  If <literal>false</literal>, they will be
+    deleted unless explicitly registered as a root (or reachable from
+    other roots).</para>
+
+    <para>Keeping derivation around is useful for querying and
+    traceability (e.g., it allows you to ask with what dependencies or
+    options a store path was built), so by default this option is on.
+    Turn it off to safe a bit of disk space (or a lot if
+    <literal>gc-keep-outputs</literal> is also turned on).</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><literal>env-keep-derivations</literal></term>
+
+    <listitem><para>If <literal>false</literal> (default), derivations
+    are not stored in Nix user environments.  That is, the derivation
+    any build-time-only dependencies may be garbage-collected.</para>
+
+    <para>If <literal>true</literal>, when you add a Nix derivation to
+    a user environment, the path of the derivation is stored in the
+    user environment.  Thus, the derivation will not be
+    garbage-collected until the user environment generation is deleted
+    (<command>nix-env --delete-generations</command>).  To prevent
+    build-time-only dependencies from being collected, you should also
+    turn on <literal>gc-keep-outputs</literal>.</para>
+
+    <para>The difference between this option and
+    <literal>gc-keep-derivations</literal> is that this one is
+    “sticky”: it applies to any user environment created while this
+    option was enabled, while <literal>gc-keep-derivations</literal>
+    only applies at the moment the garbage collector is
+    run.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+</sect1>

doc/manual/env-common.xml

@@ -0,0 +1,274 @@
+<sect1 id="sec-common-env"><title>Common environment variables</title>
+
+<para>Most Nix commands interpret the following environment variables:</para>
+
+<variablelist>
+
+  
+<varlistentry><term><envar>NIX_ROOT</envar></term>
+
+  <listitem><para>If <envar>NIX_ROOT</envar> is set, the Nix command
+  will on startup perform a <function>chroot()</function> to the
+  specified directory.  This is useful in certain bootstrapping
+  situations (e.g., when installing a Nix installation onto a hard
+  disk from CD-ROM).</para></listitem>
+
+</varlistentry>
+
+  
+<varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term>
+
+  <listitem>
+
+  <para>Normally, the Nix store directory (typically
+  <filename>/nix/store</filename>) is not allowed to contain any
+  symlink components.  This is to prevent “impure” builds.  Builders
+  sometimes “canonicalise” paths by resolving all symlink components.
+  Thus, builds on different machines (with
+  <filename>/nix/store</filename> resolving to different locations)
+  could yield different results.  This is generally not a problem,
+  except when builds are deployed to machines where
+  <filename>/nix/store</filename> resolves differently.  If you are
+  sure that you’re not going to do that, you can set
+  <envar>NIX_IGNORE_SYMLINK_STORE</envar> to <envar>1</envar>.</para>
+
+  <para>Note that if you’re symlinking the Nix store so that you can
+  put it on another file system than the root file system, on Linux
+  you’re better off using <literal>bind</literal> mount points, e.g.,
+
+  <screen>
+$ mkdir /nix   
+$ mount -o bind /mnt/otherdisk/nix /nix</screen>
+
+  Consult the <citerefentry><refentrytitle>mount</refentrytitle>
+  <manvolnum>8</manvolnum></citerefentry> manual page for details.</para>
+
+  </listitem>
+
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STORE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix store (default
+  <filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DATA_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix static data
+  directory (default
+  <filename><replaceable>prefix</replaceable>/share</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix log directory
+  (default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_STATE_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix state directory
+  (default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_DB_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix database (default
+  <filename><replaceable>$NIX_STATE_DIR</replaceable>/db</filename>, i.e.,
+  <filename><replaceable>prefix</replaceable>/var/nix/db</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_CONF_DIR</envar></term>
+
+  <listitem><para>Overrides the location of the Nix configuration
+  directory (default
+  <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><envar>NIX_LOG_TYPE</envar></term>
+
+  <listitem><para>Equivalent to the <link
+  linkend="opt-log-type"><option>--log-type</option>
+  option</link>.</para></listitem>
+
+</varlistentry>
+  
+
+<varlistentry><term><envar>TMPDIR</envar></term>
+
+  <listitem><para>Use the specified directory to store temporary
+  files.  In particular, this includes temporary build directories;
+  these can take up substantial amounts of disk space.  The default is
+  <filename>/tmp</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term>
+
+  <listitem>
+
+  <para>Specifies the location of the <emphasis>build hook</emphasis>,
+  which is a program (typically some script) that Nix will call
+  whenever it wants to build a derivation.  This is used to implement
+  distributed builds (see <xref linkend="sec-distributed-builds"
+  />).  The protocol by which the calling Nix process and the build
+  hook communicate is as follows.</para>
+
+  <para>The build hook is called with the following command-line
+  arguments:
+
+  <orderedlist>
+
+    <listitem><para>A boolean value <literal>0</literal> or
+    <literal>1</literal> specifying whether Nix can locally execute
+    more builds, as per the <link
+    linkend="opt-max-jobs"><option>--max-jobs</option> option</link>.
+    The purpose of this argument is to allow the hook to not have to
+    maintain bookkeeping for the local machine.</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the local machine
+    (e.g., <literal>i686-linux</literal>).</para></listitem>
+
+    <listitem><para>The Nix platform identifier for the derivation,
+    i.e., its <link linkend="attr-system"><varname>system</varname>
+    attribute</link>.</para></listitem>
+
+    <listitem><para>The store path of the derivation.</para></listitem>
+
+  </orderedlist>
+
+  </para>
+
+  <para>On the basis of this information, and whatever persistent
+  state the build hook keeps about other machines and their current
+  load, it has to decide what to do with the build.  It should print
+  out on file descriptor 3 one of the following responses (terminated
+  by a newline, <literal>"\n"</literal>):
+
+  <variablelist>
+
+    <varlistentry><term><literal>decline</literal></term>
+
+      <listitem><para>The build hook is not willing or able to perform
+      the build; the calling Nix process should do the build itself,
+      if possible.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>postpone</literal></term>
+
+      <listitem><para>The build hook cannot perform the build now, but
+      can do so in the future (e.g., because all available build slots
+      on remote machines are in use).  The calling Nix process should
+      postpone this build until at least one currently running build
+      has terminated.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>accept</literal></term>
+
+      <listitem><para>The build hook has accepted the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the build hook accepts the build, it is possible that it is
+  no longer necessary to do the build because some other process has
+  performed the build in the meantime.  To prevent races, the hook
+  must read from file descriptor 4 a single line that tells it whether
+  to continue:
+
+  <variablelist>
+
+    <varlistentry><term><literal>cancel</literal></term>
+
+      <listitem><para>The build has already been done, so the hook
+      should exit.</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><literal>okay</literal></term>
+
+      <listitem><para>The hook should proceed with the build.  At this
+      point, the calling Nix process has acquired locks on the output
+      path, so no other Nix process will perform the
+      build.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>If the hook has been told to proceed, Nix will store in the
+  hook’s current directory a number of text files that contain
+  information about the derivation:
+
+  <variablelist>
+
+    <varlistentry><term><filename>inputs</filename></term>
+
+      <listitem><para>The set of store paths that are inputs to the
+      build process (one per line).  These have to be copied
+      <emphasis>to</emphasis> the remote machine (in addition to the
+      store derivation itself).</para></listitem>
+
+    </varlistentry>
+  
+    <varlistentry><term><filename>outputs</filename></term>
+
+      <listitem><para>The set of store paths that are outputs of the
+      derivation (one per line).  These have to be copied
+      <emphasis>from</emphasis> the remote machine if the build
+      succeeds.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><filename>references</filename></term>
+
+      <listitem><para>The reference graph of the inputs, in the format
+      accepted by the command <command>nix-store
+      --register-validity</command>.  It is necessary to run this
+      command on the remote machine after copying the inputs to inform
+      Nix on the remote machine that the inputs are valid
+      paths.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>
+
+  </para>
+
+  <para>The hook should copy the inputs to the remote machine,
+  register the validity of the inputs, perform the remote build, and
+  copy the outputs back to the local machine.  An exit code other than
+  <literal>0</literal> indicates that the hook has failed.</para>
+
+  </listitem>
+
+
+</varlistentry>
+
+
+</variablelist>
+
+</sect1>

doc/manual/glossary.xml

@@ -0,0 +1,163 @@
+<appendix><title>Glossary</title>
+
+<glosslist>
+
+
+<glossentry id="gloss-derivation"><glossterm>derivation</glossterm>
+
+  <glossdef><para>A description of a build action.  The result of a
+  derivation is a store object.  Derivations are typically specified
+  in Nix expressions using the <link
+  linkend="ssec-derivation"><function>derivation</function>
+  primitive</link>.  These are translated into low-level
+  <emphasis>store derivations</emphasis> (implicitly by
+  <command>nix-env</command> and <command>nix-build</command>, or
+  explicitly by <command>nix-instantiate</command>).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store</glossterm>
+
+  <glossdef><para>The location in the file system where store objects
+  live.  Typically <filename>/nix/store</filename>.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store path</glossterm>
+
+  <glossdef><para>The location in the file system of a store object,
+  i.e., an immediate child of the Nix store
+  directory.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>store object</glossterm>
+
+  <glossdef><para>A file that is an immediate child of the Nix store
+  directory.  These can be regular files, but also entire directory
+  trees.  Store objects can be sources (objects copied from outside of
+  the store), derivation outputs (objects produced by running a build
+  action), or derivations (files describing a build
+  action).</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-substitute"><glossterm>substitute</glossterm>
+
+  <glossdef><para>A substitute is a command invocation stored in the
+  Nix database that describes how to build a store object, bypassing
+  normal the build mechanism (i.e., derivations).  Typically, the
+  substitute builds the store object by downloading a pre-built
+  version of the store object from some server.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>purity</glossterm>
+
+  <glossdef><para>The assumption that equal Nix derivations when run
+  always produce the same output.  This cannot be guaranteed in
+  general (e.g., a builder can rely on external inputs such as the
+  network or the system time) but the Nix model assumes
+  it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry><glossterm>Nix expression</glossterm>
+
+  <glossdef><para>A high-level description of software components and
+  compositions thereof.  Deploying software using Nix entails writing
+  Nix expressions for your components.  Nix expressions are translated
+  to derivations that are stored in the Nix store.  These derivations
+  can then be built.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-reference"><glossterm>reference</glossterm>
+
+  <glossdef><para>A store path <varname>P</varname> is said to have a
+  reference to a store path <varname>Q</varname> if the store object
+  at <varname>P</varname> contains the path <varname>Q</varname>
+  somewhere.  This implies than an execution involving
+  <varname>P</varname> potentially needs <varname>Q</varname> to be
+  present.  The <emphasis>references</emphasis> of a store path are
+  the set of store paths to which it has a reference.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-closure"><glossterm>closure</glossterm>
+
+  <glossdef><para>The closure of a store path is the set of store
+  paths that are directly or indirectly “reachable” from that store
+  path; that is, it’s the closure of the path under the <link
+  linkend="gloss-reference">references</link> relation.  For instance,
+  if the store object at path <varname>P</varname> contains a
+  reference to path <varname>Q</varname>, then <varname>Q</varname> is
+  in the closure of <varname>P</varname>.  For correct deployment it
+  is necessary to deploy whole closures, since otherwise at runtime
+  files could be missing.  The command <command>nix-store
+  -qR</command> prints out closures of store paths.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-output-path"><glossterm>output path</glossterm>
+
+  <glossdef><para>A store path produced by a derivation.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-deriver"><glossterm>deriver</glossterm>
+
+  <glossdef><para>The deriver of an <link
+  linkend="gloss-output-path">output path</link> is the store
+  derivation that built it.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-validity"><glossterm>validity</glossterm>
+
+  <glossdef><para>A store path is considered
+  <emphasis>valid</emphasis> if it exists in the file system, is
+  listed in the Nix database as being valid, and if all paths in its
+  closure are also valid.</para></glossdef>
+
+</glossentry>
+
+
+<glossentry id="gloss-user-env"><glossterm>user environment</glossterm>
+
+  <glossdef><para>An automatically generated store object that
+  consists of a set of symlinks to “active” applications, i.e., other
+  store paths.  These are generated automatically by <link
+  linkend="sec-nix-env"><command>nix-env</command></link>.  See <xref
+  linkend="sec-profiles" />.</para>
+
+  </glossdef>
+  
+</glossentry>
+
+
+<glossentry id="gloss-profile"><glossterm>profile</glossterm>
+
+  <glossdef><para>A symlink to the current <link
+  linkend="gloss-user-env">user environment</link> of a user, e.g.,
+  <filename>/nix/var/nix/profiles/default</filename>.</para></glossdef>
+
+</glossentry>
+
+
+
+</glosslist>
+
+
+</appendix>

doc/manual/installation.xml

@@ -1,211 +1,212 @@
-<chapter id='chap-installation'>
-  <title>Installation</title>
+<chapter id='chap-installation'><title>Installation</title>
 
-  <sect1>
-    <title>Obtaining Nix</title>
 
-    <para>
-      The easiest way to obtain Nix is to download a <ulink
-      url='http://www.cs.uu.nl/groups/ST/Trace/Nix'>source
-      distribution</ulink>.  RPMs for SuSE and Red Hat are also
-      available.  These distributions are generated automatically.
-    </para>
+<sect1><title>Obtaining Nix</title>
 
-    <para>
-      Alternatively, the most recent sources of Nix can be obtained from its
-      <ulink url='https://svn.cs.uu.nl:12443/repos/trace/nix/trunk'>Subversion 
-        repository</ulink>.  For example, the following command will check out
-      the latest revision into a directory called <filename>nix</filename>:
-    </para>
+<para>The easiest way to obtain Nix is to download a <ulink
+url='http://www.cs.uu.nl/groups/ST/Trace/Nix'>source
+distribution</ulink>.  RPMs for Red Hat, SuSE, and Fedore Core are
+also available.</para>
 
-    <screen>
+<para>Alternatively, the most recent sources of Nix can be obtained
+from its <ulink
+url='https://svn.cs.uu.nl:12443/repos/trace/nix/trunk'>Subversion
+repository</ulink>.  For example, the following command will check out
+the latest revision into a directory called <filename>nix</filename>:</para>
+
+<screen>
 $ svn checkout https://svn.cs.uu.nl:12443/repos/trace/nix/trunk nix</screen>
 
-    <para>
-      Likewise, specific releases can be obtained from the <ulink
-        url='https://svn.cs.uu.nl:12443/repos/trace/nix/tags'>tags
-        directory</ulink> of the repository.  If you don't have Subversion, you
-      can also download an automatically generated <ulink
-        url='https://svn.cs.uu.nl:12443/dist/trace/'>compressed
-        tar-file</ulink> of the head revision of the trunk.
+<para>Likewise, specific releases can be obtained from the <ulink
+url='https://svn.cs.uu.nl:12443/repos/trace/nix/tags'>tags
+directory</ulink> of the repository.  If you don't have Subversion,
+you can also download an automatically generated <ulink
+url='https://svn.cs.uu.nl:12443/dist/trace/'>compressed
+tar-file</ulink> of the head revision of the trunk.</para>
+
+</sect1>
+
+
+<sect1><title>Prerequisites</title>
+
+<para>The following prerequisites only apply when you build from
+source.  Binary releases (e.g., RPMs) have no prerequisites.</para>
+
+<para>A fairly recent version of GCC/G++ is required.  Version 2.95
+and higher should work.</para>
+
+<para>To build this manual and the man-pages you need the
+<command>xmllint</command> and <command>xsltproc</command> programs,
+which are part of the <literal>libxml2</literal> and
+<literal>libxslt</literal> packages, respectively.  You also need the
+<ulink url='http://docbook.sourceforge.net/projects/xsl/'>DocBook XSL
+stylesheets</ulink> and optionally the <ulink
+url='http://www.oasis-open.org/docbook/xml/4.2/docbook-xml-4.2.zip'>
+DocBook XML 4.2 DTD</ulink>.  Note that these are only required if you
+modify the manual sources or when you are building from the Subversion
+repository.</para>
+
+<para>To build the parser, very <emphasis>recent</emphasis> versions
+of Bison and Flex are required.  (This is because Nix needs GLR
+support in Bison and reentrancy support in Flex.)  For Bison, you need
+version 1.875c or higher (1.875 does <emphasis>not</emphasis> work),
+which can be obtained from the <ulink
+url='ftp://alpha.gnu.org/pub/gnu/bison'>GNU FTP server</ulink>.  For
+Flex, you need version 2.5.31, which is available on <ulink
+url='http://lex.sourceforge.net/'>SourceForge</ulink>.  Slightly older
+versions may also work, but ancient versions like the ubiquitous
+2.5.4a won't.  Note that these are only required if you modify the
+parser or when you are building from the Subversion repository.</para>
+
+<para>Nix uses Sleepycat's Berkeley DB and CWI's ATerm library.  These
+are included in the Nix source distribution.  If you build from the
+Subversion repository, you must download them yourself and place them
+in the <filename>externals/</filename> directory.  See
+<filename>externals/Makefile.am</filename> for the precise URLs of
+these packages.  Alternatively, if you already have them installed,
+you can use <command>configure</command>'s <option>--with-bdb</option>
+and <option>--with-aterm</option> options to point to their respective
+locations.  Note that Berkeley DB <emphasis>must</emphasis> be version
+4.2; other versions may not have compatible database formats.</para>
+
+</sect1>
+
+
+<sect1><title>Building Nix from source</title>
+
+<para>After unpacking or checking out the Nix sources, issue the
+following commands:
     </para>
 
-  </sect1>
-
-  <sect1>
-    <title>Prerequisites</title>
-
-    <para>
-      The following prerequisites only apply when you build from
-      source.  Binary releases (e.g., RPMs) have no prerequisites.
-    </para>
-
-    <para>
-      A fairly recent version of GCC/G++ is required.  Version 2.95
-      and higher should work.
-    </para>
-
-    <para>
-      To build this manual and the man-pages you need the
-      <command>xmllint</command> and <command>xsltproc</command>
-      programs, which are part of the <literal>libxml2</literal> and
-      <literal>libxslt</literal> packages, respectively.  You also
-      need the <ulink
-      url='http://docbook.sourceforge.net/projects/xsl/'>DocBook XSL
-      stylesheets</ulink> and optionally the <ulink
-      url='http://www.oasis-open.org/docbook/xml/4.2/docbook-xml-4.2.zip'>
-      DocBook XML 4.2 DTD</ulink>.  Note that these are only required
-      if you modify the manual sources or when you are building from
-      the Subversion repository.
-    </para>
-
-    <para>
-      To build the parser, very <emphasis>recent</emphasis> versions
-      of Bison and Flex are required.  (This is because Nix needs GLR
-      support in Bison and reentrancy support in Flex.)  For Bison,
-      you need version 1.875c or higher (1.875 does
-      <emphasis>not</emphasis> work), which can be obtained from the
-      <ulink url='ftp://alpha.gnu.org/pub/gnu/bison'>GNU FTP
-      server</ulink>.  For Flex, you need version 2.5.31, which is
-      available on <ulink
-      url='http://lex.sourceforge.net/'>SourceForge</ulink>.  Slightly
-      older versions may also work, but ancient versions like the
-      ubiquitous 2.5.4a won't.  Note that these are only required if
-      you modify the parser or when you are building from the
-      Subversion repository.
-    </para>
-
-    <para>
-      Nix uses Sleepycat's Berkeley DB and CWI's ATerm library.  These
-      are included in the Nix source distribution.  If you build from
-      the Subversion repository, you must download them yourself and
-      place them in the <filename>externals/</filename> directory.
-      See <filename>externals/Makefile.am</filename> for the precise
-      URLs of these packages.
-    </para>
-  </sect1>
-
-  <sect1>
-    <title>Building Nix from source</title>
-
-    <para>
-      After unpacking or checking out the Nix sources, issue the following
-      commands:
-    </para>
-
-    <screen>
+<screen>
 $ ./configure <replaceable>options...</replaceable>
 $ make
 $ make install</screen>
 
-    <para>
-      When building from the Subversion repository, these should be preceded by
-      the command:
+<para>When building from the Subversion repository, these should be
+preceded by the command:
     </para>
 
-    <screen>
+<screen>
 $ autoreconf -i</screen>
 
-    <para>
-      The installation path can be specified by passing the
-      <option>--prefix=<replaceable>prefix</replaceable></option> to
-      <command>configure</command>.  The default installation directory is
-      <filename>/nix</filename>.  You can change this to any location you like.
-      You must have write permission to the <replaceable>prefix</replaceable>
-      path.
-    </para>
+<para>The installation path can be specified by passing the
+<option>--prefix=<replaceable>prefix</replaceable></option> to
+<command>configure</command>.  The default installation directory is
+<filename>/nix</filename>.  You can change this to any location you
+like.  You must have write permission to the
+<replaceable>prefix</replaceable> path.</para>
 
-    <warning>
-      <para>
-        It is advisable <emphasis>not</emphasis> to change the installation
-        prefix from its default, since doing so will in all likelihood make it
-        impossible to use derivations built on other systems.
-      </para>
-    </warning>
+<warning><para>It is advisable <emphasis>not</emphasis> to change the
+installation prefix from its default, since doing so will in all
+likelihood make it impossible to use derivations built on other
+systems.</para></warning>
 
-    <para>
-      If you want to rebuilt the documentation, pass the full path to the
-      DocBook XML catalog file (<filename>docbook.cat</filename>) and to the
-      DocBook XSL stylesheets using the
-      <option>--with-docbook-catalog=<replaceable>path</replaceable></option>
-      and <option>--with-docbook-xsl=<replaceable>path</replaceable></option>
-      options.
-    </para>
+<para>If you want to rebuilt the documentation, pass the full path to
+the DocBook XML catalog file (<filename>docbook.cat</filename>) and to
+the DocBook XSL stylesheets using the
+<option>--with-docbook-catalog=<replaceable>path</replaceable></option>
+and
+<option>--with-docbook-xsl=<replaceable>path</replaceable></option>
+options.</para>
 
-  </sect1>
+</sect1>
 
 
-  <sect1>
-    <title>Installing from RPMs</title>
+<sect1><title>Installing from RPMs</title>
 
-    <para>
-      RPM packages of Nix can be downloaded from <ulink
-      url='http://www.cs.uu.nl/groups/ST/Trace/Nix' />.  These RPMs
-      should work for most fairly recent releases of SuSE and Red Hat
-      Linux.  They have been known to work work on SuSE Linux 8.1 and
-      9.0, and Red Hat 9.0.  In fact, it should work on any RPM-based
-      Linux distribution based on <literal>glibc</literal> 2.3 or
-      later.
-    </para>
+<para>RPM packages of Nix can be downloaded from <ulink
+url='http://www.cs.uu.nl/groups/ST/Trace/Nix' />.  These RPMs should
+work for most fairly recent releases of SuSE and Red Hat Linux.  They
+have been known to work work on SuSE Linux 8.1 and 9.0, and Red Hat
+9.0.  In fact, it should work on any RPM-based Linux distribution
+based on <literal>glibc</literal> 2.3 or later.</para>
 
-    <para>
-      Once downloaded, the RPMs can be installed or upgraded using
-      <command>rpm -U</command>.  For example,
-    </para>
+<para>Once downloaded, the RPMs can be installed or upgraded using
+<command>rpm -U</command>.  For example,</para>
 
-    <screen>
-rpm -U nix-0.5pre664-1.i386.rpm</screen>
+<screen>
+$ rpm -U nix-0.5pre664-1.i386.rpm</screen>
 
-    <para>
-      The RPMs install into the directory <filename>/nix</filename>.
-      Nix can be uninstalled using <command>rpm -e nix</command>.
-      After this it will be necessary to manually remove the Nix store
-      and other auxiliary data:
-    </para>
+<para>The RPMs install into the directory <filename>/nix</filename>.
+Nix can be uninstalled using <command>rpm -e nix</command>.  After
+this it will be necessary to manually remove the Nix store and other
+auxiliary data:</para>
 
-    <screen>
-rm -rf /nix/store
-rm -rf /nix/var</screen>
+<screen>
+$ rm -rf /nix/store
+$ rm -rf /nix/var</screen>
 
-  </sect1>
+</sect1>
 
-  <sect1>
-    <title>Permissions</title>
 
-    <para>
-      All Nix operations must be performed under the user ID that owns
-      the Nix store and database
-      (<filename><replaceable>prefix</replaceable>/store</filename>
-      and
-      <filename><replaceable>prefix</replaceable>/var/nix/db</filename>,
-      respectively).  When installed from the RPM packages, these
-      directories are owned by <systemitem
-      class='username'>root</systemitem>.
-    </para>
+<sect1><title>Permissions</title>
 
-  </sect1>
+<para>All Nix operations must be performed under the user ID that owns
+the Nix store and database
+(<filename><replaceable>prefix</replaceable>/store</filename> and
+<filename><replaceable>prefix</replaceable>/var/nix/db</filename>,
+respectively).  When installed from the RPM packages, these
+directories are owned by <systemitem
+class='username'>root</systemitem>.</para>
 
-  <sect1>
-    <title>Using Nix</title>
+<sect2><title>Setuid installation</title>
 
-    <para>
-      To use Nix, some environment variables should be set.  In
-      particular, <envar>PATH</envar> should contain the directories
-      <filename><replaceable>prefix</replaceable>/bin</filename> and
-      <filename>~/.nix-profile/bin</filename>.  The first directory
-      contains the Nix tools themselves, while
-      <filename>~/.nix-profile</filename> is a symbolic link to the
-      current <emphasis>user environment</emphasis> (an automatically
-      generated package consisting of symlinks to installed packages).
-      The simplest way to set the required environment variables is to
-      include the file
-      <filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
-      in your <filename>~/.bashrc</filename> (or similar), like this:
-    </para>
+<para>As a somewhat <emphasis>ad hoc</emphasis> hack, you can also
+install the Nix binaries <quote>setuid</quote> so that a Nix store can
+be shared among several users.  To do this, configure Nix with the
+<emphasis>--enable-setuid</emphasis> option.  Nix will be installed as
+owned by a user and group specified by the
+<option>--with-nix-user=<parameter>user</parameter></option> and
+<option>--with-nix-group=<parameter>group</parameter></option>
+options.  E.g.,
 
-    <screen>
-. <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen>
+<screen>
+$ ./configure --enable-setuid --with-nix-user=my_nix_user --with-nix-group=my_nix_group</screen>
+
+The user and group default to <literal>nix</literal>.  You should make
+sure that both the user and the group exist.  Any <quote>real</quote>
+users that you want to allow access should be added to the Nix
+group.</para>
+
+<warning><para>A setuid installation should only by used if the users
+in the Nix group are mutually trusted, since any user in that group
+has the ability to change anything in the Nix store or database.  For
+instance, they could install a trojan horse in executables used by
+other users.</para></warning>
+
+<warning><para>On some platforms, the Nix binaries will be installed
+as setuid <literal>root</literal>.  They drop root privileges
+immediately after startup and switch to the Nix user.  The reason for
+this is that both the real and effective user must be set to the Nix
+user, and POSIX has no system call to do this.  This is not the case
+on systems that have the <function>setresuid()</function> system call
+(such as Linux and FreeBSD), so on those systems the binaries are
+simply owned by the Nix user.</para></warning>
+
+</sect2>
+
+</sect1>
+
+
+<sect1><title>Using Nix</title>
+
+<para>To use Nix, some environment variables should be set.  In
+particular, <envar>PATH</envar> should contain the directories
+<filename><replaceable>prefix</replaceable>/bin</filename> and
+<filename>~/.nix-profile/bin</filename>.  The first directory contains
+the Nix tools themselves, while <filename>~/.nix-profile</filename> is
+a symbolic link to the current <emphasis>user environment</emphasis>
+(an automatically generated package consisting of symlinks to
+installed packages).  The simplest way to set the required environment
+variables is to include the file
+<filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
+in your <filename>~/.bashrc</filename> (or similar), like this:</para>
+
+<screen>
+source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen>
+
+</sect1>
 
-  </sect1>
 
 </chapter>

doc/manual/introduction.xml

@@ -1,93 +1,150 @@
-<chapter>
-  <title>Introduction</title>
+<chapter><title>Introduction</title>
 
-  <epigraph>
-    <para><quote>The number of Nix installations in the world has grown to 5,
-        with more expected.</quote></para>
-  </epigraph>
+<!--
+<epigraph><para><quote>The number of Nix installations in the world
+has grown to 5, with more expected.</quote></para></epigraph>
+-->
 
-  <para>
-    Nix is a system for software deployment.  It supports the
-    creation and distribution of software packages, as well as the installation
-    and subsequent management of these on target machines (i.e., it is also a
-    package manager).
-  </para>
+<para>Nix is a system for the deployment of software.  Software
+deployment is concerned with the creation, distribution, and
+management of software components (<quote>packages</quote>).  Its main
+features are:
 
-  <para>
-    Nix solves some large problems that exist in most current deployment and
-    package management systems.  <emphasis>Dependency determination</emphasis>
-    is a big one: the correct installation of a software component requires
-    that all dependencies of that component (i.e., other components used by it)
-    are also installed.  Most systems have no way to verify that the specified
-    dependencies of a component are actually sufficient.
-  </para>
+<itemizedlist>
 
-  <para>
-    Another big problem is the lack of support for concurrent availability of
-    multiple <emphasis>variants</emphasis> of a component.  It must be possible
-    to have several versions of a component installed at the same time, or
-    several instances of the same version built with different parameters.
-    Unfortunately, components are in general not properly isolated from each
-    other.  For instance, upgrading a component that is a dependency for some
-    other component might break the latter.
-  </para>
+<listitem><para>It helps you make sure that dependency specifications
+are complete.  In general in a deployment system you have to specify
+for each component what its dependencies are, but there are no
+guarantees that this specification is complete.  If you forget a
+dependency, then the component will build and work correctly on
+<emphasis>your</emphasis> machine if you have the dependency
+installed, but not on the end user's machine if it's not
+there.</para></listitem>
 
-  <para>
-    Nix solves these problems by building and storing packages in paths that
-    are infeasible to predict in advance.  For example, the artifacts of a
-    package <literal>X</literal> might be stored in
-    <filename>/nix/store/d58a0606ed616820de291d594602665d-X</filename>, rather
-    than in, say, <filename>/usr/lib</filename>.  The path component
-    <filename>d58a...</filename> is actually a cryptographic hash of all the
-    inputs (i.e., sources, requisites, and build flags) used in building
-    <literal>X</literal>, and as such is very fragile: any change to the inputs
-    will change the hash.  Therefore it is not sensible to
-    <emphasis>hard-code</emphasis> such a path into the build scripts of a
-    package <literal>Y</literal> that uses <literal>X</literal> (as does happen
-    with <quote>fixed</quote> paths such as <filename>/usr/lib</filename>).
-    Rather, the build script of package <literal>Y</literal> is parameterised
-    with the actual location of <literal>X</literal>, which is supplied by the
-    Nix system.
-  </para>
+<listitem><para>It is possible to have <emphasis>multiple versions or
+variants</emphasis> of a component installed at the same time.  In
+contrast, in systems such as RPM different versions of the same
+package tend to install to the same location in the file system, so
+installing one version will remove the other.  This is especially
+important if you want to use applications that have conflicting
+requirements on different versions of a component (e.g., application A
+requires version 1.0 of library X, while application B requires a
+non-backwards compatible version 1.1).</para></listitem>
 
-  <para>
-    As stated above, the path name of a file system object contain a
-    cryptographic hash of all inputs involved in building it.  A change to any
-    of the inputs will cause the hash to change--and by extension, the path
-    name.  These inputs include both sources (variation in time) and
-    configuration options (variation in space).  Therefore variants of the same
-    package don't clash---they can co-exist peacefully within the same file
-    system.
-  </para>
+<listitem><para>Users can have different <quote>views</quote>
+(<quote>profiles</quote> in Nix parlance) on the set of installed
+applications in a system.  For instance, one user can have version 1.0
+of some package visible, while another is using version 1.1, and a
+third doesn't use it at all.</para></listitem>
 
-  <para>
-    Other features:
-  </para>
+<listitem><para>It is possible to atomically
+<emphasis>upgrade</emphasis> software.  I.e., there is no time window
+during an upgrade in which part of the old version and part of the new
+version are simultaneously visible (which might well cause the
+component to fail).</para></listitem>
 
-  <para>
-    <emphasis>Transparent source/binary deployment.</emphasis>
-  </para>
+<listitem><para>Likewise, it is possible to atomically roll back after
+an install, upgrade, or uninstall action.  That is, in a fast (O(1))
+operation the previous configuration of the system can be restored.
+This is because upgrade or uninstall actions don't actually remove
+components from the system.</para></listitem>
 
-  <para>
-    <emphasis>Unambiguous identification of configuration.</emphasis>
-  </para>
+<listitem><para>Unused components can be
+<emphasis>garbage-collected</emphasis> automatically and safely: when
+you remove an application from a profile, its dependencies will be
+deleted by the garbage collector only if there are no other active
+applications using them.</para></listitem>
 
-  <para>
-    <emphasis>Automatic storage management.</emphasis>
-  </para>
+<listitem><para>Nix supports both source-based deployment models
+(where you distribute <emphasis>Nix expressions</emphasis> that tell
+Nix how to build software from source) and binary-based deployment
+models.  The latter is more-or-less transparent: installation of
+components is always based on Nix expressions, but if the expressions
+have been built before and Nix knows that the resulting binaries are
+available somewhere, it will use those instead.</para></listitem>
 
-  <para>
-    <emphasis>Atomic upgrades and rollbacks.</emphasis>
-  </para>
+<listitem><para>Nix is flexible in the deployment policies that it
+supports.  There is a clear separation between the tools that
+implement basic Nix <emphasis>mechanisms</emphasis> (e.g., building
+Nix expressions), and the tools that implement various deployment
+<emphasis>policies</emphasis>.  For instance, there is a concept of
+<quote>Nix channels</quote> that can be used to keep software
+installations up-to-date automatically from a network source.  This is
+a policy that is implemented by a fairly short Perl script, which can
+be adapted easily to achieve similar policies.</para></listitem>
 
-  <para>
-    <emphasis>Support for many simultaneous configurations.</emphasis>
-  </para>
+<listitem><para>Nix component builds aim to be <quote>pure</quote>;
+that is, unaffected by anything other than the declared dependencies.
+This means that if a component was built successfully once, it can be
+rebuilt again on another machine and the result will be the same.  We
+cannot <emphasis>guarantee</emphasis> this (e.g., if the build depends
+on the time-of-day), but Nix (and the tools in the Nix Packages
+collection) takes special care to help achieve this.</para></listitem>
 
-  <para>
-    <emphasis>Portability.</emphasis>  Nix is quite portable.  Contrary to
-    build systems like those in, e.g., Vesta and ClearCase, it does not rely on
-    operating system extensions.
-  </para>
+<listitem><para>Nix expressions (the things that tell Nix how to build
+components) are self-contained: they describe not just components but
+complete compositions.  In other words, Nix expressions also describe
+how to build all the dependencies.  This is in contrast to component
+specification languages like RPM spec files, which might say that a
+component X depends on some other component Y, but since it does not
+describe <emphasis>exactly</emphasis> what Y is, the result of
+building or running X might be different on different machines.
+Combined with purity, self-containedness ensures that a component that
+<quote>works</quote> on one machine also works on another, when
+deployed using Nix.</para></listitem>
+
+<listitem><para>The Nix expression language makes it easy to describe
+variability in components (e.g., optional features or
+dependencies).</para></listitem>
+
+<listitem><para>Nix is ideal for building build farms that do
+continuous builds of software from a version management system, since
+it can take care of building all the dependencies as well.  Also, Nix
+only rebuilds components that have changed, so there are no
+unnecessary builds.  In addition, Nix can transparently distribute
+build jobs over different machines, including different
+platforms.</para></listitem>
+
+<listitem><para>Nix can be used not only for software deployment, but
+also for <emphasis>service deployment</emphasis>, such as the
+deployment of a complete web server with all its configuration files,
+static pages, software dependencies, and so on.  Nix's advantages for
+software deployment also apply here: for instance, the ability
+trivially to have multiple configurations at the same time, or the
+ability to do rollbacks.</para></listitem>
+
+<listitem><para>Nix can efficiently upgrade between different versions
+of a component through <emphasis>binary patching</emphasis>.  If
+patches are available on a server, and you try to install a new
+version of some component, Nix will automatically apply a patch (or
+sequence of patches), if available, to transform the installed
+component into the new version.</para></listitem> 
+
+</itemizedlist>
+
+</para>
+
+<para>This manual tells you how to install and use Nix and how to
+write Nix expressions for software not already in the Nix Packages
+collection.  It also discusses some advanced topics, such as setting
+up a Nix-based build farm, and doing service deployment using
+Nix.</para>
+
+<note><para>Some background information on Nix can be found in three
+papers.  The ICSE 2004 paper <ulink
+url='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'><citetitle>Imposing
+a Memory Management Discipline on Software
+Deployment</citetitle></ulink> discusses the hashing mechanism used to
+ensure reliable dependency identification and non-interference between
+different versions and variants of packages.  The LISA 2004 paper
+<ulink
+url='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'><citetitle>Nix:
+A Safe and Policy-Free System for Software
+Deployment</citetitle></ulink> gives a more general discussion of Nix
+from a system-administration perspective.  The CBSE 2005 paper <ulink
+url='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'><citetitle>Efficient
+Upgrading in a Purely Functional Component Deployment Model
+</citetitle></ulink> is about transparent patch deployment in
+Nix.</para></note>
 
 </chapter>

doc/manual/manual.xml

@@ -1,34 +1,16 @@
 <?xml version="1.0"?>
 <!DOCTYPE book
-    PUBLIC "-//OASIS//DTD DocBook EBNF Module V1.0//EN"
-    "http://www.oasis-open.org/docbook/xml/ebnf/1.0/dbebnf.dtd"
-[
-<!-- <!DOCTYPE book
     PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
-    "http://www.docbook.org/xml/4.3/docbook-xml-4.3.zip" -->
-<!ENTITY introduction SYSTEM "introduction.xml">
-<!ENTITY quick-start SYSTEM "quick-start.xml">
-<!ENTITY installation SYSTEM "installation.xml">
-<!ENTITY overview SYSTEM "overview.xml">
-<!ENTITY opt-common SYSTEM "opt-common.xml">
-<!ENTITY opt-common-syn SYSTEM "opt-common-syn.xml">
-<!ENTITY nix-env SYSTEM "nix-env.xml">
-<!ENTITY nix-store SYSTEM "nix-store.xml">
-<!ENTITY nix-instantiate SYSTEM "nix-instantiate.xml">
-<!ENTITY nix-collect-garbage SYSTEM "nix-collect-garbage.xml">
-<!ENTITY nix-push SYSTEM "nix-push.xml">
-<!ENTITY nix-pull SYSTEM "nix-pull.xml">
-<!ENTITY nix-prefetch-url SYSTEM "nix-prefetch-url.xml">
-<!ENTITY nix-lang-ref SYSTEM "nix-lang-ref.xml">
-<!ENTITY troubleshooting SYSTEM "troubleshooting.xml">
-<!ENTITY bugs SYSTEM "bugs.xml">
-<!ENTITY version SYSTEM "version.xml">
+    "http://www.docbook.org/xml/4.3/docbook-xml-4.3.zip"
+[
 ]>
 
 <book>
-  <title>Nix: A System for Software Deployment</title>
+  <title>Nix User's Guide</title>
 
-  <subtitle>Draft (Version &version;)</subtitle>
+  <subtitle>Draft (Version <xi:include
+  xmlns:xi="http://www.w3.org/2001/XInclude"
+  href="version.txt" parse="text" />)</subtitle>
   
   <bookinfo>
     <author>
@@ -37,50 +19,65 @@
     </author>
     <copyright>
       <year>2004</year>
+      <year>2005</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
   </bookinfo>
-
-  &introduction;
-  &quick-start;
-  &installation;
-  &overview;
+  
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="introduction.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="quick-start.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="installation.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="package-management.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="writing-nix-expressions.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="build-farm.xml" />
 
   <appendix>
     <title>Command Reference</title>
-    <sect1>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="env-common.xml" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="conf-file.xml" />
+    <sect1 id="sec-nix-env">
       <title>nix-env</title>
-      &nix-env;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-env.xml" />
+    </sect1>
+    <sect1 id="sec-nix-build">
+      <title>nix-build</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-build.xml" />
     </sect1>
     <sect1>
       <title>nix-store</title>
-      &nix-store;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-store.xml" />
     </sect1>
-    <sect1>
+    <sect1 id="sec-nix-instantiate">
       <title>nix-instantiate</title>
-      &nix-instantiate;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-instantiate.xml" />
     </sect1>
     <sect1>
       <title>nix-collect-garbage</title>
-      &nix-collect-garbage;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-collect-garbage.xml" />
+    </sect1>
+    <sect1 id="sec-nix-channel">
+      <title>nix-channel</title>
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-channel.xml" />
     </sect1>
     <sect1>
       <title>nix-push</title>
-      &nix-push;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-push.xml" />
     </sect1>
     <sect1>
       <title>nix-pull</title>
-      &nix-pull;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-pull.xml" />
     </sect1>
     <sect1>
       <title>nix-prefetch-url</title>
-      &nix-prefetch-url;
+      <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-prefetch-url.xml" />
     </sect1>
   </appendix>
 
-  &nix-lang-ref;
+  <!--  &nix-lang-ref; -->
 
-  &troubleshooting;
-  &bugs;
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="troubleshooting.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="bugs.xml" />
+  <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="glossary.xml" />
 
 </book>

doc/manual/nix-build.xml

@@ -0,0 +1,74 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-build</refname>
+  <refpurpose>build a Nix expression</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-build</command>
+    <arg><option>--add-drv-link</option></arg>
+    <arg><option>--no-link</option></arg>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>The <command>nix-build</command> command builds the derivations
+described by the Nix expressions in <replaceable>paths</replaceable>.
+If the build succeeds, it places a symlink to the result in the
+current directory.  The symlink is called <filename>result</filename>.
+If there are multiple Nix expressions, or the Nix expressions evaluate
+to multiple derivations, multiple sequentially numbered symlinks are
+created (<filename>result</filename>, <filename>result-2</filename>,
+and so on).</para>
+
+<note><para><command>nix-build</command> is essentially a wrapper
+around <link
+linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>
+(to translate a high-level Nix expression to a low-level store
+derivation) and <link
+linkend="rsec-nix-store-realise"><command>nix-store
+--realise</command></link> (to build the store
+derivation).</para></note>
+
+<warning><para>The result of the build is automatically registered as
+a root of the Nix garbage collector.  This root disappears
+automatically when the <filename>result</filename> symlink is deleted
+or renamed.  So don’t rename the symlink.</para></warning>
+
+</refsection>
+
+
+<refsection><title>Options</title>
+
+<variablelist>
+
+  <varlistentry><term><option>--add-drv-link</option></term>
+  
+    <listitem><para>Add a symlink in the current directory to the
+    store derivation produced by <command>nix-instantiate</command>.
+    The symlink is called <filename>derivation</filename> (which is
+    numbered in the case of multiple derivations).  The derivation is
+    a root of the garbage collector until the symlink is deleted or
+    renamed.</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--no-link</option></term>
+  
+    <listitem><para>Do not create a symlink to the output path.  Note
+    that as a result the output does not become a root of the garbage
+    collector, and so might be deleted by <command>nix-store
+    --gc</command>.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</refsection>
+
+
+</refentry>

doc/manual/nix-channel.xml

@@ -0,0 +1,83 @@
+<refentry>
+  
+<refnamediv>
+  <refname>nix-channel</refname>
+  <refpurpose>manage Nix channels</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-channel</command>
+    <group choice='req'>
+      <arg choice='plain'><option>--add</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--remove</option> <replaceable>url</replaceable></arg>
+      <arg choice='plain'><option>--list</option></arg>
+      <arg choice='plain'><option>--update</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+<refsection><title>Description</title>
+
+<para>A Nix channel is mechanism that allows you to automatically stay
+up-to-date with a set of pre-built Nix expressions.  A Nix channel is
+just a URL that points to a place that contains a set of Nix
+expressions, as well as a <command>nix-push</command> manifest.  See
+also <xref linkend="sec-channels" />.</para>
+
+<para>This command has the following operations:
+
+<variablelist>
+
+  <varlistentry><term><option>--add</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Adds <replaceable>url</replaceable> to the list of
+    subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--remove</option> <replaceable>url</replaceable></term>
+
+    <listitem><para>Removes <replaceable>url</replaceable> from the
+    list of subscribed channels.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--list</option></term>
+
+    <listitem><para>Prints the URLs of all subscribed channels on
+    standard output.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--update</option></term>
+
+    <listitem><para>Downloads the Nix expressions of all subscribed
+    channels, makes the conjunction of these the default for
+    <command>nix-env</command> operations (by calling <command>nix-env
+    -I</command>), and performs a <command>nix-pull</command> on the
+    manifests of all channels to make pre-built binaries
+    available.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</para>
+
+<para>Note that <option>--add</option> and <option>--remove</option>
+do not automatically perform an update.</para>
+
+<para>The list of subscribed channels is stored in
+<filename>~/.nix-channels</filename>.</para>
+
+<para>A channel consists of two elements: a bzipped Tar archive
+containing the Nix expressions, and a manifest created by
+<command>nix-push</command>.  These must be stored under
+<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal> and
+<literal><replaceable>url</replaceable>/MANIFEST</literal>,
+respectively.</para>
+
+</refsection>
+
+</refentry>

doc/manual/nix-collect-garbage.xml

@@ -1,75 +1,29 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-collect-garbage</refname>
-    <refpurpose>determine the set of unreachable store paths</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-collect-garbage</refname>
+  <refpurpose>delete unreachable store paths</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-collect-garbage</command>
-      <arg><option>--invert</option></arg>
-      <arg><option>--no-successors</option></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-collect-garbage</command>
+    <group choice='opt'>
+      <arg choice='plain'><option>--print-roots</option></arg>
+      <arg choice='plain'><option>--print-live</option></arg>
+      <arg choice='plain'><option>--print-dead</option></arg>
+      <arg choice='plain'><option>--delete</option></arg>
+    </group>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
+<refsection><title>Description</title>
 
-    <para>
-      The command <command>nix-collect-garbage</command> determines
-      the paths in the Nix store that are garbage, that is, not
-      reachable from outside of the store.  These paths can be safely
-      deleted without affecting the integrity of the system.
-    </para>
+<para>The command <command>nix-collect-garbage</command> is an
+obsolete wrapper around <link
+linkend="rsec-nix-store-gc"><command>nix-store
+--gc</command></link>.</para>
 
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Options</title>
-
-    <variablelist>
-
-      <varlistentry>
-        <term><option>--invert</option></term>
-        <listitem>
-          <para>
-            Causes the set of <emphasis>reachable</emphasis> paths to
-            be printed, rather than the unreachable paths.  These are
-            the paths that may <emphasis>not</emphasis> be deleted.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-      <varlistentry>
-        <term><option>--no-successors</option></term>
-        <listitem>
-          <para>
-            Causes <command>nix-collect-garbage</command> not to
-            follow successor relations.  By default, if a derivation
-            store expression is reachable, its successor (i.e., a
-            closure store expression) is also considered to be
-            reachable.  This option is always safe, but garbage
-            collecting successors may cause undesirable rebuilds later
-            on.
-          </para>
-        </listitem>
-      </varlistentry>
-      
-    </variablelist>
-
-  </refsection>
-
-  <refsection>
-    <title>Examples</title>
-
-    <para>
-      To delete all unreachable paths, do the following:
-    
-      <screen>
-$ nix-collect-garbage | xargs nix-store --delete</screen>
-
-    </para>
-
-  </refsection>
-    
 </refentry>

doc/manual/nix-instantiate.xml

@@ -1,65 +1,97 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-instantiate</refname>
-    <refpurpose>instantiate store expressions from Nix expressions</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-instantiate</refname>
+  <refpurpose>instantiate store derivations from Nix expressions</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-instantiate</command>
-      &opt-common-syn;
-      <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-instantiate</command>
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
+    <arg><option>--indirect</option></arg>
+    <group choice='opt'>
+      <arg choice='plain'><option>--parse-only</option></arg>
+      <arg choice='plain'><option>--eval-only</option></arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-instantiate</command> generates
-      (low-level) store expressions from (high-level) Nix expressions.
-      It loads and evaluates the Nix expressions in each of
-      <replaceable>files</replaceable>.  Each top-level expression
-      should evaluate to a derivation, a list of derivations, or a set
-      of derivations.  The paths of the resulting store expressions
-      are printed on standard output.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is generally used for testing Nix expression before
-      they are used with <command>nix-env</command>.
-    </para>
+<para>The command <command>nix-instantiate</command> generates <link
+linkend="gloss-derivation">store derivations</link> from (high-level)
+Nix expressions.  It loads and evaluates the Nix expressions in each
+of <replaceable>files</replaceable>.  Each top-level expression should
+evaluate to a derivation, a list of derivations, or a set of
+derivations.  The paths of the resulting store derivations are printed
+on standard output.</para>
 
-  </refsection>
+<para>Most users and developers don’t need to use this command
+(<command>nix-env</command> and <command>nix-build</command> perform
+store derivation instantiation from Nix expressions automatically).
+It is most commonly used for implementing new deployment
+policies.</para>
 
-  <refsection>
-    <title>Options</title>
+<para>See also <xref linkend="sec-common-options" /> for a list of
+common options.</para>
 
-    <variablelist>
+</refsection>
 
-      &opt-common;
 
-    </variablelist>
+<refsection><title>Options</title>
 
-  </refsection>
+<variablelist>
 
-  <refsection>
-    <title>Examples</title>
+  <varlistentry>
+    <term><option>--add-root</option> <replaceable>path</replaceable></term>
+    <term><option>--indirect</option></term>
 
-    <screen>
-$ nix-instantiate gcc.nix <lineannotation>(instantiate)</lineannotation>
-/nix/store/468abdcb93aa22bb721142615b97698b-d-gcc-3.3.2.store
+    <listitem><para>See the <link linkend="opt-add-root">corresponding
+    options</link> in <command>nix-store</command>.</para></listitem>
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(build)</lineannotation>
+  </varlistentry>
 
-$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(print output path)</lineannotation>
-/nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
+    
+  <varlistentry><term><option>--parse-only</option></term>
+  
+    <listitem><para>Just parse the input files, and print their
+    abstract syntax trees on standard output in ATerm
+    format.</para></listitem>
+    
+  </varlistentry>
+      
+  <varlistentry><term><option>--eval-only</option></term>
+  
+    <listitem><para>Just parse and evaluate the input files, and print
+    the resulting values on standard output.  No instantiation of
+    store derivations takes place.</para></listitem>
+    
+  </varlistentry>
 
-$ ls -l /nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
-dr-xr-xr-x    2 eelco    users         360 2003-12-01 16:12 bin
-dr-xr-xr-x    3 eelco    users          72 2003-12-01 16:12 include
+</variablelist>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<screen>
+$ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation>
+/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
+
+$ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation>
+<replaceable>...</replaceable>
+/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation>
+
+$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
+dr-xr-xr-x    2 eelco    users        4096 1970-01-01 01:00 lib
 ...</screen>
 
-  </refsection>
-    
+</refsection>
+
+
 </refentry>

doc/manual/nix-lang-ref.xml

@@ -1,4 +1,4 @@
-<chapter>
+<appendix>
   <title>Nix Language Reference</title>
 
   <sect1>
@@ -274,4 +274,4 @@
   </sect1>
   
 
-</chapter>
+</appendix>

doc/manual/nix-prefetch-url.xml

@@ -1,54 +1,69 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-prefetch-url</refname>
-    <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
-  </refnamediv>
+  
+<refnamediv>
+  <refname>nix-prefetch-url</refname>
+  <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
+</refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-prefetch-url</command>
-      <arg choice='plain'><replaceable>url</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-prefetch-url</command>
+    <arg choice='plain'><replaceable>url</replaceable></arg>
+    <arg><replaceable>hash</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-  <refsection>
-    <title>Description</title>
 
-    <para>
-      The command <command>nix-prefetch-url</command> downloads the
-      file referenced by the URL <replaceable>url</replaceable>,
-      prints its MD5 cryptographic hash code, and copies it into the
-      Nix store.  The file name in the store is
-      <filename><replaceable>hash</replaceable>-<replaceable>basename</replaceable></filename>,
-      where <replaceable>basename</replaceable> is everything
-      following the final slash in <replaceable>url</replaceable>.
-    </para>
+<refsection><title>Description</title>
 
-    <para>
-      This command is just a convenience to Nix expression writers.
-      Often a Nix expressions fetch some source distribution from the
-      network using the <literal>fetchurl</literal> expression
-      contained in <literal>nixpkgs</literal>.  However,
-      <literal>fetchurl</literal> requires an MD5 hash.  If you don't
-      know the hash, you would have to download the file first, and
-      then <literal>fetchurl</literal> would download it again when
-      you build your Nix expression.  Since
-      <literal>fetchurl</literal> uses the same name for the
-      downloaded file as <command>nix-prefetch-url</command>, the
-      redundant download can be avoided.
-    </para>
+<para>The command <command>nix-prefetch-url</command> downloads the
+file referenced by the URL <replaceable>url</replaceable>, prints its
+cryptographic hash, and copies it into the Nix store.  The file name
+in the store is
+<filename><replaceable>hash</replaceable>-<replaceable>baseName</replaceable></filename>,
+where <replaceable>baseName</replaceable> is everything following the
+final slash in <replaceable>url</replaceable>.</para>
 
-  </refsection>
+<para>This command is just a convenience for Nix expression writers.
+Often a Nix expression fetches some source distribution from the
+network using the <literal>fetchurl</literal> expression contained in
+Nixpkgs.  However, <literal>fetchurl</literal> requires a
+cryptographic hash.  If you don't know the hash, you would have to
+download the file first, and then <literal>fetchurl</literal> would
+download it again when you build your Nix expression.  Since
+<literal>fetchurl</literal> uses the same name for the downloaded file
+as <command>nix-prefetch-url</command>, the redundant download can be
+avoided.</para>
 
-  <refsection>
-    <title>Examples</title>
+<para>The environment variable <envar>NIX_HASH_ALGO</envar> specifies
+which hash algorithm to use.  It can be either <literal>md5</literal>,
+<literal>sha1</literal>, or <literal>sha256</literal>.  The default is
+<literal>md5</literal>.</para>
 
-    <screen>
+<para>If <replaceable>hash</replaceable> is specified, then a download
+is not performed if the Nix store already contains a file with the
+same hash and base name.  Otherwise, the file is downloaded, and an
+error if signaled if the actual hash of the file does not match the
+specified hash.</para>
+
+<para>This command prints the hash on standard output.  Additionally,
+if the environment variable <envar>PRINT_PATH</envar> is set, the path
+of the downloaded file in the Nix store is also printed.</para>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<screen>
 $ nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
-...
-file has hash 0bbd1df101bc0294d440471e50feca71
-...</screen>
+0bbd1df101bc0294d440471e50feca71
+
+$ PRINT_PATH=1 nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
+0bbd1df101bc0294d440471e50feca71
+/nix/store/wvyz8ifdn7wyz1p3pqyn0ra45ka2l492-make-3.80.tar.bz2</screen>
+
+</refsection>
 
-  </refsection>
     
 </refentry>

doc/manual/nix-push.xml

@@ -1,138 +1,116 @@
 <refentry>
-  <refnamediv>
-    <refname>nix-push</refname>
-    <refpurpose>push store paths onto a network cache</refpurpose>
-  </refnamediv>
 
-  <refsynopsisdiv>
-    <cmdsynopsis>
-      <command>nix-push</command>
-      <arg choice='plain'><replaceable>archives-put-url</replaceable></arg>
-      <arg choice='plain'><replaceable>archives-get-url</replaceable></arg>
-      <arg choice='plain'><replaceable>manifest-put-url</replaceable></arg>
-      <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
-    </cmdsynopsis>
-  </refsynopsisdiv>
+<refnamediv>
+  <refname>nix-push</refname>
+  <refpurpose>push store paths onto a network cache</refpurpose>
+</refnamediv>
 
-  <refsection>
-    <title>Description</title>
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-push</command>
+    <group choice='req'>
+      <arg choice='req'>
+        <arg choice='plain'><replaceable>archivesPutURL</replaceable></arg>
+        <arg choice='plain'><replaceable>archivesGetURL</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestPutURL</replaceable></arg>
+      </arg>
+      <arg choice='req'>
+        <arg choice='plain'><option>--copy</option></arg>
+        <arg choice='plain'><replaceable>archivesDir</replaceable></arg>
+        <arg choice='plain'><replaceable>manifestFile</replaceable></arg>
+      </arg>
+    </group>
+    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
 
-    <para>
-      The command <command>nix-push</command> builds a set of store
-      expressions (if necessary), and then packages and uploads all
-      store paths in the resulting closures to a server.  A network
-      cache thus populated can subsequently be used to speed up
-      software deployment on other machines using the
-      <command>nix-pull</command> command.
-    </para>
 
-    <para>
-      <command>nix-push</command> performs the following actions.
+<refsection><title>Description</title>
+
+<para>The command <command>nix-push</command> builds a set of store
+paths (if necessary), and then packages and uploads all store paths in
+the resulting closures to a server.  A network cache thus populated
+can subsequently be used to speed up software deployment on other
+machines using the <command>nix-pull</command> command.</para>
+
+<para><command>nix-push</command> performs the following actions.
       
-      <orderedlist>
+<orderedlist>
 
-        <listitem>
-          <para>
-            The store expressions stored in
-            <replaceable>paths</replaceable> are realised (using
-            <literal>nix-store --realise</literal>).
-          </para>
-        </listitem>
+  <listitem><para>Each path in <replaceable>paths</replaceable> is
+  realised (using <link
+  linkend='rsec-nix-store-realise'><literal>nix-store
+  --realise</literal></link>).</para></listitem>
 
-        <listitem>
-          <para>
-            All paths in the closure of the store expressions stored
-            in <replaceable>paths</replaceable> are determined (using
-            <literal>nix-store --query --requisites
-            --include-successors</literal>).  It should be noted that
-            since the <option>--include-successors</option> flag is
-            used, if you specify a derivation store expression, you
-            get a combined source/binary distribution.  If you only
-            want a binary distribution, you should specify the closure
-            store expression that result from realising these (see
-            below).
-          </para>
-        </listitem>
+  <listitem><para>All paths in the closure of the store expressions
+  stored in <replaceable>paths</replaceable> are determined (using
+  <literal>nix-store --query --requisites
+  --include-outputs</literal>).  It should be noted that since the
+  <option>--include-outputs</option> flag is used, you get a combined
+  source/binary distribution.</para></listitem>
 
-        <listitem>
-          <para>
-            All store paths determined in the previous step are
-            packaged and compressed into a <command>bzip</command>ped
-            NAR archive (extension <filename>.nar.bz2</filename>).
-          </para>
-        </listitem>
+  <listitem><para>All store paths determined in the previous step are
+  packaged and compressed into a <command>bzip</command>ped NAR
+  archive (extension <filename>.nar.bz2</filename>).</para></listitem>
 
-        <listitem>
-          <para>
-            A <emphasis>manifest</emphasis> is created that contains
-            information on the store paths, their eventual URLs in the
-            cache, and cryptographic hashes of the contents of the NAR
-            archives.
-          </para>
-        </listitem>
+  <listitem><para>A <emphasis>manifest</emphasis> is created that
+  contains information on the store paths, their eventual URLs in the
+  cache, and cryptographic hashes of the contents of the NAR
+  archives.</para></listitem>
 
-        <listitem>
-          <para>
-            Each store path is uploaded to the remote directory
-            specified by <replaceable>archives-put-url</replaceable>.
-            HTTP PUT requests are used to do this.  However, before a
-            file <varname>x</varname> is uploaded to
-            <literal><replaceable>archives-put-url</replaceable>/<varname>x</varname></literal>,
-            <command>nix-push</command> first determines whether this
-            upload is unnecessary by issuing a HTTP HEAD request on
-            <literal><replaceable>archives-get-url</replaceable>/<varname>x</varname></literal>.
-            This allows a cache to be shared between many partially
-            overlapping <command>nix-push</command> invocations.
-            (We use two URLs because the upload URL typically
-            refers to a CGI script, while the download URL just refers
-            to a file system directory on the server.)
-          </para>
-        </listitem>
+  <listitem><para>Each store path is uploaded to the remote directory
+  specified by <replaceable>archivesPutURL</replaceable>.  HTTP PUT
+  requests are used to do this.  However, before a file
+  <varname>x</varname> is uploaded to
+  <literal><replaceable>archivesPutURL</replaceable>/<varname>x</varname></literal>,
+  <command>nix-push</command> first determines whether this upload is
+  unnecessary by issuing a HTTP HEAD request on
+  <literal><replaceable>archivesGetURL</replaceable>/<varname>x</varname></literal>.
+  This allows a cache to be shared between many partially overlapping
+  <command>nix-push</command> invocations.  (We use two URLs because
+  the upload URL typically refers to a CGI script, while the download
+  URL just refers to a file system directory on the server.)</para></listitem>
 
-        <listitem>
-          <para>
-            The manifest is uploaded using an HTTP PUT request to
-            <replaceable>manifest-put-url</replaceable>.  The
-            corresponding URL to download the manifest can then be
-            used by <command>nix-pull</command>.
-          </para>
-        </listitem>
+  <listitem><para>The manifest is uploaded using an HTTP PUT request
+  to <replaceable>manifestPutURL</replaceable>.  The corresponding
+  URL to download the manifest can then be used by
+  <command>nix-pull</command>.</para></listitem>
         
-      </orderedlist>
-    </para>
+</orderedlist>
+
+</para>
+
+<para>TODO: <option>--copy</option></para>
             
-  </refsection>
+</refsection>
 
-  <refsection>
-    <title>Examples</title>
 
-    <para>
-      To upload files there typically is some CGI script on the server
-      side.  This script should be be protected with a password.  The
-      following example uploads the store paths resulting from
-      building the Nix expressions in <filename>foo.nix</filename>,
-      passing appropriate authentication information:
+<refsection><title>Examples</title>
+
+<para>To upload files there typically is some CGI script on the server
+side.  This script should be be protected with a password.  The
+following example uploads the store paths resulting from building the
+Nix expressions in <filename>foo.nix</filename>, passing appropriate
+authentication information:
     
-      <screen>
+<screen>
 $ nix-push \
     http://foo@bar:server.domain/cgi-bin/upload.pl/cache \
     http://server.domain/cache \
     http://foo@bar:server.domain/cgi-bin/upload.pl/MANIFEST \
     $(nix-instantiate foo.nix)</screen>
 
-    This will push both sources and binaries (and any build-time
-    dependencies used in the build, such as compilers).
-    </para>
+This will push both sources and binaries (and any build-time
+dependencies used in the build, such as compilers).</para>
 
-    <para>
-      If we just want to push binaries, not sources and build-time
-      dependencies, we can do:
+<para>If we just want to push binaries, not sources and build-time
+dependencies, we can do:
       
-      <screen>
+<screen>
 $ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
     
-    </para>
+</para>
 
-  </refsection>
+</refsection>
     
 </refentry>

doc/manual/opt-common-syn.xml

@@ -1,8 +1,24 @@
+<nop>
+  
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>
 <arg rep='repeat'><option>--verbose</option></arg>
 <arg rep='repeat'><option>-v</option></arg>
-<arg><option>--build-output</option></arg>
-<arg><option>-B</option></arg>
+<arg><option>--no-build-output</option></arg>
+<arg><option>-Q</option></arg>
+<arg>
+  <group choice='req'>
+    <arg choice='plain'><option>--max-jobs</option></arg>
+    <arg choice='plain'><option>-j</option></arg>
+  </group>
+  <replaceable>number</replaceable>
+</arg>
+<arg><option>--keep-going</option></arg>
+<arg><option>-k</option></arg>
 <arg><option>--keep-failed</option></arg>
 <arg><option>-K</option></arg>
+<arg><option>--fallback</option></arg>
+<arg><option>--readonly-mode</option></arg>
+<arg><option>--log-type</option> <replaceable>type</replaceable></arg>
+
+</nop>

doc/manual/opt-common.xml

@@ -1,122 +1,216 @@
-<varlistentry>
-  <term><option>--help</option></term>
+<sect1 id="sec-common-options"><title>Common options</title>
+
+<para>Most Nix commands accept the following command-line options:</para>
+
+<variablelist>
+
+<varlistentry><term><option>--help</option></term>
+  
+  <listitem><para>Prints out a summary of the command syntax and
+  exits.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--version</option></term>
+  
+  <listitem><para>Prints out the Nix version number on standard output
+  and exits.</para></listitem>
+</varlistentry>
+
+
+<varlistentry><term><option>--verbose</option></term>
+  <term><option>-v</option></term>
+
   <listitem>
-    <para>
-      Prints out a summary of the command syntax and exits.
+    
+  <para>Increases the level of verbosity of diagnostic messages
+  printed on standard error.  For each Nix operation, the information
+  printed on standard output is well-defined; any diagnostic
+  information is printed on standard error, never on standard
+  output.</para>
+
+  <para>This option may be specified repeatedly.  Currently, the
+  following verbosity levels exist:</para>
+
+  <variablelist>
+    
+    <varlistentry><term>0</term>
+    <listitem><para>“Errors only”: only print messages
+    explaining why the Nix invocation failed.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>1</term>
+    <listitem><para>“Informational”: print
+    <emphasis>useful</emphasis> messages about what Nix is doing.
+    This is the default.</para></listitem>
+    </varlistentry>
+      
+    <varlistentry><term>2</term>
+    <listitem><para>“Talkative”: print more informational
+    messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>3</term>
+    <listitem><para>“Chatty”: print even more
+    informational messages.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>4</term>
+    <listitem><para>“Debug”: print debug
+    information.</para></listitem>
+    </varlistentry>
+
+    <varlistentry><term>5</term>
+    <listitem><para>“Vomit”: print vast amounts of debug
+    information.</para></listitem>
+    </varlistentry>
+    
+  </variablelist>
+
+  </listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--no-build-output</option></term>
+  <term><option>-Q</option></term>
+
+  <listitem><para>By default, output written by builders to standard
+  output and standard error is echoed to the Nix command's standard
+  error.  This option suppresses this behaviour.  Note that the
+  builder's standard output and error are always written to a log file
+  in
+  <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry id="opt-max-jobs"><term><option>--max-jobs</option></term>
+  <term><option>-j</option></term>
+
+  <listitem><para>Sets the maximum number of build jobs that Nix will
+  perform in parallel to the specified number.  The default is 1.  A
+  higher value is useful on SMP systems or to exploit I/O latency.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-going</option></term>
+  <term><option>-k</option></term>
+
+  <listitem><para>Keep going in case of failed builds, to the
+  greatest extent possible.  That is, if building an input of some
+  derivation fails, Nix will still build the other inputs, but not the
+  derivation itself.  Without this option, Nix stops if any build
+  fails (except for builds of substitutes), possibly killing builds in
+  progress (in case of parallel or distributed builds).</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry><term><option>--keep-failed</option></term>
+  <term><option>-K</option></term>
+
+  <listitem><para>Specifies that in case of a build failure, the
+  temporary directory (usually in <filename>/tmp</filename>) in which
+  the build takes place should not be deleted.  The path of the build
+  directory is printed as an informational message.
     </para>
   </listitem>
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--version</option></term>
+<varlistentry><term><option>--fallback</option></term>
+
   <listitem>
-    <para>
-      Prints out the Nix version number on standard output and exits.
-    </para>
+
+  <para>Whenever Nix attempts to build a derivation for which
+  substitutes are known for each output path, but realising the output
+  paths through the substitutes fails, fall back on building the
+  derivation.</para>
+
+  <para>The most common scenario in which this is useful is when we
+  have registered substitutes in order to perform binary distribution
+  from, say, a network repository.  If the repository is down, the
+  realisation of the derivation will fail.  When this option is
+  specified, Nix will build the derivation instead.  Thus,
+  installation from binaries falls back on nstallation from source.
+  This option is not the default since it is generally not desirable
+  for a transient failure in obtaining the substitutes to lead to a
+  full build from source (with the related consumption of
+  resources).</para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--verbose</option> / <option>-v</option></term>
-  <listitem>
-    <para>
-      Increases the level of verbosity of diagnostic messages printed
-      on standard error.  For each Nix operation, the information
-      printed on standard output is well-defined; any diagnostic
-      information is printed on standard error, never on standard
-      output.
-    </para>
+<varlistentry><term><option>--readonly-mode</option></term>
 
-    <para>
-      This option may be specified repeatedly.  Currently, the
-      following verbosity levels exist:
-    </para>
-
-    <variablelist>
-      <varlistentry>
-        <term>0</term>
-        <listitem>
-          <para>
-            <quote>Errors only</quote>: only print messages explaining
-            why the Nix invocation failed.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>1</term>
-        <listitem>
-          <para>
-            <quote>Informational</quote>: print
-            <emphasis>useful</emphasis> messages about what Nix is
-            doing.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>2</term>
-        <listitem>
-          <para>
-            <quote>Talkative</quote>: print more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>3</term>
-        <listitem>
-          <para>
-            <quote>Chatty</quote>: print even more informational messages.
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>4</term>
-        <listitem>
-          <para>
-            <quote>Debug</quote>: print debug information:
-          </para>
-        </listitem>
-      </varlistentry>
-      <varlistentry>
-        <term>5</term>
-        <listitem>
-          <para>
-            <quote>Vomit</quote>: print vast amounts of debug
-            information.
-          </para>
-        </listitem>
-      </varlistentry>
-    </variablelist>
-
-  </listitem>
+  <listitem><para>When this option is used, no attempt is made to open
+  the Nix database.  Most Nix operations do need database access, so
+  those operations will fail.</para></listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--build-output</option> / <option>-B</option></term>
+<varlistentry id="opt-log-type"><term><option>--log-type</option>
+<replaceable>type</replaceable></term>
+
   <listitem>
-    <para>
-      Causes the output written by build actions to standard output
-      and standard error to be echoed to standard error, regardless of
-      verbosity level.  By default, it is only echoed at a verbosity
-      level of at least 4 (<quote>Debug</quote>), and is suppressed at
-      lower levels.  Note that it is always written to a log file in
-      <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.
-    </para>
+
+  <para>This option determines how the output written to standard
+  error is formatted.  Nix’s diagnostic messages are typically
+  <emphasis>nested</emphasis>.  For instance, when tracing Nix
+  expression evaluation (<command>nix-env -vvvvv</command>, messages
+  from subexpressions are nested inside their parent expressions.  Nix
+  builder output is also often nested.  For instance, the Nix Packages
+  generic builder nests the various build tasks (unpack, configure,
+  compile, etc.), and the GNU Make in <literal>stdenv-linux</literal>
+  has been patched to provide nesting for recursive Make
+  invocations.</para>
+
+  <para><replaceable>type</replaceable> can be one of the
+  following:
+
+  <variablelist>
+
+    <varlistentry><term><literal>pretty</literal></term>
+
+      <listitem><para>Pretty-print the output, indicating different
+      nesting levels using spaces.  This is the
+      default.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>escapes</literal></term>
+
+      <listitem><para>Indicate nesting using escape codes that can be
+      interpreted by the <command>log2xml</command> tool in the Nix
+      source distribution.  The resulting XML file can be fed into the
+      <command>log2html.xsl</command> stylesheet to create an HTML
+      file that can be browsed interactively, using Javascript to
+      expand and collapse parts of the output.</para></listitem>
+
+    </varlistentry>
+
+    <varlistentry><term><literal>flat</literal></term>
+
+      <listitem><para>Remove all nesting.</para></listitem>
+
+    </varlistentry>
+
+  </variablelist>    
+  
+  </para>
+
   </listitem>
+  
 </varlistentry>
 
 
-<varlistentry>
-  <term><option>--keep-failed</option> / <option>-K</option></term>
-  <listitem>
-    <para>
-      Specifies that in case of a build failure, the temporary
-      directory (usually in <filename>/tmp</filename>) in which the
-      build takes place should not be deleted.  The path of the build
-      directory is printed as an informational message.
-    </para>
-  </listitem>
-</varlistentry>
+</variablelist>
 
+
+</sect1>

doc/manual/overview.xml

@@ -1,450 +0,0 @@
-<chapter id='chap-overview'>
-  <title>Overview</title>
-
-  <para>
-    This chapter provides a guided tour of Nix.
-  </para>
-
-
-  
-  <!--######################################################################-->
-
-  <sect1>
-    <title>Basic package management</title>
-
-    <para>
-      Let's start from the perspective of an end user.  Common operations at
-      this level are to install and remove packages, ask what packages are
-      installed or available for installation, and so on.  These are operations
-      on the <emphasis>user environment</emphasis>: the set of packages that a
-      user <quote>sees</quote>.  In a command line Unix environment, this means
-      the set of programs that are available through the <envar>PATH</envar>
-      environment variable.  (In other environments it might mean the set of
-      programs available on the desktop, through the start menu, and so on.)
-    </para>
-
-    <para>
-      The terms <quote>installation</quote> and <quote>uninstallation</quote>
-      are used in this context to denote the act of adding or removing packages
-      from the user environment.  In Nix, these operations are dissociated from
-      the physical copying or deleting of files.  Installation requires that
-      the files constituting the package are present, but they may be present
-      beforehand.  Likewise, uninstallation does not actually delete any files;
-      this is done automatically by running a garbage collector.
-    </para>
-
-    <para>
-      User environments are manipulated through the <command>nix-env</command>
-      command.  The query operation can be used to see what packages are
-      currently installed.
-    </para>
-
-    <screen>
-$ nix-env -q
-MozillaFirebird-0.7
-sylpheed-0.9.7
-pan-0.14.2</screen>
-
-    <para>
-      (<option>-q</option> is actually short for <option>--query
-        --installed</option>.)  The package names are symbolic: they don't have
-      any particular significance to Nix (as they shouldn't, since they are not
-      unique&mdash;there can be many derivations with the same name).  Note that
-      these packages have many dependencies (e.g., Mozilla uses the
-      <literal>gtk+</literal> package) but these have not been installed in the
-      user environment, though they are present on the system.  Generally,
-      there is no need to install such packages; only packages containing
-      programs should be installed.
-    </para>
-      
-    <para>
-      To install packages, a <emphasis>Nix expression</emphasis> is required
-      that tells Nix how to build that package.  There is a <ulink
-        url='https://svn.cs.uu.nl:12443/dist/trace/trace-nixpkgs-trunk.tar.bz2'>set 
-        of standard of Nix expressions</ulink> for many common packages.
-      Assuming that you have downloaded and unpacked these, you can view the
-      set of available packages:
-    </para>
-
-    <screen>
-$ nix-env -qaf pkgs/system/i686-linux.nix
-gettext-0.12.1
-sylpheed-0.9.7
-aterm-2.0
-gtk+-1.2.10
-apache-httpd-2.0.48
-pan-0.14.2
-...</screen>
-
-    <para>
-      The Nix expression in the file <filename>i686-linux.nix</filename> yields
-      the set of packages for a Linux system running on x86 hardware.  For
-      other platforms, copy and modify this file for your platform as
-      appropriate. [TODO: improve this]
-    </para>
-
-    <para>
-      It is also possible to see the <emphasis>status</emphasis> of available
-      packages, i.e., whether they are installed into the user environment
-      and/or present in the system:
-    </para>
-
-    <screen>
-$ nix-env -qasf pkgs/system/i686-linux.nix
--P gettext-0.12.1
-IP sylpheed-0.9.7
--- aterm-2.0
--P gtk+-1.2.10</screen>
-
-    <para>
-      This reveals that the <literal>sylpheed</literal> package is already
-      installed, or more precisely, that exactly the same instantiation of
-      <literal>sylpheed</literal> is installed.  This guarantees that the
-      available package is exactly the same as the installed package with
-      regard to sources, dependencies, build flags, and so on.  Similarly, we
-      see that the <literal>gettext</literal> and <literal>gtk+</literal>
-      packages are present but not installed in the user environment, while the
-      <literal>aterm</literal> package is not installed or present at all (so,
-      if we were to install it, it would have to be built or downloaded first).
-    </para>
-
-    <para>
-      The install operation is used install available packages from a Nix
-      environment.  To install the <literal>pan</literal> package (a
-      newsreader), you would do:
-    </para>
-
-    <screen>
-$ nix-env -if pkgs/system/i686-linux.nix pan</screen>
-
-    <para>
-      Since installation may take a long time, depending on whether any
-      packages need to be built or downloaded, it's a good idea to make
-      <command>nix-env</command> run verbosely by using the <option>-v</option>
-      (<option>--verbose</option>) option.  This option may be repeated to
-      increase the level of verbosity.  A good value is 3
-      (<option>-vvv</option>).
-    </para>
-
-    <para>
-      In fact, if you run this command verbosely you will observe that Nix
-      starts to build many packages, including large and fundamental ones such
-      as <literal>glibc</literal> and <literal>gcc</literal>.  I.e., you are
-      performing a source installation.  This is generally undesirable, since
-      installation from sources may require large amounts of disk and CPU
-      resources.  Therefore a <quote>binary</quote> installation is generally
-      preferable.
-    </para>
-
-    <para>
-      Rather than provide different mechanisms to create and perform
-      the installation of binary packages, Nix supports binary deployment
-      <emphasis>transparently</emphasis> through a generic mechanism of
-      <emphasis>substitute expressions</emphasis>.  If an request is made to
-      build some Nix expression, Nix will first try to build any substitutes
-      for that expression.  These substitutes presumably perform an identical
-      build operation with respect to the result, but require less resources.
-      For instance, a substitute that downloads a pre-built package from the
-      network requires less CPU and disk resources, and possibly less time.
-    </para>
-
-    <para>
-      Nix's use of cryptographic hashes makes this entirely safe.  It is not
-      possible, for instance, to accidentally substitute a build of some
-      package for a Solaris or Windows system for a build on a SuSE/x86 system.
-    </para>
-
-    <para>
-      While the substitute mechanism is a generic mechanism, Nix provides two
-      standard tools called <command>nix-pull</command> and
-      <command>nix-push</command> that maintain and use a shared cache of
-      prebuilt derivations on some network site (reachable through HTTP).  If
-      you attempt to install some package that someone else has previously
-      built and <quote>pushed</quote> into the cache, and you have done a
-      <quote>pull</quote> to register substitutes that download these prebuilt
-      packages, then the installation will automatically use these.
-    </para>
-
-    <para>
-      For example, to pull from our <ulink
-        url='http://losser.st-lab.cs.uu.nl/~eelco/nix-dist/'>cache</ulink> of
-      prebuilt packages (at the time of writing, for SuSE Linux/x86), use the
-      following command:
-    </para>
-
-    <screen>
-$ nix-pull http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-<replaceable>version</replaceable>/MANIFEST
-obtaining list of Nix archives at http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-<replaceable>version</replaceable>/MANIFEST...
-...</screen>
-
-    <para>
-      If <command>nix-pull</command> is run without any arguments, it will pull
-      from the URLs specified in the file
-      <filename><replaceable>prefix</replaceable>/etc/nix/prebuilts.conf</filename>.
-    </para>
-
-    <para>
-      Assuming that the <literal>pan</literal> installation produced no errors,
-      it can be used immediately, that is, it now appears in a directory in the
-      <envar>PATH</envar> environment variable.  Specifically,
-      <envar>PATH</envar> includes the entry
-      <filename><replaceable>prefix</replaceable>/var/nix/profiles/default/bin</filename>, 
-      where
-      <filename><replaceable>prefix</replaceable>/var/nix/profiles/default</filename> 
-      is just a symlink to the current user environment:
-    </para>
-
-    <screen>
-$ ls -l /nix/var/nix/profiles/
-...
-lrwxrwxrwx  1 eelco ... default-15-link -> /nix/store/1871...12b0-user-environment
-lrwxrwxrwx  1 eelco ... default-16-link -> /nix/store/59ba...df6b-user-environment
-lrwxrwxrwx  1 eelco ... default -> default-16-link</screen>
-
-    <para>
-      That is, <filename>default</filename> in this example is a link
-      to <filename>default-16-link</filename>, which is the current
-      user environment.  Before the installation, it pointed to
-      <filename>default-15-link</filename>.  Note that this means that
-      you can atomically roll-back to the previous user environment by
-      pointing the symlink <filename>default</filename> at
-      <filename>default-15-link</filename> again.  This also shows
-      that operations such as installation are atomic in the Nix
-      system: any arbitrarily complex set of installation,
-      uninstallation, or upgrade actions eventually boil down to the
-      single operation of pointing a symlink somewhere else (which can
-      be implemented atomically in Unix).
-    </para>
-
-    <para>
-      What's in a user environment? It's just a set of symlinks to the files
-      that constitute the installed packages.  For instance:
-    </para>
-
-    <screen>
-$ ls -l /nix/var/nix/profiles/default-16-link/bin
-lrwxrwxrwx  1 eelco ... MozillaFirebird -> /nix/store/35f8...4ae6-MozillaFirebird-0.7/bin/MozillaFirebird
-lrwxrwxrwx  1 eelco ... svn -> /nix/store/3829...fb5d-subversion-0.32.1/bin/svn
-...</screen>
-
-    <para>
-      Note that, e.g., <filename>svn</filename> =
-      <filename>/nix/var/nix/profiles/default/bin/svn</filename> =
-      <filename>/nix/var/nix/profiles/default-16-link/bin/svn</filename> =
-      <filename>/nix/store/59ba...df6b-user-environment/bin/svn</filename> =
-      <filename>/nix/store/3829...fb5d-subversion-0.32.1/bin/svn</filename>.
-    </para>
-
-    <para>
-      Naturally, packages can also be uninstalled:
-    </para>
-
-    <screen>
-$ nix-env -e pan</screen>
-
-    <para>
-      This means that the package is removed from the user
-      environment.  It is <emphasis>not</emphasis> yet removed from
-      the system.  When a package is uninstalled from a user
-      environment, it may still be used by other packages, or may
-      still be present in other user environments.  Deleting it under
-      such conditions would break those other packages or user
-      environments.  To prevent this, packages are only
-      <quote>physically</quote> deleted by running the Nix garbage
-      collector, which searches for all packages in the Nix store that
-      are no longer <quote>reachable</quote> from outside the store.
-      Thus, uninstalling a package is always safe: it cannot break
-      other packages.
-    </para>
-
-    <para>
-      Upgrading packages is easy.  Given a Nix expression that
-      contains newer versions of installed packages (that is, packages
-      with the same package name, but a higher version number),
-      <command>nix-env -u</command> will replace the installed package
-      in the user environment with the newer package.  For example,
-      
-      <screen>
-$ nix-env -uf pkgs/system/i686-linux.nix pan</screen>
-
-      looks for a newer version of Pan, and installs it if found.
-      Also useful is the ability to upgrade <emphasis>all</emphasis>
-      packages:
-      
-      <screen>
-$ nix-env -uf pkgs/system/i686-linux.nix '*'</screen>
-
-      The asterisk matches all installed packages<footnote><para>No,
-      we don't support arbitrary regular
-      expressions</para></footnote>.  Note that <literal>*</literal>
-      must be quoted to prevent shell globbing.
-    </para>
-
-  </sect1>
-
-
-  
-  <!--######################################################################-->
-
-  <sect1>
-    <title>Writing Nix expressions</title>
-
-    <sect2>
-      <title>A simple Nix expression</title>
-
-      <para>
-        This section shows how to write simple Nix expressions&mdash;the things
-        that describe how to build a package.
-      </para>
-
-      <example id='ex-hello-nix'>
-        <title>Nix expression for GNU Hello</title>
-        <programlisting>
-{stdenv, fetchurl, perl}: <co id='ex-hello-nix-co-1' />
-
-derivation { <co id='ex-hello-nix-co-2' />
-  name = "hello-2.1.1"; <co id='ex-hello-nix-co-3' />
-  system = stdenv.system; <co id='ex-hello-nix-co-4' />
-  builder = ./builder.sh; <co id='ex-hello-nix-co-5' />
-  src = fetchurl { <co id='ex-hello-nix-co-6' />
-    url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz;
-    md5 = "70c9ccf9fac07f762c24f2df2290784d";
-  };
-  stdenv = stdenv; <co id='ex-hello-nix-co-7' />
-  perl = perl;
-}</programlisting>
-      </example>
-
-      <para>
-        A simple Nix expression is shown in <xref linkend='ex-hello-nix' />. It
-        describes how to the build the <ulink
-          url='http://www.gnu.org/directory/GNU/hello.html'>GNU Hello
-          package</ulink>.  This package has several dependencies.  First, it
-        requires a number of other packages, such as a C compiler, standard
-        Unix shell tools, and Perl.  Rather than have this Nix expression refer
-        to and use specific versions of these packages, it should be generic;
-        that is, it should be a <emphasis>function</emphasis> that takes the
-        required packages as inputs and yield a build of the GNU Hello package
-        as a result.  This Nix expression defines a function with three
-        arguments <xref linkend='ex-hello-nix-co-1' />, namely:
-        <orderedlist>
-          <listitem><para><varname>stdenv</varname>, which should be a
-              <emphasis>standard environment package</emphasis>.  The standard
-              environment is a set of tools and other components that would be
-              expected in a fairly minimal Unix-like environment: a C compiler
-              and linker, Unix shell tools, and so on.</para>
-          </listitem>
-          <listitem><para><varname>fetchurl</varname>, which should be a
-              function that given parameters <varname>url</varname> and
-              <varname>md5</varname>, will fetch a file from the specified
-              location and check that this file has the given MD5 hash code.
-              The hash is required because build operations must be
-              <emphasis>pure</emphasis>: given the same inputs they should
-              always yield the same output.  Since network resources can change
-              at any time, we must in some way guarantee what the result will
-              be.</para>
-          </listitem>
-          <listitem><para><varname>perl</varname>, which should be a Perl
-              interpreter.</para>
-          </listitem>
-        </orderedlist>
-      </para>
-
-      <para>
-        The remainder of the file is the body of the function, which happens to
-        be a <emphasis>derivation</emphasis> <xref
-        linkend='ex-hello-nix-co-2' />, which is the built-in function
-        <varname>derivation</varname> applied to a set of attributes that
-        encode all the necessary information for building the GNU Hello
-        package. 
-      </para>
-
-      <example>
-        <title>Build script (<filename>builder.sh</filename>) for GNU
-          Hello</title>
-        <programlisting>
-#! /bin/sh
-
-buildinputs="$perl"
-. $stdenv/setup || exit 1
-
-tar xvfz $src || exit 1
-cd hello-* || exit 1
-./configure --prefix=$out || exit 1
-make || exit 1
-make install || exit 1</programlisting>
-      </example>
-
-    </sect2>
-
-    <sect2>
-      <title>A more complex Nix expression</title>
-
-      <example id='ex-svn-nix'>
-        <title>Nix expression for Subversion</title>
-        <programlisting>
-{ localServer ? false <co id='ex-svn-nix-co-1' />
-, httpServer ? false
-, sslSupport ? false
-, swigBindings ? false
-, stdenv, fetchurl
-, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null
-}:
-
-assert !isNull expat; <co id='ex-svn-nix-co-2' />
-assert localServer -> !isNull db4;
-assert httpServer -> !isNull httpd &amp;&amp; httpd.expat == expat; <co id='ex-svn-nix-co-3' />
-assert sslSupport -> !isNull openssl &amp;&amp; (httpServer -> httpd.openssl == openssl);
-assert swigBindings -> !isNull swig;
-
-derivation {
-  name = "subversion-0.32.1";
-  system = stdenv.system;
-
-  builder = ./builder.sh;
-  src = fetchurl {
-    url = http://svn.collab.net/tarballs/subversion-0.32.1.tar.gz;
-    md5 = "b06717a8ef50db4b5c4d380af00bd901";
-  };
-
-  localServer = localServer;
-  httpServer = httpServer;
-  sslSupport = sslSupport;
-  swigBindings = swigBindings;
-
-  stdenv = stdenv;
-  openssl = if sslSupport then openssl else null; <co id='ex-svn-nix-co-4' />
-  httpd = if httpServer then httpd else null;
-  expat = expat;
-  db4 = if localServer then db4 else null;
-  swig = if swigBindings then swig else null;
-}</programlisting>
-      </example>
-
-      <para>
-        This example shows several features.  Default parameters <xref
-          linkend='ex-svn-nix-co-1'/> can be used to simplify call sites: if an
-        argument that has a default is omitted, its default value is used.
-      </para>
-
-      <para>
-        You can use <emphasis>assertions</emphasis> to test whether arguments
-        satisfy certain constraints.  The simple assertion <xref
-          linkend='ex-svn-nix-co-2'/> tests whether the
-        <varname>expat</varname> argument is not a null value.  The more
-        complex assertion <xref linkend='ex-svn-nix-co-3'/> says that if
-        Subversion is built with Apache support, then <varname>httpd</varname>
-        (the Apache package) must not be null and it must have been built using
-        the same instance of the <varname>expat</varname> library as was passed
-        to the Subversion expression.  This is since the Subversion code is
-        dynamically linked against the Apache code and they both use Expat,
-        they must be linked against the same instance&mdash;otherwise a
-        conflict might occur. 
-      </para>
-
-    </sect2>
-
-  </sect1>
-
-
-</chapter>

doc/manual/package-management.xml

@@ -0,0 +1,462 @@
+<chapter id='chap-package-management'><title>Package Management</title>
+
+<para>This chapter discusses how to do package management with Nix,
+i.e., how to obtain, install, upgrade, and erase components.  This is
+the “user’s” perspective of the Nix system — people
+who want to <emphasis>create</emphasis> components should consult
+<xref linkend='chap-writing-nix-expressions' />.</para>
+
+
+<sect1><title>Basic package management</title>
+
+<para>The main command for package management is <link
+linkend="sec-nix-env"><command>nix-env</command></link>.  You can use
+it to install, upgrade, and erase components, and to query what
+components are installed or are available for installation.</para>
+
+<para>In Nix, different users can have different “views”
+on the set of installed applications.  That is, there might be lots of
+applications present on the system (possibly in many different
+versions), but users can have a specific selection of those active —
+where “active” just means that it appears in a directory
+in the user’s <envar>PATH</envar>.  Such a view on the set of
+installed applications is called a <emphasis>user
+environment</emphasis>, which is just a directory tree consisting of
+symlinks to the files of the active applications.  </para>
+
+<para>Components are installed from a set of <emphasis>Nix
+expressions</emphasis> that tell Nix how to build those components,
+including, if necessary, their dependencies.  There is a collection of
+Nix expressions called the Nix Package collection that contains
+components ranging from basic development stuff such as GCC and Glibc,
+to end-user applications like Mozilla Firefox.  (Nix is however not
+tied to the Nix Package collection; you could write your own Nix
+expressions based on it, or completely new ones.)  You can download
+the latest version from <ulink
+url='http://catamaran.labs.cs.uu.nl/dist/nix' />.</para>
+
+<para>Assuming that you have downloaded and unpacked a release of Nix
+Packages, you can view the set of available components in the release:
+
+<screen>
+$ nix-env -qaf nixpkgs-<replaceable>version</replaceable>
+ant-blackdown-1.4.2
+aterm-2.2
+bash-3.0
+binutils-2.15
+bison-1.875d
+blackdown-1.4.2
+bzip2-1.0.2
+...</screen>
+
+where <literal>nixpkgs-<replaceable>version</replaceable></literal> is
+where you’ve unpacked the release.</para>
+
+<para>It is also possible to see the <emphasis>status</emphasis> of
+available components, i.e., whether they are installed into the user
+environment and/or present in the system:
+
+<screen>
+$ nix-env -qasf nixpkgs-<replaceable>version</replaceable>
+...
+-PS bash-3.0
+--S binutils-2.15
+IPS bison-1.875d
+...</screen>
+
+The first character (<literal>I</literal>) indicates whether the
+component is installed in your current user environment.  The second
+(<literal>P</literal>) indicates whether it is present on your system
+(in which case installing it into your user environment would be a
+very quick operation).  The last one (<literal>S</literal>) indicates
+whether there is a so-called <emphasis>substitute</emphasis> for the
+component, which is Nix’s mechanism for doing binary deployment.  It
+just means that Nix know that it can fetch a pre-built component from
+somewhere (typically a network server) instead of building it
+locally.</para>
+
+<para>So now that we have a set of Nix expressions we can build the
+components contained in them.  This is done using <literal>nix-env
+-i</literal>.  For instance,
+
+<screen>
+$ nix-env -f nixpkgs-<replaceable>version</replaceable> -i subversion</screen>
+
+will install the component called <literal>subversion</literal> (which
+is, of course, the <ulink
+url='http://subversion.tigris.org/'>Subversion version management
+system</ulink>).</para>
+
+<para>When you do this for the first time, Nix will start building
+Subversion and all its dependencies.  This will take quite a while —
+typically an hour or two on modern machines.  Fortunately, there is a
+faster way (so do a Ctrl-C on that install operation!): you just need
+to tell Nix that pre-built binaries of all those components are
+available somewhere.  This is done using the
+<command>nix-pull</command> command, which must be supplied with a URL
+containing a <emphasis>manifest</emphasis> describing what binaries
+are available.  This URL should correspond to the Nix Packages release
+that you’re using.  For instance, if you obtained a release from
+<ulink
+url='http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />,
+then you should do:
+
+<screen>
+$ nix-pull http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/MANIFEST</screen>
+
+If you then issue the installation command, it should start
+downloading binaries from <systemitem
+class='fqdomainname'>catamaran.labs.cs.uu.nl</systemitem>, instead of
+building them from source.  This might still take a while since all
+dependencies must be downloaded, but on a reasonably fast connection
+such as an DSL line it’s on the order of a few minutes.</para>
+
+<para>Naturally, packages can also be uninstalled:
+
+<screen>
+$ nix-env -e subversion</screen>
+
+</para>
+
+<para>Upgrading to a new version is just as easy.  If you have a new
+release of Nix Packages, you can do:
+
+<screen>
+$ nix-env -f nixpkgs-<replaceable>version</replaceable> -u subversion</screen>
+
+This will <emphasis>only</emphasis> upgrade Subversion if there is a
+“newer” version in the new set of Nix expressions, as
+defined by some pretty arbitrary rules regarding ordering of version
+numbers (which generally do what you’d expect of them).  To just
+unconditionally replace Subversion with whatever version is in the Nix
+expressions, use <parameter>-i</parameter> instead of
+<parameter>-u</parameter>; <parameter>-i</parameter> will remove
+whatever version is already installed.</para>
+
+<para>You can also upgrade all components for which there are newer
+versions:
+
+<screen>
+$ nix-env -f nixpkgs-<replaceable>version</replaceable> -u '*'</screen>
+
+</para>
+
+<para>Sometimes it’s useful to be able to ask what
+<command>nix-env</command> would do, without actually doing it.  For
+instance, to find out what packages would be upgraded by
+<literal>nix-env -u '*'</literal>, you can do
+
+<screen>
+$ nix-env ... -u '*' --dry-run
+(dry run; not doing anything)
+upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
+upgrading `graphviz-1.10' to `graphviz-1.12'
+upgrading `coreutils-5.0' to `coreutils-5.2.1'</screen>
+
+</para>
+
+<para>If you grow bored of specifying the Nix expressions using
+<parameter>-f</parameter> all the time, you can set a default
+location:
+
+<screen>
+$ nix-env -I nixpkgs-<replaceable>version</replaceable></screen>
+
+After this you can just say, for instance, <literal>nix-env -u
+'*'</literal>.<footnote><para>Setting a default using
+<parameter>-I</parameter> currently clashes with using Nix channels,
+since <literal>nix-channel --update</literal> calls <literal>nix-env
+-I</literal> to set the default to the Nix expressions it downloaded
+from the channel, replacing whatever default you had
+set.</para></footnote></para>
+
+</sect1>
+
+
+<sect1 id="sec-profiles"><title>Profiles</title>
+
+<para>Profiles and user environments are Nix’s mechanism for
+implementing the ability to allow differens users to have different
+configurations, and to do atomic upgrades and rollbacks.  To
+understand how they work, it’s useful to know a bit about how Nix
+works.  In Nix, components are stored in unique locations in the
+<emphasis>Nix store</emphasis> (typically,
+<filename>/nix/store</filename>).  For instance, a particular version
+of the Subversion component might be stored in a directory
+<filename>/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/</filename>,
+while another version might be stored in
+<filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>.
+The long strings prefixed to the directory names are cryptographic
+hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in
+a base-32 notation, to be precise.</para></footnote> of
+<emphasis>all</emphasis> inputs involved in building the component —
+sources, dependencies, compiler flags, and so on.  So if two
+components differ in any way, they end up in different locations in
+the file system, so they don’t interfere with each other.  <xref
+linkend='fig-user-environments' /> shows a part of a typical Nix
+store.</para>
+
+<figure id='fig-user-environments'><title>User environments</title>
+  <mediaobject>
+    <imageobject>
+      <imagedata fileref='figures/user-environments.png' format='PNG' />
+    </imageobject>
+  </mediaobject>
+</figure>
+
+<para>Of course, you wouldn’t want to type
+
+<screen>
+$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen>
+
+every time you want to run Subversion.  Of course we could set up the
+<envar>PATH</envar> environment variable to include the
+<filename>bin</filename> directory of every component we want to use,
+but this is not very convenient since changing <envar>PATH</envar>
+doesn’t take effect for already existing processes.  The solution Nix
+uses is to create directory trees of symlinks to
+<emphasis>activated</emphasis> components.  These are called
+<emphasis>user environments</emphasis> and they are components
+themselves (though automatically generated by
+<command>nix-env</command>), so they too reside in the Nix store.  For
+instance, in <xref linkend='fig-user-environments' /> the user
+environment <filename>/nix/store/5mq2jcn36ldl...-user-env</filename>
+contains a symlink to just Subversion 1.1.2 (arrows in the figure
+indicate symlinks).  This would be what we would obtain if we had done
+
+<screen>
+$ nix-env -i subversion</screen>
+
+on a set of Nix expressions that contained Subversion 1.1.2.</para>
+
+<para>This doesn’t in itself solve the problem, of course; you
+wouldn’t want to type
+<filename>/nix/store/0c1p5z4kda11...-user-env/bin/svn</filename>
+either.  That’s why there are symlinks outside of the store that point
+to the user environments in the store; for instance, the symlinks
+<filename>default-42-link</filename> and
+<filename>default-43-link</filename> in the example.  These are called
+<emphasis>generations</emphasis> since every time you perform a
+<command>nix-env</command> operation, a new user environment is
+generated based on the current one.  For instance, generation 43 was
+created from generation 42 when we did
+
+<screen>
+$ nix-env -i subversion mozilla</screen>
+
+on a set of Nix expressions that contained Mozilla and a new version
+of Subversion.</para>
+
+<para>Generations are grouped together into
+<emphasis>profiles</emphasis> so that different users don’t interfere
+with each other if they don’t want to.  For example:
+
+<screen>
+$ ls -l /nix/var/nix/profiles/
+...
+lrwxrwxrwx  1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
+lrwxrwxrwx  1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
+lrwxrwxrwx  1 eelco ... default -> default-43-link</screen>
+
+This shows a profile called <filename>default</filename>.  The file
+<filename>default</filename> itself is actually a symlink that points
+to the current generation.  When we do a <command>nix-env</command>
+operation, a new user environment and generation link are created
+based on the current one, and finally the <filename>default</filename>
+symlink is made to point at the new generation.  This last step is
+atomic on Unix, which explains how we can do atomic upgrades.  (Note
+that the building/installing of new components doesn’t interfere in
+any way with old components, since they are stored in different
+locations in the Nix store.)</para>
+
+<para>If you find that you want to undo a <command>nix-env</command>
+operation, you can just do
+
+<screen>
+$ nix-env --rollback</screen>
+
+which will just make the current generation link point at the previous
+link.  E.g., <filename>default</filename> would be made to point at
+<filename>default-42-link</filename>.  You can also switch to a
+specific generation:
+
+<screen>
+$ nix-env --switch-generation 43</screen>
+
+which in this example would roll forward to generation 43 again.  You
+can also see all available generations:
+
+<screen>
+$ nix-env --list-generations</screen></para>
+
+<para>Actually, there is another level of indirection not shown in the
+figure above.  You generally wouldn’t have
+<filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
+in your <envar>PATH</envar>.  Rather, there is a symlink
+<filename>~/.nix-profile</filename> that points to your current
+profile.  This means that you should put
+<filename>~/.nix-profile/bin</filename> in your <envar>PATH</envar>
+(and indeed, that’s what the initialisation script
+<filename>/nix/etc/profile.d/nix.sh</filename> does).  This makes it
+easier to switch to a different profile.  You can do that using the
+command <command>nix-env --switch-profile</command>:
+
+<screen>
+$ nix-env --switch-profile /nix/var/nix/profiles/my-profile
+
+$ nix-env --switch-profile /nix/var/nix/profiles/default</screen>
+
+These commands switch to the <filename>my-profile</filename> and
+default profile, respectively.  If the profile doesn’t exist, it will
+be created automatically.  You should be careful about storing a
+profile in another location than the <filename>profiles</filename>
+directory, since otherwise it might not be used as a root of the
+garbage collector (see section <xref linkend='sec-garbage-collection'
+/>).</para>
+
+<para>All <command>nix-env</command> operations work on the profile
+pointed to by <command>~/.nix-profile</command>, but you can override
+this using the <option>--profile</option> option (abbreviation
+<option>-p</option>):
+
+<screen>
+$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion</screen>
+
+This will <emphasis>not</emphasis> change the
+<command>~/.nix-profile</command> symlink.</para>
+
+</sect1>
+
+
+<sect1 id='sec-garbage-collection'><title>Garbage collection</title>
+
+<para><command>nix-env</command> operations such as upgrades
+(<option>-u</option>) and uninstall (<option>-e</option>) never
+actually delete components from the system.  All they do (as shown
+above) is to create a new user environment that no longer contains
+symlinks to the “deleted” components.</para>
+
+<para>Of course, since disk space is not infinite, unused components
+should be removed at some point.  You can do this by running the Nix
+garbage collector.  It will remove from the Nix store any component
+not used (directly or indirectly) by any generation of any
+profile.</para>
+
+<para>Note however that as long as old generations reference a
+component, it will not be deleted.  After all, we wouldn’t be able to
+do a rollback otherwise.  So in order for garbage collection to be
+effective, you should also delete (some) old generations.  Of course,
+this should only be done if you are certain that you will not need to
+roll back.</para>
+
+<para>To delete all old (non-current) generations of your current
+profile:
+
+<screen>
+$ nix-env --delete-generations old</screen>
+
+Instead of <literal>old</literal> you can also specify a list of
+generations, e.g.,
+
+<screen>
+$ nix-env --delete-generations 10 11 14</screen>
+
+</para>
+
+<para>After removing appropriate old generations you can run the
+garbage collector as follows:
+
+<screen>
+$ nix-store --gc</screen>
+
+If you are feeling uncertain, you can also first view what files would
+be deleted:
+
+<screen>
+$ nix-store --gc --print-dead</screen>
+
+Likewise, the option <option>--print-live</option> will show the paths
+that <emphasis>won’t</emphasis> be deleted.</para>
+
+
+<sect2 id="ssec-gc-roots"><title>Garbage collector roots</title>
+
+<para>The roots of the garbage collector are all store paths to which
+there are symlinks in the directory
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>.
+For instance, the following command makes the path
+<filename>/nix/store/d718ef...-foo</filename> a root of the collector:
+
+<screen>
+$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar</screen>
+	
+That is, after this command, the garbage collector will not remove
+<filename>/nix/store/d718ef...-foo</filename> or any of its
+dependencies.</para>
+
+<para>Subdirectories of
+<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>
+are also searched for symlinks.  Symlinks to non-store paths are
+followed and searched for roots, but symlinks to non-store paths
+<emphasis>inside</emphasis> the paths reached in that way are not
+followed to prevent infinite recursion.</para>
+
+</sect2>
+
+</sect1>
+
+
+<sect1 id="sec-channels"><title>Channels</title>
+
+<para>If you want to stay up to date with a set of packages, it’s not
+very convenient to manually download the latest set of Nix expressions
+for those packages, use <command>nix-pull</command> to register
+pre-built binaries (if available), and upgrade using
+<command>nix-env</command>.  Fortunately, there’s a better way:
+<emphasis>Nix channels</emphasis>.</para>
+
+<para>A Nix channel is just a URL that points to a place that contains
+a set of Nix expressions and a manifest.  Using the command <link
+linkend="sec-nix-channel"><command>nix-channel</command></link> you
+can automatically stay up to date with whatever is available at that
+URL.</para>
+
+<para>You can “subscribe” to a channel using
+<command>nix-channel --add</command>, e.g.,
+
+<screen>
+$ nix-channel --add http://catamaran.labs.cs.uu.nl/dist/nix/channels/nixpkgs-unstable</screen>
+
+subscribes you to a channel that always contains that latest version
+of the Nix Packages collection.  (Instead of
+<literal>nixpkgs-unstable</literal> you could also subscribe to
+<literal>nixpkgs-stable</literal>, which should have a higher level of
+stability, but right now is just outdated.)  Subscribing really just
+means that the URL is added to the file
+<filename>~/.nix-channels</filename>.  Right now there is no command
+to “unsubscribe”; you should just edit that file manually
+and delete the offending URL.</para>
+
+<para>To obtain the latest Nix expressions available in a channel, do
+
+<screen>
+$ nix-channel --update</screen>
+
+This downloads the Nix expressions in every channel (downloaded from
+<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal>)
+and registers any available pre-built binaries in every channel
+(by <command>nix-pull</command>ing
+<literal><replaceable>url</replaceable>/MANIFEST</literal>).  It also
+makes the union of each channel’s Nix expressions the default for
+<command>nix-env</command> operations.  Consequently, you can then say
+
+<screen>
+$ nix-env -u '*'</screen>
+
+to upgrade all components in your profile to the latest versions
+available in the subscribed channels.</para>
+
+</sect1>
+
+
+</chapter>

doc/manual/quick-start.xml

@@ -1,141 +1,118 @@
-<chapter>
-  <title>Quick Start</title>
+<chapter><title>Quick Start</title>
 
-  <para>
-    This chapter is for impatient people who don't like reading
-    documentation.  For more in-depth information you are kindly
-    referred to <xref linkend='chap-installation' /> and <xref
-    linkend='chap-overview' />.
-  </para>
+<para>This chapter is for impatient people who don't like reading
+documentation.  For more in-depth information you are kindly referred
+to the following chapters.</para>
 
-  <orderedlist>
+<orderedlist>
 
-    <listitem>
-      <para>
-        Download a source tarball or RPM from <ulink
-        url='http://www.cs.uu.nl/groups/ST/Trace/Nix'/>.
-        Build source distributions using the regular sequence:
+<listitem><para>Download a source tarball or RPM from <ulink
+url='http://www.cs.uu.nl/groups/ST/Trace/Nix'/>.  Build source
+distributions using the regular sequence:
         
-        <screen>
+<screen>
 $ tar xvfj nix-<replaceable>version</replaceable>.tar.bz2
 $ ./configure
 $ make
 $ make install <lineannotation>(as root)</lineannotation></screen>
 
-        This will install Nix in <filename>/nix</filename>.  You
-        should also add <filename>/nix/etc/profile.d/nix.sh</filename>
-        to your <filename>~/.bashrc</filename> (or some other login
-        file).
-      </para>
-    </listitem>
+This will install Nix in <filename>/nix</filename>.  You shouldn't
+change the prefix if at all possible since that will make it
+impossible to use our pre-built components.  Alternatively, you could
+grab an RPM if you're on an RPM-based system.  You should also add
+<filename>/nix/etc/profile.d/nix.sh</filename> to your
+<filename>~/.bashrc</filename> (or some other login
+file).</para></listitem>
 
-    <listitem>
-      <para>
-        Get some Nix expressions for pre-built packages by downloading
-        the latest <literal>nixpkgs</literal> distribution (from the
-        same location), and unpack them.
+<listitem><para>Subscribe to the Nix Packages channel.
 
-        <screen>
-$ wget http://<replaceable>...</replaceable>/nix/nixpkgs-<replaceable>version</replaceable>/nixpkgs-<replaceable>version</replaceable>.tar.bz2
-$ tar xfj nixpkgs-<replaceable>version</replaceable>.tar.bz2</screen>
+<screen>
+$ nix-channel --add http://catamaran.labs.cs.uu.nl/dist/nix/channels/nixpkgs-unstable</screen>
 
-        This will unpack the distribution into a directory
-        <filename>nixpkgs-<replaceable>version</replaceable>/</filename>.
-      </para>
-    </listitem>
+</para></listitem>
 
-    <listitem>
-      <para>
-        Pull the Nix cache.  This ensures that when you install
-        packages they are downloaded in pre-built form from the
-        network, rather than built from source.
+<listitem><para>Download the latest Nix expressions available in the channel.
+<screen>
+$ nix-channel --update</screen>
 
-        <screen>
-$ nix-pull http://<replaceable>...</replaceable>/nix/nixpkgs-<replaceable>version</replaceable>/MANIFEST</screen>
+Note that this in itself doesn't download any components, it just
+downloads the Nix expressions that build them and stores them
+somewhere (under <filename>~/.nix-defexpr</filename>, in case you're
+curious).  Also, it registers the fact that pre-built binaries are
+available remotely.</para></listitem>
 
-      </para>
+<listitem><para>See what installable components are currently
+available in the channel:
 
-      <para>
-        Note that currently we only pre-build for Linux on x86
-        platforms.
-      </para>
-        
-    </listitem>
-        
-    <listitem>
-      <para>
-        See what's available:
-
-        <screen>
-$ nix-env -qaf nixpkgs-<replaceable>version</replaceable>/
-MozillaFirebird-0.7
-hello-2.1.1
+<screen>
+$ nix-env -qa
 docbook-xml-4.2
+firefox-1.0pre-PR-0.10.1
+hello-2.1.1
 libxslt-1.1.0
 <replaceable>...</replaceable></screen>
 
-      </para>
-    </listitem>
+</para></listitem>
 
-    <listitem>
-      <para>
-        Install some packages:
+<listitem><para>Install some components from the channel:
         
-        <screen>
-$ nix-env -iBf nixpkgs-<replaceable>version</replaceable>/ hello MozillaFirebird <replaceable>...</replaceable> </screen>
+<screen>
+$ nix-env -i hello firefox <replaceable>...</replaceable> </screen>
 
-      </para>
-    </listitem>
+This should download the pre-built components; it should not build
+them locally (if it does, something went wrong).</para></listitem>
 
-    <listitem>
-      <para>
-        Test that they work:
+<listitem><para>Test that they work:
 
-        <screen>
+<screen>
 $ which hello
 /home/eelco/.nix-profile/bin/hello
 $ hello
 Hello, world!
-$ MozillaFirebird
+$ firefox
 <lineannotation>(read Slashdot or something)</lineannotation></screen>
 
-      </para>
-    </listitem>
+</para></listitem>
     
-    <listitem>
-      <para>
-        Uninstall a package:
+<listitem><para>Uninstall a package:
 
-        <screen>
+<screen>
 $ nix-env -e hello</screen>
 
-      </para>
-    </listitem>
+</para></listitem>
 
-    <listitem>
-      <para>
-        If a new release of <literal>nixpkgs</literal> comes along,
-        you can upgrade all installed packages to the latest versions
-        by downloading and unpacking the new release and doing:
+<listitem><para>To keep up-to-date with the channel, do:
 
-        <screen>
-$ nix-env -uBf nixpkgs-<replaceable>version</replaceable>/ '*'</screen>
+<screen>
+$ nix-channel --update
+$ nix-env -u '*'</screen>
 
-      </para>
-    </listitem>
+The latter command will upgrade each installed component for which
+there is a “newer” version (as determined by comparing the version
+numbers).</para></listitem>
 
-    <listitem>
-      <para>
-        You should periodically run the Nix garbage collector to get
-        rid of unused packages, since uninstalls or upgrades don't
-        actual delete them:
+<listitem><para>If you're unhappy with the result of a
+<command>nix-env</command> action (e.g., an upgraded component turned
+out not to work properly), you can go back:
 
-        <screen>
-$ nix-collect-garbage | xargs nix-store --delete</screen>
+<screen>
+$ nix-env --rollback</screen>
 
-      </para>
-    </listitem>
+</para></listitem>
 
-  </orderedlist>
+<listitem><para>You should periodically run the Nix garbage collector
+to get rid of unused packages, since uninstalls or upgrades don't
+actually delete them:
 
-</chapter>
+<screen>
+$ nix-env --delete-generations old
+$ nix-store --gc</screen>
+
+The first command deletes old “generations” of your profile (making
+rollbacks impossible, but also making the components in those old
+generations available for garbage collection), while the second
+command actually deletes them.</para></listitem>
+
+</orderedlist>
+
+</chapter>

doc/manual/schemas.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0"?>
 <locatingRules xmlns="http://thaiopensource.com/ns/locating-rules/1.0">
-  <uri pathSuffix=".xml" typeId="DocBook"/>
+  <uri pattern="*.xml" typeId="DocBook"/>
 </locatingRules>

doc/manual/style.css

@@ -102,6 +102,8 @@ pre.screen
 
 .note,.warning
 {
+    margin-top: 1em;
+    margin-bottom: 1em;
     border: 1px solid #6185a0;
     padding: 0px 1em;
     background: #fffff5;
@@ -112,10 +114,16 @@ div.note,div.warning
     font-style: italic;
 }
 
-div.note h3,div.warning h3
+div.warning h3
 {
     color: red;
-    text-decoration: underline;
+    font-size: 100%;
+}
+
+div.note h3
+{
+    color: blue;
+    font-size: 100%;
 }
 
 div.navfooter *
@@ -148,7 +156,7 @@ a:hover { background: #ffffcd; }
                                Special elements:
  ***************************************************************************/
 
-tt
+tt, code
 {
     color: #400000;
 }
@@ -223,4 +231,4 @@ div.epigraph
 table.productionset table.productionset
 {
     font-family: monospace;
-}
+}

doc/manual/troubleshooting.xml

@@ -1,14 +1,71 @@
-<appendix>
-  <title>Troubleshooting</title>
+<appendix><title>Troubleshooting</title>
+
+<para>This section provides solutions for some common problems.</para>
+
+
+<sect1><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
+
+<para>Symptom: Nix operations (in particular the
+<command>nix-store</command> operations <option>--gc</option>,
+<option>--verify</option>, and <option>--clear-substitutes</option> —
+the latter being called by <command>nix-channel --update</command>)
+failing:
+
+<screen>
+$ nix-store --verify
+error: Db::del: Cannot allocate memory</screen>
+
+</para>
+
+<para>Possible solution: make sure that no Nix processes are running,
+then do:
+
+<screen>
+$ cd /nix/var/nix/db
+$ rm __db.00*</screen>
+
+</para>
+
+</sect1>
+
+
+<sect1><title>Collisions in <command>nix-env</command></title>
+
+<para>Symptom: when installing or upgrading, you get an error message such as
+
+<screen>
+$ nix-env -i docbook-xml
+...
+adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
+collission between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
+  and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
+  at /nix/store/...-builder.pl line 62.</screen>
+
+</para>
+
+<para>The cause is that two installed packages in the user environment
+have overlapping filenames (e.g.,
+<filename>xml/dtd/docbook/calstblx.dtd</filename>.  This usually
+happens when you accidentally try to install two versions of the same
+package.  For instance, in the example above, the Nix Packages
+collection contains two versions of <literal>docbook-xml</literal>, so
+<command>nix-env -i</command> will try to install both.  The default
+user environment builder has no way to way to resolve such conflicts,
+so it just gives up.</para>
+
+<para>Solution: remove one of the offending packages from the user
+environment (if already installed) using <command>nix-env
+-u</command>, or specify exactly which version should be installed
+(e.g., <literal>nix-env -i docbook-xml-4.2</literal>).</para>
+
+<para>Alternatively, you can modify the user environment builder
+script (in
+<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>)
+to implement some conflict resolution policy.  E.g., the script could
+be modified to rename conflicting file names, or to pick one over the
+other.</para>
+
+</sect1>
+
 
-  <para>
-    (Nothing.)
-  </para>
-	
 </appendix>
-
-<!--
-local variables:
-sgml-parent-document: ("book.xml" "appendix")
-end:
--->

externals/Makefile.am

@@ -1,11 +1,11 @@
 # Berkeley DB
 
-DB = db-4.2.52
+DB = db-4.3.28.NC
 
 $(DB).tar.gz:
 	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.2.52 from"
-	@echo "  http://www.sleepycat.com/update/snapshot/db-4.2.52.tar.gz"
+	@echo "Please download version 4.3.28 from"
+	@echo "  http://downloads.sleepycat.com/db-4.3.28.NC.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -26,20 +26,20 @@ build-db: have-db
 	 ../dist/configure --prefix=$$pfx/inst-bdb \
 	 --enable-cxx --disable-shared --disable-cryptography \
          --disable-replication --disable-verify && \
-	make && \
-	make install)
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-db
 endif
 
 
 # CWI ATerm
 
-ATERM = aterm-2.0.5
+ATERM = aterm-2.3.1
 
 $(ATERM).tar.gz:
 	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.0.5 from"
-	@echo "  http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.0.5.tar.gz"
+	@echo "Please download version 2.3.1 from"
+	@echo "  http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.3.1.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -56,10 +56,9 @@ else
 build-aterm: have-aterm
 	(pfx=`pwd` && \
 	cd $(ATERM) && \
-	CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm \
-     --with-cflags="-DNDEBUG -DXGC_VERBOSE -DXHASHPEM -DWITH_STATS $(CFLAGS)" && \
-	make && \
-	make install)
+	CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm && \
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-aterm
 endif
 

make/examples/aterm/aterm/default.nix

@@ -0,0 +1,34 @@
+{sharedLib ? true}:
+
+rec {
+
+  inherit (import ../../../lib) compileC makeLibrary;
+
+  sources = [
+    ./afun.c
+    ./aterm.c
+    ./bafio.c
+    ./byteio.c
+    ./gc.c
+    ./hash.c
+    ./list.c
+    ./make.c
+    ./md5c.c
+    ./memory.c
+    ./tafio.c
+    ./version.c
+  ];
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    forSharedLib = sharedLib;
+  };
+
+  libATerm = makeLibrary {
+    libraryName = "ATerm";
+    objects = map compile sources;
+    inherit sharedLib;
+  };
+
+}

make/examples/aterm/default.nix

@@ -0,0 +1 @@
+import test/default.nix

make/examples/aterm/test/default.nix

@@ -0,0 +1,18 @@
+let {
+
+  inherit (import ../../../lib) compileC link;
+
+  inherit (import ../aterm {}) libATerm;
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    cFlags = "-I../aterm";
+  };
+
+  fib = link {objects = compile ./fib.c; libraries = libATerm;};
+
+  primes = link {objects = compile ./primes.c; libraries = libATerm;};
+  
+  body = [fib primes];
+}

make/examples/default.nix

@@ -0,0 +1,6 @@
+[ (import ./trivial)
+  (import ./simple-header)
+  (import ./not-so-simple-header)
+  (import ./not-so-simple-header-auto)
+  (import ./aterm)
+]

make/examples/not-so-simple-header-auto/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header-auto/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC findIncludes link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = "auto";
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header-auto/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header-auto/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/not-so-simple-header/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header/default.nix

@@ -0,0 +1,14 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = [
+      [./foo/fnord/indirect.h "fnord/indirect.h"]
+      [./bar/hello.h "fnord/../../bar/hello.h"]
+    ];
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {
+    main = ./hello.c;
+    localIncludes = [ [./hello.h "hello.h"] ];
+  };};
+
+  body = [hello];
+}

make/examples/simple-header/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "hello.h"
+
+int main(int argc, char * * argv)
+{
+    printf("Hello " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/trivial/default.nix

@@ -0,0 +1,8 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {main = ./hello.c;};};
+
+  body = [hello];
+}

make/examples/trivial/hello.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(int argc, char * * argv)
+{
+    printf("Hello World\n");
+    return 0;
+}

make/lib/compile-c.sh

@@ -0,0 +1,73 @@
+. $stdenv/setup
+
+mainName=$(basename $main | cut -c34-)
+
+echo "compiling \`$mainName'..."
+
+# Turn $localIncludes into an array.
+localIncludes=($localIncludes)
+
+# Determine how many `..' levels appear in the header file references.
+# E.g., if there is some reference `../../foo.h', then we have to
+# insert two extra levels in the directory structure, so that `a.c' is
+# stored at `dotdot/dotdot/a.c', and a reference from it to
+# `../../foo.h' resolves to `dotdot/dotdot/../../foo.h' == `foo.h'.
+n=0
+maxDepth=0
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    target=${localIncludes[$((n + 1))]}
+
+    # Split the target name into path components using some IFS magic.
+    savedIFS="$IFS"
+    IFS=/
+    components=($target)
+    depth=0
+    for ((m = 0; m < ${#components[*]}; m++)); do
+        c=${components[m]}
+        if test "$c" = ".."; then
+            depth=$((depth + 1))
+        fi
+    done
+    IFS="$savedIFS"
+
+    if test $depth -gt $maxDepth; then
+        maxDepth=$depth;
+    fi
+done
+
+# Create the extra levels in the directory hierarchy.
+prefix=
+for ((n = 0; n < maxDepth; n++)); do
+    prefix="dotdot/$prefix"
+done
+
+# Create symlinks to the header files.
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    source=${localIncludes[n]}
+    target=${localIncludes[$((n + 1))]}
+
+    # Create missing directories.  We use IFS magic to split the path
+    # into path components.
+    savedIFS="$IFS"
+    IFS=/
+    components=($prefix$target)
+    fullPath=(.)
+    for ((m = 0; m < ${#components[*]} - 1; m++)); do
+        fullPath=("${fullPath[@]}" ${components[m]})
+        if ! test -d "${fullPath[*]}"; then
+            mkdir "${fullPath[*]}"
+        fi
+    done
+    IFS="$savedIFS"
+    
+    ln -sf $source $prefix$target
+done
+
+# Create a symlink to the main file.
+if ! test "$(readlink $prefix$mainName)" = $main; then
+    ln -s $main $prefix$mainName
+fi
+
+mkdir $out
+test "$prefix" && cd $prefix
+gcc -Wall $cFlags -c $mainName -o $out/$mainName.o

make/lib/default.nix

@@ -0,0 +1,59 @@
+rec {
+
+  # Should point at your Nixpkgs installation.
+  pkgPath = ./pkgs;
+
+  pkgs = import (pkgPath + /system/all-packages.nix) {};
+
+  stdenv = pkgs.stdenv;
+  
+
+  compileC = {main, localIncludes ? [], cFlags ? "", forSharedLib ? false}:
+  stdenv.mkDerivation {
+    name = "compile-c";
+    builder = ./compile-c.sh;
+    localIncludes =
+      if localIncludes == "auto" then
+        import (findIncludes {
+          main = toString main;
+          hack = __currentTime;
+          inherit cFlags;
+        })
+      else
+        localIncludes;
+    inherit main;
+    cFlags = [
+      cFlags
+      (if forSharedLib then ["-fpic"] else [])
+    ];
+  };
+
+  /*
+  runCommand = {command}: {
+    name = "run-command";
+    builder = ./run-command.sh;
+    inherit command;
+  };
+  */
+
+  findIncludes = {main, hack, cFlags ? ""}: stdenv.mkDerivation {
+    name = "find-includes";
+    builder = ./find-includes.sh;
+    inherit main hack cFlags;
+  };
+  
+  link = {objects, programName ? "program", libraries ? []}: stdenv.mkDerivation {
+    name = "link";
+    builder = ./link.sh;
+    inherit objects programName libraries;
+  };
+
+  makeLibrary = {objects, libraryName ? [], sharedLib ? false}:
+  # assert sharedLib -> fold (obj: x: assert obj.sharedLib && x) false objects
+  stdenv.mkDerivation {
+    name = "library";
+    builder = ./make-library.sh;
+    inherit objects libraryName sharedLib;
+  };
+
+}

make/lib/find-includes.sh

@@ -0,0 +1,20 @@
+. $stdenv/setup
+
+echo "finding includes of \`$(basename $main)'..."
+
+makefile=$NIX_BUILD_TOP/makefile
+
+mainDir=$(dirname $main)
+(cd $mainDir && gcc $cFlags -MM $(basename $main) -MF $makefile) || false
+
+echo "[" >$out
+
+while read line; do
+    line=$(echo "$line" | sed 's/.*://')
+    for i in $line; do
+        fullPath=$(readlink -f $mainDir/$i)
+        echo "  [ $fullPath \"$i\" ]" >>$out
+    done
+done < $makefile
+
+echo "]" >>$out

make/lib/link.sh

@@ -0,0 +1,21 @@
+. $stdenv/setup
+
+shopt -s nullglob
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+libs=
+for i in $libraries; do
+    lib=$(echo $i/*.a; echo $i/*.so)
+    name=$(echo $(basename $lib) | sed -e 's/^lib//' -e 's/.a$//' -e 's/.so$//')
+    libs="$libs -L$(dirname $lib) -l$name" 
+done
+
+echo "linking object files into \`$programName'..."
+
+mkdir $out
+gcc -o $out/$programName $objs $libs

make/lib/make-library.sh

@@ -0,0 +1,28 @@
+. $stdenv/setup
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+echo "archiving object files into library \`$libraryName'..."
+
+ensureDir $out
+
+if test -z "$sharedLib"; then
+
+    outPath=$out/lib${libraryName}.a
+
+    ar crs $outPath $objs
+    ranlib $outPath
+
+else
+
+    outPath=$out/lib${libraryName}.so
+
+    gcc -shared -o $outPath $objs
+
+fi    
+
+    

misc/Makefile.am

@@ -0,0 +1 @@
+SUBDIRS = emacs

misc/emacs/Makefile.am

@@ -0,0 +1,5 @@
+EXTRA_DIST = nix-mode.el
+
+install-data-local:
+	$(INSTALL) -d $(DESTDIR)$(datadir)/emacs/site-lisp
+	$(INSTALL_DATA) nix-mode.el $(DESTDIR)$(datadir)/emacs/site-lisp

misc/emacs/README

@@ -0,0 +1,10 @@
+The Nix Emacs mode supports syntax highlighting, somewhat sensible
+indenting, and refilling of comments.
+
+To enable Nix mode in Emacs, add something like this to your ~/.emacs
+file:
+
+  (load "/nix/share/emacs/site-lisp/nix-mode.el")
+
+This automatically causes Nix mode to be activated for all files with
+extension `.nix'.

misc/emacs/nix-mode.el

@@ -0,0 +1,109 @@
+(defun nix-mode ()
+  "Major mode for editing Nix expressions.
+
+The following commands may be useful:
+
+  '\\[newline-and-indent]'
+    Insert a newline and move the cursor to align with the previous
+    non-empty line.
+
+  '\\[fill-paragraph]'
+    Refill a paragraph so that all lines are at most `fill-column'
+    lines long.  This should do the right thing for comments beginning
+    with `#'.  However, this command doesn't work properly yet if the
+    comment is adjacent to code (i.e., no intervening empty lines).
+    In that case, select the text to be refilled and use
+    `\\[fill-region]' instead.
+
+The hook `nix-mode-hook' is run when Nix mode is started.
+
+\\{nix-mode-map}
+"
+
+  (interactive)
+  
+  (kill-all-local-variables)
+  
+  (setq major-mode 'nix-mode)
+  (setq mode-name "Nix")
+
+  (use-local-map nix-mode-map)
+
+  (set-syntax-table nix-mode-syntax-table)
+
+  ;; Font lock support.
+  (setq font-lock-defaults '(nix-keywords nil nil nil nil))
+
+  ;; Automatic indentation [C-j].
+  (make-local-variable 'indent-line-function)
+  (setq indent-line-function 'nix-indent-line)
+
+  ;; Indenting of comments.
+  (make-local-variable 'comment-start)
+  (setq comment-start "# ")
+  (make-local-variable 'comment-end)
+  (setq comment-end "")
+  (make-local-variable 'comment-start-skip)
+  (setq comment-start-skip "\\(^\\|\\s-\\);?#+ *")
+
+  ;; Filling of comments.
+  (make-local-variable 'adaptive-fill-mode)
+  (setq adaptive-fill-mode t)
+  (make-local-variable 'paragraph-start)
+  (setq paragraph-start "[ \t]*\\(#+[ \t]*\\)?$")
+  (make-local-variable 'paragraph-separate)
+  (setq paragraph-separate paragraph-start)
+  
+  (run-hooks 'nix-mode-hook)
+  )
+
+
+(defvar nix-mode-map nil
+  "Keymap for Nix mode.")
+
+(setq nix-mode-map (make-sparse-keymap))
+;(define-key nix-mode-map [tab] 'tab-to-tab-stop)
+
+
+(defvar nix-keywords
+  '("\\<if\\>" "\\<then\\>" "\\<else\\>" "\\<assert\\>"
+    "\\<let\\>" "\\<rec\\>" "\\<inherit\\>"
+    ("\\<true\\>" . font-lock-builtin-face)
+    ("\\<false\\>" . font-lock-builtin-face)
+    ("\\<null\\>" . font-lock-builtin-face)
+    ("\\<import\\>" . font-lock-builtin-face)
+    ("\\<derivation\\>" . font-lock-builtin-face)
+    ("\\<baseNameOf\\>" . font-lock-builtin-face)
+    ("\\<toString\\>" . font-lock-builtin-face)
+    ("\\<isNull\\>" . font-lock-builtin-face)
+    ("\\<\\([a-zA-Z_][a-zA-Z0-9_']*\\)[ \t]*="
+     (1 font-lock-variable-name-face nil nil))
+    ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+"
+     . font-lock-constant-face)
+    ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+"
+     . font-lock-constant-face)
+    ))
+
+
+(defvar nix-mode-syntax-table nil
+  "Syntax table for Nix mode.")
+
+(if nix-mode-syntax-table
+    nil
+    (progn
+      (setq nix-mode-syntax-table (make-syntax-table))
+      (modify-syntax-entry ?/ ". 14" nix-mode-syntax-table)
+      (modify-syntax-entry ?* ". 23" nix-mode-syntax-table)
+      (modify-syntax-entry ?# "< b" nix-mode-syntax-table)
+      (modify-syntax-entry ?\n "> b" nix-mode-syntax-table)
+      ))
+
+
+(defun nix-indent-line ()
+  "Indent current line in a Nix expression."
+  (interactive)
+  (indent-relative-maybe))
+
+
+(setq auto-mode-alist (cons '("\\.nix\\'" . nix-mode) auto-mode-alist))
+(setq auto-mode-alist (cons '("\\.nix.in\\'" . nix-mode) auto-mode-alist))

nix.conf.example

@@ -0,0 +1,49 @@
+### Option `gc-keep-outputs'
+#
+# If `true', the garbage collector will keep the outputs of
+# non-garbage derivations.  If `false' (default), outputs will be
+# deleted unless they are GC roots themselves (or reachable from other
+# roots).
+# 
+# In general, outputs must be registered as roots separately.
+# However, even if the output of a derivation is registered as a root,
+# the collector will still delete store paths that are used only at
+# build time (e.g., the C compiler, or source tarballs downloaded from
+# the network).  To prevent it from doing so, set this option to
+# `true'.
+gc-keep-outputs = false
+
+
+### Option `gc-keep-derivations'
+#
+# If `true' (default), the garbage collector will keep the derivations
+# from which non-garbage store paths were built.  If `false', they
+# will be deleted unless explicitly registered as a root (or reachable
+# from other roots).
+#
+# Keeping derivation around is useful for querying and traceability
+# (e.g., it allows you to ask with what dependencies or options a
+# store path was built), so by default this option is on.  Turn it off
+# to safe a bit of disk space (or a lot if `gc-keep-outputs' is also
+# turned on).
+gc-keep-derivations = true
+
+
+### Option `env-keep-derivations'
+#
+# If `false' (default), derivations are not stored in Nix user
+# environments.  That is, the derivation any build-time-only
+# dependencies may be garbage-collected.
+#
+# If `true', when you add a Nix derivation to a user environment, the
+# path of the derivation is stored in the user environment.  Thus, the
+# derivation will not be garbage-collected until the user environment
+# generation is deleted (`nix-env --delete-generations').  To prevent
+# build-time-only dependencies from being collected, you should also
+# turn on `gc-keep-outputs'.
+#
+# The difference between this option and `gc-keep-derivations' is that
+# this one is `sticky': it applies to any user environment created
+# while this option was enabled, while `gc-keep-derivations' only
+# applies at the moment the garbage collector is run.
+env-keep-derivations = false

nix.spec.in

@@ -1,26 +1,49 @@
+%define enable_setuid ""
+%define nix_user "nix"
+%define nix_group "nix"
+
+# If set, the Nix user and group will be created by the RPM
+# pre-install script.
+%define nix_user_uid ""
+%define nix_group_gid ""
+
 Summary: The Nix software deployment system
 Name: nix
 Version: @version@
 Release: 1
 License: GPL
-Group: WeetNiet
-URL: http://www.cs.uu.nl/groups/ST/twiki/bin/view/Trace/NixDeploymentSystem
-Source0: %{name}-@version@.tar.gz
+Group: Software Deployment
+URL: http://www.cs.uu.nl/groups/ST/Trace/Nix
+Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
 %define _prefix /nix
 Prefix: %{_prefix}
 Requires: /usr/bin/perl
 
+# Hack to make that shitty RPM scanning hack shut up.
+Provides: perl(readmanifest)
+                                                               
 %description
 
-Nix is a software deployment system.
+Nix is a system for software deployment.
 
 %prep
 %setup -q
 
 %build
-./configure --prefix=%{_prefix}
+extraFlags=
+if test -n "%{enable_setuid}"; then
+    extraFlags="$extraFlags --enable-setuid"
+    if test -n "%{nix_user}"; then
+        extraFlags="$extraFlags --with-nix-user=%{nix_user}"
+    fi
+    if test -n "%{nix_group}"; then
+        extraFlags="$extraFlags --with-nix-group=%{nix_group}"
+    fi
+fi
+./configure --prefix=%{_prefix} $extraFlags
 make
+make check
 
 %install
 rm -rf $RPM_BUILD_ROOT
@@ -30,8 +53,18 @@ strip $RPM_BUILD_ROOT/%{_prefix}/bin/* || true
 %clean
 rm -rf $RPM_BUILD_ROOT
 
+%pre
+if test -n "%{nix_group_gid}"; then
+    /usr/sbin/groupadd -g %{nix_group_gid} %{nix_group} || true
+fi
+if test -n "%{nix_user_uid}"; then
+    /usr/sbin/useradd -c "Nix" -u %{nix_user_uid} \
+        -s /sbin/nologin -r -d /var/empty %{nix_user} \
+        -g %{nix_group} || true
+fi
+
 %files
-%defattr(-,root,root)
+#%defattr(-,root,root)
 %{_prefix}/bin
 %{_prefix}/libexec
 %{_prefix}/var

scripts/Makefile.am

@@ -1,24 +1,26 @@
 bin_SCRIPTS = nix-collect-garbage \
- nix-pull nix-push nix-prefetch-url \
- nix-install-package nix-channel
+  nix-pull nix-push nix-prefetch-url \
+  nix-install-package nix-channel nix-build
 
-noinst_SCRIPTS = nix-profile.sh
+noinst_SCRIPTS = nix-profile.sh generate-patches.pl
 
-nix-pull nix-push: readmanifest.pm
+nix-pull nix-push: readmanifest.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm
+install-exec-local: readmanifest.pm download-using-manifests.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
-# !!! don't overwrite local modifications
-	$(INSTALL_DATA) prebuilts.conf $(DESTDIR)$(sysconfdir)/nix/prebuilts.conf
 
 include ../substitute.mk
 
 EXTRA_DIST = nix-collect-garbage.in \
- nix-pull.in nix-push.in nix-profile.sh.in \
- nix-prefetch-url.in nix-install-package.in \
- nix-channel.in \
- prebuilts.conf readmanifest.pm.in
+  nix-pull.in nix-push.in nix-profile.sh.in \
+  nix-prefetch-url.in nix-install-package.in \
+  nix-channel.in \
+  readmanifest.pm.in \
+  nix-build.in \
+  download-using-manifests.pl.in \
+  generate-patches.pl.in

scripts/copying-collector.pl

@@ -0,0 +1,111 @@
+#! /usr/bin/perl -w
+
+use strict;
+
+my @paths = `nix-store -qR /home/eelco/.nix-profile/bin/firefox`;
+
+my %copyMap;
+my %rewriteMap;
+
+
+my $counter = 0;
+
+foreach my $path (@paths) {
+    chomp $path;
+
+    $path =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$path'";
+    my $hash = $2;
+
+#    my $newHash = "deadbeef" . (sprintf "%024d", $counter++);
+    my $newHash = "deadbeef" . substr($hash, 0, 24);
+    my $newPath = "/home/eelco/chroot/$1/$newHash-$3";
+
+    die unless length $newHash == length $hash;
+
+    $copyMap{$path} = $newPath;
+    $rewriteMap{$hash} = $newHash;
+}
+
+
+my %rewriteMap2;
+
+
+sub rewrite;
+sub rewrite {
+    my $src = shift;
+    my $dst = shift;
+
+    if (-l $dst) {
+
+        my $target = readlink $dst or die;
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+            $target =~ s/$srcHash/$dstHash/g;
+        }
+
+        unlink $dst or die;
+
+        symlink $target, $dst;
+        
+    }
+
+    elsif (-f $dst) {
+
+        print "$dst\n";
+
+        foreach my $srcHash (keys %rewriteMap2) {
+            my $dstHash = $rewriteMap{$srcHash};
+            print "  $srcHash -> $dstHash\n";
+
+            my @stats = lstat $dst or die;
+            
+            system "sed s/$srcHash/$dstHash/g < '$dst' > '$dst.tmp'";
+            die if $? != 0;
+            rename "$dst.tmp", $dst or die;
+
+            chmod $stats[2], $dst or die;
+        }
+
+    }
+
+    elsif (-d $dst) {
+
+        chmod 0755, $dst;
+        
+        opendir(DIR, "$dst") or die "cannot open `$dst': $!";
+        my @files = readdir DIR;
+        closedir DIR;
+        
+        foreach my $file (@files) {
+            next if $file eq "." || $file eq "..";
+            rewrite "$src/$file", "$dst/$file";
+        }
+    }
+}
+
+
+foreach my $src (keys %copyMap) {
+    my $dst = $copyMap{$src};
+    print "$src -> $dst\n";
+
+    if (!-e $dst) {
+        system "cp -prd $src $dst";
+        die if $? != 0;
+
+        my @refs = `nix-store -q --references $src`;
+
+        %rewriteMap2 = ();
+        foreach my $ref (@refs) {
+            chomp $ref;
+
+            $ref =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$ref'";
+            my $hash = $2;
+
+            $rewriteMap2{$hash} = $rewriteMap{$hash};
+        }
+
+        rewrite $src, $dst;
+    }
+}

scripts/download-using-manifests.pl.in

@@ -0,0 +1,279 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use readmanifest;
+
+my $manifestDir = "@localstatedir@/nix/manifests";
+my $logFile = "@localstatedir@/log/nix/downloads";
+
+open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
+
+
+# Check the arguments.
+die unless scalar @ARGV == 1;
+my $targetPath = $ARGV[0];
+
+my $date = `date` or die;
+chomp $date;
+print LOGFILE "$$ get $targetPath $date\n";
+
+print "\n*** Trying to download/patch `$targetPath'\n";
+
+
+# Load all manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+for my $manifest (glob "$manifestDir/*.nixmanifest") {
+#    print STDERR "reading $manifest\n";
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
+        exit 1;
+    }
+}
+
+
+# Build a graph of all store paths that might contribute to the
+# construction of $targetPath, and the special node "start".  The
+# edges are either patch operations, or downloads of full NAR files.
+# The latter edges only occur between "start" and a store path.
+
+my %graph;
+
+$graph{"start"} = {d => 0, pred => undef, edges => []};
+
+my @queue = ();
+my $queueFront = 0;
+my %done;
+
+sub addToQueue {
+    my $v = shift;
+    return if defined $done{$v};
+    $done{$v} = 1;
+    push @queue, $v;
+}
+
+sub addNode {
+    my $u = shift;
+    $graph{$u} = {d => 999999999999, pred => undef, edges => []}
+        unless defined $graph{$u};
+}
+
+sub addEdge {
+    my $u = shift;
+    my $v = shift;
+    my $w = shift;
+    my $type = shift;
+    my $info = shift;
+    addNode $u;
+    push @{$graph{$u}->{edges}},
+        {weight => $w, start => $u, end => $v, type => $type, info => $info};
+    my $n = scalar @{$graph{$u}->{edges}};
+}
+
+addToQueue $targetPath;
+
+sub isValidPath {
+    my $p = shift;
+    system "@bindir@/nix-store --check-validity '$p' 2> /dev/null";
+    return $? == 0;
+}
+
+sub parseHash {
+    my $hash = shift;
+    if ($hash =~ /^(.+):(.+)$/) {
+        return ($1, $2);
+    } else {
+        return ("md5", $hash);
+    }
+}
+
+while ($queueFront < scalar @queue) {
+    my $u = $queue[$queueFront++];
+#    print "$u\n";
+
+    addNode $u;
+
+    # If the path already exists, it has distance 0 from the "start"
+    # node.
+    if (isValidPath($u)) {
+        addEdge "start", $u, 0, "present", undef;
+    }
+
+    else {
+
+        # Add patch edges.
+        my $patchList = $patches{$u};
+        foreach my $patch (@{$patchList}) {
+            if (isValidPath($patch->{basePath})) {
+                # !!! this should be cached
+                my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
+                my $format = "--base32";
+                $format = "" if $baseHashAlgo eq "md5";
+                my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
+                chomp $hash;
+#                print "  MY HASH is $hash\n";
+                if ($hash ne $baseHash) {
+                    print LOGFILE "$$ rejecting $patch->{basePath}\n";
+                    next;
+                }
+            }
+#            print "  PATCH from $patch->{basePath}\n";
+            addToQueue $patch->{basePath};
+            addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
+        }
+
+        # Add NAR file edges to the start node.
+        my $narFileList = $narFiles{$u};
+        foreach my $narFile (@{$narFileList}) {
+#            print "  NAR from $narFile->{url}\n";
+            addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
+            if ($u eq $targetPath) {
+                print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
+            }
+        }
+
+    }
+}
+
+
+# Run Dijkstra's shortest path algorithm to determine the shortest
+# sequence of download and/or patch actions that will produce
+# $targetPath.
+
+sub byDistance { # sort by distance, reversed
+    return -($graph{$a}->{d} <=> $graph{$b}->{d});
+}
+
+my @todo = keys %graph;
+
+while (scalar @todo > 0) {
+
+    # Remove the closest element from the todo list.
+    @todo = sort byDistance @todo;
+    my $u = pop @todo;
+
+    my $u_ = $graph{$u};
+
+#    print "IN $u $u_->{d}\n";
+
+    foreach my $edge (@{$u_->{edges}}) {
+        my $v_ = $graph{$edge->{end}};
+        if ($v_->{d} > $u_->{d} + $edge->{weight}) {
+            $v_->{d} = $u_->{d} + $edge->{weight};
+            # Store the edge; to edge->start is actually the
+            # predecessor.
+            $v_->{pred} = $edge; 
+#            print "  RELAX $edge->{end} $v_->{d}\n";
+        }
+    }
+}
+
+
+# Retrieve the shortest path from "start" to $targetPath.
+my @path = ();
+my $cur = $targetPath;
+die "don't know how to produce $targetPath\n"
+    unless defined $graph{$targetPath}->{pred};
+while ($cur ne "start") {
+    push @path, $graph{$cur}->{pred};
+    $cur = $graph{$cur}->{pred}->{start};
+}
+
+
+# Traverse the shortest path, perform the actions described by the
+# edges.
+my $curStep = 1;
+my $maxStep = scalar @path;
+
+sub downloadFile {
+    my $url = shift;
+    my ($hashAlgo, $hash) = parseHash(shift);
+    $ENV{"PRINT_PATH"} = 1;
+    $ENV{"QUIET"} = 1;
+    $ENV{"NIX_HASH_ALGO"} = $hashAlgo;
+    my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
+    die "download of `$url' failed" unless $? == 0;
+    chomp $hash2;
+    chomp $path;
+    die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
+    return $path;
+}
+
+while (scalar @path > 0) {
+    my $edge = pop @path;
+    my $u = $edge->{start};
+    my $v = $edge->{end};
+
+    print "\n*** Step $curStep/$maxStep: ";
+
+    if ($edge->{type} eq "present") {
+        print "using already present path `$v'\n";
+        print LOGFILE "$$ present $v\n";
+
+        if ($curStep < $maxStep) {
+            # Since this is not the last step, the path will be used
+            # as a base to one or more patches.  So turn the base path
+            # into a NAR archive, to which we can apply the patch.
+            print "  packing base path...\n";
+            system "@bindir@/nix-store --dump $v > /tmp/nar";
+            die "cannot dump `$v'" if ($? != 0);
+        }
+    }
+
+    elsif ($edge->{type} eq "patch") {
+        my $patch = $edge->{info};
+        print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
+
+        print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n";
+
+        # Download the patch.
+        print "  downloading patch...\n";
+        my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
+
+        # Apply the patch to the NAR archive produced in step 1 (for
+        # the already present path) or a later step (for patch sequences).
+        print "  applying patch...\n";
+        system "@libexecdir@/bspatch /tmp/nar /tmp/nar2 $patchPath";
+        die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
+
+        if ($curStep < $maxStep) {
+            # The archive will be used as the base of the next patch.
+            rename "/tmp/nar2", "/tmp/nar" or die "cannot rename NAR archive: $!";
+        } else {
+            # This was the last patch.  Unpack the final NAR archive
+            # into the target path.
+            print "  unpacking patched archive...\n";
+            system "@bindir@/nix-store --restore $v < /tmp/nar2";
+            die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
+        }
+    }
+
+    elsif ($edge->{type} eq "narfile") {
+        my $narFile = $edge->{info};
+        print "downloading `$narFile->{url}' into `$v'\n";
+
+        print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
+        
+        # Download the archive.
+        print "  downloading archive...\n";
+        my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
+
+        if ($curStep < $maxStep) {
+            # The archive will be used a base to a patch.
+            system "@bunzip2@ < '$narFilePath' > /tmp/nar";
+        } else {
+            # Unpack the archive into the target path.
+            print "  unpacking archive...\n";
+            system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
+            die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
+        }
+    }
+
+    $curStep++;
+}
+
+
+print LOGFILE "$$ success\n";
+close LOGFILE;

scripts/gc-releases.pl

@@ -0,0 +1,75 @@
+#! /usr/bin/perl -w
+
+use strict;
+use readmanifest;
+
+
+# Read the archive directories.
+my @archives = ();
+my %archives;
+
+sub readDir {
+    my $dir = shift;
+    opendir(DIR, "$dir") or die "cannot open `$dir': $!";
+    my @as = readdir DIR;
+    foreach my $archive (@as) {
+        push @archives, $archive;
+        $archives{$archive} = "$dir/$archive";
+    }
+    closedir DIR;
+}
+
+readDir "/mnt/scratchy/eelco/public_html/nix-cache";
+readDir "/mnt/scratchy/eelco/public_html/patches";
+
+print STDERR scalar @archives, "\n";
+
+
+# Read the manifests.
+my %narFiles;
+my %patches;
+my %successors;
+
+foreach my $manifest (@ARGV) {
+    print STDERR "loading $manifest\n";
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors, 1) < 3) {
+#        die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n";
+    }
+}
+
+
+# Find the live archives.
+my %usedFiles;
+
+foreach my $narFile (keys %narFiles) {
+    foreach my $file (@{$narFiles{$narFile}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+foreach my $patch (keys %patches) {
+    foreach my $file (@{$patches{$patch}}) {
+        $file->{url} =~ /\/([^\/]+)$/;
+        my $basename = $1;
+        die unless defined $basename;
+#        print $basename, "\n";
+        $usedFiles{$basename} = 1;
+        die "missing archive `$basename'"
+            unless defined $archives{$basename};
+    }
+}
+
+
+# Print out the dead archives.
+foreach my $archive (@archives) {
+    next if $archive eq "." || $archive eq "..";
+    if (!defined $usedFiles{$archive}) {
+        print $archives{$archive}, "\n";
+    }
+}

scripts/generate-patches.pl.in

@@ -0,0 +1,376 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use POSIX qw(tmpnam);
+use readmanifest;
+
+die unless scalar @ARGV == 5;
+
+my $hashAlgo = "sha256";
+
+my $cacheDir = $ARGV[0];
+my $patchesDir = $ARGV[1];
+my $patchesURL = $ARGV[2];
+my $srcDir = $ARGV[3];
+my $dstDir = $ARGV[4];
+
+my $tmpdir;
+do { $tmpdir = tmpnam(); }
+until mkdir $tmpdir, 0777;
+
+print "TEMP = $tmpdir\n";
+
+#END { rmdir $tmpdir; }
+
+my %srcNarFiles;
+my %srcPatches;
+my %srcSuccessors;
+
+my %dstNarFiles;
+my %dstPatches;
+my %dstSuccessors;
+
+readManifest "$srcDir/MANIFEST",
+    \%srcNarFiles, \%srcPatches, \%srcSuccessors;
+
+readManifest "$dstDir/MANIFEST",
+    \%dstNarFiles, \%dstPatches, \%dstSuccessors;
+
+
+sub findOutputPaths {
+    my $narFiles = shift;
+    my $successors = shift;
+
+    my %outPaths;
+    
+    foreach my $p (keys %{$narFiles}) {
+
+        # Ignore store expressions.
+        next if ($p =~ /\.store$/);
+        next if ($p =~ /\.drv$/);
+        
+        # Ignore builders (too much ambiguity -- they're all called
+        # `builder.sh').
+        next if ($p =~ /\.sh$/);
+        next if ($p =~ /\.patch$/);
+        
+        # Don't bother including tar files etc.
+        next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
+
+        $outPaths{$p} = 1;
+    }
+
+    return %outPaths;
+}
+
+print "finding src output paths...\n";
+my %srcOutPaths = findOutputPaths \%srcNarFiles, \%srcSuccessors;
+
+print "finding dst output paths...\n";
+my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
+
+
+sub getNameVersion {
+    my $p = shift;
+    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
+    my $name = $1;
+    my $version = $2;
+    $name =~ s/^-//;
+    $version =~ s/^-//;
+    return ($name, $version);
+}
+
+
+# A quick hack to get a measure of the `distance' between two
+# versions: it's just the position of the first character that differs
+# (or 999 if they are the same).
+sub versionDiff {
+    my $s = shift;
+    my $t = shift;
+    my $i;
+    return 999 if $s eq $t;
+    for ($i = 0; $i < length $s; $i++) {
+        return $i if $i >= length $t or
+            substr($s, $i, 1) ne substr($t, $i, 1);
+    }
+    return $i;
+}
+
+
+sub getNarBz2 {
+    my $narFiles = shift;
+    my $storePath = shift;
+    
+    my $narFileList = $$narFiles{$storePath};
+    die "missing store expression $storePath" unless defined $narFileList;
+
+    my $narFile = @{$narFileList}[0];
+    die unless defined $narFile;
+
+    $narFile->{url} =~ /\/([^\/]+)$/;
+    die unless defined $1;
+    return "$cacheDir/$1";
+}
+
+
+sub containsPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $basePath = shift;
+    my $patchList = $$patches{$storePath};
+    return 0 if !defined $patchList;
+    my $found = 0;
+    foreach my $patch (@{$patchList}) {
+        # !!! baseHash might differ
+        return 1 if $patch->{basePath} eq $basePath;
+    }
+    return 0;
+}
+
+
+# Compute the "weighted" number of uses of a path in the build graph.
+sub computeUses {
+    my $narFiles = shift;
+    my $path = shift;
+
+    # Find the deriver of $path.
+    return 1 unless defined $$narFiles{$path};
+    my $deriver = @{$$narFiles{$path}}[0]->{deriver};
+    return 1 unless defined $deriver && $deriver ne "";
+
+#    print "  DERIVER $deriver\n";
+
+    # Optimisation: build the referers graph from the references
+    # graph.
+    my %referers;
+    foreach my $q (keys %{$narFiles}) {
+        my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
+        foreach my $r (@refs) {
+            $referers{$r} = [] unless defined $referers{$r};
+            push @{$referers{$r}}, $q;
+        }
+    }
+
+    # Determine the shortest path from $deriver to all other reachable
+    # paths in the `referers' graph.
+
+    my %dist;
+    $dist{$deriver} = 0;
+
+    my @queue = ($deriver);
+    my $pos = 0;
+    
+    while ($pos < scalar @queue) {
+        my $p = $queue[$pos];
+        $pos++;
+
+        foreach my $q (@{$referers{$p}}) {
+            if (!defined $dist{$q}) {
+                $dist{$q} = $dist{$p} + 1;
+#                print "    $q $dist{$q}\n";
+                push @queue, $q;
+            }
+        }
+    }
+
+    my $wuse = 1.0;
+    foreach my $user (keys %dist) {
+        next if $user eq $deriver;
+#        print "    $user $dist{$user}\n";
+        $wuse += 1.0 / 2.0**$dist{$user};
+    }
+
+#    print "    XXX $path $wuse\n";
+    
+    return $wuse;
+}
+
+
+# For each output path in the destination, see if we need to / can
+# create a patch.
+
+print "creating patches...\n";
+
+foreach my $p (keys %dstOutPaths) {
+
+    # If exactly the same path already exists in the source, skip it.
+    next if defined $srcOutPaths{$p};
+    
+    print "  $p\n";
+
+    # If not, then we should find the paths in the source that are
+    # `most' likely to be present on a system that wants to install
+    # this path.
+
+    (my $name, my $version) = getNameVersion $p;
+
+    my @closest = ();
+    my $closestVersion;
+    my $minDist = -1; # actually, larger means closer
+
+    # Find all source paths with the same name.
+
+    foreach my $q (keys %srcOutPaths) {
+        (my $name2, my $version2) = getNameVersion $q;
+        if ($name eq $name2) {
+
+            # If the sizes differ too much, then skip.  This
+            # disambiguates between, e.g., a real component and a
+            # wrapper component (cf. Firefox in Nixpkgs).
+            my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
+            my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
+            my $ratio = $srcSize / $dstSize;
+            $ratio = 1 / $ratio if $ratio < 1;
+#            print "  SIZE $srcSize $dstSize $ratio $q\n";
+
+            if ($ratio >= 3) {
+                print "  SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
+                next;
+            }
+
+            # If the numbers of weighted uses differ too much, then
+            # skip.  This disambiguates between, e.g., the bootstrap
+            # GCC and the final GCC in Nixpkgs.
+            my $srcUses = computeUses \%srcNarFiles, $q;
+            my $dstUses = computeUses \%dstNarFiles, $p;
+            $ratio = $srcUses / $dstUses;
+            $ratio = 1 / $ratio if $ratio < 1;
+            print "  USE $srcUses $dstUses $ratio $q\n";
+            
+#            if ($ratio >= 2) {
+#                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
+#                next;
+#            }
+
+            # If there are multiple matching names, include the ones
+            # with the closest version numbers.
+            my $dist = versionDiff $version, $version2;
+            if ($dist > $minDist) {
+                $minDist = $dist;
+                @closest = ($q);
+                $closestVersion = $version2;
+            } elsif ($dist == $minDist) {
+                push @closest, $q;
+            }
+        }
+    }
+
+    if (scalar(@closest) == 0) {
+        print "  NO BASE: $p\n";
+        next;
+    }
+
+    foreach my $closest (@closest) {
+
+        # Generate a patch between $closest and $p.
+        print "  $p <- $closest\n";
+
+        # If the patch already exists, skip it.
+        if (containsPatch(\%srcPatches, $p, $closest) ||
+            containsPatch(\%dstPatches, $p, $closest))
+        {
+            print "    skipping, already exists\n";
+            next;
+        }
+
+#        next;
+        
+        my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
+        my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
+        
+        system("@bunzip2@ < $srcNarBz2 > $tmpdir/A") == 0
+            or die "cannot unpack $srcNarBz2";
+
+        system("@bunzip2@ < $dstNarBz2 > $tmpdir/B") == 0
+            or die "cannot unpack $dstNarBz2";
+
+        system("@libexecdir@/bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
+            or die "cannot compute binary diff";
+
+        my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/A` or die;
+        chomp $baseHash;
+
+        my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/B` or die;
+        chomp $narHash;
+
+        my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/DIFF` or die;
+        chomp $narDiffHash;
+
+        my $narDiffSize = (stat "$tmpdir/DIFF")[7];
+        my $dstNarBz2Size = (stat $dstNarBz2)[7];
+
+        if ($narDiffSize >= $dstNarBz2Size) {
+            print "    rejecting; patch bigger than full archive\n";
+            next;
+        }
+    
+        my $finalName =
+            "$narDiffHash.nar-bsdiff";
+
+        print "    size $narDiffSize; full size $dstNarBz2Size\n";
+        
+        if (-e "$patchesDir/$finalName") {
+            print "    not copying, already exists\n";
+        }
+
+        else {
+
+            system("cp '$tmpdir/DIFF' '$patchesDir/$finalName.tmp'") == 0
+                or die "cannot copy diff";
+            
+            rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
+                or die "cannot rename $patchesDir/$finalName.tmp";
+            
+        }
+        
+        # Add the patch to the manifest.
+        addPatch \%dstPatches, $p,
+            { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
+            , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
+            , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
+            }, 0;
+    }
+}
+
+
+# Add in any potentially useful patches in the source (namely, those
+# patches that produce either paths in the destination or paths that
+# can be used as the base for other useful patches).
+
+my $changed;
+do {
+    # !!! we repeat this to reach the transitive closure; inefficient
+    $changed = 0;
+
+    foreach my $p (keys %srcPatches) {
+        my $patchList = $srcPatches{$p};
+
+        my $include = 0;
+
+        # Is path $p included in the destination?  If so, include
+        # patches that produce it.
+        $include = 1 if (defined $dstNarFiles{$p});
+
+        # Is path $p a path that serves as a base for paths in the
+        # destination?  If so, include patches that produce it.
+        foreach my $q (keys %dstPatches) {
+            foreach my $patch (@{$dstPatches{$q}}) {
+                # !!! check baseHash
+                $include = 1 if ($p eq $patch->{basePath});
+            }
+        }
+
+        if ($include) {
+            foreach my $patch (@{$patchList}) {
+                $changed = 1 if addPatch \%dstPatches, $p, $patch;
+            }
+        }            
+        
+    }
+    
+} while $changed;
+
+
+# Rewrite the manifest of the destination (with the new patches).
+writeManifest "$dstDir/MANIFEST",
+    \%dstNarFiles, \%dstPatches, \%dstSuccessors;

scripts/nix-build.in

@@ -0,0 +1,59 @@
+#! @shell@ -e
+
+nixExpr=$1
+
+if test -z "$nixExpr"; then
+    echo "syntax: $0 NIX-EXPR..." >&2
+    exit 1
+fi
+
+extraArgs=
+addDrvLink=0
+addOutLink=1
+
+
+trap 'rm -f ./.nix-build-tmp-*' EXIT
+
+
+# Process the arguments.
+for i in "$@"; do
+    case "$i" in
+    
+        --add-drv-link)
+            addDrvLink=1
+            ;;
+            
+        --no-link)
+            addOutLink=0
+            ;;
+            
+        -*)
+            extraArgs="$extraArgs $i"
+            ;;
+            
+        *)
+            # Instantiate the Nix expression.
+            prefix=
+            if test "$addDrvLink" = 0; then prefix=.nix-build-tmp-; fi
+            storeExprs=$(@bindir@/nix-instantiate \
+                --add-root ./${prefix}derivation --indirect \
+                "$i")
+                
+            for j in $storeExprs; do
+                echo "store expression is $(readlink "$j")" >&2
+            done
+
+            # Build the resulting store derivation.
+            prefix=
+            if test "$addOutLink" = 0; then prefix=.nix-build-tmp-; fi
+            outPaths=$(@bindir@/nix-store \
+                --add-root ./${prefix}result --indirect \
+                -rv $extraArgs $storeExprs)
+                
+            for j in $outPaths; do
+                echo "$(readlink "$j")"
+            done
+            
+            ;;
+    esac
+done

scripts/nix-channel.in

@@ -2,6 +2,8 @@
 
 use strict;
 
+my $rootsDir = "@localstatedir@/nix/gcroots/channels";
+
 
 # Figure out the name of the `.nix-channels' file to use.
 my $home = $ENV{"HOME"};
@@ -15,7 +17,7 @@ my @channels;
 # Reads the list of channels from the file $channelsList;
 sub readChannels {
     return if (!-f $channelsList);
-    open CHANNELS, "<$channelsList" or die "cannot open `$channelsList'";
+    open CHANNELS, "<$channelsList" or die "cannot open `$channelsList': $!";
     while (<CHANNELS>) {
         chomp;
         push @channels, $_;
@@ -26,7 +28,7 @@ sub readChannels {
 
 # Writes the list of channels to the file $channelsList;
 sub writeChannels {
-    open CHANNELS, ">$channelsList" or die "cannot open `$channelsList'";
+    open CHANNELS, ">$channelsList" or die "cannot open `$channelsList': $!";
     foreach my $url (@channels) {
         print CHANNELS "$url\n";
     }
@@ -46,11 +48,28 @@ sub addChannel {
 }
 
 
+# Remove a channel from the file $channelsList;
+sub removeChannel {
+    my $url = shift;
+    my @left = ();
+    readChannels;
+    foreach my $url2 (@channels) {
+        push @left, $url2 if $url ne $url2;
+    }
+    @channels = @left;
+    writeChannels;
+}
+
+
 # Fetch Nix expressions and pull cache manifests from the subscribed
 # channels.
 sub update {
     readChannels;
 
+    # Get rid of all the old substitutes.
+    system "@bindir@/nix-store --clear-substitutes";
+    die "cannot clear substitutes" if ($? != 0);
+
     # Pull cache manifests.
     foreach my $url (@channels) {
         print "pulling cache manifest from `$url'\n";
@@ -62,54 +81,94 @@ sub update {
     # expressions.
 
     my $nixExpr = "[";
-    
     foreach my $url (@channels) {
         my $fullURL = "$url/nixexprs.tar.bz2";
-        my $hash = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`
-            or die "cannot fetch `$fullURL'";
-        chomp $hash;
-        # !!! escaping
-        $nixExpr .= "((import @datadir@/nix/corepkgs/fetchurl) " .
-            "{url = $fullURL; md5 = \"$hash\"; system = \"@system@\";}) "
+        $ENV{"PRINT_PATH"} = 1;
+        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
+        die "cannot fetch `$fullURL'" if $? != 0;
+        chomp $path;
+        $nixExpr .= $path . " ";
     }
-
     $nixExpr .= "]";
 
     $nixExpr =
         "(import @datadir@/nix/corepkgs/channels/unpack.nix) " .
         "{inputs = $nixExpr; system = \"@system@\";}";
 
-    # Instantiate and realise it.
-    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
+    # Figure out a name for the GC root.
+    my $userName = getpwuid($<);
+    die "who ARE you? go away" unless defined $userName;
+
+    my $rootFile = "$rootsDir/$userName";
+    
+    # Instantiate the Nix expression.
+    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate --add-root '$rootFile'.tmp -`
         or die "cannot instantiate Nix expression";
     chomp $storeExpr;
 
-    my $outPath = `nix-store -qnfB '$storeExpr'`
+    # Build the resulting derivation.
+    my $outPath = `nix-store --add-root '$rootFile' -r '$storeExpr'`
         or die "cannot realise store expression";
     chomp $outPath;
 
+    unlink "$rootFile.tmp";
+
     # Make it the default Nix expression for `nix-env'.
     system "@bindir@/nix-env --import '$outPath'";
     die "cannot pull set default Nix expression to `$outPath'" if ($? != 0);
 }
 
 
+sub usageError {
+    print STDERR <<EOF;
+Usage:
+  nix-channel --add URL
+  nix-channel --remove URL
+  nix-channel --list
+  nix-channel --update
+EOF
+    exit 1;
+}
+
+
+usageError if scalar @ARGV == 0;
+
+
 while (scalar @ARGV) {
     my $arg = shift @ARGV;
 
     if ($arg eq "--add") {
-        die "syntax: nix-channel --add URL" if (scalar @ARGV != 1);
+        usageError if scalar @ARGV != 1;
         addChannel (shift @ARGV);
         last;
     }
 
-    elsif ($arg eq "--update") {
-        die "syntax: nix-channel --update" if (scalar @ARGV != 0);
-        update;
+    if ($arg eq "--remove") {
+        usageError if scalar @ARGV != 1;
+        removeChannel (shift @ARGV);
         last;
     }
 
+    if ($arg eq "--list") {
+        usageError if scalar @ARGV != 0;
+        readChannels;
+        foreach my $url (@channels) {
+            print "$url\n";
+        }
+        last;
+    }
+
+    elsif ($arg eq "--update") {
+        usageError if scalar @ARGV != 0;
+        update;
+        last;
+    }
+    
+    elsif ($arg eq "--help") {
+        usageError;
+    }
+
     else {
-        die "unknown argument `$arg'";
+        die "unknown argument `$arg'; try `--help'";
     }
 }

scripts/nix-collect-garbage.in

@@ -1,100 +1,2 @@
-#! @perl@ -w
-
-use strict;
-use IPC::Open2;
-
-my $rootsDir = "@localstatedir@/nix/gcroots";
-my $storeDir = "@storedir@";
-
-my %alive;
-
-my $keepSuccessors = 1;
-my $invert = 0;
-
-my @roots = ();
-
-
-# Parse the command line.
-foreach my $arg (@ARGV) {
-    if ($arg eq "--no-successors") { $keepSuccessors = 0; }
-    elsif ($arg eq "--invert") { $invert = 1; }
-    else { die "unknown argument `$arg'" };
-}
-
-
-# Read all GC roots from the given file.
-sub readRoots {
-    my $fileName = shift;
-    open ROOT, "<$fileName" or die "cannot open `$fileName': $!";
-    while (<ROOT>) {
-        chomp;
-        foreach my $root (split ' ') {
-            die "bad root `$root' in file `$fileName'"
-                unless $root =~ /^\S+$/;
-            push @roots, $root;
-        }
-    }
-    close ROOT;
-}
-
-
-# Recursively finds all *.gcroot files in the given directory.
-sub findRoots;
-sub findRoots {
-    my $followSymlinks = shift;
-    my $dir = shift;
-
-    opendir(DIR, $dir) or die "cannot open directory `$dir': $!";
-    my @names = readdir DIR or die "cannot read directory `$dir': $!";
-    closedir DIR;
-
-    foreach my $name (@names) {
-        next if $name eq "." || $name eq "..";
-        $name = $dir . "/" . $name;
-        if ($name =~ /.gcroot$/ && -f $name) {
-            readRoots $name;
-        }
-        elsif (-d $name) {
-            if ($followSymlinks || !-l $name) {
-                findRoots 0, $name;
-            }
-        }
-    }
-    
-}
-
-
-# Find GC roots, starting at $rootsDir.
-findRoots 1, $rootsDir;
-
-
-# Determine all store paths reachable from the roots.
-my $extraarg = "";
-if ($keepSuccessors) { $extraarg = "--include-successors"; };
-my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --query --requisites $extraarg @roots") 
-    or die "determining live paths";
-close WRITE;
-while (<READ>) {
-	chomp;
-	$alive{$_} = 1;
-	if ($invert) { print "$_\n"; };
-}
-close READ;
-
-waitpid $pid, 0;
-$? == 0 or die "determining live paths";
-
-exit 0 if ($invert);
-
-
-# Using that information, find all store paths *not* reachable from
-# the roots.
-opendir(DIR, $storeDir) or die "cannot open directory $storeDir: $!";
-foreach my $name (readdir DIR) {
-    next if ($name eq "." || $name eq "..");
-    $name = "$storeDir/$name";
-    if (!$alive{$name}) {
-        print "$name\n";
-    }
-}
-closedir DIR;
+#! @shell@ -e
+exec @bindir@/nix-store --gc "$@"

scripts/nix-install-package.in

@@ -3,35 +3,49 @@
 use strict;
 use POSIX qw(tmpnam);
 
-my $pkgfile = $ARGV[0];
-die unless defined $pkgfile;
+my $pkgFile = $ARGV[0];
+die unless defined $pkgFile;
 
-my $tmpdir;
-do { $tmpdir = tmpnam(); }
-until mkdir $tmpdir, 0777;
 
-# !!! remove tmpdir on exit
+# Re-execute in a terminal, if necessary, so that if we're executed
+# from a web browser, the user gets to see us.
+if (!defined $ENV{"NIX_HAVE_TERMINAL"}) {
+    $ENV{"NIX_HAVE_TERMINAL"} = "1";
+    $ENV{"LD_LIBRARY_PATH"} = "";
+    exec("xterm", "-e", "@shell@", "-c", "@bindir@/nix-install-package '$pkgFile' || read");
+    die "cannot execute `xterm'";
+}
 
-print "Unpacking $pkgfile in $tmpdir...\n";
-system "bunzip2 < $pkgfile | (cd $tmpdir && tar xf -)";
-die if $?;
 
-print "This package contains the following derivations:\n";
-system "@bindir@/nix-env -qasf $tmpdir/default.nix";
-die if $?;
+# Read and parse the package file.
+open PKGFILE, "<$pkgFile" or die "cannot open `$pkgFile': $!";
+my $contents = <PKGFILE>;
+close PKGFILE;
 
-print "Do you wish to install these (Y/N)? ";
+$contents =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ or die "invalid package contents";
+my $version = $1;
+my $manifestURL = $2;
+my $drvName = $3;
+my $system = $4;
+my $drvPath = $5;
+my $outPath = $6;
+
+die "invalid package version `$version'" unless $version eq "NIXPKG1";
+
+
+# Ask confirmation.
+print "Do you want to install `$drvName' (Y/N)? ";
 my $reply = <STDIN>;
 chomp $reply;
-exit if (!($reply eq "y"));
+exit if $reply ne "y" && $reply ne "Y";
 
-print "Pulling caches...\n";
-system "@bindir@/nix-pull `cat $tmpdir/caches`";
-die if $?;
+print "\nPulling manifests...\n";
+system "@bindir@/nix-pull '$manifestURL'";
+die if $? != 0;
 
-print "Installing package...\n";
-system "@bindir@/nix-env -if $tmpdir/default.nix '*'";
-die if $?;
+print "\nInstalling package...\n";
+system "@bindir@/nix-env -i '$outPath'";
+die if $? != 0;
 
-print "Installation succeeded! Press Enter to continue.\n";
+print "\nInstallation succeeded! Press Enter to continue.\n";
 <STDIN>;

scripts/nix-prefetch-url.in

@@ -1,37 +1,69 @@
 #! @shell@ -e
 
 url=$1
+expHash=$2
+
+hashType=$NIX_HASH_ALGO
+if test -z "$hashType"; then
+    hashType=md5
+fi
+
+hashFormat=
+if test "$hashType" != "md5"; then
+    hashFormat=--base32
+fi
 
 if test -z "$url"; then
-    echo "syntax: nix-prefetch-url URL" >&2
+    echo "syntax: nix-prefetch-url URL [EXPECTED-HASH]" >&2
     exit 1
 fi
 
-# !!! race? should be relatively safe, `svn export' barfs if $tmpPath exists.
-tmpPath1=@storedir@/nix-prefetch-url-$$
+name=$(basename "$url")
+if test -z "$name"; then echo "invalid url"; exit 1; fi
 
-# Perform the checkout.
-@curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
 
-# Compute the hash.
-hash=$(@bindir@/nix-hash --flat $tmpPath1)
-echo "hash is $hash" >&2
+# If the hash was given, a file with that hash may already be in the
+# store.
+if test -n "$expHash"; then
+    finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$expHash" "$name")
+    if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
+        finalPath=
+    fi
+    hash=$expHash
+fi
 
-# Rename it so that the fetchsvn builder can find it.
-tmpPath2=@storedir@/nix-prefetch-url-$hash
-test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
 
-# Create a Nix expression that does a fetchsvn.
-storeExpr=$( \
-  echo "(import @datadir@/nix/corepkgs/fetchurl) \
-        {url = $url; md5 = \"$hash\"; system = \"@system@\";}" \
-  | nix-instantiate -)
+# If we don't know the hash or a file with that hash doesn't exist,
+# download the file and add it to the store.
+if test -z "$finalPath"; then
 
-# Realise it.
-finalPath=$(nix-store -qnB --force-realise $storeExpr)
+    tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
+    tmpFile=$tmpPath/$name
+    mkdir $tmpPath
 
-echo "path is $finalPath" >&2
+    # Perform the download.
+    @curl@ --fail --location --max-redirs 20 "$url" > $tmpFile
 
-rm -rf $tmpPath2 || true
+    # Compute the hash.
+    hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
+    if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
+
+    # Add the downloaded file to the Nix store.
+    finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+
+    if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+
+    if test -n "$expHash" -a "$expHash" != "$hash"; then
+        echo "hash mismatch for URL \`$url'"
+        exit 1
+    fi
+fi
+
+
+if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
 
 echo $hash
+
+if test -n "$PRINT_PATH"; then
+    echo $finalPath
+fi

scripts/nix-pull.in

@@ -10,100 +10,92 @@ do { $tmpdir = tmpnam(); }
 until mkdir $tmpdir, 0777;
 
 my $manifest = "$tmpdir/manifest";
-my $conffile = "@sysconfdir@/nix/prebuilts.conf";
 
-#END { unlink $manifest; rmdir $tmpdir; }
+END { unlink $manifest; rmdir $tmpdir; }
+
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
+$libexecDir = "@libexecdir@" unless defined $libexecDir;
+
+my $stateDir = $ENV{"NIX_STATE_DIR"};
+$stateDir = "@localstatedir@/nix" unless defined $stateDir;
+
+
+# Prevent access problems in shared-stored installations.
+umask 0022;
 
 
 # Obtain URLs either from the command line or from a configuration file.
-my %storepaths2urls;
-my %urls2hashes;
+my %narFiles;
+my %patches;
 my %successors;
-sub doURL {
+
+sub processURL {
     my $url = shift;
-    processURL $manifest, $url, \%storepaths2urls, \%urls2hashes, \%successors;
-}
-if (scalar @ARGV > 0) {
-    while (@ARGV) {
-        my $url = shift @ARGV;
-	doURL $url;
+
+    $url =~ s/\/$//;
+    print "obtaining list of Nix archives at $url...\n";
+
+    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
+           "'$url' > '$manifest'") == 0
+           or die "curl failed: $?";
+
+    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
+        die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
     }
-} else {
-    open CONFFILE, "<$conffile";
-    while (<CONFFILE>) {
-        chomp;
-        if (/^\s*(\S+)\s*(\#.*)?$/) {
-            my $url = $1;
-	    doURL $url;
-        }
+
+    my $baseName = "unnamed";
+    if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
+        $baseName = $1;
     }
-    close CONFFILE;
+
+    my $hash = `$binDir/nix-hash --flat '$manifest'`
+        or die "cannot hash `$manifest'";
+    chomp $hash;
+    
+    my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
+    
+    system("mv -f '$manifest' '$finalPath'") == 0
+        or die "cannot move `$manifest' to `$finalPath";
+}
+
+while (@ARGV) {
+    my $url = shift @ARGV;
+    processURL $url;
 }
 
 
-# Create a Nix expression for the substitutes.
-my $fullexpr = "[";
-
-my @storepaths;
-foreach my $storepath (keys %storepaths2urls) {
-    # Construct a Nix expression that fetches and unpacks a
-    # Nix archive from the network.
-    my $url = $storepaths2urls{$storepath};
-    my $hash = $urls2hashes{$url};
-    my $fetch =
-	"(import @datadir@/nix/corepkgs/fetchurl) " .
-	"{url = $url; md5 = \"$hash\"; system = \"@system@\";}";
-    my $nixexpr =
-	"((import @datadir@/nix/corepkgs/nar/unnar.nix) " .
-	"{narFile = ($fetch); outPath = \"$storepath\"; system = \"@system@\";}) ";
-    $fullexpr .= $nixexpr; # !!! O(n^2)?
-    push @storepaths, $storepath;
-}
-
-$fullexpr .= "]";
-
-
-# Instantiate store expressions from the Nix expressions we created above.
-print STDERR "instantiating store expressions...\n";
-my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-instantiate -") or die "cannot run nix-instantiate";
-
-print WRITE $fullexpr;
-close WRITE;
-my $i = 0;
-my %substitutes;
-while (<READ>) {
-    chomp;
-    die unless /^\//;
-    my $subpath = $_;
-    die unless ($i < scalar @storepaths);
-    $substitutes{$storepaths[$i++]} = $subpath;
-}
-
-waitpid $pid, 0;
-$? == 0 or die "nix-instantiate failed";
+my $size = scalar (keys %narFiles);
+print "$size store paths in manifest\n";
 
 
 # Register all substitutes.
 print STDERR "registering substitutes...\n";
-my @subs = %substitutes;
-while (scalar @subs > 0) {
-    my $n = scalar @subs;
-    if ($n > 256) { $n = 256 };
-    my @subs2 = @subs[0..$n - 1];
-    @subs = @subs[$n..scalar @subs - 1];
-    system "@bindir@/nix-store --substitute @subs2";
-    if ($?) { die "`nix-store --substitute' failed"; }
+
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --register-substitutes")
+    or die "cannot run nix-store";
+
+close READ;
+
+foreach my $storePath (keys %narFiles) {
+    my $narFileList = $narFiles{$storePath};
+    foreach my $narFile (@{$narFileList}) {
+        print WRITE "$storePath\n";
+        print WRITE "$narFile->{deriver}\n";
+        print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
+        print WRITE "0\n";
+        my @references = split " ", $narFile->{references};
+        my $count = scalar @references;
+        print WRITE "$count\n";
+        foreach my $reference (@references) {
+            print WRITE "$reference\n";
+        }
+    }
 }
 
+close WRITE;
 
-# Register all successors.
-print STDERR "registering successors...\n";
-my @sucs = %successors;
-while (scalar @sucs > 0) {
-    my $n = scalar @sucs;
-    if ($n > 256) { $n = 256 };
-    my @sucs2 = @sucs[0..$n - 1];
-    @sucs = @sucs[$n..scalar @sucs - 1];
-    system "@bindir@/nix-store --successor @sucs2";
-    if ($?) { die "`nix-store --successor' failed"; }
-}
+waitpid $pid, 0;
+$? == 0 or die "nix-store failed";

scripts/nix-push.in

@@ -1,7 +1,11 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
 
 use strict;
+use IPC::Open2;
 use POSIX qw(tmpnam);
+use readmanifest;
+
+my $hashAlgo = "sha256";
 
 my $tmpdir;
 do { $tmpdir = tmpnam(); }
@@ -16,35 +20,65 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+my $dataDir = $ENV{"NIX_DATA_DIR"};
+$dataDir = "@datadir@" unless defined $dataDir;
+
 
 # Parse the command line.
-my $archives_put_url = shift @ARGV;
-my $archives_get_url = shift @ARGV;
-my $manifest_put_url = shift @ARGV;
+my $localCopy;
+my $localArchivesDir;
+my $localManifestFile;
+
+my $archivesPutURL;
+my $archivesGetURL;
+my $manifestPutURL;
+
+if ($ARGV[0] eq "--copy") {
+    die "syntax: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 1;
+    shift @ARGV;
+    $localArchivesDir = shift @ARGV;
+    $localManifestFile = shift @ARGV;
+}
+else {
+    die "syntax: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL " .
+        "MANIFEST_PUT_URL PATHS...\n" if scalar @ARGV < 3;
+    $localCopy = 0;
+    $archivesPutURL = shift @ARGV;
+    $archivesGetURL = shift @ARGV;
+    $manifestPutURL = shift @ARGV;
+}
 
 
-# From the given store expressions, determine the requisite store
-# paths.
-my %storepaths;
+# From the given store paths, determine the set of requisite store
+# paths, i.e, the paths required to realise them.
+my %storePaths;
 
-foreach my $storeexpr (@ARGV) {
-    die unless $storeexpr =~ /^\//;
+foreach my $path (@ARGV) {
+    die unless $path =~ /^\//;
 
     # Get all paths referenced by the normalisation of the given 
     # Nix expression.
-    system "@bindir@/nix-store --realise $storeexpr > /dev/null";
-    die if ($?);
-
-    open PATHS, "@bindir@/nix-store --query --requisites --include-successors $storeexpr 2> /dev/null |" or die;
-    while (<PATHS>) {
+    my $pid = open2(\*READ, \*WRITE,
+        "$binDir/nix-store --query --requisites --force-realise " .
+        "--include-outputs '$path'") or die;
+    close WRITE;
+    
+    while (<READ>) {
         chomp;
         die "bad: $_" unless /^\//;
-        $storepaths{$_} = "";
+        $storePaths{$_} = "";
     }
-    close PATHS;
+    close READ;
+    
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
-my @storepaths = keys %storepaths;
+my @storePaths = keys %storePaths;
 
 
 # For each path, create a Nix expression that turns the path into
@@ -52,14 +86,13 @@ my @storepaths = keys %storepaths;
 open NIX, ">$nixfile";
 print NIX "[";
 
-foreach my $storepath (@storepaths) {
-    die unless ($storepath =~ /\/[0-9a-z]{32}.*$/);
+foreach my $storePath (@storePaths) {
+    die unless ($storePath =~ /\/[0-9a-z]{32}.*$/);
 
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
-        "((import @datadir@/nix/corepkgs/nar/nar.nix) " .
-        # !!! $storepath should be represented as a closure
-        "{path = \"$storepath\"; system = \"@system@\";}) ";
+        "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
+        "{path = \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
     
     print NIX $nixexpr;
 }
@@ -69,112 +102,158 @@ close NIX;
 
 
 # Instantiate store expressions from the Nix expression.
-my @storeexprs;
+my @storeExprs;
 print STDERR "instantiating store expressions...\n";
-open STOREEXPRS, "@bindir@/nix-instantiate $nixfile |" or die "cannot run nix-instantiate";
-while (<STOREEXPRS>) {
+my $pid = open2(\*READ, \*WRITE, "$binDir/nix-instantiate $nixfile")
+    or die "cannot run nix-instantiate";
+close WRITE;
+while (<READ>) {
     chomp;
     die unless /^\//;
-    push @storeexprs, $_;
+    push @storeExprs, $_;
 }
-close STOREEXPRS;
+close READ;
+
+waitpid $pid, 0;
+$? == 0 or die "nix-instantiate failed";
 
 
 # Realise the store expressions.
 print STDERR "creating archives...\n";
 
-my @narpaths;
+my @narPaths;
 
-my @tmp = @storeexprs;
+my @tmp = @storeExprs;
 while (scalar @tmp > 0) {
     my $n = scalar @tmp;
     if ($n > 256) { $n = 256 };
     my @tmp2 = @tmp[0..$n - 1];
     @tmp = @tmp[$n..scalar @tmp - 1];
 
-    system "@bindir@/nix-store --realise -B @tmp2 > /dev/null";
-    if ($?) { die "`nix-store --realise' failed"; }
-
-    open NARPATHS, "@bindir@/nix-store --query --list @tmp2 |" or die "cannot run nix";
-    while (<NARPATHS>) {
+    my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --realise @tmp2")
+        or die "cannot run nix-store";
+    close WRITE;
+    while (<READ>) {
         chomp;
         die unless (/^\//);
-        push @narpaths, "$_";
+        push @narPaths, "$_";
     }
-    close NARPATHS;
+    close READ;
+
+    waitpid $pid, 0;
+    $? == 0 or die "nix-store failed";
 }
 
 
 # Create the manifest.
 print STDERR "creating manifest...\n";
 
-open MANIFEST, ">$manifest";
+my %narFiles;
+my %patches;
 
-my @nararchives;
-for (my $n = 0; $n < scalar @storepaths; $n++) {
-    my $storepath = $storepaths[$n];
-    my $nardir = $narpaths[$n];
+my @narArchives;
+for (my $n = 0; $n < scalar @storePaths; $n++) {
+    my $storePath = $storePaths[$n];
+    my $narDir = $narPaths[$n];
     
-    $storepath =~ /\/([^\/]*)$/;
+    $storePath =~ /\/([^\/]*)$/;
     my $basename = $1;
     defined $basename or die;
 
-    my $narname = "$basename.nar.bz2";
+    open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
+    my $narbz2Hash = <HASH>;
+    chomp $narbz2Hash;
+    $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
 
-    my $narfile = "$nardir/$narname";
-    (-f $narfile) or die "narfile for $storepath not found";
-    push @nararchives, $narfile;
+    open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
+    my $narHash = <HASH>;
+    chomp $narHash;
+    $narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
+    close HASH;
+    
+    my $narName = "$narbz2Hash.nar.bz2";
 
-    open MD5, "$nardir/md5" or die "cannot open hash";
-    my $hash = <MD5>;
-    chomp $hash;
-    $hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
-    close MD5;
+    my $narFile = "$narDir/$narName";
+    (-f $narFile) or die "narfile for $storePath not found";
+    push @narArchives, $narFile;
 
-    print MANIFEST "{\n";
-    print MANIFEST "  StorePath: $storepath\n";
-    print MANIFEST "  NarURL: $archives_get_url/$narname\n";
-    print MANIFEST "  MD5: $hash\n";
+    my $narbz2Size = (stat $narFile)[7];
 
-    if ($storepath =~ /\.store$/) {
-        open PREDS, "@bindir@/nix-store --query --predecessors $storepath |" or die "cannot run nix";
-        while (<PREDS>) {
-            chomp;
-            die unless (/^\//);
-            my $pred = $_;
-            # Only include predecessors that are themselves being
-            # pushed.
-            if (defined $storepaths{$pred}) {
-                print MANIFEST "  SuccOf: $pred\n";
-            }
-        }
-        close PREDS;
+    my $references = `$binDir/nix-store --query --references '$storePath'`;
+    die "cannot query references for `$storePath'" if $? != 0;
+    $references = join(" ", split(" ", $references));
+
+    my $deriver = `$binDir/nix-store --query --deriver '$storePath'`;
+    die "cannot query deriver for `$storePath'" if $? != 0;
+    chomp $deriver;
+    $deriver = "" if $deriver eq "unknown-deriver";
+
+    my $url;
+    if ($localCopy) {
+        $url = "file://$localArchivesDir/$narName";
+    } else {
+        $url = "$archivesGetURL/$narName";
     }
-
-    print MANIFEST "}\n";
+    $narFiles{$storePath} = [
+        { url => $url
+        , hash => "$hashAlgo:$narbz2Hash"
+        , size => $narbz2Size
+        , narHash => "$hashAlgo:$narHash"
+        , references => $references
+        , deriver => $deriver
+        }
+    ];
 }
 
-close MANIFEST;
+writeManifest $manifest, \%narFiles, \%patches;
+
+
+sub copyFile {
+    my $src = shift;
+    my $dst = shift;
+    system("cp '$src' '$dst.tmp'") == 0 or die "cannot copy file";
+    rename("$dst.tmp", "$dst") or die "cannot rename file";
+}
 
 
 # Upload the archives.
 print STDERR "uploading archives...\n";
-foreach my $nararchive (@nararchives) {
 
-    $nararchive =~ /\/([^\/]*)$/;
+sub archiveExists {
+    my $name = shift;
+    print STDERR "  HEAD on $archivesGetURL/$name\n";
+    return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
+}
+
+foreach my $narArchive (@narArchives) {
+
+    $narArchive =~ /\/([^\/]*)$/;
     my $basename = $1;
 
-    if (system("$curl --head $archives_get_url/$basename > /dev/null") != 0) {
-        print STDERR "  $nararchive\n";
-        system("$curl --show-error --upload-file " .
-               "'$nararchive' '$archives_put_url/$basename' > /dev/null") == 0 or
-            die "curl failed on $nararchive: $?";
+    if ($localCopy) {
+        if (! -f "$localArchivesDir/$basename") {
+            print STDERR "  $narArchive\n";
+            copyFile $narArchive, "$localArchivesDir/$basename";
+        }
+    }
+    else {
+        if (!archiveExists("$basename")) {
+            print STDERR "  $narArchive\n";
+            system("$curl --show-error --upload-file " .
+                   "'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
+                   die "curl failed on $narArchive: $?";
+        }
     }
 }
 
 
 # Upload the manifest.
 print STDERR "uploading manifest...\n";
-system("$curl  --show-error --upload-file " .
-       "'$manifest' '$manifest_put_url' > /dev/null") == 0 or
-    die "curl failed on $manifest: $?";
+if ($localCopy) {
+    copyFile $manifest, $localManifestFile;
+} else {
+    system("$curl  --show-error --upload-file " .
+           "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
+           die "curl failed on $manifest: $?";
+}

scripts/optimise-store.pl

@@ -0,0 +1,69 @@
+#! /usr/bin/perl -w
+
+use strict;
+
+{ my $ofh = select STDOUT;
+  $| = 1;
+  select $ofh;
+}
+
+#my @paths = ("/nix/store/caef3a49150506d233f474322a824e50-glibc-2.3.3", "/nix/store/a8a9d585d1ad4b1bc911be7743b3b996-glibc-2.3.3");
+my @paths = ("/nix/store");
+
+my $tmpfile = "/tmp/nix-optimise-hash-list";
+#my $tmpfile = "/data/nix-optimise-hash-list";
+
+system("find @paths -type f -print0 | xargs -0 md5sum -- > $tmpfile") == 0
+    or die "cannot hash store files";
+
+system("sort $tmpfile > $tmpfile.sorted") == 0
+    or die "cannot sort list";
+
+open LIST, "<$tmpfile.sorted" or die;
+
+my $prevFile;
+my $prevHash;
+
+my $totalSpace = 0;
+my $savedSpace = 0;
+
+my $files = 0;
+
+while (<LIST>) {
+#    print "D";
+    /^([0-9a-f]*)\s+(.*)$/ or die;
+    my $curFile = $2;
+    my $curHash = $1;
+
+#    print "A";
+    my $fileSize = (stat $curFile)[7];
+#    print "B";
+#    my $fileSize = 1;
+    $totalSpace += $fileSize;
+
+    if (defined $prevHash && $curHash eq $prevHash) {
+        
+#        print "$curFile = $prevFile\n";
+
+        $savedSpace += $fileSize;
+        
+    } else {
+        $prevFile = $curFile;
+        $prevHash = $curHash;
+    }
+
+    print "." if ($files++ % 100 == 0);
+    #print ".";
+
+#    print "C";
+}
+
+print "\n";
+
+print "total space = $totalSpace\n";
+print "saved space = $savedSpace\n";
+my $savings = ($savedSpace / $totalSpace) * 100.0;
+print "savings = $savings %\n";
+
+
+close LIST;

scripts/readmanifest.pm.in

@@ -1,27 +1,64 @@
 use strict;
 
-sub processURL {
+
+sub addPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $patch = shift;
+    my $allowConflicts = shift;
+
+    $$patches{$storePath} = []
+        unless defined $$patches{$storePath};
+
+    my $patchList = $$patches{$storePath};
+
+    my $found = 0;
+    foreach my $patch2 (@{$patchList}) {
+        if ($patch2->{url} eq $patch->{url}) {
+            if ($patch2->{hash} eq $patch->{hash}) {
+                $found = 1 if ($patch2->{basePath} eq $patch->{basePath});
+            } else {
+                die "conflicting hashes for URL $patch->{url}, " .
+                    "namely $patch2->{hash} and $patch->{hash}"
+                    unless $allowConflicts;
+            }
+        }
+    }
+    
+    push @{$patchList}, $patch if !$found;
+
+    return !$found;
+}
+
+
+sub readManifest {
     my $manifest = shift;
-    my $url = shift;
-    my $storepaths2urls = shift;
-    my $urls2hashes = shift;
+    my $narFiles = shift;
+    my $patches = shift;
     my $successors = shift;
+    my $allowConflicts = shift;
+    $allowConflicts = 0 unless defined $allowConflicts;
 
-    $url =~ s/\/$//;
-    print "obtaining list of Nix archives at $url...\n";
-
-    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
-           "'$url' > '$manifest' 2> /dev/null") == 0
-           or die "curl failed: $?";
-        
-    open MANIFEST, "<$manifest";
+    open MANIFEST, "<$manifest"
+        or die "cannot open `$manifest': $!";
 
     my $inside = 0;
+    my $type;
 
-    my $storepath;
-    my $narurl;
+    my $manifestVersion = 2;
+
+    my $storePath;
+    my $url;
     my $hash;
+    my $size;
     my @preds;
+    my $basePath;
+    my $baseHash;
+    my $patchType;
+    my $narHash;
+    my $references;
+    my $deriver;
+    my $hashAlgo;
 
     while (<MANIFEST>) {
         chomp;
@@ -29,43 +66,150 @@ sub processURL {
         next if (/^$/);
 
         if (!$inside) {
-            if (/^\{$/) { 
+
+            if (/^\s*(\w*)\s*\{$/) {
+                $type = $1;
+                $type = "narfile" if $type eq "";
                 $inside = 1;
-                undef $storepath;
-                undef $narurl;
+                undef $storePath;
+                undef $url;
                 undef $hash;
+                undef $size;
                 @preds = ();
+                undef $narHash;
+                undef $basePath;
+                undef $baseHash;
+                undef $patchType;
+                $references = "";
+                $deriver = "";
+                $hashAlgo = "md5";
 	    }
-            else { die "bad line: $_"; }
+
         } else {
+            
             if (/^\}$/) {
                 $inside = 0;
 
-		$$storepaths2urls{$storepath} = $narurl;
-		$$urls2hashes{$narurl} = $hash;
+                if ($type eq "narfile") {
 
-                foreach my $p (@preds) {
-		    $$successors{$p} = $storepath;
+                    $$narFiles{$storePath} = []
+                        unless defined $$narFiles{$storePath};
+
+                    my $narFileList = $$narFiles{$storePath};
+
+                    my $found = 0;
+                    foreach my $narFile (@{$narFileList}) {
+                        if ($narFile->{url} eq $url) {
+                            if ($narFile->{hash} eq $hash) {
+                                $found = 1;
+                            } else {
+                                die "conflicting hashes for URL $url, " .
+                                    "namely $narFile->{hash} and $hash"
+                                    unless $allowConflicts;
+                            }
+                        }
+                    }
+                    if (!$found) {
+                        push @{$narFileList},
+                            { url => $url, hash => $hash, size => $size
+                            , narHash => $narHash, references => $references
+                            , deriver => $deriver, hashAlgo => $hashAlgo
+                            };
+                    }
+                
+                    foreach my $p (@preds) {
+                        $$successors{$p} = $storePath;
+                    }
+
+                }
+
+                elsif ($type eq "patch") {
+                    addPatch $patches, $storePath,
+                        { url => $url, hash => $hash, size => $size
+                        , basePath => $basePath, baseHash => $baseHash
+                        , narHash => $narHash, patchType => $patchType
+                        , hashAlgo => $hashAlgo
+                        }, $allowConflicts;
                 }
 
             }
-            elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
-                $storepath = $1;
-            }
-            elsif (/^\s*NarURL:\s*(\S+)\s*$/) {
-                $narurl = $1;
-	    }
-            elsif (/^\s*MD5:\s*(\S+)\s*$/) {
-                $hash = $1;
-            }
-            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
-                push @preds, $1;
-            }
-            else { die "bad line: $_"; }
+            
+            elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; }
+            elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; }
+            elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; }
+            elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; }
+            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { push @preds, $1; }
+            elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
+            elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
+            elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
+            elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+            elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
+            elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
+            elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
+
+            # Compatibility;
+            elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
+            elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; }
+
         }
     }
 
     close MANIFEST;
+
+    return $manifestVersion;
+}
+
+
+sub writeManifest
+{
+    my $manifest = shift;
+    my $narFiles = shift;
+    my $patches = shift;
+
+    open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
+
+    print MANIFEST "version {\n";
+    print MANIFEST "  ManifestVersion: 3\n";
+    print MANIFEST "}\n";
+
+    foreach my $storePath (keys %{$narFiles}) {
+        my $narFileList = $$narFiles{$storePath};
+        foreach my $narFile (@{$narFileList}) {
+            print MANIFEST "{\n";
+            print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  NarURL: $narFile->{url}\n";
+            print MANIFEST "  Hash: $narFile->{hash}\n";
+            print MANIFEST "  NarHash: $narFile->{narHash}\n";
+            print MANIFEST "  Size: $narFile->{size}\n";
+            print MANIFEST "  References: $narFile->{references}\n"
+                if defined $narFile->{references} && $narFile->{references} ne "";
+            print MANIFEST "  Deriver: $narFile->{deriver}\n"
+                if defined $narFile->{deriver} && $narFile->{deriver} ne "";
+            print MANIFEST "}\n";
+        }
+    }
+    
+    foreach my $storePath (keys %{$patches}) {
+        my $patchList = $$patches{$storePath};
+        foreach my $patch (@{$patchList}) {
+            print MANIFEST "patch {\n";
+            print MANIFEST "  StorePath: $storePath\n";
+            print MANIFEST "  NarURL: $patch->{url}\n";
+            print MANIFEST "  Hash: $patch->{hash}\n";
+            print MANIFEST "  NarHash: $patch->{narHash}\n";
+            print MANIFEST "  Size: $patch->{size}\n";
+            print MANIFEST "  BasePath: $patch->{basePath}\n";
+            print MANIFEST "  BaseHash: $patch->{baseHash}\n";
+            print MANIFEST "  Type: $patch->{patchType}\n";
+            print MANIFEST "}\n";
+        }
+    }
+    
+    
+    close MANIFEST;
+
+    rename("$manifest.tmp", $manifest)
+        or die "cannot rename $manifest.tmp: $!";
 }
 
 

scripts/remove-patches.pl

@@ -0,0 +1,19 @@
+#! /usr/bin/perl -w -I/home/eelco/nix/scripts
+
+use strict;
+use readmanifest;
+
+for my $p (@ARGV) {
+
+    my %narFiles;
+    my %patches;
+    my %successors;
+
+    readManifest $p,
+        \%narFiles, \%patches, \%successors;
+
+    %patches = ();
+    
+    writeManifest $p,
+        \%narFiles, \%patches, \%successors;
+}

scripts/update-manifest.pl

@@ -0,0 +1,53 @@
+#! /usr/bin/perl -w -I.
+
+use strict;
+use readmanifest;
+
+die unless scalar @ARGV == 2;
+
+my $cache = $ARGV[0];
+my $manifest = $ARGV[1];
+my %narFiles;
+my %patches;
+my %successors;
+
+readManifest $manifest, \%narFiles, \%patches, \%successors;
+
+foreach my $storePath (keys %narFiles) {
+    my $narFileList = $narFiles{$storePath};
+
+    foreach my $narFile (@{$narFileList}) {
+        if (!defined $narFile->{size} or
+            !defined $narFile->{narHash})
+        {
+            $narFile->{url} =~ /\/([^\/]+)$/;
+            die unless defined $1;
+            my $fn = "$cache/$1";
+            
+            my @info = stat $fn or die;
+            $narFile->{size} = $info[7];
+
+            my $narHash;
+            my $hashFile = "$fn.NARHASH";
+            if (-e $hashFile) {
+                open HASH, "<$hashFile" or die;
+                $narHash = <HASH>;
+                close HASH;
+            } else {
+                print "$fn\n";
+                $narHash = `bunzip2 < '$fn' | nix-hash --flat /dev/stdin` or die;
+                open HASH, ">$hashFile" or die;
+                print HASH $narHash;
+                close HASH;
+            }
+            chomp $narHash;
+            $narFile->{narHash} = $narHash;
+        }
+    }
+}
+
+if (! -e "$manifest.backup") {
+    system "mv --reply=no '$manifest' '$manifest.backup'";
+}
+
+writeManifest $manifest, \%narFiles, \%patches, \%successors;

src/Makefile.am

@@ -1,2 +1,15 @@
 SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
- libexpr nix-instantiate nix-env log2xml
+ libexpr nix-instantiate nix-env log2xml bsdiff-4.2
+
+EXTRA_DIST = aterm-helper.pl
+
+SETUID_PROGS = nix-store nix-instantiate nix-env
+install-exec-hook:
+if SETUID_HACK
+if HAVE_SETRESUID
+	cd $(DESTDIR)$(bindir) && chown @NIX_USER@ $(SETUID_PROGS) \
+	  && chgrp @NIX_GROUP@ $(SETUID_PROGS) && chmod ug+s $(SETUID_PROGS)
+else
+	cd $(DESTDIR)$(bindir) && chown root $(SETUID_PROGS) && chmod u+s $(SETUID_PROGS)
+endif
+endif

src/aterm-helper.pl

@@ -0,0 +1,153 @@
+#! /usr/bin/perl -w
+
+# This program generates C/C++ code for efficiently manipulating
+# ATerms.  It generates functions to build and match ATerms according
+# to a set of constructor definitions defined in a file read from
+# standard input.  A constructor is defined by a line with the
+# following format:
+#
+#   SYM | ARGS | TYPE | FUN?
+#
+# where SYM is the name of the constructor, ARGS is a
+# whitespace-separated list of argument types, TYPE is the type of the
+# resulting ATerm (which should be `ATerm' or a type synonym for
+# `ATerm'), and the optional FUN is used to construct the names of the
+# build and match functions (it defaults to SYM; overriding it is
+# useful if there are overloaded constructors, e.g., with different
+# arities).  Note that SYM may be empty.
+#
+# A line of the form
+#
+#   VAR = EXPR
+#
+# causes a ATerm variable to be generated that is initialised to the
+# value EXPR.
+#
+# Finally, a line of the form
+#
+#   init NAME
+#
+# causes the initialisation function to be called `NAME'.  This
+# function must be called before any of the build/match functions or
+# the generated variables are used.
+
+die if scalar @ARGV != 2;
+
+my $syms = "";
+my $init = "";
+my $initFun = "init";
+
+open HEADER, ">$ARGV[0]";
+open IMPL, ">$ARGV[1]";
+
+while (<STDIN>) {
+    next if (/^\s*$/);
+    
+    if (/^\s*(\w*)\s*\|([^\|]*)\|\s*(\w+)\s*\|\s*(\w+)?/) {
+        my $const = $1;
+        my @types = split ' ', $2;
+        my $result = $3;
+        my $funname = $4;
+        $funname = $const unless defined $funname;
+
+        my $formals = "";
+        my $formals2 = "";
+        my $args = "";
+        my $unpack = "";
+        my $n = 1;
+        foreach my $type (@types) {
+            my $realType = $type;
+            $args .= ", ";
+            if ($type eq "string") {
+#                $args .= "(ATerm) ATmakeAppl0(ATmakeAFun((char *) e$n, 0, ATtrue))";
+#                $type = "const char *";
+                $type = "ATerm";
+                $args .= "e$n";
+                # !!! in the matcher, we should check that the
+                # argument is a string (i.e., a nullary application).
+            } elsif ($type eq "int") {
+                $args .= "(ATerm) ATmakeInt(e$n)";
+            } elsif ($type eq "ATermList" || $type eq "ATermBlob") {
+                $args .= "(ATerm) e$n";
+            } else {
+                $args .= "e$n";
+            }
+            $formals .= ", " if $formals ne "";
+            $formals .= "$type e$n";
+            $formals2 .= ", ";
+            $formals2 .= "$type & e$n";
+            my $m = $n - 1;
+            # !!! more checks here
+            if ($type eq "int") {
+                $unpack .= "    e$n = ATgetInt((ATermInt) ATgetArgument(e, $m));\n";
+            } elsif ($type eq "ATermList") {
+                $unpack .= "    e$n = (ATermList) ATgetArgument(e, $m);\n";
+            } elsif ($type eq "ATermBlob") {
+                $unpack .= "    e$n = (ATermBlob) ATgetArgument(e, $m);\n";
+            } elsif ($realType eq "string") {
+                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
+                $unpack .= "    if (ATgetType(e$n) != AT_APPL) return false;\n";
+            } else {
+                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
+            }
+            $n++;
+        }
+
+        my $arity = scalar @types;
+
+        print HEADER "extern AFun sym$funname;\n\n";
+        
+        print IMPL "AFun sym$funname = 0;\n";
+        
+        print HEADER "static inline $result make$funname($formals) {\n";
+        if ($arity <= 6) {
+            print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
+        } else {
+            $args =~ s/^,//;
+            print HEADER "    ATerm array[$arity] = {$args};\n";
+            print HEADER "    return (ATerm) ATmakeApplArray(sym$funname, array);\n";
+        }
+        print HEADER "}\n\n";
+
+        print HEADER "#ifdef __cplusplus\n";
+        print HEADER "static inline bool match$funname(ATerm e$formals2) {\n";
+        print HEADER "    if (ATgetType(e) != AT_APPL || (AFun) ATgetAFun(e) != sym$funname) return false;\n";
+        print HEADER "$unpack";
+        print HEADER "    return true;\n";
+        print HEADER "}\n";
+        print HEADER "#endif\n\n\n";
+
+        $init .= "    sym$funname = ATmakeAFun(\"$const\", $arity, ATfalse);\n";
+        $init .= "    ATprotectAFun(sym$funname);\n";
+    }
+
+    elsif (/^\s*(\w+)\s*=\s*(.*)$/) {
+        my $name = $1;
+        my $value = $2;
+        print HEADER "extern ATerm $name;\n";
+        print IMPL "ATerm $name = 0;\n";
+        $init .= "    $name = $value;\n";
+    }
+
+    elsif (/^\s*init\s+(\w+)\s*$/) {
+        $initFun = $1;
+    }
+
+    else {
+        die "bad line: `$_'";
+    }
+}
+
+print HEADER "void $initFun();\n\n";
+
+print HEADER "static inline const char * aterm2String(ATerm t) {\n";
+print HEADER "    return (const char *) ATgetName(ATgetAFun(t));\n";
+print HEADER "}\n\n";
+
+print IMPL "\n";
+print IMPL "void $initFun() {\n";
+print IMPL "$init";
+print IMPL "}\n";
+
+close HEADER;
+close IMPL;

src/boost/Makefile.am

@@ -1,3 +1,6 @@
 SUBDIRS = format
 
-EXTRA_DIST = format.hpp
+EXTRA_DIST = assert.hpp checked_delete.hpp format.hpp \
+ shared_ptr.hpp weak_ptr.hpp throw_exception.hpp \
+ enable_shared_from_this.hpp \
+ detail/shared_count.hpp detail/workaround.hpp

src/boost/assert.hpp

@@ -0,0 +1,38 @@
+//
+//  boost/assert.hpp - BOOST_ASSERT(expr)
+//
+//  Copyright (c) 2001, 2002 Peter Dimov and Multi Media Ltd.
+//
+//  Permission to copy, use, modify, sell and distribute this software
+//  is granted provided this copyright notice appears in all copies.
+//  This software is provided "as is" without express or implied
+//  warranty, and with no claim as to its suitability for any purpose.
+//
+//  Note: There are no include guards. This is intentional.
+//
+//  See http://www.boost.org/libs/utility/assert.html for documentation.
+//
+
+#undef BOOST_ASSERT
+
+#if defined(BOOST_DISABLE_ASSERTS)
+
+# define BOOST_ASSERT(expr) ((void)0)
+
+#elif defined(BOOST_ENABLE_ASSERT_HANDLER)
+
+#include <boost/current_function.hpp>
+
+namespace boost
+{
+
+void assertion_failed(char const * expr, char const * function, char const * file, long line); // user defined
+
+} // namespace boost
+
+#define BOOST_ASSERT(expr) ((expr)? ((void)0): ::boost::assertion_failed(#expr, BOOST_CURRENT_FUNCTION, __FILE__, __LINE__))
+
+#else
+# include <assert.h>
+# define BOOST_ASSERT(expr) assert(expr)
+#endif

src/boost/checked_delete.hpp

@@ -0,0 +1,71 @@
+#ifndef BOOST_CHECKED_DELETE_HPP_INCLUDED
+#define BOOST_CHECKED_DELETE_HPP_INCLUDED
+
+// MS compatible compilers support #pragma once
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1020)
+# pragma once
+#endif
+
+//
+//  boost/checked_delete.hpp
+//
+//  Copyright (c) 1999, 2000, 2001, 2002 boost.org
+//  Copyright (c) 2002, 2003 Peter Dimov
+//  Copyright (c) 2003 Daniel Frey
+//  Copyright (c) 2003 Howard Hinnant
+//
+//  Permission to copy, use, modify, sell and distribute this software
+//  is granted provided this copyright notice appears in all copies.
+//  This software is provided "as is" without express or implied
+//  warranty, and with no claim as to its suitability for any purpose.
+//
+//  See http://www.boost.org/libs/utility/checked_delete.html for documentation.
+//
+
+namespace boost
+{
+
+// verify that types are complete for increased safety
+
+template<class T> inline void checked_delete(T * x)
+{
+    // intentionally complex - simplification causes regressions
+    typedef char type_must_be_complete[ sizeof(T)? 1: -1 ];
+    (void) sizeof(type_must_be_complete);
+    delete x;
+}
+
+template<class T> inline void checked_array_delete(T * x)
+{
+    typedef char type_must_be_complete[ sizeof(T)? 1: -1 ];
+    (void) sizeof(type_must_be_complete);
+    delete [] x;
+}
+
+template<class T> struct checked_deleter
+{
+    typedef void result_type;
+    typedef T * argument_type;
+
+    void operator()(T * x) const
+    {
+        // boost:: disables ADL
+        boost::checked_delete(x);
+    }
+};
+
+template<class T> struct checked_array_deleter
+{
+    typedef void result_type;
+    typedef T * argument_type;
+
+    void operator()(T * x) const
+    {
+        boost::checked_array_delete(x);
+    }
+};
+
+} // namespace boost
+
+#endif  // #ifndef BOOST_CHECKED_DELETE_HPP_INCLUDED

src/boost/detail/shared_count.hpp

@@ -0,0 +1,555 @@
+#ifndef BOOST_DETAIL_SHARED_COUNT_HPP_INCLUDED
+#define BOOST_DETAIL_SHARED_COUNT_HPP_INCLUDED
+
+// MS compatible compilers support #pragma once
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1020)
+# pragma once
+#endif
+
+//
+//  detail/shared_count.hpp
+//
+//  Copyright (c) 2001, 2002, 2003 Peter Dimov and Multi Media Ltd.
+//
+//  Permission to copy, use, modify, sell and distribute this software
+//  is granted provided this copyright notice appears in all copies.
+//  This software is provided "as is" without express or implied
+//  warranty, and with no claim as to its suitability for any purpose.
+//
+
+//#include <boost/config.hpp>
+
+#if defined(BOOST_SP_USE_STD_ALLOCATOR) && defined(BOOST_SP_USE_QUICK_ALLOCATOR)
+# error BOOST_SP_USE_STD_ALLOCATOR and BOOST_SP_USE_QUICK_ALLOCATOR are incompatible.
+#endif
+
+#include <boost/checked_delete.hpp>
+#include <boost/throw_exception.hpp>
+//#include <boost/detail/lightweight_mutex.hpp>
+
+#if defined(BOOST_SP_USE_QUICK_ALLOCATOR)
+#include <boost/detail/quick_allocator.hpp>
+#endif
+
+#include <memory>           // std::auto_ptr, std::allocator
+#include <functional>       // std::less
+#include <exception>        // std::exception
+#include <new>              // std::bad_alloc
+#include <typeinfo>         // std::type_info in get_deleter
+#include <cstddef>          // std::size_t
+
+#ifdef __BORLANDC__
+# pragma warn -8026     // Functions with excep. spec. are not expanded inline
+# pragma warn -8027     // Functions containing try are not expanded inline
+#endif
+
+namespace boost
+{
+
+// Debug hooks
+
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+
+void sp_scalar_constructor_hook(void * px, std::size_t size, void * pn);
+void sp_array_constructor_hook(void * px);
+void sp_scalar_destructor_hook(void * px, std::size_t size, void * pn);
+void sp_array_destructor_hook(void * px);
+
+#endif
+
+
+// The standard library that comes with Borland C++ 5.5.1
+// defines std::exception and its members as having C calling
+// convention (-pc). When the definition of bad_weak_ptr
+// is compiled with -ps, the compiler issues an error.
+// Hence, the temporary #pragma option -pc below. The version
+// check is deliberately conservative.
+
+#if defined(__BORLANDC__) && __BORLANDC__ == 0x551
+# pragma option push -pc
+#endif
+
+class bad_weak_ptr: public std::exception
+{
+public:
+
+    virtual char const * what() const throw()
+    {
+        return "boost::bad_weak_ptr";
+    }
+};
+
+#if defined(__BORLANDC__) && __BORLANDC__ == 0x551
+# pragma option pop
+#endif
+
+namespace detail
+{
+
+class sp_counted_base
+{
+private:
+
+    //    typedef detail::lightweight_mutex mutex_type;
+
+public:
+
+    sp_counted_base(): use_count_(1), weak_count_(1)
+    {
+    }
+
+    virtual ~sp_counted_base() // nothrow
+    {
+    }
+
+    // dispose() is called when use_count_ drops to zero, to release
+    // the resources managed by *this.
+
+    virtual void dispose() = 0; // nothrow
+
+    // destruct() is called when weak_count_ drops to zero.
+
+    virtual void destruct() // nothrow
+    {
+        delete this;
+    }
+
+    virtual void * get_deleter(std::type_info const & ti) = 0;
+
+    void add_ref_copy()
+    {
+#if defined(BOOST_HAS_THREADS)
+        mutex_type::scoped_lock lock(mtx_);
+#endif
+        ++use_count_;
+    }
+
+    void add_ref_lock()
+    {
+#if defined(BOOST_HAS_THREADS)
+        mutex_type::scoped_lock lock(mtx_);
+#endif
+        if(use_count_ == 0) boost::throw_exception(boost::bad_weak_ptr());
+        ++use_count_;
+    }
+
+    void release() // nothrow
+    {
+        {
+#if defined(BOOST_HAS_THREADS)
+            mutex_type::scoped_lock lock(mtx_);
+#endif
+            long new_use_count = --use_count_;
+
+            if(new_use_count != 0) return;
+        }
+
+        dispose();
+        weak_release();
+    }
+
+    void weak_add_ref() // nothrow
+    {
+#if defined(BOOST_HAS_THREADS)
+        mutex_type::scoped_lock lock(mtx_);
+#endif
+        ++weak_count_;
+    }
+
+    void weak_release() // nothrow
+    {
+        long new_weak_count;
+
+        {
+#if defined(BOOST_HAS_THREADS)
+            mutex_type::scoped_lock lock(mtx_);
+#endif
+            new_weak_count = --weak_count_;
+        }
+
+        if(new_weak_count == 0)
+        {
+            destruct();
+        }
+    }
+
+    long use_count() const // nothrow
+    {
+#if defined(BOOST_HAS_THREADS)
+        mutex_type::scoped_lock lock(mtx_);
+#endif
+        return use_count_;
+    }
+
+private:
+
+    sp_counted_base(sp_counted_base const &);
+    sp_counted_base & operator= (sp_counted_base const &);
+
+    long use_count_;        // #shared
+    long weak_count_;       // #weak + (#shared != 0)
+
+#if defined(BOOST_HAS_THREADS) || defined(BOOST_LWM_WIN32)
+    mutable mutex_type mtx_;
+#endif
+};
+
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+
+template<class T> void cbi_call_constructor_hook(sp_counted_base * pn, T * px, checked_deleter<T> const &, int)
+{
+    boost::sp_scalar_constructor_hook(px, sizeof(T), pn);
+}
+
+template<class T> void cbi_call_constructor_hook(sp_counted_base *, T * px, checked_array_deleter<T> const &, int)
+{
+    boost::sp_array_constructor_hook(px);
+}
+
+template<class P, class D> void cbi_call_constructor_hook(sp_counted_base *, P const &, D const &, long)
+{
+}
+
+template<class T> void cbi_call_destructor_hook(sp_counted_base * pn, T * px, checked_deleter<T> const &, int)
+{
+    boost::sp_scalar_destructor_hook(px, sizeof(T), pn);
+}
+
+template<class T> void cbi_call_destructor_hook(sp_counted_base *, T * px, checked_array_deleter<T> const &, int)
+{
+    boost::sp_array_destructor_hook(px);
+}
+
+template<class P, class D> void cbi_call_destructor_hook(sp_counted_base *, P const &, D const &, long)
+{
+}
+
+#endif
+
+//
+// Borland's Codeguard trips up over the -Vx- option here:
+//
+#ifdef __CODEGUARD__
+# pragma option push -Vx-
+#endif
+
+template<class P, class D> class sp_counted_base_impl: public sp_counted_base
+{
+private:
+
+    P ptr; // copy constructor must not throw
+    D del; // copy constructor must not throw
+
+    sp_counted_base_impl(sp_counted_base_impl const &);
+    sp_counted_base_impl & operator= (sp_counted_base_impl const &);
+
+    typedef sp_counted_base_impl<P, D> this_type;
+
+public:
+
+    // pre: initial_use_count <= initial_weak_count, d(p) must not throw
+
+    sp_counted_base_impl(P p, D d): ptr(p), del(d)
+    {
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        detail::cbi_call_constructor_hook(this, p, d, 0);
+#endif
+    }
+
+    virtual void dispose() // nothrow
+    {
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        detail::cbi_call_destructor_hook(this, ptr, del, 0);
+#endif
+        del(ptr);
+    }
+
+    virtual void * get_deleter(std::type_info const & ti)
+    {
+        return ti == typeid(D)? &del: 0;
+    }
+
+#if defined(BOOST_SP_USE_STD_ALLOCATOR)
+
+    void * operator new(std::size_t)
+    {
+        return std::allocator<this_type>().allocate(1, static_cast<this_type *>(0));
+    }
+
+    void operator delete(void * p)
+    {
+        std::allocator<this_type>().deallocate(static_cast<this_type *>(p), 1);
+    }
+
+#endif
+
+#if defined(BOOST_SP_USE_QUICK_ALLOCATOR)
+
+    void * operator new(std::size_t)
+    {
+        return quick_allocator<this_type>::alloc();
+    }
+
+    void operator delete(void * p)
+    {
+        quick_allocator<this_type>::dealloc(p);
+    }
+
+#endif
+};
+
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+
+int const shared_count_id = 0x2C35F101;
+int const   weak_count_id = 0x298C38A4;
+
+#endif
+
+class weak_count;
+
+class shared_count
+{
+private:
+
+    sp_counted_base * pi_;
+
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+    int id_;
+#endif
+
+    friend class weak_count;
+
+public:
+
+    shared_count(): pi_(0) // nothrow
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(shared_count_id)
+#endif
+    {
+    }
+
+    template<class P, class D> shared_count(P p, D d): pi_(0)
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(shared_count_id)
+#endif
+    {
+#ifndef BOOST_NO_EXCEPTIONS
+
+        try
+        {
+            pi_ = new sp_counted_base_impl<P, D>(p, d);
+        }
+        catch(...)
+        {
+            d(p); // delete p
+            throw;
+        }
+
+#else
+
+        pi_ = new sp_counted_base_impl<P, D>(p, d);
+
+        if(pi_ == 0)
+        {
+            d(p); // delete p
+            boost::throw_exception(std::bad_alloc());
+        }
+
+#endif
+    }
+
+#ifndef BOOST_NO_AUTO_PTR
+
+    // auto_ptr<Y> is special cased to provide the strong guarantee
+
+    template<class Y>
+    explicit shared_count(std::auto_ptr<Y> & r): pi_(new sp_counted_base_impl< Y *, checked_deleter<Y> >(r.get(), checked_deleter<Y>()))
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(shared_count_id)
+#endif
+    {
+        r.release();
+    }
+
+#endif 
+
+    ~shared_count() // nothrow
+    {
+        if(pi_ != 0) pi_->release();
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        id_ = 0;
+#endif
+    }
+
+    shared_count(shared_count const & r): pi_(r.pi_) // nothrow
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(shared_count_id)
+#endif
+    {
+        if(pi_ != 0) pi_->add_ref_copy();
+    }
+
+    explicit shared_count(weak_count const & r); // throws bad_weak_ptr when r.use_count() == 0
+
+    shared_count & operator= (shared_count const & r) // nothrow
+    {
+        sp_counted_base * tmp = r.pi_;
+        if(tmp != 0) tmp->add_ref_copy();
+        if(pi_ != 0) pi_->release();
+        pi_ = tmp;
+
+        return *this;
+    }
+
+    void swap(shared_count & r) // nothrow
+    {
+        sp_counted_base * tmp = r.pi_;
+        r.pi_ = pi_;
+        pi_ = tmp;
+    }
+
+    long use_count() const // nothrow
+    {
+        return pi_ != 0? pi_->use_count(): 0;
+    }
+
+    bool unique() const // nothrow
+    {
+        return use_count() == 1;
+    }
+
+    friend inline bool operator==(shared_count const & a, shared_count const & b)
+    {
+        return a.pi_ == b.pi_;
+    }
+
+    friend inline bool operator<(shared_count const & a, shared_count const & b)
+    {
+        return std::less<sp_counted_base *>()(a.pi_, b.pi_);
+    }
+
+    void * get_deleter(std::type_info const & ti) const
+    {
+        return pi_? pi_->get_deleter(ti): 0;
+    }
+};
+
+#ifdef __CODEGUARD__
+# pragma option pop
+#endif
+
+
+class weak_count
+{
+private:
+
+    sp_counted_base * pi_;
+
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+    int id_;
+#endif
+
+    friend class shared_count;
+
+public:
+
+    weak_count(): pi_(0) // nothrow
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(weak_count_id)
+#endif
+    {
+    }
+
+    weak_count(shared_count const & r): pi_(r.pi_) // nothrow
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(shared_count_id)
+#endif
+    {
+        if(pi_ != 0) pi_->weak_add_ref();
+    }
+
+    weak_count(weak_count const & r): pi_(r.pi_) // nothrow
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(shared_count_id)
+#endif
+    {
+        if(pi_ != 0) pi_->weak_add_ref();
+    }
+
+    ~weak_count() // nothrow
+    {
+        if(pi_ != 0) pi_->weak_release();
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        id_ = 0;
+#endif
+    }
+
+    weak_count & operator= (shared_count const & r) // nothrow
+    {
+        sp_counted_base * tmp = r.pi_;
+        if(tmp != 0) tmp->weak_add_ref();
+        if(pi_ != 0) pi_->weak_release();
+        pi_ = tmp;
+
+        return *this;
+    }
+
+    weak_count & operator= (weak_count const & r) // nothrow
+    {
+        sp_counted_base * tmp = r.pi_;
+        if(tmp != 0) tmp->weak_add_ref();
+        if(pi_ != 0) pi_->weak_release();
+        pi_ = tmp;
+
+        return *this;
+    }
+
+    void swap(weak_count & r) // nothrow
+    {
+        sp_counted_base * tmp = r.pi_;
+        r.pi_ = pi_;
+        pi_ = tmp;
+    }
+
+    long use_count() const // nothrow
+    {
+        return pi_ != 0? pi_->use_count(): 0;
+    }
+
+    friend inline bool operator==(weak_count const & a, weak_count const & b)
+    {
+        return a.pi_ == b.pi_;
+    }
+
+    friend inline bool operator<(weak_count const & a, weak_count const & b)
+    {
+        return std::less<sp_counted_base *>()(a.pi_, b.pi_);
+    }
+};
+
+inline shared_count::shared_count(weak_count const & r): pi_(r.pi_)
+#if defined(BOOST_SP_ENABLE_DEBUG_HOOKS)
+        , id_(shared_count_id)
+#endif
+{
+    if(pi_ != 0)
+    {
+        pi_->add_ref_lock();
+    }
+    else
+    {
+        boost::throw_exception(boost::bad_weak_ptr());
+    }
+}
+
+} // namespace detail
+
+} // namespace boost
+
+#ifdef __BORLANDC__
+# pragma warn .8027     // Functions containing try are not expanded inline
+# pragma warn .8026     // Functions with excep. spec. are not expanded inline
+#endif
+
+#endif  // #ifndef BOOST_DETAIL_SHARED_COUNT_HPP_INCLUDED

src/boost/detail/workaround.hpp

@@ -0,0 +1,74 @@
+// Copyright David Abrahams 2002. Permission to copy, use,
+// modify, sell and distribute this software is granted provided this
+// copyright notice appears in all copies. This software is provided
+// "as is" without express or implied warranty, and with no claim as
+// to its suitability for any purpose.
+#ifndef WORKAROUND_DWA2002126_HPP
+# define WORKAROUND_DWA2002126_HPP
+
+// Compiler/library version workaround macro
+//
+// Usage:
+//
+//     #if BOOST_WORKAROUND(BOOST_MSVC, <= 1200)
+//        ... // workaround code here
+//     #endif
+//
+// When BOOST_STRICT_CONFIG is defined, expands to 0. Otherwise, the
+// first argument must be undefined or expand to a numeric
+// value. The above expands to:
+//
+//     (BOOST_MSVC) != 0 && (BOOST_MSVC) <= 1200
+//
+// When used for workarounds that apply to the latest known version 
+// and all earlier versions of a compiler, the following convention 
+// should be observed:
+//
+//     #if BOOST_WORKAROUND(BOOST_MSVC, BOOST_TESTED_AT(1301))
+//
+// The version number in this case corresponds to the last version in
+// which the workaround was known to have been required. When
+// BOOST_DETECT_OUTDATED_WORKAROUNDS is not the defined, the macro
+// BOOST_TESTED_AT(x) expands to "!= 0", which effectively activates
+// the workaround for any version of the compiler. When
+// BOOST_DETECT_OUTDATED_WORKAROUNDS is defined, a compiler warning or
+// error will be issued if the compiler version exceeds the argument
+// to BOOST_TESTED_AT().  This can be used to locate workarounds which
+// may be obsoleted by newer versions.
+
+# ifndef BOOST_STRICT_CONFIG
+
+#  define BOOST_WORKAROUND(symbol, test)                \
+        ((symbol != 0) && (1 % (( (symbol test) ) + 1)))
+//                              ^ ^           ^ ^
+// The extra level of parenthesis nesting above, along with the
+// BOOST_OPEN_PAREN indirection below, is required to satisfy the
+// broken preprocessor in MWCW 8.3 and earlier.
+//
+// The basic mechanism works as follows:
+//      (symbol test) + 1        =>   if (symbol test) then 2 else 1
+//      1 % ((symbol test) + 1)  =>   if (symbol test) then 1 else 0
+//
+// The complication with % is for cooperation with BOOST_TESTED_AT().
+// When "test" is BOOST_TESTED_AT(x) and
+// BOOST_DETECT_OUTDATED_WORKAROUNDS is #defined,
+//
+//      symbol test              =>   if (symbol <= x) then 1 else -1
+//      (symbol test) + 1        =>   if (symbol <= x) then 2 else 0
+//      1 % ((symbol test) + 1)  =>   if (symbol <= x) then 1 else divide-by-zero
+//
+
+#  ifdef BOOST_DETECT_OUTDATED_WORKAROUNDS
+#   define BOOST_OPEN_PAREN (
+#   define BOOST_TESTED_AT(value)  > value) ?(-1): BOOST_OPEN_PAREN 1
+#  else
+#   define BOOST_TESTED_AT(value) != ((value)-(value))
+#  endif
+
+# else
+
+#  define BOOST_WORKAROUND(symbol, test) 0
+
+# endif 
+
+#endif // WORKAROUND_DWA2002126_HPP

src/boost/enable_shared_from_this.hpp

@@ -0,0 +1,68 @@
+#ifndef BOOST_ENABLE_SHARED_FROM_THIS_HPP_INCLUDED
+#define BOOST_ENABLE_SHARED_FROM_THIS_HPP_INCLUDED
+
+//
+//  enable_shared_from_this.hpp
+//
+//  Copyright (c) 2002 Peter Dimov
+//
+//  Permission to copy, use, modify, sell and distribute this software
+//  is granted provided this copyright notice appears in all copies.
+//  This software is provided "as is" without express or implied
+//  warranty, and with no claim as to its suitability for any purpose.
+//
+//  http://www.boost.org/libs/smart_ptr/enable_shared_from_this.html
+//
+
+#include <boost/weak_ptr.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/assert.hpp>
+//#include <boost/config.hpp>
+
+namespace boost
+{
+
+template<class T> class enable_shared_from_this
+{
+protected:
+
+    enable_shared_from_this()
+    {
+    }
+
+    enable_shared_from_this(enable_shared_from_this const &)
+    {
+    }
+
+    enable_shared_from_this & operator=(enable_shared_from_this const &)
+    {
+        return *this;
+    }
+
+    ~enable_shared_from_this()
+    {
+    }
+
+public:
+
+    shared_ptr<T> shared_from_this()
+    {
+        shared_ptr<T> p(_internal_weak_this);
+        BOOST_ASSERT(p.get() == this);
+        return p;
+    }
+
+    shared_ptr<T const> shared_from_this() const
+    {
+        shared_ptr<T const> p(_internal_weak_this);
+        BOOST_ASSERT(p.get() == this);
+        return p;
+    }
+
+    typedef T _internal_element_type; // for bcc 5.5.1
+    weak_ptr<_internal_element_type> _internal_weak_this;
+};
+
+} // namespace boost
+
+#endif  // #ifndef BOOST_ENABLE_SHARED_FROM_THIS_HPP_INCLUDED