* Option to turn off position information to test the impact on

maximal sharing.

Makefile.am

@@ -35,11 +35,11 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
-	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
-	ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
+	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
 	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(prefix)/store
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
+	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
 #	$(bindir)/nix-store --init
 
 else

configure.ac

@@ -9,14 +9,14 @@ STABLE=0
 
 # Put the revision number in the version.
 if test "$STABLE" != "1"; then
-    if REVISION=`test -d $srcdir/.svn && svnversion $srcdir 2> /dev/null`; then
+    if REVISION=`test -d $srcdir/.svn && svnversion -n $srcdir 2> /dev/null`; then
         VERSION=${VERSION}pre${REVISION}
     elif REVISION=`cat $srcdir/svn-revision 2> /dev/null`; then
         VERSION=${VERSION}pre${REVISION}
     fi
 fi
 
-AC_DEFINE_UNQUOTED(NIX_VERSION, ["$(echo $VERSION)"], [version])
+AC_DEFINE_UNQUOTED(NIX_VERSION, ["$VERSION"], [version])
 
 AC_PREFIX_DEFAULT(/nix)
 
@@ -139,6 +139,7 @@ fi
 AC_MSG_RESULT(yes)
 
 NEED_PROG(cat, cat)
+NEED_PROG(tr, tr)
 AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
   [path of cat, mkdir, etc.]),
   coreutils=$withval, coreutils=$(dirname $cat))

corepkgs/buildenv/builder.pl.in

@@ -12,13 +12,15 @@ mkdir "$out", 0755 || die "error creating $out";
 
 my $symlinks = 0;
 
+my %priorities;
+
 
 # For each activated package, create symlinks.
 
 sub createLinks {
     my $srcDir = shift;
     my $dstDir = shift;
-    my $ignoreCollisions = shift;
+    my $priority = shift;
 
     my @srcFiles = glob("$srcDir/*");
 
@@ -42,7 +44,7 @@ sub createLinks {
             lstat $dstFile;
             
             if (-d _) {
-                createLinks($srcFile, $dstFile, $ignoreCollisions);
+                createLinks($srcFile, $dstFile, $priority);
             }
 
             elsif (-l _) {
@@ -53,27 +55,35 @@ sub createLinks {
                 unlink $dstFile or die "error unlinking `$dstFile': $!";
                 mkdir $dstFile, 0755 || 
                     die "error creating directory `$dstFile': $!";
-                createLinks($target, $dstFile, $ignoreCollisions);
-                createLinks($srcFile, $dstFile, $ignoreCollisions);
+                createLinks($target, $dstFile, $priorities{$dstFile});
+                createLinks($srcFile, $dstFile, $priority);
             }
 
             else {
                 symlink($srcFile, $dstFile) ||
                     die "error creating link `$dstFile': $!";
+                $priorities{$dstFile} = $priority;
                 $symlinks++;
             }
         }
 
-        elsif (-l $dstFile) {
-            if (!$ignoreCollisions) {
-                my $target = readlink $dstFile;
-                die "collission between `$srcFile' and `$target'";
-            }
-        }
-
         else {
+
+            if (-l $dstFile) {
+                my $target = readlink $dstFile;
+                my $prevPriority = $priorities{$dstFile};
+                die ( "Collission between `$srcFile' and `$target'. "
+                    . "Suggested solution: use `nix-env --set-flag "
+                    . "priority NUMBER PKGNAME' to change the priority of "
+                    . "one of the conflicting packages.\n" )
+                    if $prevPriority == $priority;
+                next if $prevPriority < $priority;
+                unlink $dstFile or die;
+            }
+            
             symlink($srcFile, $dstFile) ||
                 die "error creating link `$dstFile': $!";
+            $priorities{$dstFile} = $priority;
             $symlinks++;
         }
     }
@@ -86,13 +96,13 @@ my %postponed;
 sub addPkg;
 sub addPkg {
     my $pkgDir = shift;
-    my $ignoreCollisions = shift;
+    my $priority = shift;
 
     return if (defined $done{$pkgDir});
     $done{$pkgDir} = 1;
 
 #    print "symlinking $pkgDir\n";
-    createLinks("$pkgDir", "$out", $ignoreCollisions);
+    createLinks("$pkgDir", "$out", $priority);
 
     my $propagatedFN = "$pkgDir/nix-support/propagated-user-env-packages";
     if (-e $propagatedFN) {
@@ -107,11 +117,29 @@ sub addPkg {
 }
 
 
-# Symlink to the packages that have been installed explicitly by the user.
-my @args = split ' ', $ENV{"derivations"};
+# Convert the stuff we get from the environment back into a coherent
+# data type.
+my @paths = split ' ', $ENV{"paths"};
+my @active = split ' ', $ENV{"active"};
+my @priority = split ' ', $ENV{"priority"};
 
-foreach my $pkgDir (sort @args) {
-    addPkg($pkgDir, 0);
+die if scalar @paths != scalar @active;
+die if scalar @paths != scalar @priority;
+
+my %pkgs;
+
+for (my $n = 0; $n < scalar @paths; $n++) {
+    $pkgs{$paths[$n]} =
+        { active => $active[$n]
+        , priority => $priority[$n] };
+}
+
+
+# Symlink to the packages that have been installed explicitly by the
+# user.
+foreach my $pkg (sort (keys %pkgs)) {
+    #print $pkg, " ", $pkgs{$pkg}->{priority}, "\n";
+    addPkg($pkg, $pkgs{$pkg}->{priority}) if $pkgs{$pkg}->{active} ne "false";
 }
 
 
@@ -119,11 +147,12 @@ foreach my $pkgDir (sort @args) {
 # installed by the user (i.e., package X declares that it want Y
 # installed as well).  We do these later because they have a lower
 # priority in case of collisions.
+my $priorityCounter = 1000; # don't care about collisions
 while (scalar(keys %postponed) > 0) {
     my @pkgDirs = keys %postponed;
     %postponed = ();
     foreach my $pkgDir (sort @pkgDirs) {
-        addPkg($pkgDir, 1);
+        addPkg($pkgDir, $priorityCounter++);
     }
 }
 

corepkgs/buildenv/default.nix

@@ -4,6 +4,11 @@ derivation {
   name = "user-environment";
   system = system;
   builder = ./builder.pl;
-  derivations = derivations;
+  
   manifest = manifest;
+
+  # !!! grmbl, need structured data for passing this in a clean way.
+  paths = derivations;
+  active = map (x: if x ? meta && x.meta ? active then x.meta.active else "true") derivations;
+  priority = map (x: if x ? meta && x.meta ? priority then x.meta.priority else "5") derivations;
 }

corepkgs/channels/unpack.sh.in

@@ -4,19 +4,23 @@
 @coreutils@/mkdir $out/tmp
 cd $out/tmp
 
-expr=$out/default.nix
-echo '[' > $expr
+inputs=($inputs)
+for ((n = 0; n < ${#inputs[*]}; n += 2)); do
+    channelName=${inputs[n]}
+    channelTarball=${inputs[n+1]}
+    echo "unpacking channel $channelName"
+    @bunzip2@ < $channelTarball | @tar@ xf -
 
-nr=0
-for i in $inputs; do
-    echo "unpacking $i"
-    @bunzip2@ < $i | @tar@ xf -
-    @coreutils@/mv * ../$nr # !!! hacky
-    echo "(import ./$nr)" >> $expr
-    nr=$(($nr + 1))
+    nr=1
+    attrName=$(echo $channelName | @tr@ -- '- ' '__')
+    dirName=$attrName
+    while test -e ../$dirName; do
+        nr=$((nr+1))
+        dirName=$attrName-$nr
+    done
+
+    @coreutils@/mv * ../$dirName # !!! hacky
 done
 
-echo ']' >> $expr
-
 cd ..
 @coreutils@/rmdir tmp

doc/manual/Makefile.am

@@ -15,7 +15,7 @@ man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
  nix-prefetch-url.1 nix-channel.1 \
  nix-pack-closure.1 nix-unpack-closure.1 \
- nix-install-package.1 nix-hash.1
+ nix-install-package.1 nix-hash.1 nix-copy-closure.1
 
 FIGURES = figures/user-environments.png
 
@@ -31,7 +31,7 @@ MANUAL_SRCS = manual.xml introduction.xml installation.xml \
 manual.is-valid: $(MANUAL_SRCS) version.txt
 #	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --relaxng $(docbookrng)/docbook.rng -
 	if test "$(jing)" != "false"; then \
-		$(XMLLINT) --xinclude $< | $(jing) $(docbookrng)/docbook.rng /dev/stdin; \
+		$(XMLLINT) --xinclude $< | $(jing) $(docbookrng)/docbook.rng /dev/fd/0; \
 	else \
 		echo "Not validating."; \
 	fi

doc/manual/installation.xml

@@ -6,6 +6,39 @@
 <title>Installation</title>
 
 
+<section><title>Supported platforms</title>
+
+<para>Nix is currently supported on the following platforms:
+
+<itemizedlist>
+
+  <listitem><para>Linux (particularly on x86, x86_64, and
+  PowerPC).</para></listitem>
+
+  <listitem><para>Mac OS X, both on Intel and
+  PowerPC.</para></listitem>
+
+  <listitem><para>FreeBSD (only tested on Intel).</para></listitem>
+
+  <listitem><para>Windows through <link
+  xlink:href="http://www.cygwin.com/">Cygwin</link>.</para>
+
+  <warning><para>On Cygwin, Nix <emphasis>must</emphasis> be installed
+  on an NTFS partition.  It will not work correctly on a FAT
+  partition.</para></warning>
+
+  </listitem>
+
+</itemizedlist>
+
+</para>
+
+<para>Nix is pretty portable, so it should work on most other Unix
+platforms as well.</para>
+
+</section>
+
+
 <section><title>Obtaining Nix</title>
 
 <para>The easiest way to obtain Nix is to download a <link
@@ -57,14 +90,15 @@ repository.</para>
 <para>To build the parser, very <emphasis>recent</emphasis> versions
 of Bison and Flex are required.  (This is because Nix needs GLR
 support in Bison and reentrancy support in Flex.)  For Bison, you need
-version 1.875c or higher (1.875 does <emphasis>not</emphasis> work),
-which can be obtained from the <link
-xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP server</link>.
-For Flex, you need version 2.5.31, which is available on <link
-xlink:href="http://lex.sourceforge.net/">SourceForge</link>.  Slightly
-older versions may also work, but ancient versions like the ubiquitous
-2.5.4a won't.  Note that these are only required if you modify the
-parser or when you are building from the Subversion repository.</para>
+version 2.3 or higher (1.875 does <emphasis>not</emphasis> work),
+which can be obtained from
+the <link xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP
+server</link>.  For Flex, you need version 2.5.33, which is available
+on <link xlink:href="http://lex.sourceforge.net/">SourceForge</link>.
+Slightly older versions may also work, but ancient versions like the
+ubiquitous 2.5.4a won't.  Note that these are only required if you
+modify the parser or when you are building from the Subversion
+repository.</para>
 
 <para>Nix uses Sleepycat's Berkeley DB and CWI's ATerm library.  These
 are included in the Nix source distribution.  If you build from the
@@ -75,7 +109,7 @@ these packages.  Alternatively, if you already have them installed,
 you can use <command>configure</command>'s <option>--with-bdb</option>
 and <option>--with-aterm</option> options to point to their respective
 locations.  Note that Berkeley DB <emphasis>must</emphasis> be version
-4.4; other versions may not have compatible database formats.</para>
+4.5; other versions may not have compatible database formats.</para>
 
 </section>
 
@@ -96,7 +130,7 @@ preceded by the command:
     </para>
 
 <screen>
-$ autoreconf -i</screen>
+$ ./boostrap</screen>
 
 <para>The installation path can be specified by passing the
 <option>--prefix=<replaceable>prefix</replaceable></option> to
@@ -162,49 +196,44 @@ xlink:href="http://nix.cs.uu.nl/dist/nix/" />.</para>
 </section>
 
 
-<section><title>Permissions</title>
+<section><title>Security</title>
 
-<para>All Nix operations must be performed under the user ID that owns
-the Nix store and database
-(<filename><replaceable>prefix</replaceable>/store</filename> and
-<filename><replaceable>prefix</replaceable>/var/nix/db</filename>,
-respectively).  When installed from the RPM packages, these
-directories are owned by <systemitem class="username">root</systemitem>.</para>
+<para>Nix has two basic security models.  First, it can be used in
+“single-user mode”, which is similar to what most other package
+management tools do: there is a single user (typically <systemitem
+class="username">root</systemitem>) who performs all package
+management operations.  All other users can then use the installed
+packages, but they cannot perform package management operations
+themselves.</para>
 
-<section><title>Setuid installation</title>
+<para>Alternatively, you can configure Nix in “multi-user mode”.  In
+this model, all users can perform package management operations — for
+instance, every user can install software without requiring root
+privileges.  Nix ensures that this is secure.  For instance, it’s not
+possible for one user to overwrite a package used by another user with
+a Trojan horse.</para>
 
-<para>As a somewhat <emphasis>ad hoc</emphasis> hack, you can also
-install the Nix binaries <quote>setuid</quote> so that a Nix store can
-be shared among several users.  To do this, configure Nix with the
-<emphasis>--enable-setuid</emphasis> option.  Nix will be installed as
-owned by a user and group specified by the
-<option>--with-nix-user=</option><parameter>user</parameter> and
-<option>--with-nix-group=</option><parameter>group</parameter>
-options.  E.g.,
 
-<screen>
-$ ./configure --enable-setuid --with-nix-user=my_nix_user --with-nix-group=my_nix_group</screen>
+<section><title>Single-user mode</title>
+  
+<para>In single-user mode, all Nix operations that access the database
+in <filename><replaceable>prefix</replaceable>/var/nix/db</filename>
+or modify the Nix store in
+<filename><replaceable>prefix</replaceable>/store</filename> must be
+performed under the user ID that owns those directories.  This is
+typically <systemitem class="username">root</systemitem>.  (If you
+install from RPM packages, that’s in fact the default ownership.)
+However, on single-user machines, it is often convenient to
+<command>chown</command> those directories to your normal user account
+so that you don’t have to <command>su</command> to <systemitem
+class="username">root</systemitem> all the time.</para>
 
-The user and group default to <literal>nix</literal>.  You should make
-sure that both the user and the group exist.  Any <quote>real</quote>
-users that you want to allow access should be added to the Nix
-group.</para>
+</section>
 
-<warning><para>A setuid installation should only by used if the users
-in the Nix group are mutually trusted, since any user in that group
-has the ability to change anything in the Nix store or database.  For
-instance, they could install a trojan horse in executables used by
-other users.</para></warning>
 
-<warning><para>On some platforms, the Nix binaries will be installed
-as setuid <literal>root</literal>.  They drop root privileges
-immediately after startup and switch to the Nix user.  The reason for
-this is that both the real and effective user must be set to the Nix
-user, and POSIX has no system call to do this.  This is not the case
-on systems that have the <function>setresuid()</function> system call
-(such as Linux and FreeBSD), so on those systems the binaries are
-simply owned by the Nix user.</para></warning>
+<section><title>Multi-user mode</title>
 
+<para></para>
 
 
 <!--
@@ -218,11 +247,17 @@ one.
 
 -->
 
-
+<note><para>Multi-user mode has one important limitation: only
+<systemitem class="username">root</systemitem> can run <command
+linkend="sec-nix-pull">nix-pull</command> to register the availability
+of pre-built binaries.  However, those registrations
+<emphasis>are</emphasis> used by all users to speed up
+builds.</para></note>
 
 </section>
 
-</section>
+
+</section> <!-- end of permissions section -->
 
 
 <section><title>Using Nix</title>

doc/manual/manual.xml

@@ -13,14 +13,21 @@
         <firstname>Eelco</firstname>
         <surname>Dolstra</surname>
       </personname>
+      <affiliation>
+        <orgname>Utrecht University</orgname>
+        <orgdiv>Faculty of Science, Department of Information and Computing Sciences</orgdiv>
+      </affiliation>
     </author>
 
     <copyright>
       <year>2004</year>
       <year>2005</year>
       <year>2006</year>
+      <year>2007</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
+
+    <date>September 2007</date>
     
   </info>
 
@@ -49,7 +56,7 @@
         <title>nix-instantiate</title>
         <xi:include href="nix-instantiate.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-store">
         <title>nix-store</title>
         <xi:include href="nix-store.xml" />
       </section>
@@ -65,10 +72,14 @@
         <title>nix-channel</title>
         <xi:include href="nix-channel.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-collect-garbage">
         <title>nix-collect-garbage</title>
         <xi:include href="nix-collect-garbage.xml" />
       </section>
+      <section xml:id="sec-nix-copy-closure">
+        <title>nix-copy-closure</title>
+        <xi:include href="nix-copy-closure.xml" />
+      </section>
       <section xml:id="sec-nix-hash">
         <title>nix-hash</title>
         <xi:include href="nix-hash.xml" />
@@ -77,15 +88,15 @@
         <title>nix-install-package</title>
         <xi:include href="nix-install-package.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-pack-closure">
         <title>nix-pack-closure</title>
         <xi:include href="nix-pack-closure.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-prefetch-url">
         <title>nix-prefetch-url</title>
         <xi:include href="nix-prefetch-url.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-pull">
         <title>nix-pull</title>
         <xi:include href="nix-pull.xml" />
       </section>
@@ -93,7 +104,7 @@
         <title>nix-push</title>
         <xi:include href="nix-push.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-unpack-closure">
         <title>nix-unpack-closure</title>
         <xi:include href="nix-unpack-closure.xml" />
       </section>

doc/manual/nix-build.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-build</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-build</refname>

doc/manual/nix-channel.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-channel</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-channel</refname>
   <refpurpose>manage Nix channels</refpurpose>

doc/manual/nix-collect-garbage.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-collect-garbage</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-collect-garbage</refname>
   <refpurpose>delete unreachable store paths</refpurpose>

doc/manual/nix-copy-closure.xml

@@ -0,0 +1,151 @@
+<refentry xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-copy-closure</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
+<refnamediv>
+  <refname>nix-copy-closure</refname>
+  <refpurpose>copy a closure to or from a remote machine via SSH</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-copy-closure</command>
+    <group>
+      <arg choice='plain'><option>--to</option></arg>
+      <arg choice='plain'><option>--from</option></arg>
+    </group>
+    <arg><option>--sign</option></arg>
+    <arg><option>--gzip</option></arg>
+    <arg choice='plain'>
+      <arg><replaceable>user@</replaceable></arg><replaceable>machine</replaceable>
+    </arg>
+    <arg choice='plain'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+
+<refsection><title>Description</title>
+
+<para><command>nix-copy-closure</command> gives you an easy and
+efficient way to exchange software between machines.  Given one or
+more Nix store paths <replaceable>paths</replaceable> on the local
+machine, <command>nix-copy-closure</command> computes the closure of
+those paths (i.e. all their dependencies in the Nix store), and copies
+all paths in the closure to the remote machine via the
+<command>ssh</command> (Secure Shell) command.  With the
+<option>--from</option>, the direction is reversed:
+the closure of <replaceable>paths</replaceable> on a remote machine is
+copied to the Nix store on the local machine.</para>
+
+<para>This command is efficient because it only sends the store paths
+that are missing on the target machine.</para>
+
+<para>Since <command>nix-copy-closure</command> calls
+<command>ssh</command>, you may be asked to type in the appropriate
+password or passphrase.  In fact, you may be asked
+<emphasis>twice</emphasis> because <command>nix-copy-closure</command>
+currently connects twice to the remote machine, first to get the set
+of paths missing on the target machine, and second to send the dump of
+those paths.  If this bothers you, use
+<command>ssh-agent</command>.</para>
+
+
+<refsection><title>Options</title>
+
+<variablelist>
+  
+  <varlistentry><term><option>--to</option></term>
+
+    <listitem><para>Copy the closure of
+    <replaceable>paths</replaceable> from the local Nix store to the
+    Nix store on <replaceable>machine</replaceable>.  This is the
+    default.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--from</option></term>
+
+    <listitem><para>Copy the closure of
+    <replaceable>paths</replaceable> from the Nix store on
+    <replaceable>machine</replaceable> to the local Nix
+    store.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--sign</option></term>
+
+    <listitem><para>Let the sending machine cryptographically sign the
+    dump of each path with the key in
+    <filename>/nix/etc/nix/signing-key.sec</filename>.  If the user on
+    the target machine does not have direct access to the Nix store
+    (i.e., if the target machine has a multi-user Nix installation),
+    then the target machine will check the dump against
+    <filename>/nix/etc/nix/signing-key.pub</filename> before unpacking
+    it in its Nix store.  This allows secure sharing of store paths
+    between untrusted users on two machines, provided that there is a
+    trust relation between the Nix installations on both machines
+    (namely, they have matching public/secret keys).</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--gzip</option></term>
+
+    <listitem><para>Compress the dump of each path with
+    <command>gzip</command> before sending it.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</refsection>
+
+
+<refsection><title>Environment variables</title>
+
+<variablelist>
+
+  <varlistentry><term><envar>NIX_SSHOPTS</envar></term>
+
+    <listitem><para>Additional options to be passed to
+    <command>ssh</command> on the command line.</para></listitem>
+
+  </varlistentry>
+  
+</variablelist>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<para>Copy Firefox with all its dependencies to a remote machine:
+
+<screen>
+$ nix-copy-closure alice@itchy.labs $(type -tP firefox)</screen>
+
+</para>
+
+<para>Copy Subversion from a remote machine and then install it into a
+user environment:
+
+<screen>
+$ nix-copy-closure --from alice@itchy.labs \
+    /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+</screen>
+
+</para>
+
+</refsection>
+
+
+</refsection>
+
+</refentry>

doc/manual/nix-env.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-env</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-env</refname>
   <refpurpose>manipulate or query Nix user environments</refpurpose>
@@ -142,14 +150,14 @@ linkend="sec-common-options" />.</para>
 <variablelist>
 
   <varlistentry><term><filename>~/.nix-defexpr</filename></term>
+
+  <!-- !!! .nix-defexpr can be a directory now -->
   
     <listitem><para>The default Nix expression used by the
     <option>--install</option>, <option>--upgrade</option>, and
     <option>--query --available</option> operations to obtain
-    derivations.  It is generally a symbolic link to some other
-    location set using the <option>--import</option> operation.  The
-    <option>--file</option> option may be used to override this
-    default.</para></listitem>
+    derivations.  The <option>--file</option> option may be used to
+    override this default.</para></listitem>
     
   </varlistentry>
 
@@ -1061,43 +1069,4 @@ error: no generation older than the current (91) exists</screen>
 
 
   
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--import</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--import</option></arg>
-    <arg choice='plain'><option>-I</option></arg>
-  </group>
-  <arg choice='req'><replaceable>path</replaceable></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-            
-<para>This operation makes <replaceable>path</replaceable> the default
-active Nix expression for the user.  That is, the symlink
-<filename>~/.nix-userenv</filename> is made to point to
-<replaceable>path</replaceable>.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env -I ~/nixpkgs-0.5/</screen>
-
-</refsection>
-    
-</refsection>
-
-
-  
 </refentry>

doc/manual/nix-hash.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-hash</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-hash</refname>
   <refpurpose>compute the cryptographic hash of a path</refpurpose>

doc/manual/nix-install-package.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-install-package</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-install-package</refname>
   <refpurpose>install a Nix Package file</refpurpose>

doc/manual/nix-instantiate.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-instantiate</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-instantiate</refname>
   <refpurpose>instantiate store derivations from Nix expressions</refpurpose>

doc/manual/nix-pack-closure.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-pack-closure</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-pack-closure</refname>

doc/manual/nix-prefetch-url.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-prefetch-url</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-prefetch-url</refname>
   <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
@@ -39,7 +47,7 @@ avoided.</para>
 <para>The environment variable <envar>NIX_HASH_ALGO</envar> specifies
 which hash algorithm to use.  It can be either <literal>md5</literal>,
 <literal>sha1</literal>, or <literal>sha256</literal>.  The default is
-<literal>md5</literal>.</para>
+<literal>sha256</literal>.</para>
 
 <para>If <replaceable>hash</replaceable> is specified, then a download
 is not performed if the Nix store already contains a file with the

doc/manual/nix-pull.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-pull</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-pull</refname>

doc/manual/nix-push.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-push</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-push</refname>

doc/manual/nix-store.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-store</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-store</refname>
@@ -649,36 +657,6 @@ $ gv graph.ps</screen>
 
 
 
-<!--######################################################################-->
-
-<!--
-<refsection><title>Operation <option>-XXX-substitute</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>-XXX-substitute</option></arg>
-  <arg choice='plain'
-       rep='repeat'><replaceable>srcpath</replaceable> <replaceable>subpath</replaceable></arg>
-</cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-            
-<para>The operation <option>-XXX-substitute</option> registers that the
-store path <replaceable>srcpath</replaceable> can be built by
-realising the derivation expression in
-<replaceable>subpath</replaceable>.  This is used to implement binary
-deployment.</para>
-
-</refsection>
-            
-</refsection>
--->
-
-
-
 <!--######################################################################-->
 
 <refsection xml:id='refsec-nix-store-verify'><title>Operation <option>--verify</option></title>

doc/manual/nix-unpack-closure.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-unpack-closure</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-unpack-closure</refname>

doc/manual/release-notes.xml

@@ -1,5 +1,6 @@
 <article xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink">
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xml:id="sec-relnotes">
 
 <title>Nix Release Notes</title>
 
@@ -7,17 +8,20 @@
 
 <!--==================================================================-->
 
-<section><title>Release 0.11 (TBA)</title>
+<section xml:id="ssec-relnotes-0.11"><title>Release 0.11 (TBA)</title>
 
 <itemizedlist>
 
 
-  <listitem><para>TODO: multi-user support.</para></listitem>
+  <listitem><para>TODO: multi-user support.  The old setuid method for
+  sharing a store between multiple users has been
+  removed.</para></listitem>
 
 
-  <listitem><para><command>nix-copy-closure</command> copies the
-  missing parts of a closure to or from a remote
-  machine.</para></listitem>
+  <listitem><para>The new command <command>nix-copy-closure</command>
+  gives you an easy and efficient way to exchange software between
+  machines.  It copies the missing parts of the closure of a set of
+  store path to or from a remote machine.</para></listitem>
 
 
   <listitem><para><command>nix-prefetch-url</command> now by default
@@ -67,11 +71,28 @@
 
 
   <listitem><para>TODO: <command>nix-env</command> now maintains meta
-  info about installed packages in user
-  environments.</para></listitem>
+  info about installed packages in user environments.  <option>-q
+  --xml --meta</option> to show all meta info.</para></listitem>
 
+  
   <listitem><para>TODO: <command>nix-env</command>
-  <option>--set-flag</option>.</para></listitem>
+  <option>--set-flag</option>.  Specific flags:
+  <literal>active</literal>, <literal>priority</literal>,
+  <literal>keep</literal>.</para></listitem>
+
+  
+  <listitem><para>TODO: <command>nix-env</command> <option>-i</option>
+  / <option>-u</option> take package priorities into
+  account.</para></listitem>
+  
+
+  <listitem><para><command>nix-env -q</command> now has a flag
+  <option>--prebuilt-only</option> (<option>-b</option>) that causes
+  <command>nix-env</command> to show only those derivations whose
+  output is already in the Nix store or that can be substituted (i.e.,
+  downloaded from somewhere).  In other words, it shows the packages
+  that can be installed “quickly”, i.e., don’t need to be built from
+  source.</para></listitem>
   
 
   <listitem><para>TODO: new built-ins
@@ -80,6 +101,21 @@
   <function>builtins.sub</function>,
   <function>builtins.stringLength</function>,
   <function>builtins.substring</function>.</para></listitem>
+
+
+  <listitem><para>TODO: each subscribed channel is its own attribute
+  in the top-level expression generated for the channel, this allows
+  disambiguation (<command>nix-env -qaA</command>).</para></listitem>
+
+
+  <listitem><para>TODO: substitutes table is gone, registering
+  substitutes is now much faster.</para></listitem>
+
+
+  <listitem><para><command>nix-prefetch-url</command> now has a
+  limited form of caching.  This is used by
+  <command>nix-channel</command> to prevent unnecessary downloads when
+  the channel hasn’t changed.</para></listitem>
   
 
 </itemizedlist>

doc/manual/writing-nix-expressions.xml

@@ -1217,6 +1217,10 @@ set, the attributes of which specify the inputs of the build.</para>
       They are simply concatenated, separated by
       spaces.</para></listitem>
 
+      <listitem><para><emphasis>true</emphasis> is passed as 
+      <emphasis>1</emphasis>, <emphasis>false</emphasis>
+      and <emphasis>null</emphasis> are passed as empty string.
+      </para></listitem>
     </itemizedlist>
 
   </para></listitem>

externals/Makefile.am

@@ -35,12 +35,12 @@ endif
 
 # CWI ATerm
 
-ATERM = aterm-2.4.2-fixes
+ATERM = aterm-2.4.2-fixes-r2
 
 $(ATERM).tar.bz2:
 	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.4.2-fixes from"
-	@echo "  http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes.tar.bz2"
+	@echo "Please download version 2.4.2-fixes-r2 from"
+	@echo "  http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes-r2.tar.bz2"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -67,12 +67,12 @@ endif
 
 # bzip2
 
-BZIP2 = bzip2-1.0.3
+BZIP2 = bzip2-1.0.4
 
 $(BZIP2).tar.gz:
 	@echo "Nix requires bzip2 to build."
-	@echo "Please download version 1.0.3 from"
-	@echo "  http://www.bzip.org/1.0.3/bzip2-1.0.3.tar.gz"
+	@echo "Please download version 1.0.4 from"
+	@echo "  http://www.bzip.org/1.0.4/bzip2-1.0.4.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 

scripts/download-using-manifests.pl.in

@@ -22,16 +22,6 @@ my $tmpNar2 = "$tmpDir/nar2";
 END { unlink $tmpNar; unlink $tmpNar2; rmdir $tmpDir; }
 
 
-# Check the arguments.
-die unless scalar @ARGV == 1;
-my $targetPath = $ARGV[0];
-
-my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
-print LOGFILE "$$ get $targetPath $date\n";
-
-print "\n*** Trying to download/patch `$targetPath'\n";
-
-
 # Load all manifests.
 my %narFiles;
 my %localPaths;
@@ -46,6 +36,54 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") {
 }
 
 
+# Parse the arguments.
+
+if ($ARGV[0] eq "--query-paths") {
+    foreach my $storePath (keys %narFiles) { print "$storePath\n"; }
+    foreach my $storePath (keys %localPaths) { print "$storePath\n"; }
+    exit 0;
+}
+
+elsif ($ARGV[0] eq "--query-info") {
+    shift @ARGV;
+    foreach my $storePath (@ARGV) {
+        my $info;
+        if (defined $narFiles{$storePath}) {
+            $info = @{$narFiles{$storePath}}[0];
+        }
+        elsif (defined $localPaths{$storePath}) {
+            $info = @{$localPaths{$storePath}}[0];
+        }
+        else {
+            next; # not an error
+        }
+        print "$storePath\n";
+        print "$info->{deriver}\n";
+        my @references = split " ", $info->{references};
+        my $count = scalar @references;
+        print "$count\n";
+        foreach my $reference (@references) {
+            print "$reference\n";
+        }
+    }
+    exit 0;
+}
+
+elsif ($ARGV[0] ne "--substitute") {
+    die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n";
+}
+
+
+die unless scalar @ARGV == 2;
+my $targetPath = $ARGV[1];
+
+
+my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
+print LOGFILE "$$ get $targetPath $date\n";
+
+print "\n*** Trying to download/patch `$targetPath'\n";
+
+
 # If we can copy from a local path, do that.
 my $localPathList = $localPaths{$targetPath};
 foreach my $localPath (@{$localPathList}) {

scripts/nix-build.in

@@ -122,7 +122,7 @@ foreach my $expr (@exprs) {
     close DRVPATHS or exit 1;
 
     foreach my $drvPath (@drvPaths) {
-        my $target = readlink $drvPath;
+        my $target = readlink $drvPath or die "cannot read symlink `$drvPath'";
         print STDERR "store derivation is $target\n";
     }
 
@@ -134,7 +134,7 @@ foreach my $expr (@exprs) {
     close OUTPATHS or exit 1;
 
     foreach my $outPath (@outPaths) {
-        my $target = readlink $outPath;
+        my $target = readlink $outPath or die "cannot read symlink `$outPath'";
         print "$target\n";
     }
 }

scripts/nix-channel.in

@@ -2,16 +2,24 @@
 
 use strict;
 
-my $rootsDir = "@localstatedir@/nix/gcroots/channels";
+my $rootsDir = "@localstatedir@/nix/gcroots";
 
 my $stateDir = $ENV{"NIX_STATE_DIR"};
 $stateDir = "@localstatedir@/nix" unless defined $stateDir;
 
 
+# Turn on caching in nix-prefetch-url.
+my $channelCache = "$stateDir/channel-cache";
+mkdir $channelCache, 0755 unless -e $channelCache;
+$ENV{'NIX_DOWNLOAD_CACHE'} = $channelCache if -W $channelCache;
+
+
 # Figure out the name of the `.nix-channels' file to use.
 my $home = $ENV{"HOME"};
 die '$HOME not set' unless defined $home;
 my $channelsList = "$home/.nix-channels";
+
+my $nixDefExpr = "$home/.nix-defexpr";
     
 
 my @channels;
@@ -70,50 +78,56 @@ sub removeChannel {
 sub update {
     readChannels;
 
-    # Get rid of all the old substitutes.
-    system("@bindir@/nix-store", "--clear-substitutes") == 0
-        or die "cannot clear substitutes";
+    # Do we have write permission to the manifests directory?  If not,
+    # then just skip pulling the manifest and just download the Nix
+    # expressions.  If the user is a non-privileged user in a
+    # multi-user Nix installation, he at least gets installation from
+    # source.
+    if (-W "$stateDir/manifests") {
 
-    # Remove all the old manifests.
-    for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
-        unlink $manifest or die "cannot remove `$manifest': $!";
-    }
+        # Remove all the old manifests.
+        for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
+            unlink $manifest or die "cannot remove `$manifest': $!";
+        }
+
+        # Pull cache manifests.
+        foreach my $url (@channels) {
+            #print "pulling cache manifest from `$url'\n";
+            system("@bindir@/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
+                or die "cannot pull cache manifest from `$url'";
+        }
 
-    # Pull cache manifests.
-    foreach my $url (@channels) {
-        #print "pulling cache manifest from `$url'\n";
-        system("@bindir@/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
-            or die "cannot pull cache manifest from `$url'";
     }
 
     # Create a Nix expression that fetches and unpacks the channel Nix
     # expressions.
 
-    my $nixExpr = "[";
+    my $inputs = "[";
     foreach my $url (@channels) {
+        $url =~ /\/([^\/]+)\/?$/;
+        my $channelName = $1;
+        $channelName = "unnamed" unless defined $channelName;
+
         my $fullURL = "$url/nixexprs.tar.bz2";
         print "downloading Nix expressions from `$fullURL'...\n";
         $ENV{"PRINT_PATH"} = 1;
-        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
+        $ENV{"QUIET"} = 1;
+        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL'`;
         die "cannot fetch `$fullURL'" if $? != 0;
         chomp $path;
-        $nixExpr .= $path . " ";
+        $inputs .= '"' . $channelName . '"' . " " . $path . " ";
     }
-    $nixExpr .= "]";
-
-    $nixExpr =
-        "(import @datadir@/nix/corepkgs/channels/unpack.nix) " .
-        "{inputs = $nixExpr; system = \"@system@\";}";
+    $inputs .= "]";
 
     # Figure out a name for the GC root.
     my $userName = getpwuid($<);
     die "who ARE you? go away" unless defined $userName;
 
-    my $rootFile = "$rootsDir/$userName";
+    my $rootFile = "$rootsDir/per-user/$userName/channels";
     
     # Instantiate the Nix expression.
     print "unpacking channel Nix expressions...\n";
-    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate --add-root '$rootFile'.tmp -`
+    my $storeExpr = `@bindir@/nix-instantiate --add-root '$rootFile'.tmp @datadir@/nix/corepkgs/channels/unpack.nix --argstr system @system@ --arg inputs '$inputs'`
         or die "cannot instantiate Nix expression";
     chomp $storeExpr;
 
@@ -124,9 +138,12 @@ sub update {
 
     unlink "$rootFile.tmp";
 
-    # Make it the default Nix expression for `nix-env'.
-    system("@bindir@/nix-env", "--import", "$outPath") == 0
-         or die "cannot pull set default Nix expression to `$outPath'";
+    # Make the channels appear in nix-env.
+    unlink $nixDefExpr if -l $nixDefExpr; # old-skool ~/.nix-defexpr
+    mkdir $nixDefExpr or die "cannot create directory `$nixDefExpr'" if !-e $nixDefExpr;
+    my $channelLink = "$nixDefExpr/channels";
+    unlink $channelLink; # !!! not atomic
+    symlink($outPath, $channelLink) or die "cannot symlink `$channelLink' to `$outPath'";
 }
 
 

scripts/nix-copy-closure.in

@@ -120,7 +120,6 @@ else { # Copy FROM the remote machine.
         if (!defined $storePathsSeen{$_}) {
             push @allStorePaths, $_;
             $storePathsSeen{$_} = 1;
-            print "GOT $_\n";
         }
     }
 

scripts/nix-install-package.in

@@ -13,7 +13,7 @@ downloading it from URL.
 
 Flags:
   --profile / -p LINK: install into the specified profile
-  --non-interactive: don't run inside a new terminal XXX
+  --non-interactive: don't run inside a new terminal
 EOF
     ; # '
     exit 1;

scripts/nix-pack-closure.in

@@ -17,9 +17,9 @@ $binDir = "@bindir@" unless defined $binDir;
 my $tmpDir = tempdir("nix-pack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-mkdir "$tmpDir/contents", 0777 or die;
-mkdir "$tmpDir/references", 0777 or die;
-mkdir "$tmpDir/derivers", 0777 or die;
+mkdir "$tmpDir/contents", 0755 or die;
+mkdir "$tmpDir/references", 0755 or die;
+mkdir "$tmpDir/derivers", 0755 or die;
 
 open TOPLEVEL, ">$tmpDir/top-level" or die;
 

scripts/nix-prefetch-url.in

@@ -36,30 +36,75 @@ if test -n "$expHash"; then
 fi
 
 
+doDownload() {
+    @curl@ $cacheFlags --fail -# --location --max-redirs 20 --disable-epsv \
+        --cookie-jar $tmpPath/cookies "$url" -o $tmpFile
+}
+
+
 # If we don't know the hash or a file with that hash doesn't exist,
 # download the file and add it to the store.
 if test -z "$finalPath"; then
 
     tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
     tmpFile=$tmpPath/$name
-    mkdir $tmpPath
+    mkdir $tmpPath # !!! retry if tmpPath already exists
+
+    # Optionally do timestamp-based caching of the download.
+    # Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is
+    # the hash and the timestamp of the file at $url.  The caching of
+    # the file *contents* is done in Nix store, where it can be
+    # garbage-collected independently.
+    if test -n "$NIX_DOWNLOAD_CACHE"; then
+        echo -n "$url" > $tmpPath/url
+        urlHash=$(nix-hash --type sha256 --base32 --flat $tmpPath/url)
+        echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url"
+        cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType"
+        cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp"
+        cacheFlags="--remote-time"
+        if test -e "$cachedTimestampFN" -a -e "$cachedHashFN"; then
+            # Only download the file if it is newer than the cached version.
+            cacheFlags="$cacheFlags --time-cond $cachedTimestampFN"
+        fi
+    fi
 
     # Perform the download.
-    @curl@ --fail --location --max-redirs 20 --disable-epsv \
-        --cookie-jar $tmpPath/cookies "$url" > $tmpFile
+    doDownload
 
-    # Compute the hash.
-    hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
-    if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
+    if test -n "$NIX_DOWNLOAD_CACHE" -a ! -e $tmpFile; then
+        # Curl didn't create $tmpFile, so apparently there's no newer
+        # file on the server.
+        hash=$(cat $cachedHashFN)
+        finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$hash" "$name") 
+        if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
+            echo "cached contents of \`$url' disappeared, redownloading..." >&2
+            finalPath=
+            cacheFlags="--remote-time"
+            doDownload
+        fi
+    fi
 
-    # Add the downloaded file to the Nix store.
-    finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+    if test -z "$finalPath"; then
 
-    if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+        # Compute the hash.
+        hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
+        if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
 
-    if test -n "$expHash" -a "$expHash" != "$hash"; then
-        echo "hash mismatch for URL \`$url'" >&2
-        exit 1
+        if test -n "$NIX_DOWNLOAD_CACHE"; then
+            echo $hash > $cachedHashFN
+            touch -r $tmpFile $cachedTimestampFN
+        fi
+
+        # Add the downloaded file to the Nix store.
+        finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+
+        if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+
+        if test -n "$expHash" -a "$expHash" != "$hash"; then
+            echo "hash mismatch for URL \`$url'" >&2
+            exit 1
+        fi
+        
     fi
 fi
 

scripts/nix-pull.in

@@ -7,8 +7,6 @@ use readmanifest;
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-my $manifest = "$tmpDir/manifest";
-
 my $binDir = $ENV{"NIX_BIN_DIR"};
 $binDir = "@bindir@" unless defined $binDir;
 
@@ -33,18 +31,47 @@ my %patches;
 
 my $skipWrongStore = 0;
 
+sub downloadFile {
+    my $url = shift;
+    $ENV{"PRINT_PATH"} = 1;
+    $ENV{"QUIET"} = 1;
+    my ($dummy, $path) = `$binDir/nix-prefetch-url '$url'`;
+    die "cannot fetch `$url'" if $? != 0;
+    die "nix-prefetch-url did not return a path" unless defined $path;
+    chomp $path;
+    return $path;
+}
+
 sub processURL {
     my $url = shift;
 
     $url =~ s/\/$//;
-    print "obtaining list of Nix archives at $url...\n";
 
-    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
-           "'$url' > '$manifest'") == 0
-           or die "curl failed: $?";
+    my $manifest;
 
+    # First see if a bzipped manifest is available.
+    if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) {
+        print "obtaining list of Nix archives at `$url.bz2'...\n";
+        my $bzipped = downloadFile "$url.bz2";
+
+        $manifest = "$tmpDir/MANIFEST";
+
+        system("@bunzip2@ < $bzipped > $manifest") == 0
+            or die "cannot decompress manifest";
+
+        $manifest = (`$binDir/nix-store --add $manifest`
+                     or die "cannot copy $manifest to the store");
+        chomp $manifest;
+    }
+
+    # Otherwise, just get the uncompressed manifest.
+    else {
+        print "obtaining list of Nix archives at `$url'...\n";
+        $manifest = downloadFile $url;
+    }
+    
     if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
-        die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
+        die "`$url' is not manifest or it is too old (i.e., for Nix <= 0.7)\n";
     }
 
     if ($skipWrongStore) {
@@ -67,8 +94,8 @@ sub processURL {
     
     my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
     
-    system ("@coreutils@/mv", "-f", "$manifest", "$finalPath") == 0
-        or die "cannot move `$manifest' to `$finalPath";
+    system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
+        or die "cannot link `$finalPath to `$manifest'";
 }
 
 while (@ARGV) {
@@ -83,41 +110,3 @@ while (@ARGV) {
 
 my $size = scalar (keys %narFiles) + scalar (keys %localPaths);
 print "$size store paths in manifest\n";
-
-
-# Register all substitutes.
-print STDERR "registering substitutes...\n";
-
-my $pid = open(WRITE, "|$binDir/nix-store --register-substitutes")
-    or die "cannot run nix-store";
-
-sub writeRegistration {
-    my $storePath = shift;
-    my $object = shift;
-    print WRITE "$storePath\n";
-    print WRITE "$object->{deriver}\n";
-    print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
-    print WRITE "0\n";
-    my @references = split " ", $object->{references};
-    my $count = scalar @references;
-    print WRITE "$count\n";
-    foreach my $reference (@references) {
-        print WRITE "$reference\n";
-    }
-}
-
-foreach my $storePath (keys %narFiles) {
-    my $narFileList = $narFiles{$storePath};
-    foreach my $narFile (@{$narFileList}) {
-        writeRegistration $storePath, $narFile;
-    }
-}
-
-foreach my $storePath (keys %localPaths) {
-    my $localPathList = $localPaths{$storePath};
-    foreach my $localPath (@{$localPathList}) {
-        writeRegistration $storePath, $localPath;
-    }
-}
-
-close WRITE or die "nix-store failed: $?";

scripts/nix-push.in

@@ -264,8 +264,12 @@ foreach my $narArchive (@narArchives) {
 print STDERR "uploading manifest...\n";
 if ($localCopy) {
     copyFile $manifest, $localManifestFile;
+    copyFile "$manifest.bz2", "$localManifestFile.bz2";
 } else {
-    system("$curl  --show-error --upload-file " .
+    system("$curl --show-error --upload-file " .
            "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
            die "curl failed on $manifest: $?";
+    system("$curl --show-error --upload-file " .
+           "'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or
+           die "curl failed on $manifest: $?";
 }

scripts/optimise-store.pl

@@ -1,91 +0,0 @@
-#! /usr/bin/perl -w
-
-use strict;
-use File::Basename;
-
-
-my @paths = ("/nix/store");
-
-
-print "hashing...\n";
-
-my $hashList = "/tmp/nix-optimise-hash-list";
-
-system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0
-    or die "cannot hash store files";
-
-
-print "sorting by hash...\n";
-
-system("sort $hashList > $hashList.sorted") == 0
-    or die "cannot sort list";
-
-
-sub atomicLink {
-    my $target = shift;
-    my $new = shift;
-    my $tmpNew = "${new}_optimise.$$";
-
-    # Make the directory writable temporarily.
-    my $dir = dirname $new;
-    my @st = stat $dir or die;
-
-    chmod ($st[2] | 0200, $dir) or die "cannot make `$dir' writable: $!";
-    
-    link $target, $tmpNew or die "cannot create hard link `$tmpNew': $!";
-
-    rename $tmpNew, $new or die "cannot rename `$tmpNew' to `$new': $!";
-
-    chmod ($st[2], $dir) or die "cannot restore permission on `$dir': $!";
-    utime ($st[8], $st[9], $dir) or die "cannot restore timestamp on `$dir': $!";
-}
-
-
-print "hard-linking...\n";
-
-open LIST, "<$hashList.sorted" or die;
-
-my $prevFile;
-my $prevHash;
-my $prevInode;
-my $prevExec;
-
-my $totalSpace = 0;
-my $savedSpace = 0;
-
-while (<LIST>) {
-    /^([0-9a-f]*)\s+(.*)$/ or die;
-    my $curFile = $2;
-    my $curHash = $1;
-
-    my @st = stat $curFile or die;
-    next if ($st[2] & 0222) != 0; # skip writable files
-
-    my $fileSize = $st[7];
-    $totalSpace += $fileSize;
-    my $isExec = ($st[2] & 0111) == 0111;
-
-    if (defined $prevHash && $curHash eq $prevHash
-        && $prevExec == $isExec)
-    {
-        
-        if ($st[1] != $prevInode) {
-            print "$curFile = $prevFile\n";
-            atomicLink $prevFile, $curFile;
-            $savedSpace += $fileSize;
-        }
-        
-    } else {
-        $prevFile = $curFile;
-        $prevHash = $curHash;
-        $prevInode = $st[1];
-        $prevExec = ($st[2] & 0111) == 0111;
-    }
-}
-
-print "total space = $totalSpace\n";
-print "saved space = $savedSpace\n";
-my $savings = ($savedSpace / $totalSpace) * 100.0;
-print "savings = $savings %\n";
-
-close LIST;

scripts/readmanifest.pm.in

@@ -222,6 +222,14 @@ sub writeManifest
 
     rename("$manifest.tmp", $manifest)
         or die "cannot rename $manifest.tmp: $!";
+
+
+    # Create a bzipped manifest.
+    system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0
+        or die "cannot compress manifest";
+
+    rename("$manifest.bz2.tmp", "$manifest.bz2")
+        or die "cannot rename $manifest.bz2.tmp: $!";
 }
 
 

src/libexpr/eval.cc

@@ -1,3 +1,5 @@
+#include <iostream>
+
 #include "eval.hh"
 #include "parser.hh"
 #include "hash.hh"
@@ -13,6 +15,19 @@
 
 
 namespace nix {
+
+
+int cacheTerms;
+
+bool shortCircuit;
+bool closedTerms; // don't substitute under terms known to be closed
+bool substCache; // memoization of the term substitution function
+bool posInfo; // attach position info to functions, assertions, attributes
+
+#define maxActiveCalls 4096
+
+ATerm activeCalls[maxActiveCalls];
+unsigned int activeCallsCount = 0;
     
 
 EvalState::EvalState()
@@ -23,6 +38,15 @@ EvalState::EvalState()
     initNixExprHelpers();
 
     addPrimOps();
+
+    if (!string2Int(getEnv("NIX_TERM_CACHE"), cacheTerms)) cacheTerms = 1;
+    shortCircuit = getEnv("NIX_SHORT_CIRCUIT", "0") == "1";
+    strictMode = getEnv("NIX_STRICT", "0") == "1";
+    closedTerms = getEnv("NIX_CLOSED_TERMS", "1") == "1";
+    substCache = getEnv("NIX_SUBST_CACHE", "1") == "1";
+    posInfo = getEnv("NIX_POS_INFO", "1") == "1";
+
+    ATprotectMemory(activeCalls, maxActiveCalls);
 }
 
 
@@ -69,6 +93,19 @@ LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2,
 }
 
 
+Expr speculativeEval(EvalState & state, Expr e)
+{
+    if (!state.strictMode) return e;
+    try {
+        return evalExpr(state, e);
+    } catch (EvalError & err) {
+        /* ignore, pass the original arg and depend on
+           laziness */
+        return e;
+    }
+}
+
+
 /* Substitute an argument set into the body of a function. */
 static Expr substArgs(EvalState & state,
     Expr body, ATermList formals, Expr arg)
@@ -80,7 +117,7 @@ static Expr substArgs(EvalState & state,
     ATermMap args;
     queryAllAttrs(arg, args);
     for (ATermMap::const_iterator i = args.begin(); i != args.end(); ++i)
-        subs.set(i->key, i->value);
+        subs.set(i->key, speculativeEval(state, i->value));
     
     /* Get the formal arguments. */
     ATermVector defsUsed;
@@ -389,7 +426,7 @@ Expr autoCallFunction(Expr e, const ATermMap & args)
             Expr name, def, value; ATerm values, def2;
             if (!matchFormal(*i, name, values, def2)) abort();
             if ((value = args.get(name)))
-                actualArgs.set(name, makeAttrRHS(value, makeNoPos()));
+                actualArgs.set(name, makeAttrRHS(allocCell(value), makeNoPos()));
             else if (!matchDefaultValue(def2, def))
                 throw TypeError(format("cannot auto-call a function that has an argument without a default value (`%1%')")
                     % aterm2String(name));
@@ -457,7 +494,7 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
     else if (matchFunction(fun, formals, body, pos)) {
         arg = evalExpr(state, arg);
         try {
-            return evalExpr(state, substArgs(state, body, formals, arg));
+            return evalExpr(state, substArgs(state, allocCells(body), formals, arg));
         } catch (Error & e) {
             addErrorPrefix(e, "while evaluating the function at %1%:\n",
                 showPos(pos));
@@ -467,9 +504,10 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
         
     else if (matchFunction1(fun, name, body, pos)) {
         try {
+            arg = speculativeEval(state, arg);
             ATermMap subs(1);
-            subs.set(name, arg);
-            return evalExpr(state, substitute(Substitution(0, &subs), body));
+            subs.set(name, allocCell(arg));
+            return evalExpr(state, substitute(Substitution(0, &subs), allocCells(body)));
         } catch (Error & e) {
             addErrorPrefix(e, "while evaluating the function at %1%:\n",
                 showPos(pos));
@@ -478,7 +516,7 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
     }
 
     else throwTypeError(
-        "the left-hand side of the function call is neither a function nor a primop (built-in operation) but %1%",
+        "attempt to call something which is neither a function nor a primop (built-in operation) but %1%",
         showType(fun));
 }
 
@@ -616,11 +654,18 @@ static char * deepestStack = (char *) -1; /* for measuring stack usage */
 
 Expr evalExpr2(EvalState & state, Expr e)
 {
+    /* When changing this function, make sure that you don't cause a
+       (large) increase in stack consumption! */
+    
     char x;
     if (&x < deepestStack) deepestStack = &x;
     
     Expr e1, e2, e3;
     ATerm name, pos;
+
+    int bla;
+    if (matchCell(e, bla, e1)) e = e1;
+
     AFun sym = ATgetAFun(e);
 
     /* Normal forms. */
@@ -712,41 +757,141 @@ Expr evalExpr2(EvalState & state, Expr e)
     if (matchOpConcat(e, e1, e2)) return evalOpConcat(state, e1, e2);
 
     /* Barf. */
+    //printMsg(lvlError, format("%1%") % e);
     abort();
 }
 
 
+class ShortCircuit
+{
+};
+
+
+unsigned int fnord;
+
+
+void maybeShortCircuit(EvalState & state, Expr e, Expr nf)
+{
+    for (unsigned int i = 0; i < activeCallsCount; ++i) {
+        Expr fun, arg;
+        if (!matchCall(activeCalls[i], fun, arg)) abort();
+        if (arg == e) {
+            //printMsg(lvlError, format("blaat"));
+            //printMsg(lvlError, format("blaat %1% %2% %3%") % fun % arg % e);
+            Expr res = state.normalForms.get(makeCall(fun, nf));
+            if (res) {
+                fnord++;
+                //printMsg(lvlError, format("blaat"));
+                throw ShortCircuit();
+            }
+        }
+    }
+}
+
+
 Expr evalExpr(EvalState & state, Expr e)
 {
     checkInterrupt();
 
-#if 0
+#if 1
     startNest(nest, lvlVomit,
         format("evaluating expression: %1%") % e);
 #endif
 
     state.nrEvaluated++;
 
+    if (cacheTerms == 0) return evalExpr2(state, e);
+
+    if (cacheTerms == 2) {
+        int pseudoAddr;
+        Expr e2;
+        if (!matchCell(e, pseudoAddr, e2)) return evalExpr2(state, e);
+    }
+    
     /* Consult the memo table to quickly get the normal form of
        previously evaluated expressions. */
     Expr nf = state.normalForms.get(e);
     if (nf) {
-        if (nf == makeBlackHole())
-            throwEvalError("infinite recursion encountered");
+        //if (nf == makeBlackHole())
+        //    throwEvalError("infinite recursion encountered");
         state.nrCached++;
         return nf;
     }
 
-    /* Otherwise, evaluate and memoize. */
-    state.normalForms.set(e, makeBlackHole());
-    try {
-        nf = evalExpr2(state, e);
-    } catch (Error & err) {
-        state.normalForms.remove(e);
-        throw;
+    Expr fun, arg;
+    if (shortCircuit && matchCall(e, fun, arg)) {
+
+#if 0        
+        Expr arg2 = state.normalForms.get(arg);
+        if (arg2) { /* the evaluated argument is now known */
+            //printMsg(lvlError, "foo");
+            /* do we know the result of the same function called
+               with the evaluated argument? */
+            Expr res = state.normalForms.get(makeCall(fun, arg2));
+            if (res) { /* woohoo! */
+                printMsg(lvlError, "dingdong");
+                state.normalForms.set(e, res);
+                return res;
+            }
+        }
+#endif
+
+        assert(activeCallsCount < maxActiveCalls);
+        activeCalls[activeCallsCount++] = e;
+
+        //state.normalForms.set(e, makeBlackHole());
+        try {
+            nf = evalExpr2(state, e);
+        }
+        catch (ShortCircuit & exception) {
+            //printMsg(lvlError, "catch!");
+            Expr arg2 = state.normalForms.get(arg);
+            if (arg2) { /* the evaluated argument is now known */
+                /* do we know the result of the same function called
+                   with the evaluated argument? */
+                Expr res = state.normalForms.get(makeCall(fun, arg2));
+                if (res) { /* woohoo! */
+                    //printMsg(lvlError, "woohoo!");
+                    //printMsg(lvlError, format("woohoo! %1% %2% %3% %4%") % fun % arg % arg2 % res);
+                    activeCallsCount--;
+                    state.normalForms.set(e, res);
+                    maybeShortCircuit(state, e, res);
+                    return res;
+                }
+            }
+            activeCallsCount--;
+            state.normalForms.remove(e);
+            throw; /* not for us */
+        }
+        catch (...) {
+            activeCallsCount--;
+            state.normalForms.remove(e);
+            throw;
+        }
+        activeCallsCount--;
+        state.normalForms.set(e, nf);
+        Expr arg2 = state.normalForms.get(arg);
+        if (arg2) state.normalForms.set(makeCall(fun, arg2), nf);
+        maybeShortCircuit(state, e, nf);
+        return nf;
+        
+    }
+
+    else {
+        
+        /* Otherwise, evaluate and memoize. */
+        //state.normalForms.set(e, makeBlackHole());
+        try {
+            nf = evalExpr2(state, e);
+        } catch (...) {
+            state.normalForms.remove(e);
+            throw;
+        }
+        state.normalForms.set(e, nf);
+        if (shortCircuit) maybeShortCircuit(state, e, nf);
+        return nf;
+
     }
-    state.normalForms.set(e, nf);
-    return nf;
 }
 
 
@@ -842,16 +987,24 @@ extern "C" {
     unsigned long AT_calcAllocatedSize();
 }
 
+
+unsigned int substs = 0;
+unsigned int substsCached = 0;
+
+
 void printEvalStats(EvalState & state)
 {
     char x;
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
+    printMsg(lvlError, format("FNORD %1%") % fnord);
     printMsg(showStats ? lvlInfo : lvlDebug,
-        format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space")
+        format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space, %6% substitutions (%7% cached)")
         % state.nrEvaluated % state.nrCached
         % ((float) state.nrCached / (float) state.nrEvaluated * 100)
         % AT_calcAllocatedSize()
-        % (&x - deepestStack));
+        % (&x - deepestStack)
+        % substs
+        % substsCached);
     if (showStats)
         printATermMapStats();
 }

src/libexpr/eval.hh

@@ -27,6 +27,9 @@ struct EvalState;
 typedef Expr (* PrimOp) (EvalState &, const ATermVector & args);
 
 
+extern int cacheTerms; // 0 = don't, 1 = do, 2 = "cell" terms only
+
+
 struct EvalState 
 {
     ATermMap normalForms;
@@ -38,6 +41,10 @@ struct EvalState
     unsigned int nrEvaluated;
     unsigned int nrCached;
 
+    bool strictMode;
+
+    ATermMap parsings; /* path -> expr mapping */
+
     EvalState();
 
     void addPrimOps();

src/libexpr/get-drvs.cc

@@ -60,6 +60,15 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
 }
 
 
+string DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
+{
+    /* !!! evaluates all meta attributes => inefficient */
+    MetaInfo meta = queryMetaInfo(state);
+    MetaInfo::iterator i = meta.find(name);
+    return i == meta.end() ? "" : i->second;
+}
+
+
 void DrvInfo::setMetaInfo(const MetaInfo & meta)
 {
     ATermMap metaAttrs;
@@ -161,12 +170,21 @@ static void getDerivations(EvalState & state, Expr e,
     if (matchAttrs(e, es)) {
         ATermMap drvMap(ATgetLength(es));
         queryAllAttrs(e, drvMap);
+
+        /* !!! undocumented hackery to support combining channels in
+           nix-env.cc. */
+        Expr e2 = drvMap.get(toATerm("_combineChannels"));
+        bool combineChannels = e2 && evalBool(state, e2);
         
         for (ATermMap::const_iterator i = drvMap.begin(); i != drvMap.end(); ++i) {
             startNest(nest, lvlDebug,
                 format("evaluating attribute `%1%'") % aterm2String(i->key));
             string pathPrefix2 = addToPath(pathPrefix, aterm2String(i->key));
-            if (getDerivation(state, i->value, pathPrefix2, drvs, doneExprs)) {
+            if (combineChannels) {
+                if (((string) aterm2String(i->key)) != "_combineChannels")
+                    getDerivations(state, i->value, pathPrefix2, autoArgs, drvs, doneExprs);
+            }
+            else if (getDerivation(state, i->value, pathPrefix2, drvs, doneExprs)) {
                 /* If the value of this attribute is itself an
                    attribute set, should we recurse into it?  => Only
                    if it has a `recurseForDerivations = true'
@@ -176,8 +194,8 @@ static void getDerivations(EvalState & state, Expr e,
                 if (matchAttrs(e, es)) {
                     ATermMap attrs(ATgetLength(es));
                     queryAllAttrs(e, attrs, false);
-                    Expr e2 = attrs.get(toATerm("recurseForDerivations"));
-                    if (e2 && evalBool(state, e2))
+                    if (((e2 = attrs.get(toATerm("recurseForDerivations")))
+                            && evalBool(state, e2)))
                         getDerivations(state, e, pathPrefix2, autoArgs, drvs, doneExprs);
                 }
             }

src/libexpr/get-drvs.hh

@@ -34,6 +34,7 @@ public:
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
     MetaInfo queryMetaInfo(EvalState & state) const;
+    string queryMetaInfo(EvalState & state, const string & name) const;
 
     void setDrvPath(const string & s)
     {

src/libexpr/nixexpr-ast.def

@@ -73,6 +73,8 @@ Inherit | Expr ATermList Pos | ATerm |
 
 Scope | | Expr |
 
+Cell | int Expr | Expr |
+
 Formal | string ValidValues DefaultValue | ATerm |
 
 ValidValues | ATermList | ValidValues |

src/libexpr/nixexpr.cc

@@ -2,6 +2,7 @@
 #include "derivations.hh"
 #include "util.hh"
 #include "aterm.hh"
+#include "eval.hh" // !!! urgh
 
 #include "nixexpr-ast.hh"
 #include "nixexpr-ast.cc"
@@ -108,7 +109,16 @@ Expr makeAttrs(const ATermMap & attrs)
 }
 
 
-Expr substitute(const Substitution & subs, Expr e)
+extern unsigned int substs;
+extern unsigned int substsCached;
+extern bool closedTerms;
+extern bool substCache;
+
+
+static Expr substitute(ATermMap & done, const Substitution & subs, Expr e);
+
+
+static Expr substitute2(ATermMap & done, const Substitution & subs, Expr e)
 {
     checkInterrupt();
 
@@ -116,19 +126,20 @@ Expr substitute(const Substitution & subs, Expr e)
 
     ATerm name, pos, e2;
 
+    substs++;
+
     /* As an optimisation, don't substitute in subterms known to be
        closed. */
-    if (matchClosed(e, e2)) return e;
+    if (closedTerms && matchClosed(e, e2)) return e;
 
     if (matchVar(e, name)) {
         Expr sub = subs.lookup(name);
-        if (sub == makeRemoved()) sub = 0;
         Expr wrapped;
         /* Add a "closed" wrapper around terms that aren't already
            closed.  The check is necessary to prevent repeated
            wrapping, e.g., closed(closed(closed(...))), which kills
            caching. */
-        return sub ? (matchClosed(sub, wrapped) ? sub : makeClosed(sub)) : e;
+        return sub ? ((!closedTerms || matchClosed(sub, wrapped)) ? sub : makeClosed(sub)) : e;
     }
 
     /* In case of a function, filter out all variables bound by this
@@ -140,18 +151,30 @@ Expr substitute(const Substitution & subs, Expr e)
         for (ATermIterator i(formals); i; ++i) {
             ATerm d1, d2;
             if (!matchFormal(*i, name, d1, d2)) abort();
-            map.set(name, makeRemoved());
+            if (subs.lookup(name))
+                map.set(name, constRemoved);
+        }
+        if (map.size() == 0)
+            return makeFunction(
+                (ATermList) substitute(done, subs, (ATerm) formals),
+                substitute(done, subs, body), pos);
+        else {
+            Substitution subs2(&subs, &map);
+            ATermMap done2(128);
+            return makeFunction(
+                (ATermList) substitute(done2, subs2, (ATerm) formals),
+                substitute(done2, subs2, body), pos);
         }
-        Substitution subs2(&subs, &map);
-        return makeFunction(
-            (ATermList) substitute(subs2, (ATerm) formals),
-            substitute(subs2, body), pos);
     }
 
     if (matchFunction1(e, name, body, pos)) {
-        ATermMap map(1);
-        map.set(name, makeRemoved());
-        return makeFunction1(name, substitute(Substitution(&subs, &map), body), pos);
+        if (subs.lookup(name)) {
+            ATermMap map(1);
+            map.set(name, constRemoved);
+            ATermMap done2(128);
+            return makeFunction1(name, substitute(done2, Substitution(&subs, &map), body), pos);
+        } else
+            return makeFunction1(name, substitute(done, subs, body), pos);
     }
         
     /* Idem for a mutually recursive attribute set. */
@@ -159,14 +182,21 @@ Expr substitute(const Substitution & subs, Expr e)
     if (matchRec(e, rbnds, nrbnds)) {
         ATermMap map(ATgetLength(rbnds) + ATgetLength(nrbnds));
         for (ATermIterator i(rbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
+            if (matchBind(*i, name, e2, pos) && subs.lookup(name))
+                map.set(name, constRemoved);
         for (ATermIterator i(nrbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
-        return makeRec(
-            (ATermList) substitute(Substitution(&subs, &map), (ATerm) rbnds),
-            (ATermList) substitute(subs, (ATerm) nrbnds));
+            if (matchBind(*i, name, e2, pos) && subs.lookup(name))
+                map.set(name, constRemoved);
+        if (map.size() == 0)
+            return makeRec(
+                (ATermList) substitute(done, subs, (ATerm) rbnds),
+                (ATermList) substitute(done, subs, (ATerm) nrbnds));
+        else {
+            ATermMap done2(128);
+            return makeRec(
+                (ATermList) substitute(done2, Substitution(&subs, &map), (ATerm) rbnds),
+                (ATermList) substitute(done, subs, (ATerm) nrbnds));
+        }
     }
 
     if (ATgetType(e) == AT_APPL) {
@@ -177,7 +207,73 @@ Expr substitute(const Substitution & subs, Expr e)
 
         for (int i = 0; i < arity; ++i) {
             ATerm arg = ATgetArgument(e, i);
-            args[i] = substitute(subs, arg);
+            args[i] = substitute(done, subs, arg);
+            if (args[i] != arg) changed = true;
+        }
+        
+        return changed ? (ATerm) ATmakeApplArray(fun, args) : e;
+    }
+
+    if (ATgetType(e) == AT_LIST) {
+        unsigned int len = ATgetLength((ATermList) e);
+        ATerm es[len];
+        ATermIterator i((ATermList) e);
+        bool changed = false;
+        for (unsigned int j = 0; i; ++i, ++j) {
+            es[j] = substitute(done, subs, *i);
+            if (es[j] != *i) changed = true;
+        }
+        if (!changed) return e;
+        ATermList out = ATempty;
+        for (unsigned int j = len; j; --j)
+            out = ATinsert(out, es[j - 1]);
+        return (ATerm) out;
+    }
+
+    return e;
+}
+
+
+static Expr substitute(ATermMap & done, const Substitution & subs, Expr e)
+{
+    Expr res = done[e];
+    if (substCache && res) {
+        substsCached++;
+        return res;
+    }
+    res = substitute2(done, subs, e);
+    done.set(e, res);
+    return res;
+}
+
+
+Expr substitute(const Substitution & subs, Expr e)
+{
+    ATermMap done(256);
+    return substitute(done, subs, e);
+}
+
+
+Expr allocCells(Expr e)
+{
+    checkInterrupt();
+
+    ATerm e2;
+    if (matchClosed(e, e2)) return e;
+
+    int i;
+    if (matchCell(e, i, e2))
+        return allocCell(allocCells(e2));
+
+    if (ATgetType(e) == AT_APPL) {
+        AFun fun = ATgetAFun(e);
+        int arity = ATgetArity(fun);
+        ATerm args[arity];
+        bool changed = false;
+
+        for (int i = 0; i < arity; ++i) {
+            ATerm arg = ATgetArgument(e, i);
+            args[i] = allocCells(arg);
             if (args[i] != arg) changed = true;
         }
         
@@ -189,7 +285,7 @@ Expr substitute(const Substitution & subs, Expr e)
         ATerm es[len];
         ATermIterator i((ATermList) e);
         for (unsigned int j = 0; i; ++i, ++j)
-            es[j] = substitute(subs, *i);
+            es[j] = allocCells(*i);
         ATermList out = ATempty;
         for (unsigned int j = len; j; --j)
             out = ATinsert(out, es[j - 1]);
@@ -399,5 +495,18 @@ string showValue(Expr e)
     return "<unknown>";
 }
 
+
+static unsigned int cellCount = 0;
+
+
+Expr allocCell(Expr e)
+{
+    if (cacheTerms != 2) return e;
+    int i;
+    Expr e2;
+    if (matchCell(e, i, e2)) return e;
+    return makeCell(cellCount++, e);
+}
+
  
 }

src/libexpr/nixexpr.hh

@@ -12,6 +12,7 @@ namespace nix {
 
 MakeError(EvalError, Error)
 MakeError(AssertionError, EvalError)
+MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
 MakeError(TypeError, EvalError)
 
@@ -33,6 +34,9 @@ typedef ATerm Pos;
 typedef vector<ATerm> ATermVector;
 
 
+extern Expr constRemoved;
+
+
 /* A substitution is a linked list of ATermMaps that map names to
    identifiers.  We use a list of ATermMaps rather than a single to
    make it easy to grow or shrink a substitution when entering a
@@ -52,7 +56,8 @@ struct Substitution
     {
         Expr x;
         for (const Substitution * s(this); s; s = s->prev)
-            if ((x = s->map->get(name))) return x;
+            if ((x = s->map->get(name)))
+                return x == constRemoved ? 0 : x;
         return 0;
     }
 };
@@ -115,6 +120,11 @@ string showType(Expr e);
 
 string showValue(Expr e);
 
+
+Expr allocCell(Expr e); // make an updateable cell (for simulating conventional laziness)
+
+Expr allocCells(Expr e); // re-allocate all cells in e
+
  
 }
 

src/libexpr/parser.y

@@ -57,7 +57,7 @@ static Expr fixAttrs(int recursive, ATermList as)
             bool fromScope = matchScope(src);
             for (ATermIterator j(names); j; ++j) {
                 Expr rhs = fromScope ? makeVar(*j) : makeSelect(src, *j);
-                *is = ATinsert(*is, makeBind(*j, rhs, pos));
+                *is = ATinsert(*is, makeBind(*j, allocCell(rhs), pos));
             }
         } else bs = ATinsert(bs, *i);
     }
@@ -70,11 +70,13 @@ static Expr fixAttrs(int recursive, ATermList as)
 
 void backToString(yyscan_t scanner);
 
+
+extern bool posInfo;
  
 static Pos makeCurPos(YYLTYPE * loc, ParseData * data)
 {
-    return makePos(toATerm(data->path),
-        loc->first_line, loc->first_column);
+    return posInfo ? makePos(toATerm(data->path),
+        loc->first_line, loc->first_column) : makeNoPos();
 }
 
 #define CUR_POS makeCurPos(yylocp, data)
@@ -226,7 +228,7 @@ binds
 
 bind
   : ID '=' expr ';'
-    { $$ = makeBind($1, $3, CUR_POS); }
+    { $$ = makeBind($1, allocCell($3), CUR_POS); }
   | INHERIT inheritsrc ids ';'
     { $$ = makeInherit($2, $3, CUR_POS); }
   ;
@@ -249,6 +251,7 @@ expr_list
 formals
   : formal ',' formals { $$ = ATinsert($3, $1); } /* idem - right recursive */
   | formal { $$ = ATinsert(ATempty, $1); }
+  | { $$ = ATempty; }
   ;
 
 formal
@@ -382,20 +385,13 @@ Expr parseExprFromFile(EvalState & state, Path path)
     if (S_ISDIR(st.st_mode))
         path = canonPath(path + "/default.nix");
 
-    /* Read the input file.  We can't use SGparseFile() because it's
-       broken, so we read the input ourselves and call
-       SGparseString(). */
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY);
-    if (fd == -1) throw SysError(format("opening `%1%'") % path);
+    Expr cached = state.parsings.get(toATerm(path));
+    if (cached) return cached;
 
-    if (fstat(fd, &st) == -1)
-        throw SysError(format("statting `%1%'") % path);
-
-    char text[st.st_size + 1];
-    readFull(fd, (unsigned char *) text, st.st_size);
-    text[st.st_size] = 0;
-
-    return parse(state, text, path, dirOf(path));
+    /* Read and parse the input file. */
+    cached = parse(state, readFile(path).c_str(), path, dirOf(path));
+    state.parsings.set(toATerm(path), cached);
+    return cached;
 }
 
 

src/libexpr/primops.cc

@@ -114,6 +114,18 @@ static Expr prim_isNull(EvalState & state, const ATermVector & args)
 }
 
 
+/* Determine whether the argument is a function. */
+static Expr prim_isFunction(EvalState & state, const ATermVector & args)
+{
+    Expr e = evalExpr(state, args[0]);
+    ATermList formals;
+    ATerm name, body, pos;
+    return makeBool(
+        matchFunction(e, formals, body, pos) ||
+        matchFunction1(e, name, body, pos));
+}
+
+
 static Path findDependency(Path dir, string dep)
 {
     if (dep[0] == '/') throw EvalError(
@@ -269,6 +281,14 @@ static Expr prim_abort(EvalState & state, const ATermVector & args)
 }
 
 
+static Expr prim_throw(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    throw ThrownError(format("user-thrown exception: `%1%'") %
+        evalString(state, args[0], context));
+}
+
+
 /* Return an environment variable.  Use with care. */
 static Expr prim_getEnv(EvalState & state, const ATermVector & args)
 {
@@ -276,6 +296,17 @@ static Expr prim_getEnv(EvalState & state, const ATermVector & args)
     return makeStr(getEnv(name));
 }
 
+/* for debugging purposes. print the first arg on stdout (perhaps stderr should be used?)
+ * and return the second
+ */
+static Expr prim_trace(EvalState & state, const ATermVector & args)
+{
+    //string str = evalStringNoCtx(state, args[0]);
+
+    Expr a = evalExpr(state, args[0]);
+    printf("traced value: %s\n", atPrint(a).c_str());
+    return evalExpr(state, args[1]);
+}
 
 static Expr prim_relativise(EvalState & state, const ATermVector & args)
 {
@@ -533,7 +564,7 @@ static Expr prim_derivationLazy(EvalState & state, const ATermVector & args)
     attrs.set(toATerm("type"),
         makeAttrRHS(makeStr("derivation"), makeNoPos()));
 
-    Expr drvStrict = makeCall(makeVar(toATerm("derivation!")), eAttrs);
+    Expr drvStrict = allocCell(makeCall(makeVar(toATerm("derivation!")), eAttrs));
 
     attrs.set(toATerm("outPath"),
         makeAttrRHS(makeSelect(drvStrict, toATerm("outPath")), makeNoPos()));
@@ -724,6 +755,39 @@ static Expr prim_hasAttr(EvalState & state, const ATermVector & args)
 }
 
 
+/* Builds an attribute set from a list specifying (name, value)
+   pairs.  To be precise, a list [{name = "name1"; value = value1;}
+   ... {name = "nameN"; value = valueN;}] is transformed to {name1 =
+   value1; ... nameN = valueN;}. */
+static Expr prim_listToAttrs(EvalState & state, const ATermVector & args)
+{
+    try {
+        ATermMap res = ATermMap();
+        ATermList list;
+        list = evalList(state, args[0]);
+        for (ATermIterator i(list); i; ++i){
+            // *i should now contain a pointer to the list item expression
+            ATermList attrs;
+            Expr evaledExpr = evalExpr(state, *i);
+            if (matchAttrs(evaledExpr, attrs)){
+                Expr e = evalExpr(state, makeSelect(evaledExpr, toATerm("name")));
+                string attr = evalStringNoCtx(state,e);
+                Expr r = makeSelect(evaledExpr, toATerm("value"));
+                res.set(toATerm(attr), makeAttrRHS(allocCell(r), makeNoPos()));
+            }
+            else
+                throw TypeError(format("list element in `listToAttrs' is %s, expected a set { name = \"<name>\"; value = <value>; }")
+                    % showType(evaledExpr));
+        }
+    
+        return makeAttrs(res);
+    
+    } catch (Error & e) {
+        e.addPrefix(format("in `listToAttrs':\n"));
+        throw;
+    }
+}
+
 static Expr prim_removeAttrs(EvalState & state, const ATermVector & args)
 {
     ATermMap attrs;
@@ -738,6 +802,12 @@ static Expr prim_removeAttrs(EvalState & state, const ATermVector & args)
     return makeAttrs(attrs);
 }
 
+/* Determine whether the argument is a list. */
+static Expr prim_isAttrs(EvalState & state, const ATermVector & args)
+{
+    ATermList list;
+    return makeBool(matchAttrs(evalExpr(state, args[0]), list));
+}
 
 /*************************************************************
  * Lists
@@ -857,6 +927,17 @@ static Expr prim_stringLength(EvalState & state, const ATermVector & args)
 }
 
 
+/*************************************************************
+ * Strictness
+ *************************************************************/
+
+
+static Expr prim_strict(EvalState & state, const ATermVector & args)
+{
+    return evalExpr(state, makeCall(args[0], evalExpr(state, args[1])));
+}
+
+
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -876,9 +957,12 @@ void EvalState::addPrimOps()
     // Miscellaneous
     addPrimOp("import", 1, prim_import);
     addPrimOp("isNull", 1, prim_isNull);
+    addPrimOp("__isFunction", 1, prim_isFunction);
     addPrimOp("dependencyClosure", 1, prim_dependencyClosure);
     addPrimOp("abort", 1, prim_abort);
+    addPrimOp("throw", 1, prim_throw);
     addPrimOp("__getEnv", 1, prim_getEnv);
+    addPrimOp("__trace", 2, prim_trace);
 
     addPrimOp("relativise", 2, prim_relativise);
 
@@ -901,7 +985,9 @@ void EvalState::addPrimOps()
     addPrimOp("__attrNames", 1, prim_attrNames);
     addPrimOp("__getAttr", 2, prim_getAttr);
     addPrimOp("__hasAttr", 2, prim_hasAttr);
+    addPrimOp("__isAttrs", 1, prim_isAttrs);
     addPrimOp("removeAttrs", 2, prim_removeAttrs);
+    addPrimOp("__listToAttrs", 1, prim_listToAttrs);
 
     // Lists
     addPrimOp("__isList", 1, prim_isList);
@@ -918,6 +1004,9 @@ void EvalState::addPrimOps()
     addPrimOp("toString", 1, prim_toString);
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
+
+    // Strictness
+    addPrimOp("strict", 2, prim_strict);
 }
 
 

src/libmain/shared.cc

@@ -7,6 +7,7 @@
 
 #include <iostream>
 #include <cctype>
+#include <exception>
 
 #include <sys/stat.h>
 #include <unistd.h>
@@ -81,6 +82,23 @@ struct RemoveTempRoots
 void initDerivationsHelpers();
 
 
+static void closeStore()
+{
+    try {
+        throw;
+    } catch (std::exception & e) {
+        printMsg(lvlError,
+            format("FATAL: unexpected exception (closing store and aborting): %1%") % e.what());
+    }
+    try {
+        store.reset((StoreAPI *) 0);
+    } catch (...) {
+        ignoreException();
+    }
+    abort();
+}
+
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
@@ -95,6 +113,12 @@ static void initAndRun(int argc, char * * argv)
     nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
     nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
 
+    string subs = getEnv("NIX_SUBSTITUTERS", "default");
+    if (subs == "default")
+        substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl");
+    else
+        substituters = tokenizeString(subs, ":");
+
     /* Get some settings from the configuration file. */
     thisSystem = querySetting("system", SYSTEM);
     maxBuildJobs = queryIntSetting("build-max-jobs", 1);
@@ -195,6 +219,12 @@ static void initAndRun(int argc, char * * argv)
        exit. */
     RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
 
+    /* Make sure that the database gets closed properly, even if
+       terminate() is called (which happens sometimes due to bugs in
+       destructor/exceptions interaction, but that needn't preclude a
+       clean shutdown of the database). */
+    std::set_terminate(closeStore);
+    
     run(remaining);
 
     /* Close the Nix database. */
@@ -296,7 +326,7 @@ int main(int argc, char * * argv)
                 "Try `%2% --help' for more information.")
             % e.what() % programId);
         return 1;
-    } catch (Error & e) {
+    } catch (BaseError & e) {
         printMsg(lvlError, format("error: %1%") % e.msg());
         return 1;
     } catch (std::exception & e) {

src/libstore/build.cc

@@ -164,6 +164,11 @@ private:
     /* Goals waiting for a build slot. */
     WeakGoals wantingToBuild;
 
+    /* Goals waiting for info from substituters (using --query-info),
+       and the info they're (collectively) waiting for. */
+    WeakGoals waitingForInfo;
+    map<Path, PathSet> requestedInfo;
+
     /* Child processes currently running. */
     Children children;
 
@@ -176,6 +181,9 @@ private:
     WeakGoalMap derivationGoals;
     WeakGoalMap substitutionGoals;
 
+    /* Goals waiting for busy paths to be unlocked. */
+    WeakGoals waitingForAnyGoal;
+    
 public:
 
     Worker();
@@ -212,12 +220,28 @@ public:
     /* Put `goal' to sleep until a child process terminates, i.e., a
        call is made to childTerminate(..., true).  */
     void waitForChildTermination(GoalPtr goal);
+
+    /* Put `goal' to sleep until the top-level loop has run `sub' to
+       get info about `storePath' (with --query-info).  We combine
+       substituter invocations to reduce overhead. */
+    void waitForInfo(GoalPtr goal, Path sub, Path storePath);
+
+    /* Wait for any goal to finish.  Pretty indiscriminate way to
+       wait for some resource that some other goal is holding. */
+    void waitForAnyGoal(GoalPtr goal);
     
     /* Loop until the specified top-level goals have finished. */
     void run(const Goals & topGoals);
 
     /* Wait for input to become available. */
     void waitForInput();
+
+private:
+
+    /* Process the pending paths in requestedInfo and wake up the
+       goals in waitingForInfo. */
+    void getInfo();
+    
 };
 
 
@@ -625,7 +649,7 @@ private:
     void buildDone();
 
     /* Is the build hook willing to perform the build? */
-    typedef enum {rpAccept, rpDecline, rpPostpone, rpDone} HookReply;
+    typedef enum {rpAccept, rpDecline, rpPostpone, rpDone, rpRestart} HookReply;
     HookReply tryBuildHook();
 
     /* Synchronously wait for a build hook to finish. */
@@ -634,9 +658,13 @@ private:
     /* Acquires locks on the output paths and gathers information
        about the build (e.g., the input closures).  During this
        process its possible that we find out that the build is
-       unnecessary, in which case we return false (this is not an
-       error condition!). */
-    bool prepareBuild();
+       unnecessary, in which case we return prDone.  It's also
+       possible that some other goal is already building/substituting
+       the output paths, in which case we return prRestart (go back to
+       the haveDerivation() state).  Otherwise, prProceed is
+       returned. */
+    typedef enum {prProceed, prDone, prRestart} PrepareBuildReply;
+    PrepareBuildReply prepareBuild();
 
     /* Start building a derivation. */
     void startBuilder();
@@ -685,8 +713,8 @@ DerivationGoal::~DerivationGoal()
     try {
         killChild();
         deleteTmpDir(false);
-    } catch (Error & e) {
-        printMsg(lvlError, format("error (ignored): %1%") % e.msg());
+    } catch (...) {
+        ignoreException();
     }
 }
 
@@ -774,6 +802,20 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
+    /* If this is a fixed-output derivation, it is possible that some
+       other goal is already building the output paths.  (The case
+       where some other process is building it is handled through
+       normal locking mechanisms.)  So if any output paths are already
+       being built, put this goal to sleep. */
+    for (PathSet::iterator i = invalidOutputs.begin();
+         i != invalidOutputs.end(); ++i)
+        if (pathIsLockedByMe(*i)) {
+            /* Wait until any goal finishes (hopefully the one that is
+               locking *i), then retry haveDerivation(). */
+            worker.waitForAnyGoal(shared_from_this());
+            return;
+        }
+
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -783,7 +825,7 @@ void DerivationGoal::haveDerivation()
            substitutes. */
         if (store->hasSubstitutes(*i))
             addWaitee(worker.makeSubstitutionGoal(*i));
-
+    
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         outputsSubstituted();
     else
@@ -867,6 +909,12 @@ void DerivationGoal::tryToBuild()
                 /* Somebody else did it. */
                 amDone(ecSuccess);
                 return;
+            case rpRestart:
+                /* Somebody else is building this output path.
+                   Restart from haveDerivation(). */
+                state = &DerivationGoal::haveDerivation;
+                worker.waitForAnyGoal(shared_from_this());
+                return;
         }
 
         /* Make sure that we are allowed to start a build. */
@@ -877,9 +925,14 @@ void DerivationGoal::tryToBuild()
 
         /* Acquire locks and such.  If we then see that the build has
            been done by somebody else, we're done. */
-        if (!prepareBuild()) {
+        PrepareBuildReply preply = prepareBuild();
+        if (preply == prDone) {
             amDone(ecSuccess);
             return;
+        } else if (preply == prRestart) {
+            state = &DerivationGoal::haveDerivation;
+            worker.waitForAnyGoal(shared_from_this());
+            return;
         }
 
         /* Okay, we have to build. */
@@ -1065,7 +1118,7 @@ static string makeValidityRegistration(const PathSet & paths,
     for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) {
         s += *i + "\n";
 
-        Path deriver = showDerivers ? queryDeriver(noTxn, *i) : "";
+        Path deriver = showDerivers ? store->queryDeriver(*i) : "";
         s += deriver + "\n";
 
         PathSet references;
@@ -1159,11 +1212,12 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
 
         /* Acquire locks and such.  If we then see that the output
            paths are now valid, we're done. */
-        if (!prepareBuild()) {
+        PrepareBuildReply preply = prepareBuild();
+        if (preply == prDone || preply == prRestart) {
             /* Tell the hook to exit. */
             writeLine(toHook.writeSide, "cancel");
             terminateBuildHook();
-            return rpDone;
+            return preply == prDone ? rpDone : rpRestart;
         }
 
         printMsg(lvlInfo, format("running hook to build path(s) %1%")
@@ -1239,11 +1293,27 @@ void DerivationGoal::terminateBuildHook(bool kill)
 }
 
 
-bool DerivationGoal::prepareBuild()
+DerivationGoal::PrepareBuildReply DerivationGoal::prepareBuild()
 {
+    /* Check for the possibility that some other goal in this process
+       has locked the output since we checked in haveDerivation().
+       (It can't happen between here and the lockPaths() call below
+       because we're not allowing multi-threading.) */
+    for (DerivationOutputs::iterator i = drv.outputs.begin(); 
+         i != drv.outputs.end(); ++i)
+        if (pathIsLockedByMe(i->second.path)) {
+            debug(format("restarting derivation `%1%' because `%2%' is locked by another goal")
+                % drvPath % i->second.path);
+            return prRestart;
+        }
+    
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes. */
     /* !!! BUG: this could block, which is not allowed. */
+    /* !!! and once we make this non-blocking, we should carefully
+       consider the case where some but not all locks are required; we
+       should then release the acquired locks so that the other
+       processes and the pathIsLockedByMe() test don't get confused. */
     outputLocks.lockPaths(outputPaths(drv.outputs),
         (format("waiting for lock on %1%") % showPaths(outputPaths(drv.outputs))).str());
 
@@ -1259,7 +1329,7 @@ bool DerivationGoal::prepareBuild()
         debug(format("skipping build of derivation `%1%', someone beat us to it")
             % drvPath);
         outputLocks.setDeletion(true);
-        return false;
+        return prDone;
     }
 
     if (validPaths.size() > 0) {
@@ -1324,7 +1394,7 @@ bool DerivationGoal::prepareBuild()
 
     allPaths.insert(inputPaths.begin(), inputPaths.end());
 
-    return true;
+    return prProceed;
 }
 
 
@@ -1829,18 +1899,22 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid)
 
 class SubstitutionGoal : public Goal
 {
+    friend class Worker;
+    
 private:
     /* The store path that should be realised through a substitute. */
     Path storePath;
 
-    /* The remaining substitutes for this path. */
-    Substitutes subs;
+    /* The remaining substituters. */
+    Paths subs;
 
-    /* The current substitute. */
-    Substitute sub;
+    /* The current substituter. */
+    Path sub;
 
-    /* Outgoing references for this path. */
+    /* Path info returned by the substituter's --query-info operation. */
+    bool infoOkay;
     PathSet references;
+    Path deriver;
 
     /* Pipe for the substitute's standard output/error. */
     Pipe logPipe;
@@ -1864,8 +1938,9 @@ public:
 
     /* The states. */
     void init();
-    void referencesValid();
     void tryNext();
+    void gotInfo();
+    void referencesValid();
     void tryToRun();
     void finished();
 
@@ -1923,17 +1998,46 @@ void SubstitutionGoal::init()
         return;
     }
 
-    /* !!! race condition; should get the substitutes and the
-       references in a transaction (in case a clearSubstitutes() is
-       done simultaneously). */
+    subs = substituters;
 
-    /* Read the substitutes. */
-    subs = store->querySubstitutes(storePath);
+    tryNext();
+}
 
+
+void SubstitutionGoal::tryNext()
+{
+    trace("trying next substituter");
+
+    if (subs.size() == 0) {
+        /* None left.  Terminate this goal and let someone else deal
+           with it. */
+        printMsg(lvlError,
+            format("path `%1%' is required, but there is no substituter that can build it")
+            % storePath);
+        amDone(ecFailed);
+        return;
+    }
+
+    sub = subs.front();
+    subs.pop_front();
+
+    infoOkay = false;
+    state = &SubstitutionGoal::gotInfo;
+    worker.waitForInfo(shared_from_this(), sub, storePath);
+}
+
+
+void SubstitutionGoal::gotInfo()
+{
+    trace("got info");
+
+    if (!infoOkay) {
+        tryNext();
+        return;
+    }
+    
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    store->queryReferences(storePath, references);
-
     for (PathSet::iterator i = references.begin();
          i != references.end(); ++i)
         if (*i != storePath) /* ignore self-references */
@@ -1948,7 +2052,7 @@ void SubstitutionGoal::init()
 
 void SubstitutionGoal::referencesValid()
 {
-    trace("all referenced realised");
+    trace("all references realised");
 
     if (nrFailed > 0) {
         printMsg(lvlError,
@@ -1961,28 +2065,7 @@ void SubstitutionGoal::referencesValid()
          i != references.end(); ++i)
         if (*i != storePath) /* ignore self-references */
             assert(store->isValidPath(*i));
-    
-    tryNext();
-}
 
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substitute");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        printMsg(lvlError,
-            format("path `%1%' is required, but it has no (remaining) substitutes")
-            % storePath);
-        amDone(ecFailed);
-        return;
-    }
-    sub = subs.front();
-    subs.pop_front();
-
-    /* Wait until we can run the substitute program. */
     state = &SubstitutionGoal::tryToRun;
     worker.waitForBuildSlot(shared_from_this());
 }
@@ -1998,6 +2081,17 @@ void SubstitutionGoal::tryToRun()
         return;
     }
 
+    /* Maybe a derivation goal has already locked this path
+       (exceedingly unlikely, since it should have used a substitute
+       first, but let's be defensive). */
+    outputLock.reset(); // make sure this goal's lock is gone
+    if (pathIsLockedByMe(storePath)) {
+        debug(format("restarting substitution of `%1%' because it's locked by another goal")
+            % storePath);
+        worker.waitForAnyGoal(shared_from_this());
+        return; /* restart in the tryToRun() state when another goal finishes */
+    }
+    
     /* Acquire a lock on the output path. */
     outputLock = boost::shared_ptr<PathLocks>(new PathLocks);
     outputLock->lockPaths(singleton<PathSet>(storePath),
@@ -2013,7 +2107,7 @@ void SubstitutionGoal::tryToRun()
 
     printMsg(lvlInfo,
         format("substituting path `%1%' using substituter `%2%'")
-        % storePath % sub.program);
+        % storePath % sub);
     
     logPipe.create();
 
@@ -2038,14 +2132,15 @@ void SubstitutionGoal::tryToRun()
             commonChildInit(logPipe);
 
             /* Fill in the arguments. */
-            Strings args(sub.args);
-            args.push_front(storePath);
-            args.push_front(baseNameOf(sub.program));
+            Strings args;
+            args.push_back(baseNameOf(sub));
+            args.push_back("--substitute");
+            args.push_back(storePath);
             const char * * argArr = strings2CharPtrs(args);
 
-            execv(sub.program.c_str(), (char * *) argArr);
+            execv(sub.c_str(), (char * *) argArr);
             
-            throw SysError(format("executing `%1%'") % sub.program);
+            throw SysError(format("executing `%1%'") % sub);
             
         } catch (std::exception & e) {
             std::cerr << format("substitute error: %1%\n") % e.what();
@@ -2098,7 +2193,7 @@ void SubstitutionGoal::finished()
 
         printMsg(lvlInfo,
             format("substitution of path `%1%' using substituter `%2%' failed: %3%")
-            % storePath % sub.program % e.msg());
+            % storePath % sub % e.msg());
         
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
@@ -2113,7 +2208,7 @@ void SubstitutionGoal::finished()
     Transaction txn;
     createStoreTransaction(txn);
     registerValidPath(txn, storePath, contentHash,
-        references, sub.deriver);
+        references, deriver);
     txn.commit();
 
     outputLock->setDeletion(true);
@@ -2220,6 +2315,16 @@ void Worker::removeGoal(GoalPtr goal)
         if (goal->getExitCode() == Goal::ecFailed && !keepGoing)
             topGoals.clear();
     }
+
+    /* Wake up goals waiting for any goal to finish. */
+    for (WeakGoals::iterator i = waitingForAnyGoal.begin();
+         i != waitingForAnyGoal.end(); ++i)
+    {
+        GoalPtr goal = i->lock();
+        if (goal) wakeUp(goal);
+    }
+
+    waitingForAnyGoal.clear();
 }
 
 
@@ -2298,6 +2403,83 @@ void Worker::waitForChildTermination(GoalPtr goal)
 }
 
 
+void Worker::waitForInfo(GoalPtr goal, Path sub, Path storePath)
+{
+    debug("wait for info");
+    requestedInfo[sub].insert(storePath);
+    waitingForInfo.insert(goal);
+}
+
+
+void Worker::waitForAnyGoal(GoalPtr goal)
+{
+    debug("wait for any goal");
+    waitingForAnyGoal.insert(goal);
+}
+
+
+void Worker::getInfo()
+{
+    for (map<Path, PathSet>::iterator i = requestedInfo.begin();
+         i != requestedInfo.end(); ++i)
+    {
+        Path sub = i->first;
+        PathSet paths = i->second;
+
+        while (!paths.empty()) {
+
+            /* Run the substituter for at most 100 paths at a time to
+               prevent command line overflows. */
+            PathSet paths2;
+            while (!paths.empty() && paths2.size() < 100) {
+                paths2.insert(*paths.begin());
+                paths.erase(paths.begin());
+            }
+
+            /* Ask the substituter for the references and deriver of
+               the paths. */
+            debug(format("running `%1%' to get info about `%2%'") % sub % showPaths(paths2));
+            Strings args;
+            args.push_back("--query-info");
+            args.insert(args.end(), paths2.begin(), paths2.end());
+            string res = runProgram(sub, false, args);
+            std::istringstream str(res);
+
+            while (true) {
+                ValidPathInfo info = decodeValidPathInfo(str);
+                if (info.path == "") break;
+
+                /* !!! inefficient */
+                for (WeakGoals::iterator k = waitingForInfo.begin();
+                     k != waitingForInfo.end(); ++k)
+                {
+                    GoalPtr goal = k->lock();
+                    if (goal) {
+                        SubstitutionGoal * goal2 = dynamic_cast<SubstitutionGoal *>(goal.get());
+                        if (goal2->storePath == info.path) {
+                            goal2->references = info.references;
+                            goal2->deriver = info.deriver;
+                            goal2->infoOkay = true;
+                            wakeUp(goal);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    for (WeakGoals::iterator k = waitingForInfo.begin();
+         k != waitingForInfo.end(); ++k)
+    {
+        GoalPtr goal = k->lock();
+        if (goal) wakeUp(goal);
+    }
+    
+    requestedInfo.clear();
+    waitingForInfo.clear(); // !!! have we done them all?
+}
+
+
 void Worker::run(const Goals & _topGoals)
 {
     for (Goals::iterator i = _topGoals.begin();
@@ -2324,11 +2506,14 @@ void Worker::run(const Goals & _topGoals)
 
         if (topGoals.empty()) break;
 
-        /* !!! not when we're polling */
-        assert(!children.empty());
-        
+        getInfo();
+
         /* Wait for input. */
-        waitForInput();
+        if (!children.empty())
+            waitForInput();
+        else
+            /* !!! not when we're polling */
+            assert(!awake.empty());
     }
 
     /* If --keep-going is not set, it's possible that the main goal

src/libstore/db.cc

@@ -142,6 +142,15 @@ Database::~Database()
 
 void openEnv(DbEnv * & env, const string & path, u_int32_t flags)
 {
+    try {
+        createDirs(path);
+    } catch (SysError & e) {
+        if (e.errNo == EPERM || e.errNo == EACCES)
+            throw DbNoPermission(format("cannot create the Nix database in `%1%'") % path);
+        else
+            throw;
+    }
+        
     try {
         env->open(path.c_str(),
             DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN |
@@ -185,7 +194,8 @@ void Database::open2(const string & path, bool removeOldEnv)
 
     env->set_errcall(errorPrinter);
     env->set_msgcall(messagePrinter);
-    //env->set_verbose(DB_VERB_REGISTER, 1);
+    if (getEnv("NIX_DEBUG_DB_REGISTER") == "1")
+        env->set_verbose(DB_VERB_REGISTER, 1);
     env->set_verbose(DB_VERB_RECOVERY, 1);
     
     /* Smaller log files. */
@@ -445,4 +455,14 @@ void Database::enumTable(const Transaction & txn, TableId table,
 }
 
  
+void Database::clearTable(const Transaction & txn, TableId table)
+{
+    try {
+        Db * db = getDb(table);
+        u_int32_t count;
+        db->truncate(txn.txn, &count, 0);
+    } catch (DbException e) { rethrow(e); }
+}
+
+
 }

src/libstore/db.hh

@@ -89,6 +89,8 @@ public:
 
     void enumTable(const Transaction & txn, TableId table,
         Strings & keys, const string & keyPrefix = "");
+
+    void clearTable(const Transaction & txn, TableId table);
 };
 
 

src/libstore/gc.cc

@@ -61,6 +61,8 @@ void createSymlink(const Path & link, const Path & target, bool careful)
     /* Create directories up to `gcRoot'. */
     createDirs(dirOf(link));
 
+    /* !!! shouldn't removing and creating the symlink be atomic? */
+
     /* Remove the old symlink. */
     if (pathExists(link)) {
         if (careful && (!isLink(link) || !isInStore(readLink(link))))
@@ -68,7 +70,7 @@ void createSymlink(const Path & link, const Path & target, bool careful)
         unlink(link.c_str());
     }
 
-    /* And create the new own. */
+    /* And create the new one. */
     if (symlink(target.c_str(), link.c_str()) == -1)
         throw SysError(format("symlinking `%1%' to `%2%'")
             % link % target);
@@ -97,6 +99,11 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
     Path gcRoot(canonPath(_gcRoot));
     assertStorePath(storePath);
 
+    if (isInStore(gcRoot))
+        throw Error(format(
+                "creating a garbage collector root (%1%) in the Nix store is forbidden "
+                "(are you running nix-build inside the store?)") % gcRoot);
+
     if (indirect) {
         createSymlink(gcRoot, storePath, true);
         store->addIndirectRoot(gcRoot);
@@ -114,7 +121,6 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
         }
             
         createSymlink(gcRoot, storePath, false);
-
     }
 
     /* Check that the root can be found by the garbage collector. */
@@ -478,7 +484,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
             /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
-            Path deriver = queryDeriver(noTxn, *i);
+            Path deriver = store->queryDeriver(*i);
             if (deriver != "" && store->isValidPath(deriver))
                 computeFSClosure(deriver, livePaths);
         }
@@ -598,6 +604,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
             }
 #endif
 
+            if (!pathExists(*i)) continue;
+                
             printMsg(lvlInfo, format("deleting `%1%'") % *i);
             
             /* Okay, it's safe to delete. */

src/libstore/globals.cc

@@ -25,6 +25,7 @@ unsigned int maxBuildJobs = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
 unsigned int maxSilentTime = 0;
+Paths substituters;
 
 
 static bool settingsRead = false;

src/libstore/globals.hh

@@ -67,6 +67,11 @@ extern string thisSystem;
    infinity. */
 extern unsigned int maxSilentTime;
 
+/* The substituters.  There are programs that can somehow realise a
+   store path without building, e.g., by downloading it or copying it
+   from a CD. */
+extern Paths substituters;
+
 
 Strings querySetting(const string & name, const Strings & def);
 

src/libstore/local-store.cc

@@ -48,22 +48,6 @@ static TableId dbReferences = 0;
    referrer. */
 static TableId dbReferrers = 0;
 
-/* dbSubstitutes :: Path -> [[Path]]
-
-   Each pair $(p, subs)$ tells Nix that it can use any of the
-   substitutes in $subs$ to build path $p$.  Each substitute defines a
-   command-line invocation of a program (i.e., the first list element
-   is the full path to the program, the remaining elements are
-   arguments).
-
-   The main purpose of this is for distributed caching of derivates.
-   One system can compute a derivate and put it on a website (as a Nix
-   archive), for instance, and then another system can register a
-   substitute for that derivate.  The substitute in this case might be
-   a Nix derivation that fetches the Nix archive.
-*/
-static TableId dbSubstitutes = 0;
-
 /* dbDerivers :: Path -> [Path]
 
    This table lists the derivation used to build a path.  There can
@@ -72,15 +56,9 @@ static TableId dbSubstitutes = 0;
 static TableId dbDerivers = 0;
 
 
-bool Substitute::operator == (const Substitute & sub) const
-{
-    return program == sub.program
-        && args == sub.args;
-}
-
-
 static void upgradeStore07();
 static void upgradeStore09();
+static void upgradeStore11();
 
 
 void checkStoreNotSymlink()
@@ -103,6 +81,8 @@ void checkStoreNotSymlink()
 
 LocalStore::LocalStore(bool reserveSpace)
 {
+    substitutablePathsLoaded = false;
+    
     if (readOnlyMode) return;
 
     checkStoreNotSymlink();
@@ -133,7 +113,6 @@ LocalStore::LocalStore(bool reserveSpace)
     dbValidPaths = nixDB.openTable("validpaths");
     dbReferences = nixDB.openTable("references");
     dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */
-    dbSubstitutes = nixDB.openTable("substitutes");
     dbDerivers = nixDB.openTable("derivers");
 
     int curSchema = 0;
@@ -153,6 +132,8 @@ LocalStore::LocalStore(bool reserveSpace)
             upgradeStore07();
         if (curSchema == 2)
             upgradeStore09();
+        if (curSchema == 3)
+            upgradeStore11();
         writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str());
     }
 }
@@ -161,7 +142,11 @@ LocalStore::LocalStore(bool reserveSpace)
 LocalStore::~LocalStore()
 {
     /* If the database isn't open, this is a NOP. */
-    nixDB.close();
+    try {
+        nixDB.close();
+    } catch (...) {
+        ignoreException();
+    }
 }
 
 
@@ -189,7 +174,7 @@ void copyPath(const Path & src, const Path & dst, PathFilter & filter)
 }
 
 
-static void _canonicalisePathMetaData(const Path & path)
+static void _canonicalisePathMetaData(const Path & path, bool recurse)
 {
     checkInterrupt();
 
@@ -238,17 +223,17 @@ static void _canonicalisePathMetaData(const Path & path)
 
     }
 
-    if (S_ISDIR(st.st_mode)) {
+    if (recurse && S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
 	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
-	    _canonicalisePathMetaData(path + "/" + *i);
+	    _canonicalisePathMetaData(path + "/" + *i, true);
     }
 }
 
 
 void canonicalisePathMetaData(const Path & path)
 {
-    _canonicalisePathMetaData(path);
+    _canonicalisePathMetaData(path, true);
 
     /* On platforms that don't have lchown(), the top-level path can't
        be a symlink, since we can't change its ownership. */
@@ -276,17 +261,6 @@ bool LocalStore::isValidPath(const Path & path)
 }
 
 
-static Substitutes readSubstitutes(const Transaction & txn,
-    const Path & srcPath);
-
-
-static bool isRealisablePath(const Transaction & txn, const Path & path)
-{
-    return isValidPathTxn(txn, path)
-        || readSubstitutes(txn, path).size() > 0;
-}
-
-
 static string addPrefix(const string & prefix, const string & s)
 {
     return prefix + string(1, (char) 0) + s;
@@ -318,11 +292,10 @@ static PathSet getReferrers(const Transaction & txn, const Path & storePath)
 void setReferences(const Transaction & txn, const Path & storePath,
     const PathSet & references)
 {
-    /* For unrealisable paths, we can only clear the references. */
-    if (references.size() > 0 && !isRealisablePath(txn, storePath))
+    /* For invalid paths, we can only clear the references. */
+    if (references.size() > 0 && !isValidPathTxn(txn, storePath))
         throw Error(
-            format("cannot set references for path `%1%' which is invalid and has no substitutes")
-            % storePath);
+            format("cannot set references for invalid path `%1%'") % storePath);
 
     Paths oldReferences;
     nixDB.queryStrings(txn, dbReferences, storePath, oldReferences);
@@ -352,7 +325,7 @@ void queryReferences(const Transaction & txn,
     const Path & storePath, PathSet & references)
 {
     Paths references2;
-    if (!isRealisablePath(txn, storePath))
+    if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
     nixDB.queryStrings(txn, dbReferences, storePath, references2);
     references.insert(references2.begin(), references2.end());
@@ -369,7 +342,7 @@ void LocalStore::queryReferences(const Path & storePath,
 void queryReferrers(const Transaction & txn,
     const Path & storePath, PathSet & referrers)
 {
-    if (!isRealisablePath(txn, storePath))
+    if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
     PathSet referrers2 = getReferrers(txn, storePath);
     referrers.insert(referrers2.begin(), referrers2.end());
@@ -389,15 +362,15 @@ void setDeriver(const Transaction & txn, const Path & storePath,
     assertStorePath(storePath);
     if (deriver == "") return;
     assertStorePath(deriver);
-    if (!isRealisablePath(txn, storePath))
+    if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
     nixDB.setString(txn, dbDerivers, storePath, deriver);
 }
 
 
-Path queryDeriver(const Transaction & txn, const Path & storePath)
+static Path queryDeriver(const Transaction & txn, const Path & storePath)
 {
-    if (!isRealisablePath(txn, storePath))
+    if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
     Path deriver;
     if (nixDB.queryString(txn, dbDerivers, storePath, deriver))
@@ -407,119 +380,39 @@ Path queryDeriver(const Transaction & txn, const Path & storePath)
 }
 
 
-const int substituteVersion = 2;
-
-
-static Substitutes readSubstitutes(const Transaction & txn,
-    const Path & srcPath)
+Path LocalStore::queryDeriver(const Path & path)
 {
-    Strings ss;
-    nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss);
+    return nix::queryDeriver(noTxn, path);
+}
 
-    Substitutes subs;
-    
-    for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
-        if (i->size() < 4 || (*i)[3] != 0) {
-            /* Old-style substitute.  !!! remove this code
-               eventually? */
-            break;
+
+PathSet LocalStore::querySubstitutablePaths()
+{
+    if (!substitutablePathsLoaded) {
+        for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) {
+            debug(format("running `%1%' to find out substitutable paths") % *i);
+            Strings args;
+            args.push_back("--query-paths");
+            Strings ss = tokenizeString(runProgram(*i, false, args), "\n");
+            for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) {
+                if (!isStorePath(*j))
+                    throw Error(format("`%1%' returned a bad substitutable path `%2%'")
+                        % *i % *j);
+                substitutablePaths.insert(*j);
+            }
         }
-        Strings ss2 = unpackStrings(*i);
-        if (ss2.size() == 0) continue;
-        int version;
-        if (!string2Int(ss2.front(), version)) continue;
-        if (version != substituteVersion) continue;
-        if (ss2.size() != 4) throw Error("malformed substitute");
-        Strings::iterator j = ss2.begin();
-        j++;
-        Substitute sub;
-        sub.deriver = *j++;
-        sub.program = *j++;
-        sub.args = unpackStrings(*j++);
-        subs.push_back(sub);
+        substitutablePathsLoaded = true;
     }
 
-    return subs;
+    return substitutablePaths;
 }
 
 
-static void writeSubstitutes(const Transaction & txn,
-    const Path & srcPath, const Substitutes & subs)
+bool LocalStore::hasSubstitutes(const Path & path)
 {
-    Strings ss;
-
-    for (Substitutes::const_iterator i = subs.begin();
-         i != subs.end(); ++i)
-    {
-        Strings ss2;
-        ss2.push_back((format("%1%") % substituteVersion).str());
-        ss2.push_back(i->deriver);
-        ss2.push_back(i->program);
-        ss2.push_back(packStrings(i->args));
-        ss.push_back(packStrings(ss2));
-    }
-
-    nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
-}
-
-
-void registerSubstitute(const Transaction & txn,
-    const Path & srcPath, const Substitute & sub)
-{
-    assertStorePath(srcPath);
-    
-    Substitutes subs = readSubstitutes(txn, srcPath);
-
-    if (find(subs.begin(), subs.end(), sub) != subs.end())
-        return;
-
-    /* New substitutes take precedence over old ones.  If the
-       substitute is already present, it's moved to the front. */
-    remove(subs.begin(), subs.end(), sub);
-    subs.push_front(sub);
-        
-    writeSubstitutes(txn, srcPath, subs);
-}
-
-
-Substitutes querySubstitutes(const Transaction & txn, const Path & path)
-{
-    return readSubstitutes(txn, path);
-}
-
-
-Substitutes LocalStore::querySubstitutes(const Path & path)
-{
-    return nix::querySubstitutes(noTxn, path);
-}
-
-
-static void invalidatePath(Transaction & txn, const Path & path);
-
-
-void clearSubstitutes()
-{
-    Transaction txn(nixDB);
-    
-    /* Iterate over all paths for which there are substitutes. */
-    Paths subKeys;
-    nixDB.enumTable(txn, dbSubstitutes, subKeys);
-    for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
-        
-        /* Delete all substitutes for path *i. */
-        nixDB.delPair(txn, dbSubstitutes, *i);
-        
-        /* Maintain the cleanup invariant. */
-        if (!isValidPathTxn(txn, *i))
-            invalidatePath(txn, *i);
-    }
-
-    /* !!! there should be no referrers to any of the invalid
-       substitutable paths.  This should be the case by construction
-       (the only referrers can be other invalid substitutable paths,
-       which have all been removed now). */
-    
-    txn.commit();
+    if (!substitutablePathsLoaded)
+        querySubstitutablePaths(); 
+    return substitutablePaths.find(path) != substitutablePaths.end();
 }
 
 
@@ -605,17 +498,12 @@ void registerValidPaths(const Transaction & txn,
    there are no referrers. */
 static void invalidatePath(Transaction & txn, const Path & path)
 {
-    debug(format("unregistering path `%1%'") % path);
+    debug(format("invalidating path `%1%'") % path);
 
     /* Clear the `references' entry for this path, as well as the
-       inverse `referrers' entries, and the `derivers' entry; but only
-       if there are no substitutes for this path.  This maintains the
-       cleanup invariant. */
-    if (querySubstitutes(txn, path).size() == 0) {
-        setReferences(txn, path, PathSet());
-        nixDB.delPair(txn, dbDerivers, path);
-    }
-    
+       inverse `referrers' entries, and the `derivers' entry. */
+    setReferences(txn, path, PathSet());
+    nixDB.delPair(txn, dbDerivers, path);
     nixDB.delPair(txn, dbValidPaths, path);
 }
 
@@ -737,7 +625,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
        consistent metadata. */
     Transaction txn(nixDB);
     addTempRoot(path);
-    if (!isValidPath(path))
+    if (!isValidPathTxn(txn, path))
         throw Error(format("path `%1%' is not valid") % path);
 
     HashAndWriteSink hashAndWriteSink(sink);
@@ -752,7 +640,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
     nix::queryReferences(txn, path, references);
     writeStringSet(references, hashAndWriteSink);
 
-    Path deriver = queryDeriver(txn, path);
+    Path deriver = nix::queryDeriver(txn, path);
     writeString(deriver, hashAndWriteSink);
 
     if (sign) {
@@ -934,6 +822,7 @@ void verifyStore(bool checkContents)
     nixDB.enumTable(txn, dbValidPaths, paths);
 
     for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
+        checkInterrupt();
         if (!pathExists(*i)) {
             printMsg(lvlError, format("path `%1%' disappeared") % *i);
             invalidatePath(txn, *i);
@@ -956,30 +845,7 @@ void verifyStore(bool checkContents)
     }
 
 
-    printMsg(lvlInfo, "checking path realisability");
-    
-    /* "Realisable" paths are those that are valid or have a
-       substitute. */
-    PathSet realisablePaths(validPaths);
-
-    /* Check that the values of the substitute mappings are valid
-       paths. */ 
-    Paths subKeys;
-    nixDB.enumTable(txn, dbSubstitutes, subKeys);
-    for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
-        Substitutes subs = readSubstitutes(txn, *i);
-        if (!isStorePath(*i)) {
-            printMsg(lvlError, format("removing substitutes for non-store path `%1%'") % *i);
-            nixDB.delPair(txn, dbSubstitutes, *i);
-        }
-        else if (subs.size() == 0)
-            nixDB.delPair(txn, dbSubstitutes, *i);
-        else
-	    realisablePaths.insert(*i);
-    }
-    
-
-    /* Check the cleanup invariant: only realisable paths can have
+    /* Check the cleanup invariant: only valid paths can have
        `references', `referrers', or `derivers' entries. */
 
 
@@ -990,8 +856,8 @@ void verifyStore(bool checkContents)
     for (Paths::iterator i = deriversKeys.begin();
          i != deriversKeys.end(); ++i)
     {
-        if (realisablePaths.find(*i) == realisablePaths.end()) {
-            printMsg(lvlError, format("removing deriver entry for unrealisable path `%1%'")
+        if (validPaths.find(*i) == validPaths.end()) {
+            printMsg(lvlError, format("removing deriver entry for invalid path `%1%'")
                 % *i);
             nixDB.delPair(txn, dbDerivers, *i);
         }
@@ -1013,13 +879,12 @@ void verifyStore(bool checkContents)
     for (Paths::iterator i = referencesKeys.begin();
          i != referencesKeys.end(); ++i)
     {
-        if (realisablePaths.find(*i) == realisablePaths.end()) {
-            printMsg(lvlError, format("removing references entry for unrealisable path `%1%'")
+        if (validPaths.find(*i) == validPaths.end()) {
+            printMsg(lvlError, format("removing references entry for invalid path `%1%'")
                 % *i);
             setReferences(txn, *i, PathSet());
         }
         else {
-            bool isValid = validPaths.find(*i) != validPaths.end();
             PathSet references;
             queryReferences(txn, *i, references);
             for (PathSet::iterator j = references.begin();
@@ -1031,7 +896,7 @@ void verifyStore(bool checkContents)
                         % *j % *i);
                     nixDB.setString(txn, dbReferrers, addPrefix(*j, *i), "");
                 }
-                if (isValid && validPaths.find(*j) == validPaths.end()) {
+                if (validPaths.find(*j) == validPaths.end()) {
                     printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'")
                         % *i % *j);
                 }
@@ -1055,14 +920,14 @@ void verifyStore(bool checkContents)
         Path to(*i, 0, nul);
         Path from(*i, nul + 1);
         
-        if (realisablePaths.find(to) == realisablePaths.end()) {
-            printMsg(lvlError, format("removing referrer entry from `%1%' to unrealisable `%2%'")
+        if (validPaths.find(to) == validPaths.end()) {
+            printMsg(lvlError, format("removing referrer entry from `%1%' to invalid `%2%'")
                 % from % to);
             nixDB.delPair(txn, dbReferrers, *i);
         }
 
-        else if (realisablePaths.find(from) == realisablePaths.end()) {
-            printMsg(lvlError, format("removing referrer entry from unrealisable `%1%' to `%2%'")
+        else if (validPaths.find(from) == validPaths.end()) {
+            printMsg(lvlError, format("removing referrer entry from invalid `%1%' to `%2%'")
                 % from % to);
             nixDB.delPair(txn, dbReferrers, *i);
         }
@@ -1085,6 +950,125 @@ void verifyStore(bool checkContents)
 }
 
 
+typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
+
+
+static void makeWritable(const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+	throw SysError(format("getting attributes of path `%1%'") % path);
+    if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+        throw SysError(format("changing writability of `%1%'") % path);
+}
+
+
+static void hashAndLink(bool dryRun, HashToPath & hashToPath,
+    OptimiseStats & stats, const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+	throw SysError(format("getting attributes of path `%1%'") % path);
+
+    /* Sometimes SNAFUs can cause files in the Nix store to be
+       modified, in particular when running programs as root under
+       NixOS (example: $fontconfig/var/cache being modified).  Skip
+       those files. */
+    if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
+        printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
+        return;
+    }
+
+    /* We can hard link regular files and symlinks. */
+    if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {
+
+        /* Hash the file.  Note that hashPath() returns the hash over
+           the NAR serialisation, which includes the execute bit on
+           the file.  Thus, executable and non-executable files with
+           the same contents *won't* be linked (which is good because
+           otherwise the permissions would be screwed up).
+
+           Also note that if `path' is a symlink, then we're hashing
+           the contents of the symlink (i.e. the result of
+           readlink()), not the contents of the target (which may not
+           even exist). */
+        Hash hash = hashPath(htSHA256, path);
+        stats.totalFiles++;
+        printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
+
+        std::pair<Path, ino_t> prevPath = hashToPath[hash];
+        
+        if (prevPath.first == "") {
+            hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
+            return;
+        }
+            
+        /* Yes!  We've seen a file with the same contents.  Replace
+           the current file with a hard link to that file. */
+        stats.sameContents++;
+        if (prevPath.second == st.st_ino) {
+            printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
+            return;
+        }
+        
+        if (!dryRun) {
+            
+            printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
+
+            Path tempLink = (format("%1%.tmp-%2%-%3%")
+                % path % getpid() % rand()).str();
+
+            /* Make the containing directory writable, but only if
+               it's not the store itself (we don't want or need to
+               mess with  its permissions). */
+            bool mustToggle = !isStorePath(path);
+            if (mustToggle) makeWritable(dirOf(path));
+        
+            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1)
+                throw SysError(format("cannot link `%1%' to `%2%'")
+                    % tempLink % prevPath.first);
+
+            /* Atomically replace the old file with the new hard link. */
+            if (rename(tempLink.c_str(), path.c_str()) == -1)
+                throw SysError(format("cannot rename `%1%' to `%2%'")
+                    % tempLink % path);
+
+            /* Make the directory read-only again and reset its
+               timestamp back to 0. */
+            if (mustToggle) _canonicalisePathMetaData(dirOf(path), false);
+            
+        } else
+            printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
+        
+        stats.filesLinked++;
+        stats.bytesFreed += st.st_size;
+    }
+
+    if (S_ISDIR(st.st_mode)) {
+        Strings names = readDirectory(path);
+	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+	    hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
+    }
+}
+
+
+void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
+{
+    HashToPath hashToPath;
+    
+    Paths paths;
+    PathSet validPaths;
+    nixDB.enumTable(noTxn, dbValidPaths, paths);
+
+    for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
+        addTempRoot(*i);
+        if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
+        startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
+        hashAndLink(dryRun, hashToPath, stats, *i);
+    }
+}
+
+
 /* Upgrade from schema 1 (Nix <= 0.7) to schema 2 (Nix >= 0.8). */
 static void upgradeStore07()
 {
@@ -1180,10 +1164,10 @@ static void upgradeStore09()
 {
     /* !!! we should disallow concurrent upgrades */
     
-    printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
-
     if (!pathExists(nixDBPath + "/referers")) return;
 
+    printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
+
     Transaction txn(nixDB);
 
     std::cerr << "converting referers to referrers...";
@@ -1220,4 +1204,29 @@ static void upgradeStore09()
 }
 
  
+/* Upgrade from schema 3 (Nix 0.10) to schema 4 (Nix >= 0.11).  The
+   only thing to do here is to delete the substitutes table and get
+   rid of invalid but substitutable references/referrers.  */
+static void upgradeStore11()
+{
+    if (!pathExists(nixDBPath + "/substitutes")) return;
+
+    printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
+
+    Transaction txn(nixDB);
+    TableId dbSubstitutes = nixDB.openTable("substitutes");
+
+    Paths subKeys;
+    nixDB.enumTable(txn, dbSubstitutes, subKeys);
+    for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
+        if (!isValidPathTxn(txn, *i))
+            invalidatePath(txn, *i);
+    }
+    
+    txn.commit();
+    nixDB.closeTable(dbSubstitutes);
+    nixDB.deleteTable("substitutes");
+}
+
+
 }

src/libstore/local-store.hh

@@ -13,16 +13,34 @@ class Transaction;
 
 
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
-   0.7.  Version 2 was Nix 0.8 and 0.8.  Version 3 is Nix 0.10 and
-   up. */
-const int nixSchemaVersion = 3;
+   0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
+   Version 4 is Nix 0.11. */
+const int nixSchemaVersion = 4;
 
 
 extern string drvsLogDir;
 
 
+struct OptimiseStats
+{
+    unsigned long totalFiles;
+    unsigned long sameContents;
+    unsigned long filesLinked;
+    unsigned long long bytesFreed;
+    OptimiseStats()
+    {
+        totalFiles = sameContents = filesLinked = 0;
+        bytesFreed = 0;
+    }
+};
+
+
 class LocalStore : public StoreAPI
 {
+private:
+    bool substitutablePathsLoaded;
+    PathSet substitutablePaths;
+    
 public:
 
     /* Open the database environment.  If `reserveSpace' is true, make
@@ -41,14 +59,18 @@ public:
     
     bool isValidPath(const Path & path);
 
-    Substitutes querySubstitutes(const Path & srcPath);
-
     Hash queryPathHash(const Path & path);
 
     void queryReferences(const Path & path, PathSet & references);
 
     void queryReferrers(const Path & path, PathSet & referrers);
 
+    Path queryDeriver(const Path & path);
+    
+    PathSet querySubstitutablePaths();
+    
+    bool hasSubstitutes(const Path & path);
+    
     Path addToStore(const Path & srcPath, bool fixed = false,
         bool recursive = false, string hashAlgo = "",
         PathFilter & filter = defaultPathFilter);
@@ -75,6 +97,10 @@ public:
 
     void collectGarbage(GCAction action, const PathSet & pathsToDelete,
         bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
+
+    /* Optimise the disk space usage of the Nix store by hard-linking
+       files with the same contents. */
+    void optimiseStore(bool dryRun, OptimiseStats & stats);
 };
 
 
@@ -84,13 +110,6 @@ void createStoreTransaction(Transaction & txn);
 /* Copy a path recursively. */
 void copyPath(const Path & src, const Path & dst);
 
-/* Register a substitute. */
-void registerSubstitute(const Transaction & txn,
-    const Path & srcPath, const Substitute & sub);
-
-/* Deregister all substitutes. */
-void clearSubstitutes();
-
 /* Register the validity of a path, i.e., that `path' exists, that the
    paths referenced by it exists, and in the case of an output path of
    a derivation, that it has been produced by a succesful execution of
@@ -101,14 +120,6 @@ void registerValidPath(const Transaction & txn,
     const Path & path, const Hash & hash, const PathSet & references,
     const Path & deriver);
 
-struct ValidPathInfo 
-{
-    Path path;
-    Path deriver;
-    Hash hash;
-    PathSet references;
-};
-
 typedef list<ValidPathInfo> ValidPathInfos;
 
 void registerValidPaths(const Transaction & txn,
@@ -136,10 +147,6 @@ void setReferences(const Transaction & txn, const Path & path,
 void setDeriver(const Transaction & txn, const Path & path,
     const Path & deriver);
 
-/* Query the deriver of a store path.  Return the empty string if no
-   deriver has been set. */
-Path queryDeriver(const Transaction & txn, const Path & path);
-
 /* Delete a value from the nixStore directory. */
 void deleteFromStore(const Path & path, unsigned long long & bytesFreed);
 

src/libstore/misc.cc

@@ -63,8 +63,7 @@ void queryMissing(const PathSet & targets,
             bool mustBuild = false;
             for (DerivationOutputs::iterator i = drv.outputs.begin();
                  i != drv.outputs.end(); ++i)
-                if (!store->isValidPath(i->second.path) &&
-                    !store->hasSubstitutes(i->second.path))
+                if (!store->isValidPath(i->second.path) && !store->hasSubstitutes(i->second.path))
                     mustBuild = true;
 
             if (mustBuild) {
@@ -83,8 +82,8 @@ void queryMissing(const PathSet & targets,
             if (store->isValidPath(p)) continue;
             if (store->hasSubstitutes(p))
                 willSubstitute.insert(p);
-            PathSet refs;
-            store->queryReferences(p, todo);
+            // XXX call the substituters
+            // store->queryReferences(p, todo);
         }
     }
 }

src/libstore/pathlocks.cc

@@ -161,10 +161,8 @@ void PathLocks::lockPaths(const PathSet & _paths, const string & waitMsg)
 
         debug(format("locking path `%1%'") % path);
 
-        if (lockedPaths.find(lockPath) != lockedPaths.end()) {
-            debug(format("already holding lock on `%1%'") % lockPath);
-            continue;
-        }
+        if (lockedPaths.find(lockPath) != lockedPaths.end())
+            throw Error("deadlock: trying to re-acquire self-held lock");
 
         AutoCloseFD fd;
         
@@ -225,5 +223,12 @@ void PathLocks::setDeletion(bool deletePaths)
     this->deletePaths = deletePaths;
 }
 
+
+bool pathIsLockedByMe(const Path & path)
+{
+    Path lockPath = path + ".lock";
+    return lockedPaths.find(lockPath) != lockedPaths.end();
+}
+
  
 }

src/libstore/pathlocks.hh

@@ -40,6 +40,9 @@ public:
 };
 
 
+bool pathIsLockedByMe(const Path & path);
+
+
 }
 
 

src/libstore/remote-store.cc

@@ -53,18 +53,24 @@ RemoteStore::RemoteStore()
     from.fd = fdSocket;
     to.fd = fdSocket;
 
-    
     /* Send the magic greeting, check for the reply. */
     try {
         writeInt(WORKER_MAGIC_1, to);
-        writeInt(verbosity, to);
         unsigned int magic = readInt(from);
         if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
+
+        unsigned int daemonVersion = readInt(from);
+        if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
+            throw Error("Nix daemon protocol version not supported");
+        writeInt(PROTOCOL_VERSION, to);
         processStderr();
+
     } catch (Error & e) {
         throw Error(format("cannot start worker (%1%)")
             % e.msg());
     }
+
+    setOptions();
 }
 
 
@@ -148,12 +154,25 @@ RemoteStore::~RemoteStore()
         fdSocket.close();
         if (child != -1)
             child.wait(true);
-    } catch (Error & e) {
-        printMsg(lvlError, format("error (ignored): %1%") % e.msg());
+    } catch (...) {
+        ignoreException();
     }
 }
 
 
+void RemoteStore::setOptions()
+{
+    writeInt(wopSetOptions, to);
+    writeInt(keepFailed, to);
+    writeInt(keepGoing, to);
+    writeInt(tryFallback, to);
+    writeInt(verbosity, to);
+    writeInt(maxBuildJobs, to);
+    writeInt(maxSilentTime, to);
+    processStderr();
+}
+
+
 bool RemoteStore::isValidPath(const Path & path)
 {
     writeInt(wopIsValidPath, to);
@@ -164,12 +183,6 @@ bool RemoteStore::isValidPath(const Path & path)
 }
 
 
-Substitutes RemoteStore::querySubstitutes(const Path & path)
-{
-    throw Error("not implemented 2");
-}
-
-
 bool RemoteStore::hasSubstitutes(const Path & path)
 {
     writeInt(wopHasSubstitutes, to);
@@ -212,6 +225,21 @@ void RemoteStore::queryReferrers(const Path & path,
 }
 
 
+Path RemoteStore::queryDeriver(const Path & path)
+{
+    writeInt(wopQueryDeriver, to);
+    writeString(path, to);
+    processStderr();
+    return readStorePath(from);
+}
+
+
+PathSet RemoteStore::querySubstitutablePaths()
+{
+    throw Error("not implemented");
+}
+
+
 Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
     bool recursive, string hashAlgo, PathFilter & filter)
 {
@@ -224,8 +252,7 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
     writeString(hashAlgo, to);
     dumpPath(srcPath, to, filter);
     processStderr();
-    Path path = readStorePath(from);
-    return path;
+    return readStorePath(from);
 }
 
 
@@ -238,8 +265,7 @@ Path RemoteStore::addTextToStore(const string & suffix, const string & s,
     writeStringSet(references, to);
     
     processStderr();
-    Path path = readStorePath(from);
-    return path;
+    return readStorePath(from);
 }
 
 
@@ -261,8 +287,7 @@ Path RemoteStore::importPath(bool requireSignature, Source & source)
        anyway. */
     
     processStderr(0, &source);
-    Path path = readStorePath(from);
-    return path;
+    return readStorePath(from);
 }
 
 

src/libstore/remote-store.hh

@@ -27,16 +27,18 @@ public:
     
     bool isValidPath(const Path & path);
 
-    Substitutes querySubstitutes(const Path & path);
-
-    bool hasSubstitutes(const Path & path);
-    
     Hash queryPathHash(const Path & path);
 
     void queryReferences(const Path & path, PathSet & references);
 
     void queryReferrers(const Path & path, PathSet & referrers);
 
+    Path queryDeriver(const Path & path);
+    
+    PathSet querySubstitutablePaths();
+    
+    bool hasSubstitutes(const Path & path);
+    
     Path addToStore(const Path & srcPath, bool fixed = false,
         bool recursive = false, string hashAlgo = "",
         PathFilter & filter = defaultPathFilter);
@@ -75,6 +77,8 @@ private:
     void forkSlave();
     
     void connectToDaemon();
+
+    void setOptions();
 };
 
 

src/libstore/store-api.cc

@@ -8,7 +8,8 @@ namespace nix {
 
 bool StoreAPI::hasSubstitutes(const Path & path)
 {
-    return !querySubstitutes(path).empty();
+    PathSet paths = querySubstitutablePaths();
+    return paths.find(path) != paths.end();
 }
 
 
@@ -130,6 +131,24 @@ Path computeStorePathForText(const string & suffix, const string & s,
 }
 
 
+ValidPathInfo decodeValidPathInfo(std::istream & str)
+{
+    ValidPathInfo info;
+    getline(str, info.path);
+    if (str.eof()) { info.path = ""; return info; }
+    getline(str, info.deriver);
+    string s; int n;
+    getline(str, s);
+    if (!string2Int(s, n)) throw Error("number expected");
+    while (n--) {
+        getline(str, s);
+        info.references.insert(s);
+    }
+    if (!str || str.eof()) throw Error("missing input");
+    return info;
+}
+
+
 }
 
 

src/libstore/store-api.hh

@@ -13,28 +13,6 @@
 namespace nix {
 
 
-/* A substitute is a program invocation that constructs some store
-   path (typically by fetching it from somewhere, e.g., from the
-   network). */
-struct Substitute
-{       
-    /* The derivation that built this store path (empty if none). */
-    Path deriver;
-    
-    /* Program to be executed to create the store path.  Must be in
-       the output path of `storeExpr'. */
-    Path program;
-
-    /* Extra arguments to be passed to the program (the first argument
-       is the store path to be substituted). */
-    Strings args;
-
-    bool operator == (const Substitute & sub) const;
-};
-
-typedef list<Substitute> Substitutes;
-
-
 typedef std::map<Path, Path> Roots;
 
 
@@ -57,13 +35,6 @@ public:
     /* Checks whether a path is valid. */ 
     virtual bool isValidPath(const Path & path) = 0;
 
-    /* Return the substitutes for the given path. */
-    virtual Substitutes querySubstitutes(const Path & path) = 0;
-
-    /* More efficient variant if we just want to know if a path has
-       substitutes. */
-    virtual bool hasSubstitutes(const Path & path);
-
     /* Queries the hash of a valid path. */ 
     virtual Hash queryPathHash(const Path & path) = 0;
 
@@ -77,6 +48,17 @@ public:
     virtual void queryReferrers(const Path & path,
         PathSet & referrers) = 0;
 
+    /* Query the deriver of a store path.  Return the empty string if
+       no deriver has been set. */
+    virtual Path queryDeriver(const Path & path) = 0;
+
+    /* Query the set of substitutable paths. */
+    virtual PathSet querySubstitutablePaths() = 0;
+
+    /* More efficient variant if we just want to know if a path has
+       substitutes. */
+    virtual bool hasSubstitutes(const Path & path);
+    
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
        If `fixed' is true, then the output of a fixed-output
@@ -105,10 +87,10 @@ public:
 
     /* Ensure that the output paths of the derivation are valid.  If
        they are already valid, this is a no-op.  Otherwise, validity
-       can be reached in two ways.  First, if the output paths have
-       substitutes, then those can be used.  Second, the output paths
-       can be created by running the builder, after recursively
-       building any sub-derivations. */
+       can be reached in two ways.  First, if the output paths is
+       substitutable, then build the path that way.  Second, the
+       output paths can be created by running the builder, after
+       recursively building any sub-derivations. */
     virtual void buildDerivations(const PathSet & drvPaths) = 0;
 
     /* Ensure that a path is valid.  If it is not currently valid, it
@@ -257,6 +239,17 @@ extern boost::shared_ptr<StoreAPI> store;
 boost::shared_ptr<StoreAPI> openStore(bool reserveSpace = true);
 
 
+struct ValidPathInfo 
+{
+    Path path;
+    Path deriver;
+    Hash hash;
+    PathSet references;
+};
+
+ValidPathInfo decodeValidPathInfo(std::istream & str);
+
+
 }
 
 

src/libstore/worker-protocol.hh

@@ -5,15 +5,17 @@
 namespace nix {
 
 
-#define WORKER_MAGIC_1 0x6e697864
-#define WORKER_MAGIC_2 0x6478696e
+#define WORKER_MAGIC_1 0x6e697863
+#define WORKER_MAGIC_2 0x6478696f
+
+#define PROTOCOL_VERSION 0x101
+#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 
 
 typedef enum {
-    wopQuit,
+    wopQuit = 0,
     wopIsValidPath,
-    wopQuerySubstitutes,
-    wopHasSubstitutes,
+    wopHasSubstitutes = 3,
     wopQueryPathHash,
     wopQueryReferences,
     wopQueryReferrers,
@@ -28,6 +30,8 @@ typedef enum {
     wopCollectGarbage,
     wopExportPath,
     wopImportPath,
+    wopQueryDeriver,
+    wopSetOptions,
 } WorkerOp;
 
 
@@ -38,9 +42,12 @@ typedef enum {
 #define STDERR_ERROR 0x63787470
 
 
-/* The default location of the daemon socket, relative to
-   nixStateDir. */
-#define DEFAULT_SOCKET_PATH "/daemon.socket"
+/* The default location of the daemon socket, relative to nixStateDir.
+   The socket is in a directory to allow you to control access to the
+   Nix daemon by setting the mode/ownership of the directory
+   appropriately.  (This wouldn't work on the socket itself since it
+   must be deleted and recreated on startup.) */
+#define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
 
 
 Path readStorePath(Source & from);

src/libutil/types.hh

@@ -19,23 +19,18 @@ using std::vector;
 using boost::format;
 
 
-class Error : public std::exception
+/* BaseError should generally not be caught, as it has Interrupted as
+   a subclass. Catch Error instead. */
+class BaseError : public std::exception 
 {
 protected:
     string err;
 public:
-    Error(const format & f);
-    ~Error() throw () { };
+    BaseError(const format & f);
+    ~BaseError() throw () { };
     const char * what() const throw () { return err.c_str(); }
     const string & msg() const throw () { return err; }
-    Error & addPrefix(const format & f);
-};
-
-class SysError : public Error
-{
-public:
-    int errNo;
-    SysError(const format & f);
+    BaseError & addPrefix(const format & f);
 };
 
 #define MakeError(newClass, superClass) \
@@ -45,6 +40,15 @@ public:
         newClass(const format & f) : superClass(f) { }; \
     };
 
+MakeError(Error, BaseError)
+
+class SysError : public Error
+{
+public:
+    int errNo;
+    SysError(const format & f);
+};
+
 
 typedef list<string> Strings;
 typedef set<string> StringSet;

src/libutil/util.cc

@@ -23,13 +23,13 @@ extern char * * environ;
 namespace nix {
 
 
-Error::Error(const format & f)
+BaseError::BaseError(const format & f)
 {
     err = f.str();
 }
 
 
-Error & Error::addPrefix(const format & f)
+BaseError & BaseError::addPrefix(const format & f)
 {
     err = f.str() + err;
     return *this;
@@ -154,6 +154,7 @@ bool pathExists(const Path & path)
 
 Path readLink(const Path & path)
 {
+    checkInterrupt();
     struct stat st;
     if (lstat(path.c_str(), &st))
         throw SysError(format("getting status of `%1%'") % path);
@@ -444,7 +445,11 @@ void warnOnce(bool & haveWarned, const format & f)
 
 static void defaultWriteToStderr(const unsigned char * buf, size_t count)
 {
-    writeFull(STDERR_FILENO, buf, count);
+    try {
+        writeFull(STDERR_FILENO, buf, count);
+    } catch (SysError & e) {
+        /* ignore EPIPE etc. */
+    }
 }
 
 
@@ -487,6 +492,7 @@ string drainFD(int fd)
     string result;
     unsigned char buffer[4096];
     while (1) {
+        checkInterrupt();
         ssize_t rd = read(fd, buffer, sizeof buffer);
         if (rd == -1) {
             if (errno != EINTR)
@@ -545,8 +551,8 @@ AutoCloseFD::~AutoCloseFD()
 {
     try {
         close();
-    } catch (Error & e) {
-        printMsg(lvlError, format("error (ignored): %1%") % e.msg());
+    } catch (...) {
+        ignoreException();
     }
 }
 
@@ -771,6 +777,8 @@ void killUser(uid_t uid)
 
 string runProgram(Path program, bool searchPath, const Strings & args)
 {
+    checkInterrupt();
+    
     /* Create a pipe. */
     Pipe pipe;
     pipe.create();
@@ -968,5 +976,15 @@ bool string2Int(const string & s, int & n)
     return str && str.get() == EOF;
 }
 
+
+void ignoreException()
+{
+    try {
+        throw;
+    } catch (std::exception & e) {
+        printMsg(lvlError, format("error (ignored): %1%") % e.what());
+    }
+}
+
  
 }

src/libutil/util.hh

@@ -256,7 +256,7 @@ void inline checkInterrupt()
     if (_isInterrupted) _interrupted();
 }
 
-MakeError(Interrupted, Error)
+MakeError(Interrupted, BaseError)
 
 
 /* String packing / unpacking. */
@@ -280,6 +280,11 @@ string int2String(int n);
 bool string2Int(const string & s, int & n);
 
 
+/* Exception handling in destructors: print an error message, then
+   ignore the exception. */
+void ignoreException();
+
+
 }
 
 

src/nix-env/help.txt

@@ -45,8 +45,14 @@ Upgrade flags:
   --eq: "upgrade" if the current version is equal
   --always: upgrade regardless of current version
 
-Query types:
+Query sources:
 
+  --installed: use installed derivations (default)
+  --available / -a: use derivations available in Nix expression
+
+Query flags:
+
+  --xml: show output in XML format
   --status / -s: print installed/present status
   --no-name: hide derivation names
   --attr / -A: shows the unambiguous attribute name of the
@@ -55,11 +61,10 @@ Query types:
   --compare-versions / -c: compare version to available or installed
   --drv-path: print path of derivation
   --out-path: print path of derivation output
-
-Query sources:
-
-  --installed: use installed derivations (default)
-  --available / -a: use derivations available in Nix expression
+  --description: print description
+  --meta: print all meta attributes (only with --xml)
+  --prebuilt-only: only show derivations whose prebuilt binaries are
+      available on this machine or are downloadable
 
 Options:
 

src/nix-env/nix-env.cc

@@ -22,6 +22,8 @@
 #include <iostream>
 #include <sstream>
 
+#include <sys/types.h>
+#include <sys/stat.h>
 #include <unistd.h>
 
 
@@ -63,7 +65,7 @@ struct Globals
 
 
 typedef void (* Operation) (Globals & globals,
-    Strings opFlags, Strings opArgs);
+    Strings args, Strings opFlags, Strings opArgs);
 
 
 void printHelp()
@@ -72,11 +74,86 @@ void printHelp()
 }
 
 
+static string needArg(Strings::iterator & i,
+    Strings & args, const string & arg)
+{
+    if (i == args.end()) throw UsageError(
+        format("`%1%' requires an argument") % arg);
+    return *i++;
+}
+
+
+static bool parseInstallSourceOptions(Globals & globals,
+    Strings::iterator & i, Strings & args, const string & arg)
+{
+    if (arg == "--from-expression" || arg == "-E")
+        globals.instSource.type = srcNixExprs;
+    else if (arg == "--from-profile") {
+        globals.instSource.type = srcProfile;
+        globals.instSource.profile = needArg(i, args, arg);
+    }
+    else if (arg == "--attr" || arg == "-A")
+        globals.instSource.type = srcAttrPath;
+    else return false;
+    return true;
+}
+
+
+static bool isNixExpr(const Path & path)
+{
+    struct stat st;
+    if (stat(path.c_str(), &st) == -1)
+        throw SysError(format("getting information about `%1%'") % path);
+
+    return !S_ISDIR(st.st_mode) || pathExists(path + "/default.nix");
+}
+
+
+static void getAllExprs(EvalState & state,
+    const Path & path, ATermMap & attrs)
+{
+    Strings names = readDirectory(path);
+
+    for (Strings::iterator i = names.begin(); i != names.end(); ++i) {
+        Path path2 = path + "/" + *i;
+        
+        struct stat st;
+        if (stat(path2.c_str(), &st) == -1)
+            continue; // ignore dangling symlinks in ~/.nix-defexpr
+        
+        if (isNixExpr(path2))
+            attrs.set(toATerm(*i), makeAttrRHS(
+                    parseExprFromFile(state, absPath(path2)), makeNoPos()));
+        else
+            getAllExprs(state, path2, attrs);
+    }
+}
+
+
+static Expr loadSourceExpr(EvalState & state, const Path & path)
+{
+    if (isNixExpr(path)) return parseExprFromFile(state, absPath(path));
+
+    /* The path is a directory.  Put the Nix expressions in the
+       directory in an attribute set, with the file name of each
+       expression as the attribute name.  Recurse into subdirectories
+       (but keep the attribute set flat, not nested, to make it easier
+       for a user to have a ~/.nix-defexpr directory that includes
+       some system-wide directory). */
+    ATermMap attrs;
+    attrs.set(toATerm("_combineChannels"), makeAttrRHS(eTrue, makeNoPos()));
+    getAllExprs(state, path, attrs);
+    return makeAttrs(attrs);
+}
+
+
 static void loadDerivations(EvalState & state, Path nixExprPath,
-    string systemFilter, const ATermMap & autoArgs, DrvInfos & elems)
+    string systemFilter, const ATermMap & autoArgs,
+    const string & pathPrefix, DrvInfos & elems)
 {
     getDerivations(state,
-        parseExprFromFile(state, absPath(nixExprPath)), "", autoArgs, elems);
+        findAlongAttrPath(state, pathPrefix, autoArgs, loadSourceExpr(state, nixExprPath)),
+        pathPrefix, autoArgs, elems);
 
     /* Filter out all derivations not applicable to the current
        system. */
@@ -232,6 +309,16 @@ static void createUserEnv(EvalState & state, const DrvInfos & elems,
 }
 
 
+static int comparePriorities(EvalState & state,
+    const DrvInfo & drv1, const DrvInfo & drv2)
+{
+    int prio1, prio2;
+    if (!string2Int(drv1.queryMetaInfo(state, "priority"), prio1)) prio1 = 0;
+    if (!string2Int(drv2.queryMetaInfo(state, "priority"), prio2)) prio2 = 0;
+    return prio2 - prio1; /* higher number = lower priority, so negate */
+}
+
+
 static DrvInfos filterBySelector(EvalState & state,
     const DrvInfos & allElems,
     const Strings & args, bool newestOnly)
@@ -258,8 +345,10 @@ static DrvInfos filterBySelector(EvalState & state,
         }
 
         /* If `newestOnly', if a selector matches multiple derivations
-           with the same name, pick the one with the highest version.
-           If there are multiple derivations with the same name *and*
+           with the same name, pick the one with the highest priority.
+           If there are multiple derivations with the same priority,
+           pick the one with the highest version.  If there are
+           multiple derivations with the same priority and name and
            version, then pick the first one. */
         if (newestOnly) {
 
@@ -270,13 +359,22 @@ static DrvInfos filterBySelector(EvalState & state,
 
             for (Matches::iterator j = matches.begin(); j != matches.end(); ++j) {
                 DrvName drvName(j->first.name);
+                int d = 1;
+
                 Newest::iterator k = newest.find(drvName.name);
+                
                 if (k != newest.end()) {
-                    int d = compareVersions(drvName.version, DrvName(k->second.first.name).version);
-                    if (d > 0) newest[drvName.name] = *j;
-                    else if (d == 0) multiple.insert(j->first.name);
-                } else
+                    d = comparePriorities(state, j->first, k->second.first);
+                    if (d == 0)
+                        d = compareVersions(drvName.version, DrvName(k->second.first.name).version);
+                }
+
+                if (d > 0) {
                     newest[drvName.name] = *j;
+                    multiple.erase(j->first.name);
+                } else if (d == 0) {
+                    multiple.insert(j->first.name);
+                }
             }
 
             matches.clear();
@@ -329,7 +427,7 @@ static void queryInstSources(EvalState & state,
                Nix expression. */
             DrvInfos allElems;
             loadDerivations(state, instSource.nixExprPath,
-                instSource.systemFilter, instSource.autoArgs, allElems);
+                instSource.systemFilter, instSource.autoArgs, "", allElems);
 
             elems = filterBySelector(state, allElems, args, newestOnly);
     
@@ -344,9 +442,7 @@ static void queryInstSources(EvalState & state,
            (import ./foo.nix)' = `(import ./foo.nix).bar'. */
         case srcNixExprs: {
                 
-
-            Expr e1 = parseExprFromFile(state,
-                absPath(instSource.nixExprPath));
+            Expr e1 = loadSourceExpr(state, instSource.nixExprPath);
 
             for (Strings::const_iterator i = args.begin();
                  i != args.end(); ++i)
@@ -408,7 +504,7 @@ static void queryInstSources(EvalState & state,
                  i != args.end(); ++i)
                 getDerivations(state,
                     findAlongAttrPath(state, *i, instSource.autoArgs,
-                        parseExprFromFile(state, instSource.nixExprPath)),
+                        loadSourceExpr(state, instSource.nixExprPath)),
                     "", instSource.autoArgs, elems);
             break;
         }
@@ -482,8 +578,10 @@ static void installDerivations(Globals & globals,
          i != installedElems.end(); ++i)
     {
         DrvName drvName(i->name);
+        MetaInfo meta = i->queryMetaInfo(globals.state);
         if (!globals.preserveInstalled &&
-            newNames.find(drvName.name) != newNames.end())
+            newNames.find(drvName.name) != newNames.end() &&
+            meta["keep"] != "true")
             printMsg(lvlInfo,
                 format("replacing old `%1%'") % i->name);
         else
@@ -505,10 +603,15 @@ static void installDerivations(Globals & globals,
 
 
 static void opInstall(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
-    if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag `%1%'") % opFlags.front());
+    for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) {
+        string arg = *i++;
+        if (parseInstallSourceOptions(globals, i, opFlags, arg)) ;
+        else if (arg == "--preserve-installed" || arg == "-P")
+            globals.preserveInstalled = true;
+        else throw UsageError(format("unknown flag `%1%'") % arg);
+    }
 
     installDerivations(globals, opArgs, globals.profile);
 }
@@ -543,10 +646,17 @@ static void upgradeDerivations(Globals & globals,
     {
         DrvName drvName(i->name);
 
+        MetaInfo meta = i->queryMetaInfo(globals.state);
+        if (meta["keep"] == "true") {
+            newElems.push_back(*i);
+            continue;
+        }
+
         /* Find the derivation in the input Nix expression with the
-           same name and satisfying the version constraints specified
+           same name that satisfies the version constraints specified
            by upgradeType.  If there are multiple matches, take the
-           one with highest version. */
+           one with the highest priority.  If there are still multiple
+           matches, take the one with the highest version. */
         DrvInfos::iterator bestElem = availElems.end();
         DrvName bestName;
         for (DrvInfos::iterator j = availElems.begin();
@@ -554,16 +664,19 @@ static void upgradeDerivations(Globals & globals,
         {
             DrvName newName(j->name);
             if (newName.name == drvName.name) {
-                int d = compareVersions(drvName.version, newName.version);
+                int d = comparePriorities(globals.state, *i, *j);
+                if (d == 0) d = compareVersions(drvName.version, newName.version);
                 if (upgradeType == utLt && d < 0 ||
                     upgradeType == utLeq && d <= 0 ||
                     upgradeType == utEq && d == 0 ||
                     upgradeType == utAlways)
                 {
-                    if ((bestElem == availElems.end() ||
-                         compareVersions(
-                             bestName.version, newName.version) < 0))
-                    {
+                    int d2 = -1;
+                    if (bestElem != availElems.end()) {
+                        d2 = comparePriorities(globals.state, *bestElem, *j);
+                        if (d2 == 0) d2 = compareVersions(bestName.version, newName.version);
+                    }
+                    if (d2 < 0) {
                         bestElem = j;
                         bestName = newName;
                     }
@@ -593,16 +706,18 @@ static void upgradeDerivations(Globals & globals,
 
 
 static void opUpgrade(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     UpgradeType upgradeType = utLt;
-    for (Strings::iterator i = opFlags.begin();
-         i != opFlags.end(); ++i)
-        if (*i == "--lt") upgradeType = utLt;
-        else if (*i == "--leq") upgradeType = utLeq;
-        else if (*i == "--eq") upgradeType = utEq;
-        else if (*i == "--always") upgradeType = utAlways;
-        else throw UsageError(format("unknown flag `%1%'") % *i);
+    for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) {
+        string arg = *i++;
+        if (parseInstallSourceOptions(globals, i, opFlags, arg)) ;
+        else if (arg == "--lt") upgradeType = utLt;
+        else if (arg == "--leq") upgradeType = utLeq;
+        else if (arg == "--eq") upgradeType = utEq;
+        else if (arg == "--always") upgradeType = utAlways;
+        else throw UsageError(format("unknown flag `%1%'") % arg);
+    }
 
     upgradeDerivations(globals, opArgs, upgradeType);
 }
@@ -618,7 +733,7 @@ static void setMetaFlag(EvalState & state, DrvInfo & drv,
 
 
 static void opSetFlag(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag `%1%'") % opFlags.front());
@@ -657,10 +772,13 @@ static void opSetFlag(Globals & globals,
 
 
 static void opSet(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
-    if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag `%1%'") % opFlags.front());
+    for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) {
+        string arg = *i++;
+        if (parseInstallSourceOptions(globals, i, opFlags, arg)) ;
+        else throw UsageError(format("unknown flag `%1%'") % arg);
+    }
 
     DrvInfos elems;
     queryInstSources(globals.state, globals.instSource, opArgs, elems, true);
@@ -713,7 +831,7 @@ static void uninstallDerivations(Globals & globals, DrvNames & selectors,
 
 
 static void opUninstall(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag `%1%'") % opFlags.front());
@@ -820,9 +938,11 @@ static string colorString(const string & s)
 
 
 static void opQuery(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     typedef vector< map<string, string> > ResultSet;
+    Strings remaining;
+    string attrPath;
         
     bool printStatus = false;
     bool printName = true;
@@ -831,6 +951,8 @@ static void opQuery(Globals & globals,
     bool printDrvPath = false;
     bool printOutPath = false;
     bool printDescription = false;
+    bool printMeta = false;
+    bool prebuiltOnly = false;
     bool compareVersions = false;
     bool xmlOutput = false;
 
@@ -838,43 +960,46 @@ static void opQuery(Globals & globals,
 
     readOnlyMode = true; /* makes evaluation a bit faster */
 
-    for (Strings::iterator i = opFlags.begin();
-         i != opFlags.end(); ++i)
-        if (*i == "--status" || *i == "-s") printStatus = true;
-        else if (*i == "--no-name") printName = false;
-        else if (*i == "--system") printSystem = true;
-        else if (*i == "--description") printDescription = true;
-        else if (*i == "--compare-versions" || *i == "-c") compareVersions = true;
-        else if (*i == "--drv-path") printDrvPath = true;
-        else if (*i == "--out-path") printOutPath = true;
-        else if (*i == "--installed") source = sInstalled;
-        else if (*i == "--available" || *i == "-a") source = sAvailable;
-        else if (*i == "--xml") xmlOutput = true;
-        else throw UsageError(format("unknown flag `%1%'") % *i);
-
-    if (globals.instSource.type == srcAttrPath) printAttrPath = true; /* hack */
-
-    if (opArgs.size() == 0) {
-        printMsg(lvlInfo, "warning: you probably meant to specify the argument '*' to show all packages");
+    for (Strings::iterator i = args.begin(); i != args.end(); ) {
+        string arg = *i++;
+        if (arg == "--status" || arg == "-s") printStatus = true;
+        else if (arg == "--no-name") printName = false;
+        else if (arg == "--system") printSystem = true;
+        else if (arg == "--description") printDescription = true;
+        else if (arg == "--compare-versions" || arg == "-c") compareVersions = true;
+        else if (arg == "--drv-path") printDrvPath = true;
+        else if (arg == "--out-path") printOutPath = true;
+        else if (arg == "--meta") printMeta = true;
+        else if (arg == "--installed") source = sInstalled;
+        else if (arg == "--available" || arg == "-a") source = sAvailable;
+        else if (arg == "--prebuilt-only" || arg == "-b") prebuiltOnly = true;
+        else if (arg == "--xml") xmlOutput = true;
+        else if (arg == "--attr-path" || arg == "-P") printAttrPath = true;
+        else if (arg == "--attr" || arg == "-A")
+            attrPath = needArg(i, args, arg);
+        else if (arg[0] == '-')
+            throw UsageError(format("unknown flag `%1%'") % arg);
+        else remaining.push_back(arg);
     }
 
+    if (remaining.size() == 0)
+        printMsg(lvlInfo, "warning: you probably meant to specify the argument '*' to show all packages");
+
     
     /* Obtain derivation information from the specified source. */
     DrvInfos availElems, installedElems;
 
-    if (source == sInstalled || compareVersions || printStatus) {
+    if (source == sInstalled || compareVersions || printStatus)
         installedElems = queryInstalled(globals.state, globals.profile);
-    }
 
-    if (source == sAvailable || compareVersions) {
+    if (source == sAvailable || compareVersions)
         loadDerivations(globals.state, globals.instSource.nixExprPath,
             globals.instSource.systemFilter, globals.instSource.autoArgs,
-            availElems);
-    }
+            attrPath, availElems);
 
     DrvInfos elems = filterBySelector(globals.state,
         source == sInstalled ? installedElems : availElems,
-        opArgs, false);
+        remaining, false);
     
     DrvInfos & otherElems(source == sInstalled ? availElems : installedElems);
 
@@ -914,6 +1039,12 @@ static void opQuery(Globals & globals,
 
             /* For XML output. */
             XMLAttrs attrs;
+
+            if (prebuiltOnly) {
+                if (!store->isValidPath(i->queryOutPath(globals.state)) &&
+                    !store->hasSubstitutes(i->queryOutPath(globals.state)))
+                    continue;
+            }
         
             if (printStatus) {
                 bool hasSubs = store->hasSubstitutes(i->queryOutPath(globals.state));
@@ -1001,14 +1132,25 @@ static void opQuery(Globals & globals,
             }
 
             if (xmlOutput)
-                xml.writeEmptyElement("item", attrs);
+                if (printMeta) {
+                    XMLOpenElement item(xml, "item", attrs);
+                    MetaInfo meta = i->queryMetaInfo(globals.state);
+                    for (MetaInfo::iterator j = meta.begin(); j != meta.end(); ++j) {
+                        XMLAttrs attrs2;
+                        attrs2["name"] = j->first;
+                        attrs2["value"] = j->second;
+                        xml.writeEmptyElement("meta", attrs2);
+                    }
+                }
+                else
+                    xml.writeEmptyElement("item", attrs);
             else
                 table.push_back(columns);
 
             cout.flush();
 
         } catch (AssertionError & e) {
-            /* !!! hm, maybe we should give some sort of warning here? */
+            printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name);
         }
     }
 
@@ -1017,7 +1159,7 @@ static void opQuery(Globals & globals,
 
 
 static void opSwitchProfile(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag `%1%'") % opFlags.front());
@@ -1065,7 +1207,7 @@ static void switchGeneration(Globals & globals, int dstGen)
 
 
 static void opSwitchGeneration(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag `%1%'") % opFlags.front());
@@ -1081,7 +1223,7 @@ static void opSwitchGeneration(Globals & globals,
 
 
 static void opRollback(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag `%1%'") % opFlags.front());
@@ -1093,7 +1235,7 @@ static void opRollback(Globals & globals,
 
 
 static void opListGenerations(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag `%1%'") % opFlags.front());
@@ -1126,7 +1268,7 @@ static void deleteGeneration2(const Path & profile, unsigned int gen)
 
 
 static void opDeleteGenerations(Globals & globals,
-    Strings opFlags, Strings opArgs)
+    Strings args, Strings opFlags, Strings opArgs)
 {
     if (opFlags.size() > 0)
         throw UsageError(format("unknown flag `%1%'") % opFlags.front());
@@ -1164,30 +1306,9 @@ static void opDeleteGenerations(Globals & globals,
 }
 
 
-static void opDefaultExpr(Globals & globals,
-    Strings opFlags, Strings opArgs)
-{
-    if (opFlags.size() > 0)
-        throw UsageError(format("unknown flag `%1%'") % opFlags.front());
-    if (opArgs.size() != 1)
-        throw UsageError(format("exactly one argument expected"));
-
-    switchLink(getDefNixExprPath(), absPath(opArgs.front()));
-}
-
-
-static string needArg(Strings::iterator & i,
-    Strings & args, const string & arg)
-{
-    if (i == args.end()) throw UsageError(
-        format("`%1%' requires an argument") % arg);
-    return *i++;
-}
-
-
 void run(Strings args)
 {
-    Strings opFlags, opArgs;
+    Strings opFlags, opArgs, remaining;
     Operation op = 0;
     
     Globals globals;
@@ -1209,14 +1330,6 @@ void run(Strings args)
 
         if (arg == "--install" || arg == "-i")
             op = opInstall;
-        else if (arg == "--from-expression" || arg == "-E")
-            globals.instSource.type = srcNixExprs;
-        else if (arg == "--from-profile") {
-            globals.instSource.type = srcProfile;
-            globals.instSource.profile = needArg(i, args, arg);
-        }
-        else if (arg == "--attr" || arg == "-A")
-            globals.instSource.type = srcAttrPath;
         else if (parseOptionArg(arg, i, args.end(),
                      globals.state, globals.instSource.autoArgs))
             ;
@@ -1232,8 +1345,6 @@ void run(Strings args)
             op = opSet;
         else if (arg == "--query" || arg == "-q")
             op = opQuery;
-        else if (arg == "--import" || arg == "-I") /* !!! bad name */
-            op = opDefaultExpr;
         else if (arg == "--profile" || arg == "-p")
             globals.profile = absPath(needArg(i, args, arg));
         else if (arg == "--file" || arg == "-f")
@@ -1252,14 +1363,15 @@ void run(Strings args)
             printMsg(lvlInfo, "(dry run; not doing anything)");
             globals.dryRun = true;
         }
-        else if (arg == "--preserve-installed" || arg == "-P")
-            globals.preserveInstalled = true;
         else if (arg == "--system-filter")
             globals.instSource.systemFilter = needArg(i, args, arg);
-        else if (arg[0] == '-')
-            opFlags.push_back(arg);
-        else
-            opArgs.push_back(arg);
+        else {
+            remaining.push_back(arg);
+            if (arg[0] == '-') 
+                opFlags.push_back(arg);
+            else
+                opArgs.push_back(arg);
+        }
 
         if (oldOp && oldOp != op)
             throw UsageError("only one operation may be specified");
@@ -1276,7 +1388,7 @@ void run(Strings args)
     
     store = openStore();
 
-    op(globals, opFlags, opArgs);
+    op(globals, remaining, opFlags, opArgs);
 
     printEvalStats(globals.state);
 }

src/nix-setuid-helper/nix-setuid-helper.cc

@@ -36,7 +36,7 @@ static void secureChown(uid_t uidFrom, gid_t gidFrom,
            to. */
         throw Error(error);
 
-    if (uidFrom != -1) {
+    if (uidFrom != (uid_t) -1) {
         assert(uidFrom != 0);
         if (st.st_uid != uidFrom)
             throw Error(error);

src/nix-store/help.txt

@@ -11,8 +11,6 @@ Operations:
   --query / -q: query information
   --read-log / -l: print build log of given store paths
 
-  --register-substitutes: register a substitute expression (dangerous!)
-  --clear-substitutes: clear all substitutes
   --register-validity: register path validity (dangerous!)
   --check-validity: check path validity
 
@@ -23,6 +21,7 @@ Operations:
 
   --init: initialise the Nix database
   --verify: verify Nix structures
+  --optimise: optimise the Nix store by hard-linking identical files
 
   --version: output version information
   --help: display help

src/nix-store/nix-store.cc

@@ -46,7 +46,7 @@ static Path fixPath(Path path)
 static Path useDeriver(Path path)
 {       
     if (!isDerivation(path)) {
-        path = queryDeriver(noTxn, path);
+        path = store->queryDeriver(path);
         if (path == "")
             throw Error(format("deriver of path `%1%' is not known") % path);
     }
@@ -330,7 +330,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
             for (Strings::iterator i = opArgs.begin();
                  i != opArgs.end(); ++i)
             {
-                Path deriver = queryDeriver(noTxn, fixPath(*i));
+                Path deriver = store->queryDeriver(fixPath(*i));
                 cout << format("%1%\n") %
                     (deriver == "" ? "unknown-deriver" : deriver);
             }
@@ -413,54 +413,6 @@ static void opReadLog(Strings opFlags, Strings opArgs)
 }
 
 
-static void opRegisterSubstitutes(Strings opFlags, Strings opArgs)
-{
-    if (!opFlags.empty()) throw UsageError("unknown flag");
-    if (!opArgs.empty()) throw UsageError("no arguments expected");
-
-    Transaction txn;
-    createStoreTransaction(txn);
-
-    while (1) {
-        Path srcPath;
-        Substitute sub;
-        PathSet references;
-        getline(cin, srcPath);
-        if (cin.eof()) break;
-        getline(cin, sub.deriver);
-        getline(cin, sub.program);
-        string s; int n;
-        getline(cin, s);
-        if (!string2Int(s, n)) throw Error("number expected");
-        while (n--) {
-            getline(cin, s);
-            sub.args.push_back(s);
-        }
-        getline(cin, s);
-        if (!string2Int(s, n)) throw Error("number expected");
-        while (n--) {
-            getline(cin, s);
-            references.insert(s);
-        }
-        if (!cin || cin.eof()) throw Error("missing input");
-        registerSubstitute(txn, srcPath, sub);
-        setReferences(txn, srcPath, references);
-    }
-
-    txn.commit();
-}
-
-
-static void opClearSubstitutes(Strings opFlags, Strings opArgs)
-{
-    if (!opFlags.empty()) throw UsageError("unknown flag");
-    if (!opArgs.empty())
-        throw UsageError("no arguments expected");
-
-    clearSubstitutes();
-}
-
-
 static void opRegisterValidity(Strings opFlags, Strings opArgs)
 {
     bool reregister = false; // !!! maybe this should be the default
@@ -475,18 +427,8 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs)
     ValidPathInfos infos;
     
     while (1) {
-        ValidPathInfo info;
-        getline(cin, info.path);
-        if (cin.eof()) break;
-        getline(cin, info.deriver);
-        string s; int n;
-        getline(cin, s);
-        if (!string2Int(s, n)) throw Error("number expected");
-        while (n--) {
-            getline(cin, s);
-            info.references.insert(s);
-        }
-        if (!cin || cin.eof()) throw Error("missing input");
+        ValidPathInfo info = decodeValidPathInfo(cin);
+        if (info.path == "") break;
         if (!store->isValidPath(info.path) || reregister) {
             /* !!! races */
             canonicalisePathMetaData(info.path);
@@ -524,6 +466,13 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
 }
 
 
+static string showBytes(unsigned long long bytes)
+{
+    return (format("%d bytes (%.2f MiB)")
+        % bytes % (bytes / (1024.0 * 1024.0))).str();
+}
+
+
 struct PrintFreed 
 {
     bool show, dryRun;
@@ -535,9 +484,9 @@ struct PrintFreed
         if (show)
             cout << format(
                 (dryRun
-                    ? "%d bytes would be freed (%.2f MiB)\n"
-                    : "%d bytes freed (%.2f MiB)\n"))
-                % bytesFreed % (bytesFreed / (1024.0 * 1024.0));
+                    ? "%1% would be freed\n"
+                    : "%1% freed\n"))
+                % showBytes(bytesFreed);
     }
 };
 
@@ -672,6 +621,46 @@ static void opVerify(Strings opFlags, Strings opArgs)
 }
 
 
+
+static void showOptimiseStats(OptimiseStats & stats)
+{
+    printMsg(lvlError,
+        format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total")
+        % showBytes(stats.bytesFreed)
+        % stats.filesLinked
+        % stats.sameContents
+        % stats.totalFiles);
+}
+
+
+/* Optimise the disk space usage of the Nix store by hard-linking
+   files with the same contents. */
+static void opOptimise(Strings opFlags, Strings opArgs)
+{
+    if (!opArgs.empty())
+        throw UsageError("no arguments expected");
+
+    bool dryRun = false;
+
+    for (Strings::iterator i = opFlags.begin();
+         i != opFlags.end(); ++i)
+        if (*i == "--dry-run") dryRun = true;
+        else throw UsageError(format("unknown flag `%1%'") % *i);
+
+    LocalStore * store2(dynamic_cast<LocalStore *>(store.get()));
+    if (!store2) throw Error("you don't have sufficient rights to use --optimise");
+
+    OptimiseStats stats;
+    try {
+        store2->optimiseStore(dryRun, stats);
+    } catch (...) {
+        showOptimiseStats(stats);
+        throw;
+    }
+    showOptimiseStats(stats);
+}
+
+
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
@@ -699,10 +688,6 @@ void run(Strings args)
             op = opQuery;
         else if (arg == "--read-log" || arg == "-l")
             op = opReadLog;
-        else if (arg == "--register-substitutes")
-            op = opRegisterSubstitutes;
-        else if (arg == "--clear-substitutes")
-            op = opClearSubstitutes;
         else if (arg == "--register-validity")
             op = opRegisterValidity;
         else if (arg == "--check-validity")
@@ -721,6 +706,8 @@ void run(Strings args)
             op = opInit;
         else if (arg == "--verify")
             op = opVerify;
+        else if (arg == "--optimise")
+            op = opOptimise;
         else if (arg == "--add-root") {
             if (i == args.end())
                 throw UsageError("`--add-root requires an argument");

src/nix-worker/nix-worker.cc

@@ -277,6 +277,15 @@ static void performOp(Source & from, Sink & to, unsigned int op)
         break;
     }
 
+    case wopQueryDeriver: {
+        Path path = readStorePath(from);
+        startWork();
+        Path deriver = store->queryDeriver(path);
+        stopWork();
+        writeString(deriver, to);
+        break;
+    }
+
     case wopAddToStore: {
         /* !!! uberquick hack */
         string baseName = readString(from);
@@ -405,6 +414,19 @@ static void performOp(Source & from, Sink & to, unsigned int op)
         
         break;
     }
+
+    case wopSetOptions: {
+        keepFailed = readInt(from) != 0;
+        keepGoing = readInt(from) != 0;
+        tryFallback = readInt(from) != 0;
+        verbosity = (Verbosity) readInt(from);
+        maxBuildJobs = readInt(from);
+        maxSilentTime = readInt(from);
+        startWork();
+        stopWork();
+        break;
+    }
+            
             
     default:
         throw Error(format("invalid operation %1%") % op);
@@ -428,14 +450,19 @@ static void processConnection()
     /* Exchange the greeting. */
     unsigned int magic = readInt(from);
     if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
-    verbosity = (Verbosity) readInt(from);
     writeInt(WORKER_MAGIC_2, to);
 
+    writeInt(PROTOCOL_VERSION, to);
+    unsigned int clientVersion = readInt(from);
+
     /* Send startup error messages to the client. */
     startWork();
 
     try {
 
+        /* If we can't accept clientVersion, then throw an error
+           *here* (not above). */
+
         /* Prevent users from doing something very dangerous. */
         if (geteuid() == 0 &&
             querySetting("build-users-group", "") == "")
@@ -508,6 +535,8 @@ static void daemonLoop()
 
     string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
 
+    createDirs(dirOf(socketPath));
+
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
     if (socketPath.size() >= sizeof(addr.sun_path))
@@ -517,7 +546,8 @@ static void daemonLoop()
     unlink(socketPath.c_str());
 
     /* Make sure that the socket is created with 0666 permission
-       (everybody can connect). */
+       (everybody can connect --- provided they have access to the
+       directory containing the socket). */
     mode_t oldMode = umask(0111);
     int res = bind(fdSocket, (struct sockaddr *) &addr, sizeof(addr));
     umask(oldMode);

substitute.mk

@@ -17,6 +17,7 @@
 	 -e "s^@perl\@^$(perl)^g" \
 	 -e "s^@coreutils\@^$(coreutils)^g" \
 	 -e "s^@tar\@^$(tar)^g" \
+	 -e "s^@tr\@^$(tr)^g" \
 	 -e "s^@dot\@^$(dot)^g" \
 	 -e "s^@xmllint\@^$(xmllint)^g" \
 	 -e "s^@xmlflags\@^$(xmlflags)^g" \

tests/Makefile.am

@@ -2,14 +2,11 @@ TESTS_ENVIRONMENT = $(SHELL) -e
 
 extra1 = $(shell pwd)/test-tmp/shared
 
-simple.sh: simple.nix
+simple.sh substitutes.sh substitutes2.sh fallback.sh: simple.nix
 dependencies.sh gc.sh nix-push.sh nix-pull.in logging.sh nix-build.sh install-package.sh check-refs.sh: dependencies.nix
 locking.sh: locking.nix
 parallel.sh: parallel.nix
 build-hook.sh: build-hook.nix
-substitutes.sh: substitutes.nix
-substitutes2.sh: substitutes2.nix
-fallback.sh: fallback.nix
 gc-concurrent.sh: gc-concurrent.nix gc-concurrent2.nix
 user-envs.sh: user-envs.nix
 fixed.sh: fixed.nix
@@ -21,7 +18,8 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
   locking.sh parallel.sh build-hook.sh substitutes.sh substitutes2.sh \
   fallback.sh nix-push.sh gc.sh gc-concurrent.sh verify.sh nix-pull.sh \
   referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
-  gc-runtime.sh install-package.sh check-refs.sh filter-source.sh
+  gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
+  remote-store.sh
 
 XFAIL_TESTS =
 
@@ -36,11 +34,9 @@ EXTRA_DIST = $(TESTS) \
   locking.nix.in locking.builder.sh \
   parallel.nix.in parallel.builder.sh \
   build-hook.nix.in build-hook.hook.sh \
-  substitutes.nix.in substituter.sh \
-  substitutes2.nix.in substituter2.sh \
+  substituter.sh substituter2.sh \
   gc-concurrent.nix.in gc-concurrent.builder.sh \
   gc-concurrent2.nix.in gc-concurrent2.builder.sh \
-  fallback.nix.in \
   user-envs.nix.in user-envs.builder.sh \
   fixed.nix.in fixed.builder1.sh fixed.builder2.sh \
   gc-runtime.nix.in \

tests/common.sh.in

@@ -2,7 +2,7 @@ set -e
 
 export TEST_ROOT=$(pwd)/test-tmp
 export NIX_STORE_DIR
-if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store); then
+if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
     # Maybe the build directory is symlinked.
     export NIX_IGNORE_SYMLINK_STORE=1
     NIX_STORE_DIR=$TEST_ROOT/store
@@ -18,7 +18,9 @@ export NIX_LIBEXEC_DIR=$TEST_ROOT/bin
 export NIX_ROOT_FINDER=
 export SHARED=$TEST_ROOT/shared
 
-export NIX_REMOTE=
+if test -z "$FORCE_NIX_REMOTE"; then
+    export NIX_REMOTE=
+fi
 
 export REAL_BIN_DIR=@bindir@
 export REAL_LIBEXEC_DIR=@libexecdir@
@@ -45,6 +47,7 @@ export nixinstantiate=$TOP/src/nix-instantiate/nix-instantiate
 export nixstore=$TOP/src/nix-store/nix-store
 export nixenv=$TOP/src/nix-env/nix-env
 export nixhash=$TOP/src/nix-hash/nix-hash
+export nixworker=$TOP/src/nix-worker/nix-worker
 export nixbuild=$NIX_BIN_DIR/nix-build
 
 readLink() {
@@ -65,3 +68,7 @@ clearProfiles() {
     profiles="$NIX_STATE_DIR"/profiles
     rm -f $profiles/*
 }
+
+clearManifests() {
+    rm -f $NIX_STATE_DIR/manifests/*
+}

tests/fallback.nix.in

@@ -1,7 +0,0 @@
-derivation {
-  name = "fall-back";
-  system = "@system@";
-  builder = "@shell@";
-  args = ["-e" "-x" ./simple.builder.sh];
-  goodPath = "@testPath@";
-}

tests/fallback.sh

@@ -1,15 +1,19 @@
 source common.sh
 
-drvPath=$($nixinstantiate fallback.nix)
+clearStore
+
+drvPath=$($nixinstantiate simple.nix)
 echo "derivation is $drvPath"
 
 outPath=$($nixstore -q --fallback "$drvPath")
 echo "output path is $outPath"
 
-# Register a non-existant substitute
-(echo $outPath && echo "" && echo $TOP/no-such-program && echo 0 && echo 0) | $nixstore --register-substitutes
+# Build with a substitute that fails.  This should fail.
+export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh
+if $nixstore -r "$drvPath"; then echo unexpected fallback; exit 1; fi
 
-# Build the derivation
+# Build with a substitute that fails.  This should fall back to a source build.
+export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh
 $nixstore -r --fallback "$drvPath"
 
 text=$(cat "$outPath"/hello)

tests/fixed.builder1.sh

@@ -1 +1,3 @@
+if test "$IMPURE_VAR1" != "foo"; then exit 1; fi
+if test "$IMPURE_VAR2" != "bar"; then exit 1; fi
 echo "Hello World!" > $out

tests/fixed.builder2.sh

@@ -1,3 +1,5 @@
+echo dummy: $dummy
+if test -n "$dummy"; then sleep 2; fi
 mkdir $out
 mkdir $out/bla
 echo "Hello World!" > $out/foo

tests/fixed.nix.in

@@ -1,6 +1,6 @@
 rec {
 
-  f = builder: mode: algo: hash: derivation {
+  f2 = dummy: builder: mode: algo: hash: derivation {
     name = "fixed";
     system = "@system@";
     builder = "@shell@";
@@ -9,8 +9,12 @@ rec {
     outputHashAlgo = algo;
     outputHash = hash;
     PATH = "@testPath@";
+    inherit dummy;
+    impureEnvVars = ["IMPURE_VAR1" "IMPURE_VAR2"];
   };
 
+  f = f2 "";
+
   good = [
     (f ./fixed.builder1.sh "flat" "md5" "8ddd8be4b179a529afa5f2ffae4b9858")
     (f ./fixed.builder1.sh "flat" "sha1" "a0b65939670bc2c010f4d5d6a0b3e4e4590fb92b")
@@ -35,4 +39,11 @@ rec {
     (f ./fixed.builder1.sh "flat" "md5" "ddd8be4b179a529afa5f2ffae4b9858")
   ];
 
-}
+  # Test for building two derivations in parallel that produce the
+  # same output path because they're fixed-output derivations.
+  parallelSame = [
+    (f2 "foo" ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42")
+    (f2 "bar" ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42")
+  ];
+
+}

tests/fixed.sh

@@ -1,18 +1,36 @@
 source common.sh
 
+clearStore
+
+export IMPURE_VAR1=foo
+export IMPURE_VAR2=bar
+
+echo 'testing good...'
 drvs=$($nixinstantiate fixed.nix -A good)
 echo $drvs
 $nixstore -r $drvs
 
+echo 'testing good2...'
 drvs=$($nixinstantiate fixed.nix -A good2)
 echo $drvs
 $nixstore -r $drvs
 
+echo 'testing bad...'
 drvs=$($nixinstantiate fixed.nix -A bad)
 echo $drvs
 if $nixstore -r $drvs; then false; fi
 
+echo 'testing reallyBad...'
 if $nixinstantiate fixed.nix -A reallyBad; then false; fi
 
 # While we're at it, check attribute selection a bit more.
+echo 'testing attribute selection...'
 test $($nixinstantiate fixed.nix -A good.1 | wc -l) = 1
+
+# Test parallel builds of derivations that produce the same output.
+# Only one should run at the same time.
+echo 'testing parallelSame...'
+clearStore
+drvs=$($nixinstantiate fixed.nix -A parallelSame)
+echo $drvs
+$nixstore -r $drvs -j2

tests/init.sh

@@ -22,10 +22,12 @@ ln -s $nixstore $NIX_BIN_DIR/
 ln -s $nixinstantiate $NIX_BIN_DIR/
 ln -s $nixhash $NIX_BIN_DIR/
 ln -s $nixenv $NIX_BIN_DIR/
+ln -s $nixworker $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-prefetch-url $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-collect-garbage $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-build $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-install-package $NIX_BIN_DIR/
+ln -s $TOP/scripts/nix-push $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-pull $NIX_BIN_DIR/
 ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/
 ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/
@@ -56,7 +58,9 @@ for i in \
     $NIX_BIN_DIR/nix-collect-garbage \
     $NIX_BIN_DIR/nix-build \
     $NIX_BIN_DIR/nix-install-package \
+    $NIX_BIN_DIR/nix-push \
     $NIX_BIN_DIR/nix-pull \
+    $NIX_BIN_DIR/nix/readmanifest.pm \
     ; do
     sed < $i > $i.tmp \
         -e "s^$REAL_BIN_DIR^$NIX_BIN_DIR^" \

tests/lang/eval-okay-empty-args.exp

@@ -0,0 +1 @@
+Str("ab",[])

tests/lang/eval-okay-empty-args.nix

@@ -0,0 +1 @@
+({}: {x,y,}: "${x}${y}") {} {x = "a"; y = "b";}

tests/lang/eval-okay-listToAttrs.nix

@@ -0,0 +1,8 @@
+# this test shows how to use listToAttrs and that evaluation is still lazy (throw isn't called)
+let 
+  asi = attr: value : { inherit attr value; };
+  list = [ ( asi "a" "A" ) ( asi "b" "B" ) ];
+  a = builtins.listToAttrs list;
+  b = builtins.listToAttrs ( list ++ list );
+  r = builtins.listToAttrs [ (asi "result" [ a b ]) ( asi "throw" (throw "this should not be thrown")) ];
+in r.result

tests/lang/eval-okay-listToAttrs.out

@@ -0,0 +1 @@
+List([Attrs([Bind("a",Str("A",[]),NoPos),Bind("b",Str("B",[]),NoPos)]),Attrs([Bind("a",Str("A",[]),NoPos),Bind("b",Str("B",[]),NoPos)])])

tests/nix-pull.sh

@@ -2,10 +2,11 @@ source common.sh
 
 pullCache () {
     echo "pulling cache..."
-    $PERL -w -I$TOP/scripts $TOP/scripts/nix-pull file://$TEST_ROOT/manifest
+    $NIX_BIN_DIR/nix-pull file://$TEST_ROOT/manifest
 }
 
 clearStore
+clearManifests
 pullCache
 
 drvPath=$($nixinstantiate dependencies.nix)
@@ -17,6 +18,7 @@ $nixstore -r $outPath
 cat $outPath/input-2/bar
 
 clearStore
+clearManifests
 pullCache
 
 echo "building $drvPath using substitutes..."
@@ -28,4 +30,4 @@ cat $outPath/input-2/bar
 test $($nixstore -q --deriver "$outPath") = "$drvPath"
 $nixstore -q --deriver $(readLink $outPath/input-2) | grep -q -- "-input-2.drv" 
 
-$nixstore --clear-substitutes
+clearManifests

tests/nix-push.sh

@@ -7,5 +7,5 @@ echo "pushing $drvPath"
 
 mkdir $TEST_ROOT/cache
 
-$PERL -w -I$TOP/scripts $TOP/scripts/nix-push \
+$NIX_BIN_DIR/nix-push \
     --copy $TEST_ROOT/cache $TEST_ROOT/manifest $drvPath

tests/remote-store.sh

@@ -0,0 +1,17 @@
+source common.sh
+
+export FORCE_NIX_REMOTE=1
+
+echo '*** testing slave mode ***'
+clearStore
+clearManifests
+NIX_REMOTE=slave $SHELL ./user-envs.sh
+
+echo '*** testing daemon mode ***'
+clearStore
+clearManifests
+$nixworker --daemon &
+pidDaemon=$!
+NIX_REMOTE=daemon $SHELL ./user-envs.sh
+kill -9 $pidDaemon
+wait $pidDaemon || true

tests/simple.builder.sh

@@ -1,7 +1,7 @@
 echo "PATH=$PATH"
 
 # Verify that the PATH is empty.
-if mkdir foo; then exit 1; fi
+if mkdir foo 2> /dev/null; then exit 1; fi
 
 # Set a PATH (!!! impure).
 export PATH=$goodPath

tests/substituter.sh

@@ -1,10 +1,19 @@
-#! /bin/sh -ex
-echo $*
-
-case $* in
-    *)
-        mkdir $1
-        echo $3 $4 > $1/hello
-        ;;
-esac        
+#! /bin/sh -e
+echo substituter args: $* >&2
 
+if test $1 = "--query-paths"; then
+    cat $TEST_ROOT/sub-paths
+elif test $1 = "--query-info"; then
+    shift
+    for i in in $@; do
+        echo $i
+        echo "" # deriver
+        echo 0 # nr of refs
+    done
+elif test $1 = "--substitute"; then
+    mkdir $2
+    echo "Hallo Wereld" > $2/hello
+else
+    echo "unknown substituter operation"
+    exit 1
+fi

tests/substituter2.sh

@@ -1,3 +1,18 @@
-#! /bin/sh -ex
-echo $*
-exit 1
+#! /bin/sh -e
+echo substituter2 args: $* >&2
+
+if test $1 = "--query-paths"; then
+    cat $TEST_ROOT/sub-paths
+elif test $1 = "--query-info"; then
+    shift
+    for i in in $@; do
+        echo $i
+        echo "" # deriver
+        echo 0 # nr of refs
+    done
+elif test $1 = "--substitute"; then
+    exit 1
+else
+    echo "unknown substituter operation"
+    exit 1
+fi

tests/substitutes.nix.in

@@ -1,6 +0,0 @@
-derivation {
-  name = "substitutes";
-  system = "@system@";
-  builder = "@shell@";
-  args = ["-e" "-x" ./simple.builder.sh];
-}

tests/substitutes.sh

@@ -1,22 +1,20 @@
 source common.sh
 
+clearStore
+
 # Instantiate.
-drvPath=$($nixinstantiate substitutes.nix)
+drvPath=$($nixinstantiate simple.nix)
 echo "derivation is $drvPath"
 
 # Find the output path.
 outPath=$($nixstore -qvv "$drvPath")
 echo "output path is $outPath"
 
-regSub() {
-    (echo $1 && echo "" && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld && echo 0) | $nixstore --register-substitutes
-}
-
-# Register a substitute for the output path.
-regSub $outPath $(pwd)/substituter.sh
+echo $outPath > $TEST_ROOT/sub-paths
 
+export NIX_SUBSTITUTERS=$(pwd)/substituter.sh
 
 $nixstore -rvv "$drvPath"
 
 text=$(cat "$outPath"/hello)
-if test "$text" != "Hallo Wereld"; then exit 1; fi
+if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi

tests/substitutes2.nix.in

@@ -1,6 +0,0 @@
-derivation {
-  name = "substitutes-2";
-  system = "@system@";
-  builder = "@shell@";
-  args = ["-e" "-x" ./simple.builder.sh];
-}

tests/substitutes2.sh

@@ -1,25 +1,21 @@
 source common.sh
 
+clearStore
+
 # Instantiate.
-drvPath=$($nixinstantiate substitutes2.nix)
+drvPath=$($nixinstantiate simple.nix)
 echo "derivation is $drvPath"
 
 # Find the output path.
 outPath=$($nixstore -qvvvvv "$drvPath")
 echo "output path is $outPath"
 
-regSub() {
-    (echo $1 && echo "" && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld && echo 0) | $nixstore --register-substitutes
-}
+echo $outPath > $TEST_ROOT/sub-paths
 
-# Register a substitute for the output path.
-regSub $outPath $(pwd)/substituter.sh
-
-# Register another substitute for the output path.  This one takes
-# precedence over the previous one.  It will fail.
-regSub $outPath $(pwd)/substituter2.sh
+# First try a substituter that fails, then one that succeeds
+export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh:$(pwd)/substituter.sh
 
 $nixstore -rvv "$drvPath"
 
 text=$(cat "$outPath"/hello)
-if test "$text" != "Hallo Wereld"; then exit 1; fi
+if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi