* Tagged Nix 0.12.

the tests) is a bad idea when $bindir = /usr and some programs (like
  perl) live there.  Fortunately it doesn't seem to be needed anymore.

Makefile.am

@@ -18,6 +18,8 @@ install-data-local: init-state
 	if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
 		$(INSTALL_DATA) $(srcdir)/nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
 	fi
+	$(INSTALL) -d $(DESTDIR)$(docdir)
+	$(INSTALL_DATA) README $(DESTDIR)$(docdir)/
 
 if INIT_STATE
 
@@ -37,10 +39,9 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
-	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(prefix)/store
+	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
-#	$(bindir)/nix-store --init
 
 else
 init-state:

README

@@ -1,9 +1,10 @@
-For installation and usage instructions, please read the manual, which
-can be found in `docs/manual/manual.html', and additionally at the Nix
-website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
+Nix is a purely functional package manager.  For installation and
+usage instructions, please read the manual, which can be found in
+`docs/manual/manual.html', and additionally at the Nix website at
+<http://nixos.org/>.
 
 
 Acknowledgments
 
 This product includes software developed by the OpenSSL Project for
-use in the OpenSSL Toolkit (http://www.OpenSSL.org/)
+use in the OpenSSL Toolkit (http://www.OpenSSL.org/).

aterm-gc.supp

@@ -1,184 +1,117 @@
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:mark_memory_young
-   fun:mark_phase_young
+   fun:*
+   fun:AT_collect_minor
 }
 
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm_young
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_markTerm_young
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:mark_phase_young
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Value4
-   fun:mark_phase_young
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
-   Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_phase_young
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
+   ATerm library conservatively scans for GC roots
    Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_phase_young
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_phase_young
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr4
+   fun:*
+   fun:AT_collect_minor
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr8
+   fun:*
+   fun:AT_collect_minor
+}
+
+{
+   ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_phase_young
-   fun:AT_collect_minor
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr4
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr8
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Cond
+   fun:*
+   fun:*
+   fun:AT_collect
 }

configure.ac

@@ -2,10 +2,10 @@ AC_INIT(nix, 0.12)
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
 AM_INIT_AUTOMAKE([dist-bzip2 foreign])
-        
+
 # Change to `1' to produce a `stable' release (i.e., the `preREVISION'
 # suffix is not added).
-STABLE=0
+STABLE=1
 
 # Put the revision number in the version.
 if test "$STABLE" != "1"; then
@@ -18,8 +18,6 @@ fi
 
 AC_DEFINE_UNQUOTED(NIX_VERSION, ["$VERSION"], [Nix version.])
 
-AC_PREFIX_DEFAULT(/nix)
-
 AC_CANONICAL_HOST
 
 
@@ -61,6 +59,10 @@ AC_SUBST(system)
 AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
 
 
+# State should be stored in /nix/var, unless the user overrides it explicitly.
+test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
+
+
 # Windows-specific stuff.
 if test "$sys_name" = "cygwin"; then
     # We cannot delete open files.
@@ -124,7 +126,7 @@ fi
 ])
 
 NEED_PROG(curl, curl)
-NEED_PROG(shell, bash)
+NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
@@ -136,6 +138,7 @@ NEED_PROG(perl, perl)
 NEED_PROG(tar, tar)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(dblatex, dblatex)
+AC_PATH_PROG(gzip, gzip)
 
 AC_PATH_PROG(openssl_prog, openssl, openssl) # if not found, call openssl in $PATH
 AC_SUBST(openssl_prog)
@@ -173,7 +176,7 @@ AC_SUBST(xmlflags)
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
   [path of the Nix store]),
-  storedir=$withval, storedir='${prefix}/store')
+  storedir=$withval, storedir='/nix/store')
 AC_SUBST(storedir)
 
 AC_ARG_ENABLE(old-db-compat, AC_HELP_STRING([--disable-old-db-compat],
@@ -239,7 +242,7 @@ if test -z "$bzip2"; then
   bzip2_lib='-L${top_builddir}/externals/inst-bzip2/lib -lbz2'
   bzip2_include='-I${top_builddir}/externals/inst-bzip2/include'
   # The binary will be copied to $libexecdir.
-  bzip2_bin='${libexecdir}'
+  bzip2_bin='${libexecdir}/nix'
   # But for testing, we have to use the temporary copy :-(
   bzip2_bin_test='${top_builddir}/externals/inst-bzip2/bin'
 else

doc/manual/Makefile.am

@@ -14,15 +14,16 @@ XSLTPROC = $(xsltproc) $(xmlflags) \
 man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
  nix-prefetch-url.1 nix-channel.1 \
- nix-pack-closure.1 nix-unpack-closure.1 \
  nix-install-package.1 nix-hash.1 nix-copy-closure.1
 
+man8_MANS = nix-worker.8
+
 FIGURES = figures/user-environments.png
 
 MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  package-management.xml writing-nix-expressions.xml builtins.xml \
  build-farm.xml \
- $(man1_MANS:.1=.xml) \
+ $(man1_MANS:.1=.xml) $(man8_MANS:.8=.xml) \
  troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
  env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
  conf-file.xml release-notes.xml \
@@ -76,12 +77,12 @@ NEWS.txt: release-notes.xml
 all-local: manual.html NEWS.html NEWS.txt
 
 install-data-local: manual.html
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/manual
-	$(INSTALL_DATA) manual.html $(DESTDIR)$(datadir)/nix/manual
-	$(INSTALL_DATA) style.css $(DESTDIR)$(datadir)/nix/manual
-	cp -r images $(DESTDIR)$(datadir)/nix/manual/images
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/manual/figures
-	$(INSTALL_DATA) $(FIGURES) $(DESTDIR)$(datadir)/nix/manual/figures
+	$(INSTALL) -d $(DESTDIR)$(docdir)/manual
+	$(INSTALL_DATA) manual.html $(DESTDIR)$(docdir)/manual
+	$(INSTALL_DATA) style.css $(DESTDIR)$(docdir)/manual
+	cp -r images $(DESTDIR)$(docdir)/manual/images
+	$(INSTALL) -d $(DESTDIR)$(docdir)/manual/figures
+	$(INSTALL_DATA) $(FIGURES) $(DESTDIR)$(docdir)/manual/figures
 
 images:
 	mkdir images

doc/manual/build-farm.xml

@@ -56,7 +56,7 @@ build farm, since:
 <para>TODO</para>
 
 <para>The sources of the Nix build farm are at <link
-xlink:href='https://svn.cs.uu.nl:12443/repos/trace/release/trunk'/>.</para>
+xlink:href='https://svn.nixos.org/repos/nix/release/trunk'/>.</para>
 
 </section>
 
@@ -85,7 +85,7 @@ nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto
 
 <para>An example build hook can be found in the Nix build farm
 sources: <link
-xlink:href='https://svn.cs.uu.nl:12443/repos/trace/release/trunk/common/distributed/build-remote.pl'
+xlink:href='https://svn.nixos.org/repos/nix/release/trunk/common/distributed/build-remote.pl'
 />.  It should be suitable for most purposes, with maybe some minor
 adjustments.  It uses <command>ssh</command> and
 <command>rsync</command> to copy the build inputs and outputs and

doc/manual/builtins.xml

@@ -93,6 +93,23 @@ if builtins ? getEnv then __getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.compareVersions</function>
+  <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
+
+    <listitem><para>Compare two strings representing versions and
+    return <literal>-1</literal> if version
+    <replaceable>s1</replaceable> is older than version
+    <replaceable>s2</replaceable>, <literal>0</literal> if they are
+    the same, and <literal>1</literal> if
+    <replaceable>s1</replaceable> is newer than
+    <replaceable>s2</replaceable>.  The version comparison algorithm
+    is the same as the one used by <link
+    linkend="ssec-version-comparisons"><command>nix-env
+    -u</command></link>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry
   xml:id='builtin-currentSystem'><term><varname>builtins.currentSystem</varname></term>
 
@@ -146,6 +163,16 @@ if builtins ? getEnv then __getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.div</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the quotient of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.filterSource</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
@@ -352,6 +379,15 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.length</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return the length of the list
+    <replaceable>e</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.lessThan</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
@@ -408,6 +444,31 @@ map (x: "foo" + x) ["bar" "bla" "abc"]</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.mul</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the product of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
+  <varlistentry><term><function>builtins.parseDrvName</function>
+  <replaceable>s</replaceable></term>
+
+    <listitem><para>Split the string <replaceable>s</replaceable> into
+    a package name and version.  The package name is everything up to
+    but not including the first dash followed by a digit, and the
+    version is everything following that dash.  The result is returned
+    in an attribute set <literal>{name, version}</literal>.  Thus,
+    <literal>builtins.parseDrvName "nix-0.12pre12876"</literal>
+    returns <literal>{name = "nix"; version =
+    "0.12pre12876";}</literal>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.pathExists</function>
   <replaceable>path</replaceable></term>
 

doc/manual/conf-file.xml

@@ -60,27 +60,6 @@ env-keep-derivations = false
   </varlistentry>
 
   
-  <varlistentry xml:id="conf-gc-reserved-space"><term><literal>gc-reserved-space</literal></term>
-
-    <listitem><para>This option specifies how much space should be
-    reserved in normal use so that the garbage collector can run
-    succesfully.  Since the garbage collector must perform Berkeley DB
-    transactions, it needs some disk space for itself.  However, when
-    the disk is full, this space is not available, so the collector
-    would not be able to run precisely when it is most needed.</para>
-
-    <para>For this reason, when Nix is run, it allocates a file
-    <filename>/nix/var/nix/db/reserved</filename> of the size
-    specified by this option.  When the garbage collector is run, this
-    file is deleted before the Berkeley DB environment is opened.
-    This should give it enough room to proceed.</para>
-
-    <para>The default is <literal>1048576</literal> (1
-    MiB).</para></listitem>
-
-  </varlistentry>
-
-  
   <varlistentry><term><literal>env-keep-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations

doc/manual/env-common.xml

@@ -274,6 +274,27 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
     
+<varlistentry xml:id="envar-other-stores"><term><envar>NIX_OTHER_STORES</envar></term>
+
+  <listitem><para>This variable contains the paths of remote Nix
+  installations from whichs paths can be copied, separated by colons.
+  See <xref linkend="sec-sharing-packages" /> for details.  Each path
+  should be the <filename>/nix</filename> directory of a remote Nix
+  installation (i.e., not the <filename>/nix/store</filename>
+  directory).  The paths are subject to globbing, so you can set it so
+  something like <literal>/var/run/nix/remote-stores/*/nix</literal>
+  and mount multiple remote filesystems in
+  <literal>/var/run/nix/remote-stores</literal>.</para>
+
+  <para>Note that if you’re building through the <link
+  linkend="sec-nix-worker">Nix daemon</link>, the only setting for
+  this variable that matters is the one that the
+  <command>nix-worker</command> process uses.  So if you want to
+  change it, you have to restart the daemon.</para></listitem>
+
+</varlistentry>
+
+    
 </variablelist>
 
 

doc/manual/installation.xml

@@ -42,25 +42,22 @@ platforms as well.</para>
 <section><title>Obtaining Nix</title>
 
 <para>The easiest way to obtain Nix is to download a <link
-xlink:href="http://nix.cs.uu.nl/">source distribution</link>.  RPMs
+xlink:href="http://nixos.org/">source distribution</link>.  RPMs
 for Red Hat, SuSE, and Fedora Core are also available.</para>
 
 <para>Alternatively, the most recent sources of Nix can be obtained
 from its <link
-xlink:href="https://svn.cs.uu.nl:12443/repos/trace/nix/trunk">Subversion
+xlink:href="https://svn.nixos.org/repos/nix/nix/trunk">Subversion
 repository</link>.  For example, the following command will check out
 the latest revision into a directory called
 <filename>nix</filename>:</para>
 
 <screen>
-$ svn checkout https://svn.cs.uu.nl:12443/repos/trace/nix/trunk nix</screen>
+$ svn checkout https://svn.nixos.org/repos/nix/nix/trunk nix</screen>
 
 <para>Likewise, specific releases can be obtained from the <link
-xlink:href="https://svn.cs.uu.nl:12443/repos/trace/nix/tags">tags
-directory</link> of the repository.  If you don't have Subversion, you
-can also download an automatically generated <link
-xlink:href="https://svn.cs.uu.nl:12443/dist/trace/">compressed
-tar-file</link> of the head revision of the trunk.</para>
+xlink:href="https://svn.nixos.org/repos/nix/nix/tags">tags
+directory</link> of the repository.</para>
 
 </section>
 
@@ -99,18 +96,23 @@ ubiquitous 2.5.4a won't.  Note that these are only required if you
 modify the parser or when you are building from the Subversion
 repository.</para>
 
-<para>Nix uses Sleepycat's Berkeley DB, CWI's ATerm library and the
-bzip2 compressor (including the bzip2 library).  These are included in
-the Nix source distribution.  If you build from the Subversion
-repository, you must download them yourself and place them in the
+<para>Nix uses CWI's ATerm library and the bzip2 compressor (including
+the bzip2 library).  These are included in the Nix source
+distribution.  If you build from the Subversion repository, you must
+download them yourself and place them in the
 <filename>externals/</filename> directory.  See
 <filename>externals/Makefile.am</filename> for the precise URLs of
 these packages.  Alternatively, if you already have them installed,
 you can use <command>configure</command>'s
-<option>--with-bdb</option>, <option>--with-aterm</option> and
-<option>--with-bzip2</option> options to point to their respective
-locations.  Note that Berkeley DB <emphasis>must</emphasis> be version
-4.5; other versions may not have compatible database formats.</para>
+<option>--with-aterm</option> and <option>--with-bzip2</option>
+options to point to their respective locations.</para>
+
+<para>If you want to be able to upgrade Nix stores from before version
+0.12pre12020, you need Sleepycat's Berkeley DB version version 4.5.
+(Other versions may not have compatible database formats.).  Berkeley
+DB 4.5 is included in the Nix source distribution.  If you do not need
+this ability, you can build Nix with the
+<option>--disable-old-db-compat</option> configure option.</para>
 
 </section>
 
@@ -161,7 +163,7 @@ options.</para>
 <section><title>Installing from RPMs</title>
 
 <para>RPM packages of Nix can be downloaded from <link
-xlink:href="http://nix.cs.uu.nl/" />.  These RPMs should work for most
+xlink:href="http://nixos.org/" />.  These RPMs should work for most
 fairly recent releases of SuSE and Red Hat Linux.  They have been
 known to work work on SuSE Linux 8.1 and 9.0, and Red Hat 9.0.  In
 fact, it should work on any RPM-based Linux distribution based on
@@ -193,12 +195,12 @@ $ rm -rf /nix/var</screen>
 
 <para>You can install the latest stable version of Nix through Nix
 itself by subscribing to the channel <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-stable" />,
+xlink:href="http://nixos.org/releases/nix/channels/nix-stable" />,
 or the latest unstable version by subscribing to the channel <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-unstable" />.
+xlink:href="http://nixos.org/releases/nix/channels/nix-unstable" />.
 You can also do a <link linkend="sec-one-click">one-click
 installation</link> by clicking on the package links at <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/" />.</para>
+xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para>
 
 </section>
 
@@ -334,8 +336,8 @@ $ chown -R root /nix/store /nix/var/nix</screen>
 
 </para>
 
-<para>The Nix daemon should be started as follows (as
-<literal>root</literal>):
+<para>The <link linkend="sec-nix-worker">Nix daemon</link> should be
+started as follows (as <literal>root</literal>):
 
 <screen>
 $ nix-worker --daemon</screen>

doc/manual/introduction.xml

@@ -240,7 +240,7 @@ configuration (e.g., to build configuration files in
 possible to easily roll back the entire configuration of the system to
 an earlier state.  Also, users can install software without root
 privileges.  For more information and downloads, see the <link
-xlink:href="http://nix.cs.uu.nl/nixos/">NixOS homepage</link>.</para>
+xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 
 </simplesect>
 
@@ -257,14 +257,15 @@ xlink:href="http://nix.cs.uu.nl/nixos/">NixOS homepage</link>.</para>
 
 <section><title>About us</title>
 
-<para>Nix was developed at the <link
+<para>Nix was originally developed at the <link
 xlink:href="http://www.cs.uu.nl/">Department of Information and
 Computing Sciences</link>, Utrecht University by the <link
 xlink:href="http://www.cs.uu.nl/wiki/Trace/WebHome">TraCE
-project</link>.  The project is funded by the Software Engineering
-Research Program <link
+project</link> (2003-2008).  The project was funded by the Software
+Engineering Research Program <link
 xlink:href="http://www.jacquard.nl/">Jacquard</link> to improve the
-support for variability in software systems.</para>
+support for variability in software systems.  Further funding is now
+provided by the NIRICT LaQuSo Build Farm project.</para>
 
 </section>
 
@@ -299,24 +300,24 @@ Lesser General Public License for more details.</para>
 
 <para>Some background information on Nix can be found in a number of
 papers.  The ICSE 2004 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'>Imposing
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/immdsd-icse2004-final.pdf'>Imposing
 a Memory Management Discipline on Software Deployment</citetitle>
 discusses the hashing mechanism used to ensure reliable dependency
 identification and non-interference between different versions and
 variants of packages.  The LISA 2004 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'>Nix:
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/nspfssd-lisa2004-final.pdf'>Nix:
 A Safe and Policy-Free System for Software Deployment</citetitle>
 gives a more general discussion of Nix from a system-administration
 perspective.  The CBSE 2005 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'>Efficient
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/eupfcdm-cbse2005-final.pdf'>Efficient
 Upgrading in a Purely Functional Component Deployment Model
 </citetitle> is about transparent patch deployment in Nix.  The SCM-12
 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/servicecm-scm12-final.pdf'>
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/servicecm-scm12-final.pdf'>
 Service Configuration Management</citetitle> shows how services (e.g.,
 web servers) can be deployed and managed through Nix.  A short
 overview of NixOS is given in the HotOS XI paper <citetitle
-xlink:href="http://www.cs.uu.nl/~eelco/pubs/hotos-final.pdf">Purely
+xlink:href="http://www.st.ewi.tudelft.nl/~dolstra/pubs/hotos-final.pdf">Purely
 Functional System Configuration Management</citetitle>.  The Nix
 homepage has <link
 xlink:href="http://nix.cs.uu.nl/docs/papers.html">an up-to-date list
@@ -328,7 +329,7 @@ Purely Functional Software Deployment Model</citetitle>, which
 contains most of the papers listed above.</para>
 
 <para>Nix has a homepage at <link
-xlink:href="http://nix.cs.uu.nl/"/>.</para>
+xlink:href="http://nixos.org/"/>.</para>
 
 </section>
 

doc/manual/manual.xml

@@ -14,8 +14,8 @@
         <surname>Dolstra</surname>
       </personname>
       <affiliation>
-        <orgname>Utrecht University</orgname>
-        <orgdiv>Faculty of Science, Department of Information and Computing Sciences</orgdiv>
+        <orgname>Delft University of Technology</orgname>
+        <orgdiv>Department of Software Technology</orgdiv>
       </affiliation>
     </author>
 
@@ -24,10 +24,11 @@
       <year>2005</year>
       <year>2006</year>
       <year>2007</year>
+      <year>2008</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>September 2007</date>
+    <date>November 2008</date>
     
   </info>
 
@@ -88,10 +89,6 @@
         <title>nix-install-package</title>
         <xi:include href="nix-install-package.xml" />
       </section>
-      <section xml:id="sec-nix-pack-closure">
-        <title>nix-pack-closure</title>
-        <xi:include href="nix-pack-closure.xml" />
-      </section>
       <section xml:id="sec-nix-prefetch-url">
         <title>nix-prefetch-url</title>
         <xi:include href="nix-prefetch-url.xml" />
@@ -104,9 +101,9 @@
         <title>nix-push</title>
         <xi:include href="nix-push.xml" />
       </section>
-      <section xml:id="sec-nix-unpack-closure">
-        <title>nix-unpack-closure</title>
-        <xi:include href="nix-unpack-closure.xml" />
+      <section xml:id="sec-nix-worker">
+        <title>nix-worker</title>
+        <xi:include href="nix-worker.xml" />
       </section>
     </section>
     

doc/manual/nix-copy-closure.xml

@@ -128,7 +128,7 @@ those paths.  If this bothers you, use
 <para>Copy Firefox with all its dependencies to a remote machine:
 
 <screen>
-$ nix-copy-closure alice@itchy.labs $(type -tP firefox)</screen>
+$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)</screen>
 
 </para>
 

doc/manual/nix-env.xml

@@ -526,7 +526,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4'</screen>
 
 </refsection>
     
-<refsection><title>Versions</title>
+<refsection xml:id="ssec-version-comparisons"><title>Versions</title>
 
 <para>The upgrade operation determines whether a derivation
 <varname>y</varname> is an upgrade of a derivation

doc/manual/nix-pack-closure.xml

@@ -1,82 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<refmeta>
-  <refentrytitle>nix-pack-closure</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-pack-closure</refname>
-  <refpurpose>pack the closure of a store path into a single file that
-  can be unpacked with
-  <command>nix-unpack-closure</command></refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-pack-closure</command>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-pack-closure</command> packs the
-contents of the store paths <replaceable>paths</replaceable> and
-<emphasis>all their dependencies</emphasis> into a single file, which
-is written to standard output.  (That is, it
-<emphasis>serialises</emphasis> <replaceable>paths</replaceable>.)
-The output can then be unpacked into the Nix store of another machine
-using <command>nix-unpack-closure</command>.</para>
-
-<para>Together, <command>nix-pack-closure</command> and
-<command>nix-unpack-closure</command> provide a quick and easy way to
-deploy a package to a different machine.  However, as the output of
-<command>nix-pack-closure</command> tends to be rather large (since it
-contains all dependencies), it’s not very efficient.
-<command>nix-push</command> and <command>nix-pull</command> are more
-efficient, but are also a bit more cumbersome to use.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>To copy some instance of Subversion with all its dependencies to
-another machine:
-
-<screen>
-$ nix-pack-closure /nix/store/hj232g1r...-subversion-1.3.0 > svn.closure
-
-<lineannotation>Copy <!-- !!! <filename> -->svn.closure to the remote machine, then on the remote machine do:</lineannotation>
-$ nix-unpack-closure &lt; svn.closure</screen>
-
-</para>
-
-<para>Copy the program <command>azureus</command> with all its
-dependencies to the machine <literal>scratchy</literal>:
-
-<screen>
-$ nix-pack-closure $(which azureus) | ssh scratchy nix-unpack-closure</screen>
-    
-</para>
-
-<para>As a variation on the previous example, copy
-<command>azureus</command>, and also install it in the user’s profile
-on the target machine:
-
-<screen>
-$ nix-pack-closure $(which azureus) | ssh scratchy 'nix-env -i $(nix-unpack-closure)'</screen>
-
-</para>
- 
-
-</refsection>
-
-
-</refentry>

doc/manual/nix-store.xml

@@ -128,6 +128,7 @@ lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134
     <arg choice='plain'><option>-r</option></arg>
   </group>
   <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg><option>--dry-run</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -166,6 +167,11 @@ the specified store paths.  Realisation is a somewhat overloaded term:
 output.  (For non-derivations argument, the argument itself is
 printed.)</para>
 
+<para>If the <option>--dry-run</option> option is used, then
+<command>nix-store</command> will print on standard error a
+description of what packages would be built or downloaded, and then
+quit.</para>
+
 </refsection>
             
 
@@ -204,6 +210,10 @@ linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
     <arg choice='plain'><option>--print-dead</option></arg>
     <arg choice='plain'><option>--delete</option></arg>
   </group>
+  <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
+  <arg><option>--max-links</option> <replaceable>nrlinks</replaceable></arg>
+  <arg><option>--max-atime</option> <replaceable>atime</replaceable></arg>
+  <arg><option>--use-atime</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -258,7 +268,70 @@ the Nix store not reachable via file system references from a set of
 
 </variablelist>
 
-<para>The behaviour of the collector is influenced by the <link
+<para>By default, all unreachable paths are deleted.  The following
+options control what gets deleted and in what order:
+
+<variablelist>
+
+  <varlistentry><term><option>--max-freed</option> <replaceable>bytes</replaceable></term>
+  
+    <listitem><para>Keep deleting paths until at least
+    <replaceable>bytes</replaceable> bytes have been
+    deleted, then stop.</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--max-links</option> <replaceable>nrlinks</replaceable></term>
+  
+    <listitem><para>Keep deleting paths until the hard link count on
+    <filename>/nix/store</filename> is less than
+    <replaceable>nrlinks</replaceable>, then stop.  This is useful for
+    very large Nix stores on filesystems with a 32000 subdirectories
+    limit (like <literal>ext3</literal>).</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--max-atime</option> <replaceable>atime</replaceable></term>
+  
+    <listitem><para>Only delete a store path if its last-accessed time
+    is less than <replaceable>atime</replaceable>.  This allows you to
+    garbage-collect only packages that haven’t been used recently.
+    The time is expressed as the number of seconds in the Unix epoch,
+    i.e., since 1970-01-01 00:00:00 UTC.  An easy way to convert to
+    this format is <literal>date +%s -d "<replaceable>date
+    specification</replaceable>"</literal>.</para>
+
+    <para>For directories, the last-accessed time is the highest
+    last-accessed time of any regular file in the directory (or in any
+    of its subdirectories).  That is, the <literal>atime</literal>
+    field maintained by the filesystem is ignored for directories.
+    This is because operations such as rebuilding the
+    <command>locate</command> database tend to update the
+    <literal>atime</literal> values of all directories, so they’re not
+    a useful indicator of whether a package was recently used.</para>
+
+    <para>Note that <command>nix-store --optimise</command> reads all
+    regular files in the Nix store, and so causes all last-accessed
+    times to be set to the present time.  This makes
+    <option>--max-atime</option> ineffective (for a while at
+    least).</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--use-atime</option></term>
+
+    <listitem><para>Delete store paths in order of ascending
+    last-accessed time.  This is useful in conjunction with the other
+    options to delete only the least recently used
+    packages.</para></listitem>
+
+  </varlistentry>
+  
+</variablelist>
+
+</para>
+
+<para>The behaviour of the collector is also influenced by the <link
 linkend="conf-gc-keep-outputs"><literal>gc-keep-outputs</literal></link>
 and <link
 linkend="conf-gc-keep-derivations"><literal>gc-keep-derivations</literal></link>
@@ -284,6 +357,20 @@ deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
 
 </para>
 
+<para>To delete unreachable paths not accessed in the last two months:
+
+<screen>
+$ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
+
+</para>
+
+<para>To delete at least 100 MiBs of unreachable paths:
+
+<screen>
+$ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
+
+</para>
+
 </refsection>
 
 
@@ -950,7 +1037,51 @@ ktorrent-2.2.1/NEWS
 </refsection>
 
 
-<!-- TODO: export, import operations -->
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--dump-db</option></title>
+
+<refsection>
+  <title>Synopsis</title>
+  <cmdsynopsis>
+    <command>nix-store</command>
+    <arg choice='plain'><option>--dump-db</option></arg>
+  </cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+            
+<para>The operation <option>--dump-db</option> writes a dump of the
+Nix database to standard output.  It can be loaded into an empty Nix
+store using <option>--load-db</option>.  This is useful for making
+backups and when migrating to different database schemas.</para>
+
+</refsection>
+            
+</refsection>
+
+
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--dump-db</option></title>
+
+<refsection>
+  <title>Synopsis</title>
+  <cmdsynopsis>
+    <command>nix-store</command>
+    <arg choice='plain'><option>--load-db</option></arg>
+  </cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+            
+<para>The operation <option>--load-db</option> reads a dump of the Nix
+database created by <option>--dump-db</option> from standard input and
+loads it into the Nix database.</para>
+
+</refsection>
+            
+</refsection>
 
 
 </refentry>

doc/manual/nix-unpack-closure.xml

@@ -1,42 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<refmeta>
-  <refentrytitle>nix-unpack-closure</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-unpack-closure</refname>
-  <refpurpose>unpack the closure of a store path created by <command>nix-pack-closure</command> into the Nix store</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-unpack-closure</command>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-unpack-closure</command> unpacks the
-closure of a set of store paths created by
-<command>nix-pack-closure</command> into the local Nix store.  The
-closure is a single file read from standard input.  See the
-description of <command>nix-pack-closure</command> for details and
-examples.</para>
-
-<para>The top-level paths in the closure (i.e., the paths passed to
-the original <command>nix-pack-closure</command> call that created the
-closure) are printed on standard output.  These paths can be passed,
-for instance, to <literal>nix-env -i</literal> to install them into a
-user environment on the target machine.</para>
-
-</refsection>
-
-
-</refentry>

doc/manual/nix-worker.xml

@@ -0,0 +1,34 @@
+<refentry xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-worker</refentrytitle>
+  <manvolnum>8</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
+<refnamediv>
+  <refname>nix-worker</refname>
+  <refpurpose>Nix multi-user support daemon</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-worker</command>
+    <arg choice="plain"><option>--daemon</option></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+
+<refsection><title>Description</title>
+
+<para>The Nix daemon is necessary in multi-user Nix installations.  It
+performs build actions and other operations on the Nix store on behalf
+of unprivileged users.</para>
+
+
+</refsection>
+
+</refentry>

doc/manual/package-management.xml

@@ -38,7 +38,7 @@ to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
 expressions based on it, or completely new ones.)  You can download
 the latest version from <link
-xlink:href='http://nix.cs.uu.nl/dist/nix' />.</para>
+xlink:href='http://nixos.org/releases/full-index-nixpkgs.html' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available packages in the release:
@@ -118,15 +118,15 @@ available somewhere.  This is done using the
 containing a <emphasis>manifest</emphasis> describing what binaries
 are available.  This URL should correspond to the Nix Packages release
 that you’re using.  For instance, if you obtained a release from <link
-xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />, then
-you should do:
+xlink:href='http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x'
+/>, then you should do:
 
 <screen>
-$ nix-pull http://nix.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/MANIFEST</screen>
+$ nix-pull http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x/MANIFEST</screen>
 
 If you then issue the installation command, it should start
 downloading binaries from <systemitem
-class='fqdomainname'>nix.cs.uu.nl</systemitem>, instead of building
+class='fqdomainname'>nixos.org</systemitem>, instead of building
 them from source.  This might still take a while since all
 dependencies must be downloaded, but on a reasonably fast connection
 such as an DSL line it’s on the order of a few minutes.</para>
@@ -458,7 +458,7 @@ URL.</para>
 <command>nix-channel --add</command>, e.g.,
 
 <screen>
-$ nix-channel --add http://nix.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable</screen>
+$ nix-channel --add http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable</screen>
 
 subscribes you to a channel that always contains that latest version
 of the Nix Packages collection.  (Instead of
@@ -496,20 +496,19 @@ available in the subscribed channels.</para>
 
 <para>Often, when you want to install a specific package (e.g., from
 the <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/">Nix
-Packages collection</link> or from our <link
-xlink:href='http://nix.cs.uu.nl/dist/'>release server</link>),
-subscribing to a channel is a bit cumbersome.  And channels don’t help
-you at all if you want to install an older version of a package than
-the one provided by the current contents of the channel, or a package
-that has been removed from the channel.  That’s when
-<emphasis>one-click installs</emphasis> come in handy: you can just go
-to the web page that contains the package, click on it, and it will be
-installed with all the necessary dependencies.</para>
+xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/">Nix
+Packages collection</link>), subscribing to a channel is a bit
+cumbersome.  And channels don’t help you at all if you want to install
+an older version of a package than the one provided by the current
+contents of the channel, or a package that has been removed from the
+channel.  That’s when <emphasis>one-click installs</emphasis> come in
+handy: you can just go to the web page that contains the package,
+click on it, and it will be installed with all the necessary
+dependencies.</para>
 
 <para>For instance, you can go to <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/" />
-— or to any older release of Nix Packages — and click on any link for
+xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/" /> —
+or to any older release of Nix Packages — and click on any link for
 the individual packages for your platform (say, <link
 xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.10pre6622/pkgs/subversion-1.4.0-i686-linux.nixpkg'><literal>subversion-1.4.0</literal>
 for <literal>i686-linux</literal></link>).  The first time you do
@@ -529,4 +528,82 @@ linkend='sec-nix-install-package' /> for details.</para>
 </section>
 
 
+<section xml:id="sec-sharing-packages"><title>Sharing packages between machines</title>
+
+<para>Sometimes you want to copy a package from one machine to
+another.  Or, you want to install some packages and you know that
+another machine already has some or all of those packages or their
+dependencies.  In that case there are mechanisms to quickly copy
+packages between machines.</para>
+
+<para>The command <command
+linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix
+store path along with all its dependencies to or from another machine
+via the SSH protocol.  It doesn’t copy store paths that are already
+present on the target machine.  For example, the following command
+copies Firefox with all its dependencies:
+
+<screen>
+$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)</screen>
+
+See <xref linkend='sec-nix-copy-closure' /> for details.</para>
+
+<para>With <command linkend='refsec-nix-store-export'>nix-store
+--export</command> and <command
+linkend='refsec-nix-store-import'>nix-store --import</command> you can
+write the closure of a store path (that is, the path and all its
+dependencies) to a file, and then unpack that file into another Nix
+store.  For example,
+
+<screen>
+$ nix-store --export $(type -p firefox) > firefox.closure</screen>
+
+writes the closure of Firefox to a file.  You can then copy this file
+to another machine and install the closure:
+
+<screen>
+$ nix-store --import &lt; firefox.closure</screen>
+
+Any store paths in the closure that are already present in the target
+store are ignored.  It is also possible to pipe the export into
+another command, e.g. to copy and install a closure directly to/on
+another machine:
+
+<screen>
+$ nix-store --export $(type -p firefox) | bzip2 | \
+    ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen>
+
+But note that <command>nix-copy-closure</command> is generally more
+efficient in this example because it only copies paths that are not
+already present in the target Nix store.</para>
+
+<para>Finally, if you can mount the Nix store of a remote machine in
+your local filesystem, Nix can copy paths from the remote Nix store to
+the local Nix store <emphasis>on demand</emphasis>.  For instance,
+suppose that you mount a remote machine containing a Nix store via
+<command
+xlink:href="http://fuse.sourceforge.net/sshfs.html">sshfs</command>:
+
+<screen>
+$ sshfs alice@itchy.example.org:/ /mnt</screen>
+
+You should then set the <envar>NIX_OTHER_STORES</envar> environment
+variable to tell Nix about this remote Nix store:
+
+<screen>
+$ export NIX_OTHER_STORES=/mnt/nix</screen>
+
+Then if you do any Nix operation, e.g.
+
+<screen>
+$ nix-env -i firefox</screen>
+
+and Nix has to build a path that it sees is already present in
+<filename>/mnt/nix</filename>, then it will just copy from there
+instead of building it from source.</para>
+ 
+
+</section>
+
+
 </chapter>

doc/manual/quick-start.xml

@@ -11,7 +11,7 @@ to the following chapters.</para>
 <orderedlist>
 
 <listitem><para>Download a source tarball or RPM from <link
-xlink:href='http://nix.cs.uu.nl/'/>.  Build source
+xlink:href='http://nixos.org/'/>.  Build source
 distributions using the regular sequence:
         
 <screen>
@@ -33,7 +33,7 @@ file).</para></listitem>
 
 <screen>
 $ nix-channel --add \
-    http://nix.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable</screen>
+    http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable</screen>
 
 </para></listitem>
 

doc/manual/release-notes.xml

@@ -8,11 +8,172 @@
 
 <!--==================================================================-->
 
-<section xml:id="ssec-relnotes-0.12"><title>Release 0.12 (TBA)</title>
+<section xml:id="ssec-relnotes-0.12"><title>Release 0.12 (November 20,
+2008)</title>
 
 <itemizedlist>
 
-  <listitem><para><command>nix-store --dump-db / --load-db</command>.</para></listitem>
+  <listitem>
+    <para>Nix no longer uses Berkeley DB to store Nix store metadata.
+    The principal advantages of the new storage scheme are: it works
+    properly over decent implementations of NFS (allowing Nix stores
+    to be shared between multiple machines); no recovery is needed
+    when a Nix process crashes; no write access is needed for
+    read-only operations; no more running out of Berkeley DB locks on
+    certain operations.</para>
+
+    <para>You still need to compile Nix with Berkeley DB support if
+    you want Nix to automatically convert your old Nix store to the
+    new schema.  If you don’t need this, you can build Nix with the
+    <filename>configure</filename> option
+    <option>--disable-old-db-compat</option>.</para>
+
+    <para>After the automatic conversion to the new schema, you can
+    delete the old Berkeley DB files:
+
+    <screen>
+$ cd /nix/var/nix/db
+$ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG</screen>
+
+    The new metadata is stored in the directories
+    <filename>/nix/var/nix/db/info</filename> and
+    <filename>/nix/var/nix/db/referrer</filename>.  Though the
+    metadata is stored in human-readable plain-text files, they are
+    not intended to be human-editable, as Nix is rather strict about
+    the format.</para>
+
+    <para>The new storage schema may or may not require less disk
+    space than the Berkeley DB environment, mostly depending on the
+    cluster size of your file system.  With 1 KiB clusters (which
+    seems to be the <literal>ext3</literal> default nowadays) it
+    usually takes up much less space.</para>
+  </listitem>
+
+  <listitem><para>There is a new substituter that copies paths
+  directly from other (remote) Nix stores mounted somewhere in the
+  filesystem.  For instance, you can speed up an installation by
+  mounting some remote Nix store that already has the packages in
+  question via NFS or <literal>sshfs</literal>.  The environment
+  variable <envar>NIX_OTHER_STORES</envar> specifies the locations of
+  the remote Nix directories,
+  e.g. <literal>/mnt/remote-fs/nix</literal>.</para></listitem>
+
+  <listitem><para>New <command>nix-store</command> operations
+  <option>--dump-db</option> and <option>--load-db</option> to dump
+  and reload the Nix database.</para></listitem>
+
+  <listitem><para>The garbage collector has a number of new options to
+  allow only some of the garbage to be deleted.  The option
+  <option>--max-freed <replaceable>N</replaceable></option> tells the
+  collector to stop after at least <replaceable>N</replaceable> bytes
+  have been deleted.  The option <option>--max-links
+  <replaceable>N</replaceable></option> tells it to stop after the
+  link count on <filename>/nix/store</filename> has dropped below
+  <replaceable>N</replaceable>.  This is useful for very large Nix
+  stores on filesystems with a 32000 subdirectories limit (like
+  <literal>ext3</literal>).  The option <option>--use-atime</option>
+  causes store paths to be deleted in order of ascending last access
+  time.  This allows non-recently used stuff to be deleted.  The
+  option <option>--max-atime <replaceable>time</replaceable></option>
+  specifies an upper limit to the last accessed time of paths that may
+  be deleted.  For instance,
+
+    <screen>
+    $ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
+
+  deletes everything that hasn’t been accessed in two months.</para></listitem>
+
+  <listitem><para><command>nix-env</command> now uses optimistic
+  profile locking when performing an operation like installing or
+  upgrading, instead of setting an exclusive lock on the profile.
+  This allows multiple <command>nix-env -i / -u / -e</command>
+  operations on the same profile in parallel.  If a
+  <command>nix-env</command> operation sees at the end that the profile
+  was changed in the meantime by another process, it will just
+  restart.  This is generally cheap because the build results are
+  still in the Nix store.</para></listitem>
+
+  <listitem><para>The option <option>--dry-run</option> is now
+  supported by <command>nix-store -r</command> and
+  <command>nix-build</command>.</para></listitem>
+
+  <listitem><para>The information previously shown by
+  <option>--dry-run</option> (i.e., which derivations will be built
+  and which paths will be substituted) is now always shown by
+  <command>nix-env</command>, <command>nix-store -r</command> and
+  <command>nix-build</command>.  The total download size of
+  substitutable paths is now also shown.  For instance, a build will
+  show something like
+
+    <screen>
+the following derivations will be built:
+  /nix/store/129sbxnk5n466zg6r1qmq1xjv9zymyy7-activate-configuration.sh.drv
+  /nix/store/7mzy971rdm8l566ch8hgxaf89x7lr7ik-upstart-jobs.drv
+  ...
+the following paths will be downloaded/copied (30.02 MiB):
+  /nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4
+  /nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6
+  ...</screen>      
+  
+  </para></listitem>
+
+  <listitem><para>Language features:
+
+    <itemizedlist>
+
+      <listitem><para>@-patterns as in Haskell.  For instance, in a
+      function definition
+
+      <programlisting>f = args @ {x, y, z}: <replaceable>...</replaceable>;</programlisting>
+
+      <varname>args</varname> refers to the argument as a whole, which
+      is further pattern-matched against the attribute set pattern
+      <literal>{x, y, z}</literal>.</para></listitem>
+
+      <listitem><para>“<literal>...</literal>” (ellipsis) patterns.
+      An attribute set pattern can now say <literal>...</literal>  at
+      the end of the attribute name list to specify that the function
+      takes <emphasis>at least</emphasis> the listed attributes, while
+      ignoring additional attributes.  For instance,
+
+      <programlisting>{stdenv, fetchurl, fuse, ...}: <replaceable>...</replaceable></programlisting>
+
+      defines a function that accepts any attribute set that includes
+      at least the three listed attributes.</para></listitem>
+
+      <listitem><para>New primops:
+      <varname>builtins.parseDrvName</varname> (split a package name
+      string like <literal>"nix-0.12pre12876"</literal> into its name
+      and version components, e.g. <literal>"nix"</literal> and
+      <literal>"0.12pre12876"</literal>),
+      <varname>builtins.compareVersions</varname> (compare two version
+      strings using the same algorithm that <command>nix-env</command>
+      uses), <varname>builtins.length</varname> (efficiently compute
+      the length of a list), <varname>builtins.mul</varname> (integer
+      multiplication), <varname>builtins.div</varname> (integer
+      division).
+      <!-- <varname>builtins.genericClosure</varname> -->
+      </para></listitem>
+      
+    </itemizedlist>
+
+  </para></listitem>
+
+  <listitem><para><command>nix-prefetch-url</command> now supports
+  <literal>mirror://</literal> URLs, provided that the environment
+  variable <envar>NIXPKGS_ALL</envar> points at a Nixpkgs
+  tree.</para></listitem>
+
+  <listitem><para>Removed the commands
+  <command>nix-pack-closure</command> and
+  <command>nix-unpack-closure</command>.   You can do almost the same
+  thing but much more efficiently by doing <literal>nix-store --export
+  $(nix-store -qR <replaceable>paths</replaceable>) > closure</literal> and
+  <literal>nix-store --import &lt;
+  closure</literal>.</para></listitem>
+
+  <listitem><para>Lots of bug fixes, including a big performance bug in
+  the handling of <literal>with</literal>-expressions.</para></listitem>
 
 </itemizedlist>
 

doc/manual/troubleshooting.xml

@@ -5,121 +5,10 @@
 
 
 <para>This section provides solutions for some common problems.  See
-the <link xlink:href="https://bugs.cs.uu.nl/browse/NIX">Nix
+the <link xlink:href="http://bugs.strategoxt.org/browse/NIX">Nix
 bug tracker</link> for a list of currently known issues.</para>
 
 
-<section><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
-
-<para>Symptom: Nix operations (in particular the
-<command>nix-store</command> operations <option>--gc</option>,
-<option>--verify</option>, and <option>--clear-substitutes</option> —
-the latter being called by <command>nix-channel --update</command>)
-failing:
-
-<screen>
-$ nix-store --verify
-error: Db::del: Cannot allocate memory</screen>
-
-</para>
-
-<para>Possible solution: make sure that no Nix processes are running,
-then do:
-
-<screen>
-$ cd /nix/var/nix/db
-$ rm __db.00*</screen>
-
-</para>
-
-</section>
-
-
-<section><title>Berkeley DB gives weird error messages</title>
-
-<para>Symptom: you get error messages such as
-
-<screen>
-Berkeley DB message: Finding last valid log LSN: file: 1 offset 28
-Berkeley DB error: file validpaths (meta pgno = 0) has LSN [483][34721].
-Berkeley DB error: end of log is [1][28]
-Berkeley DB error: /nix/var/nix/db/validpaths: unexpected file type or format</screen>
-
-or other weird Berkeley DB errors, and they don’t go away (i.e.,
-automatic recovery doesn’t work).  This may be the case after a system
-crash.</para>
-
-<para>Solution: first try to run <command>db_recover</command> and
-then <link linkend='refsec-nix-store-verify'><command>nix-store
---verify</command></link>:
-
-<screen>
-$ db_recover -h /nix/var/nix/db
-$ nix-store --verify</screen>
-
-(Make sure that you have the right version of
-<command>db_recover</command>, namely, Berkeley DB 4.4 for Nix 0.10,
-and 4.5 for Nix 0.11.)</para>
-
-<para>If that doesn’t work, it’s time to bring out the big guns:
-
-<screen>
-$ cd /nix/var/nix
-$ cp -pr db db-backup  <lineannotation>(making a backup just in case)</lineannotation>
-$ cd db
-$ rm __db.* log*  <lineannotation>(removing the Berkeley DB environment)</lineannotation>
-$ mkdir tmp
-$ for i in *; do db_dump $i | (cd tmp &amp;&amp; db_load $i); done 
-<lineannotation>(ignore error messages about non-database files like “reserved”)</lineannotation>
-$ mv tmp/* .
-$ nix-store --verify</screen>
-
-</para>
-
-</section>
-
-
-<section><title>Berkeley DB out of locks</title>
-
-<para>It is possible, especially in <command>nix-store
---verify</command> or when running the garbage collector, to run out
-of Berkeley DB locks, like this:
-
-<screen>
-$ nix-store --verify
-checking path existence
-checking path realisability
-checking the derivers table
-checking the references table
-Berkeley DB error: Lock table is out of available object entries
-error: Db::get: Cannot allocate memory</screen>
-
-</para>
-
-<para>A workaround is to increase the number of locks that Berkeley DB
-allocates.  (The real solution would be for Nix to not use so many
-locks.)  This can be done by putting the following in the file
-<filename>/nix/var/nix/db/<link
-xlink:href="http://www.oracle.com/technology/documentation/berkeley-db/db/ref/env/db_config.html">DB_CONFIG</link></filename>:
-
-<programlisting>
-set_lk_max_locks   100000
-set_lk_max_lockers 100000
-set_lk_max_objects 100000
-</programlisting>
-
-(Increase these numbers if necessary.)  Then make sure that there are
-no running Nix processes and delete the Berkeley DB environment:
-
-<screen>
-$ rm /nix/var/nix/db/__db.*</screen>
-
-The Berkeley DB environment is automatically recreated with the new
-limits when you run any Nix command.</para>
-
-</section>
-
-
 <section><title>Collisions in <command>nix-env</command></title>
 
 <para>Symptom: when installing or upgrading, you get an error message such as
@@ -187,7 +76,8 @@ Furthermore, the <literal>st_nlink</literal> field of the
 <para>This only happens on very large Nix installations (such as build
 machines).</para>
 
-<para>Quick solution: run the garbage collector.</para>
+<para>Quick solution: run the garbage collector.  You may want to use
+the <option>--max-links</option> option.</para>
 
 <para>Real solution: put the Nix store on a file system that supports
 more than 32,000 subdirectories per directory, such as ReiserFS.

doc/manual/writing-nix-expressions.xml

@@ -731,9 +731,9 @@ stdenv.mkDerivation {
     xlink:href='http://www.ietf.org/rfc/rfc2396.txt'>RFC 2396</link>
     can be written <emphasis>as is</emphasis>, without quotes.  For
     instance, the string
-    <literal>"https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2"</literal>
+    <literal>"http://example.org/foo.tar.bz2"</literal>
     can also be written as
-    <literal>https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2</literal>.</para>
+    <literal>http://example.org/foo.tar.bz2</literal>.</para>
 
   </listitem>
 
@@ -937,17 +937,84 @@ set.</para>
 <para>Functions have the following form:
 
 <programlisting>
-{<replaceable>params</replaceable>}: <replaceable>body</replaceable></programlisting>
+<replaceable>pattern</replaceable>: <replaceable>body</replaceable></programlisting>
 
-This defines a function that must be called with an attribute set
-containing the attributes listed in <replaceable>params</replaceable>,
-which is a comma-separated list of attribute names.  Optionally, for
-each parameter a <emphasis>default value</emphasis> may be specified
-by writing <literal><replaceable>param</replaceable> ?
-<replaceable>e</replaceable></literal>, where
-<replaceable>e</replaceable> is an arbitrary expression.  If a
-parameter has a default, the corresponding attribute may be omitted in
-function calls.</para>
+The pattern specifies what the argument of the function must look
+like, and binds variables in the body to (parts of) the
+argument.  There are three kinds of patterns:</para>
+
+<itemizedlist>
+
+  
+  <listitem><para>If a pattern is a single identifier, then the
+  function matches any argument.  Example:
+
+  <programlisting>
+let negate = x: !x;
+    concat = x: y: x + y;
+in if negate true then concat "foo" "bar" else ""</programlisting>
+
+  Note that <function>concat</function> is a function that takes one
+  argument and returns a function that takes another argument.  This
+  allows partial parameterisation (i.e., only filling some of the
+  arguments of a function); e.g.,
+
+  <programlisting>
+map (concat "foo") ["bar" "bla" "abc"]</programlisting>
+
+  evaluates to <literal>["foobar" "foobla"
+  "fooabc"]</literal>.</para></listitem>
+
+  
+  <listitem><para>An <emphasis>attribute set pattern</emphasis> of the
+  form <literal>{name1, name2, …, nameN}</literal>
+  matches an attribute set containing the listed attributes, and binds
+  the values of those attributes to variables in the function body.
+  For example, the function
+
+<programlisting>
+{x, y, z}: z + y + x</programlisting>
+
+  can only be called with a set containing exactly the attributes
+  <varname>x</varname>, <varname>y</varname> and
+  <varname>z</varname>.  No other attributes are allowed.  If you want
+  to allow additional arguments, you can use an ellipsis
+  (<literal>...</literal>): 
+
+<programlisting>
+{x, y, z, ....}: z + y + x</programlisting>
+
+  This works on any set that contains at least the three named
+  attributes.</para>
+
+  <para>It is possible to provide <emphasis>default values</emphasis>
+  for attributes, in which case they are allowed to be missing.  A
+  default value is specified by writing
+  <literal><replaceable>name</replaceable> ?
+  <replaceable>e</replaceable></literal>, where
+  <replaceable>e</replaceable> is an arbitrary expression.  For example,
+
+<programlisting>
+{x, y ? "foo", z ? "bar"}: z + y + x</programlisting>
+
+  specifies a function that only requires an attribute named
+  <varname>x</varname>, but optionally accepts <varname>y</varname>
+  and <varname>z</varname>.</para></listitem>
+
+
+  <listitem><para>An <literal>@</literal>-pattern requires that the
+  argument matches with the patterns on the left- and right-hand side
+  of the <literal>@</literal>-sign.  For example:
+
+<programlisting>
+args@{x, y, z, ...}: z + y + x + args.a</programlisting>
+
+  Here <varname>args</varname> is bound to the entire argument, which
+  is further matches against the pattern <literal>{x, y, z,
+  ...}</literal>.</para></listitem>
+
+
+</itemizedlist>
 
 <para>Note that functions do not have names.  If you want to give them
 a name, you can bind them to an attribute, e.g.,
@@ -958,31 +1025,6 @@ in concat {x = "foo"; y = "bar";}</programlisting>
 
 </para>
 
-<para>It is also possible to define a function that takes a single
-argument and that does not need to be called with an attribute set as
-argument.  The syntax is
-
-<programlisting>
-<replaceable>var</replaceable>: <replaceable>body</replaceable></programlisting>
-
-where <replaceable>var</replaceable> is the name of the argument.  It
-is not possible to define a default.  Example:
-
-<programlisting>
-let negate = x: !x;
-    concat = x: y: x + y;
-in if negate true then concat "foo" "bar" else ""</programlisting>
-
-Note that <function>concat</function> is a function that takes one
-arguments and returns a function that takes another argument.  This
-allows partial parameterisation (i.e., only filling some of the
-arguments of a function); e.g.,
-
-<programlisting>
-map (concat "foo") ["bar" "bla" "abc"]</programlisting>
-
-evaluates to <literal>["foobar" "foobla" "fooabc"]</literal>.</para>
-
 </simplesect>
 
 
@@ -1644,59 +1686,6 @@ impureEnvVars = ["http_proxy" "https_proxy" <replaceable>...</replaceable>];
 
 <section xml:id='sec-standard-environment'><title>The standard environment</title>
 
-<para>The standard build environment in the Nix Packages collection
-provides a basic environment for building Unix packages.  It consists
-of the following packages:
-
-<itemizedlist>
-
-  <listitem><para>The GNU C Compiler, configured with C and C++
-  support.  On Linux, the compiler has been patched to provide greater
-  <quote>purity</quote> assurance.  For instance, the compiler doesn't
-  search in locations such as <filename>/usr/include</filename>.  In
-  fact, attempts to add such directories through the
-  <option>-I</option> flag are filtered out.  Likewise, the linker
-  (from GNU binutils) doesn't search in standard locations such as
-  <filename>/usr/lib</filename>.  Programs built on Linux are linked
-  against a GNU C Library that likewise doesn't search in the default
-  system locations.</para></listitem>
-
-  <listitem><para>GNU coreutils (contains a few dozen standard Unix
-  commands).</para></listitem>
-
-  <listitem><para>GNU findutils (contains
-  <command>find</command>).</para></listitem>
-
-  <listitem><para>GNU diffutils (contains <command>diff</command>,
-  <command>cmp</command>).</para></listitem>
-
-  <listitem><para>GNU <command>sed</command>.</para></listitem>
-
-  <listitem><para>GNU <command>grep</command>.</para></listitem>
-
-  <listitem><para>GNU <command>awk</command>.</para></listitem>
-
-  <listitem><para>GNU <command>tar</command>.</para></listitem>
-
-  <listitem><para><command>gzip</command> and
-  <command>bzip2</command>.</para></listitem>
-
-  <listitem><para>GNU Make.  It has been patched to provide
-  <quote>nested</quote> output that can be fed into the
-  <command>nix-log2xml</command> command and
-  <command>log2html</command> stylesheet to create a structured,
-  readable output of the build steps performed by
-  Make.</para></listitem>
-
-  <listitem><para>Bash.  This is the shell used for all builders in
-  the Nix Packages collection.  Not using <command>/bin/sh</command>
-  removes a large source of portability problems.</para></listitem>
-
-  <listitem><para>Patch.</para></listitem>
-
-</itemizedlist>
-
-</para>
 
 <para>The standard environment is used by passing it as an input
 called <envar>stdenv</envar> to the derivation, and then doing
@@ -1765,115 +1754,6 @@ myPostInstall() {
 
 </para>
 
-<para>The generic builder has a number of <emphasis>phases</emphasis>,
-each of which can be override in its entirety by setting the indicated
-variable.  The phases are:
-
-<itemizedlist>
-
-  <listitem>
-
-    <para><function>unpackPhase</function> unpacks the source files
-    listed in the <envar>src</envar> environment variable to the
-    current directory.  It supports <filename>tar</filename> files,
-    optionally compressed with <command>gzip</command> or
-    <command>bzip2</command>; Zip files (but note that the
-    <command>unzip</command> command is not a part of the standard
-    environment; you should add it as a build input yourself); and
-    unpacked source trees (i.e., directories; they are copied
-    verbatim).  You can add support for other file types by setting
-    the <varname>findUnpacker</varname> hook.  This hook should set
-    the variable <varname>unpackCmd</varname> to contain the command
-    to be executed to unpack the file.</para>
-
-    <para>After unpacking all source files,
-    <function>unpackPhase</function> changes the current directory to
-    the directory created by unpacking the sources.  If there are
-    multiple source directories, you should set
-    <varname>sourceRoot</varname> to the name of the intended
-    directory.</para>
-
-    <para>It also calls the hook <varname>postUnpack</varname> after
-    unpacking.</para>
-
-  </listitem>
-
-  <listitem><para><function>patchPhase</function> calls the
-  <command>patch</command> command with the <option>-p1</option>
-  option for each patch file listed in the <envar>patches</envar>
-  variable.</para></listitem>
-
-  <listitem>
-
-    <para><function>configurePhase</function> runs the script called
-    <filename>configure</filename> in the current directory with a
-    <option>--prefix</option> set to the output path.  You can add
-    additional flags through the <varname>configureFlags</varname>
-    variable.  If <filename>configure</filename> does not exist,
-    nothing happens.</para>
-
-    <para>Before and after running <filename>configure</filename>, the
-    hooks <varname>preConfigure</varname> and
-    <varname>postConfigure</varname> are called, respectively.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para><function>buildPhase</function> calls
-    <command>make</command>.  You can set flags for
-    <command>make</command> through the <varname>makeFlags</varname>
-    variable.</para>
-
-    <para>Before and after running <command>make</command>, the hooks
-    <varname>preBuild</varname> and <varname>postBuild</varname> are
-    called, respectively.</para>
-    
-  </listitem>
-
-  <listitem><para><function>checkPhase</function> calls <command>make
-  check</command>, but only if the <varname>doCheck</varname> variable
-  is set to <literal>1</literal>.  Additional flags can be set through
-  the <varname>checkFlags</varname> variable.</para></listitem>
-
-  <listitem>
-
-    <para><function>installPhase</function> calls <command>make
-    install</command>.  Additional flags can be set through the
-    <varname>installFlags</varname> variable.  It also strips any
-    static libraries in the output path of debug information unless
-    <varname>dontStrip</varname> is set to
-    <literal>1</literal>.</para>
-
-    <para>Before and after running <command>make install</command>,
-    the hooks <varname>preInstall</varname> and
-    <varname>postInstall</varname> are called, respectively.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para><function>distPhase</function> calls <command>make
-    dist</command>, but only if the <varname>doDist</varname> variable
-    is set to <literal>1</literal>.  Additional flags can be set
-    through the <varname>distFlags</varname> variable.  The resulting
-    tarball is copied to the <filename>/tarballs</filename>
-    subdirectory of the output path.</para>
-
-    <para>Before and after running <command>make dist</command>, the
-    hooks <varname>preDist</varname> and <varname>postDist</varname>
-    are called, respectively.</para>
-    
-  </listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>You can change the order in which phases are executed, or add
-new phases, by setting the <varname>phases</varname> variable.  The
-default is <literal>patchPhase configurePhase buildPhase checkPhase
-installPhase distPhase</literal>.</para>
 
 </section>
 

externals/Makefile.am

@@ -75,12 +75,12 @@ endif
 
 # bzip2
 
-BZIP2 = bzip2-1.0.4
+BZIP2 = bzip2-1.0.5
 
 $(BZIP2).tar.gz:
 	@echo "Nix requires bzip2 to build."
-	@echo "Please download version 1.0.4 from"
-	@echo "  http://www.bzip.org/1.0.4/bzip2-1.0.4.tar.gz"
+	@echo "Please download version 1.0.5 from"
+	@echo "  http://www.bzip.org/1.0.5/bzip2-1.0.5.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -113,5 +113,5 @@ EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.bz2 $(BZIP2).tar.gz \
   bdb-cygwin.patch
 
 ext-clean:
-	$(RM) -f have-db build-db have-aterm build-aterm
+	$(RM) -f have-db build-db have-aterm build-aterm have-bzip2 build-bzip2
 	$(RM) -rf $(DB) $(ATERM) $(BZIP2)

nix.conf.example

@@ -29,25 +29,6 @@
 #gc-keep-derivations = true
 
 
-### Option `gc-reserved-space'
-#
-# This option specifies how much space should be reserved in normal
-# use so that the garbage collector can run succesfully.  Since the
-# garbage collector must perform Berkeley DB transactions, it needs
-# some disk space for itself.  However, when the disk is full, this
-# space is not available, so the collector would not be able to run
-# precisely when it is most needed.
-#
-# For this reason, when Nix is run, it allocates a file
-# /nix/var/nix/db/reserved of the size specified by this option.  When
-# the garbage collector is run, this file is deleted before the
-# Berkeley DB environment is opened.  This should give it enough room
-# to proceed.
-#
-# The default is "1048576" (1 MiB).
-#gc-reserved-space = 1048576
-
-
 ### Option `env-keep-derivations'
 #
 # If `false' (default), derivations are not stored in Nix user
@@ -164,13 +145,13 @@
 # (using `mount --bind' on Linux) some directories from the normal
 # file system hierarchy inside the chroot.  These are the Nix store,
 # the temporary build directory (usually /tmp/nix-<pid>-<number>) and
-# the directories listed here.  The default is "/dev /proc".  Files
-# in /dev (such as /dev/null) are needed by many builds, and some
-# files in /proc may also be needed occasionally.
+# the directories listed here.  The default is "/dev /dev/pts /proc".
+# Files in /dev (such as /dev/null) are needed by many builds, and
+# some files in /proc may also be needed occasionally.
 #
 # Example:
 #   build-use-chroot = /dev /proc /bin
-#build-chroot-dirs = /dev /proc
+#build-chroot-dirs = /dev /dev/pts /proc
 
 
 ### Option `system'

nix.spec.in

@@ -13,11 +13,10 @@ Version: @version@
 Release: 1
 License: GPL
 Group: Software Deployment
-URL: http://nix.cs.uu.nl/
+URL: http://nixos.org/
 Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
-%define _prefix /nix
-Prefix: %{_prefix}
+Prefix: /usr
 Requires: /usr/bin/perl
 Requires: curl
 
@@ -47,14 +46,14 @@ if test -n "%{enable_setuid}"; then
         extraFlags="$extraFlags --with-nix-group=%{nix_group}"
     fi
 fi
-./configure --prefix=%{_prefix} $extraFlags
+./configure --prefix=%{_prefix} --sysconfdir=/etc $extraFlags
 make
 make check
 
 %install
 rm -rf $RPM_BUILD_ROOT
 make DESTDIR=$RPM_BUILD_ROOT install
-rm $RPM_BUILD_ROOT/%{_prefix}/etc/nix/nix.conf
+rm $RPM_BUILD_ROOT/etc/nix/nix.conf
 strip $RPM_BUILD_ROOT/%{_prefix}/bin/* || true
 
 %clean
@@ -76,10 +75,9 @@ fi
 %{_prefix}/lib
 %{_prefix}/libexec
 %{_prefix}/include
-%{_prefix}/var
 %{_prefix}/share
-%{_prefix}/store
+/etc/profile.d/nix.sh
+/nix/var
+/nix/store
 %config
-%{_prefix}/etc
-#%doc
-#%{_prefix}/share/nix/manual
+/etc/nix

scripts/Makefile.am

@@ -1,24 +1,26 @@
 bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build \
-  nix-pack-closure nix-unpack-closure \
   nix-copy-closure 
 
 noinst_SCRIPTS = nix-profile.sh generate-patches.pl \
-  find-runtime-roots.pl build-remote.pl nix-reduce-build
+  find-runtime-roots.pl build-remote.pl nix-reduce-build \
+  copy-from-other-stores.pl nix-http-export.cgi
 
 nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm download-using-manifests.pl find-runtime-roots.pl
+install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
+	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
+	$(INSTALL_PROGRAM) copy-from-other-stores.pl $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 
 include ../substitute.mk
@@ -31,9 +33,10 @@ EXTRA_DIST = nix-collect-garbage.in \
   readconfig.pm.in \
   nix-build.in \
   download-using-manifests.pl.in \
+  copy-from-other-stores.pl.in \
   generate-patches.pl.in \
-  nix-pack-closure.in nix-unpack-closure.in \
   nix-copy-closure.in \
   find-runtime-roots.pl.in \
   build-remote.pl.in \
-  nix-reduce-build.in
+  nix-reduce-build.in \
+  nix-http-export.cgi.in 

scripts/copy-from-other-stores.pl.in

@@ -0,0 +1,98 @@
+#! @perl@ -w
+
+use strict;
+use File::Basename;
+use IO::Handle;
+
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
+
+STDOUT->autoflush(1);
+
+my @remoteStoresAll = split ':', ($ENV{"NIX_OTHER_STORES"} or "");
+
+my @remoteStores;
+foreach my $dir (@remoteStoresAll) {
+    push @remoteStores, glob($dir);
+}
+
+
+sub findStorePath {
+    my $storePath = shift;
+    
+    my $storePathName = basename $storePath;
+    
+    foreach my $store (@remoteStores) {
+        # Determine whether $storePath exists by looking for the
+        # existence of the info file, and if so, get store path info
+        # from that file.  This rather breaks abstraction: we should
+        # be using `nix-store' for that.  But right now there is no
+        # good way to tell nix-store to access a store mounted under a
+        # different location (there's $NIX_STORE, but that only works
+        # if the remote store is mounted under its "real" location).
+        my $infoFile = "$store/var/nix/db/info/$storePathName";
+        my $storePath2 = "$store/store/$storePathName";
+        if (-f $infoFile && -e $storePath2) {
+            return ($infoFile, $storePath2);
+        }
+    }
+}
+
+
+if ($ARGV[0] eq "--query") {
+
+    while (<STDIN>) {
+        my $cmd = $_; chomp $cmd;
+
+        if ($cmd eq "have") {
+            my $storePath = <STDIN>; chomp $storePath;
+            (my $infoFile) = findStorePath $storePath;
+            print STDOUT ($infoFile ? "1\n" : "0\n");
+        }
+
+        elsif ($cmd eq "info") {
+            my $storePath = <STDIN>; chomp $storePath;
+            (my $infoFile) = findStorePath $storePath;
+            if (!$infoFile) {
+                print "0\n";
+                next; # not an error
+            }
+            print "1\n";
+
+            my $deriver = "";
+            my @references = ();
+
+            open INFO, "<$infoFile" or die "cannot read info file $infoFile\n";
+            while (<INFO>) {
+                chomp;
+                /^([\w-]+): (.*)$/ or die "bad info file";
+                my $key = $1;
+                my $value = $2;
+                if ($key eq "Deriver") { $deriver = $value; }
+                elsif ($key eq "References") { @references = split ' ', $value; }
+            }
+            close INFO;
+
+            print "$deriver\n";
+            print scalar @references, "\n";
+            print "$_\n" foreach @references;
+            print "0\n"; # !!! showing size not supported (yet)
+        }
+
+        else { die "unknown command `$cmd'"; }
+    }
+}
+
+
+elsif ($ARGV[0] eq "--substitute") {
+    die unless scalar @ARGV == 2;
+    my $storePath = $ARGV[1];
+    (my $infoFile, my $sourcePath) = findStorePath $storePath;
+    die unless $infoFile;
+    print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
+    system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
+        or die "cannot copy `$sourcePath' to `$storePath'";
+}
+
+
+else { die; }

scripts/download-using-manifests.pl.in

@@ -5,20 +5,13 @@ use readmanifest;
 use POSIX qw(strftime);
 use File::Temp qw(tempdir);
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
+STDOUT->autoflush(1);
+
 my $manifestDir = "@localstatedir@/nix/manifests";
 my $logFile = "@localstatedir@/log/nix/downloads";
 
-open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
-
-# Create a temporary directory.
-my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-chdir $tmpDir or die "cannot change to `$tmpDir': $!";
-
-my $tmpNar = "$tmpDir/nar";
-my $tmpNar2 = "$tmpDir/nar2";
-
 
 # Load all manifests.
 my %narFiles;
@@ -26,7 +19,6 @@ my %localPaths;
 my %patches;
 
 for my $manifest (glob "$manifestDir/*.nixmanifest") {
-#    print STDERR "reading $manifest\n";
     if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
         print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
         exit 1;
@@ -36,34 +28,42 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") {
 
 # Parse the arguments.
 
-if ($ARGV[0] eq "--query-paths") {
-    foreach my $storePath (keys %narFiles) { print "$storePath\n"; }
-    foreach my $storePath (keys %localPaths) { print "$storePath\n"; }
-    exit 0;
-}
+if ($ARGV[0] eq "--query") {
 
-elsif ($ARGV[0] eq "--query-info") {
-    shift @ARGV;
-    foreach my $storePath (@ARGV) {
-        my $info;
-        if (defined $narFiles{$storePath}) {
-            $info = @{$narFiles{$storePath}}[0];
+    while (<STDIN>) {
+        my $cmd = $_; chomp $cmd;
+
+        if ($cmd eq "have") {
+            my $storePath = <STDIN>; chomp $storePath;
+            print STDOUT ((defined $narFiles{$storePath} or defined $localPaths{$storePath})
+                ? "1\n" : "0\n");
         }
-        elsif (defined $localPaths{$storePath}) {
-            $info = @{$localPaths{$storePath}}[0];
-        }
-        else {
-            next; # not an error
-        }
-        print "$storePath\n";
-        print "$info->{deriver}\n";
-        my @references = split " ", $info->{references};
-        my $count = scalar @references;
-        print "$count\n";
-        foreach my $reference (@references) {
-            print "$reference\n";
+
+        elsif ($cmd eq "info") {
+            my $storePath = <STDIN>; chomp $storePath;
+            my $info;
+            if (defined $narFiles{$storePath}) {
+                $info = @{$narFiles{$storePath}}[0];
+            }
+            elsif (defined $localPaths{$storePath}) {
+                $info = @{$localPaths{$storePath}}[0];
+            }
+            else {
+                print "0\n";
+                next; # not an error
+            }
+            print "1\n";
+            print "$info->{deriver}\n";
+            my @references = split " ", $info->{references};
+            print scalar @references, "\n";
+            print "$_\n" foreach @references;
+            my $size = $info->{size} || 0;
+            print "$size\n";
         }
+        
+        else { die "unknown command `$cmd'"; }
     }
+
     exit 0;
 }
 
@@ -76,6 +76,18 @@ die unless scalar @ARGV == 2;
 my $targetPath = $ARGV[1];
 
 
+# Create a temporary directory.
+my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+    or die "cannot create a temporary directory";
+
+chdir $tmpDir or die "cannot change to `$tmpDir': $!";
+
+my $tmpNar = "$tmpDir/nar";
+my $tmpNar2 = "$tmpDir/nar2";
+
+
+open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
+
 my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
 print LOGFILE "$$ get $targetPath $date\n";
 
@@ -88,7 +100,7 @@ foreach my $localPath (@{$localPathList}) {
     my $sourcePath = $localPath->{copyFrom};
     if (-e $sourcePath) {
         print "\n*** Step 1/1: copying from $sourcePath\n";
-        system("@bindir@/nix-store --dump $sourcePath | @bindir@/nix-store --restore $targetPath") == 0
+        system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $targetPath") == 0
             or die "cannot copy `$sourcePath' to `$targetPath'";
         exit 0;
     }
@@ -137,7 +149,7 @@ addToQueue $targetPath;
 
 sub isValidPath {
     my $p = shift;
-    return system("@bindir@/nix-store --check-validity '$p' 2> /dev/null") == 0;
+    return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
 }
 
 sub parseHash {
@@ -171,7 +183,7 @@ while ($queueFront < scalar @queue) {
                 my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
                 my $format = "--base32";
                 $format = "" if $baseHashAlgo eq "md5";
-                my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
+                my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
                 chomp $hash;
 #                print "  MY HASH is $hash\n";
                 if ($hash ne $baseHash) {
@@ -253,7 +265,7 @@ sub downloadFile {
     $ENV{"PRINT_PATH"} = 1;
     $ENV{"QUIET"} = 1;
     $ENV{"NIX_HASH_ALGO"} = $hashAlgo;
-    my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
+    my ($hash2, $path) = `$binDir/nix-prefetch-url '$url' '$hash'`;
     die "download of `$url' failed" unless $? == 0;
     chomp $hash2;
     chomp $path;
@@ -277,7 +289,7 @@ while (scalar @path > 0) {
             # as a base to one or more patches.  So turn the base path
             # into a NAR archive, to which we can apply the patch.
             print "  packing base path...\n";
-            system("@bindir@/nix-store --dump $v > $tmpNar") == 0
+            system("$binDir/nix-store --dump $v > $tmpNar") == 0
                 or die "cannot dump `$v'";
         }
     }
@@ -305,7 +317,7 @@ while (scalar @path > 0) {
             # This was the last patch.  Unpack the final NAR archive
             # into the target path.
             print "  unpacking patched archive...\n";
-            system("@bindir@/nix-store --restore $v < $tmpNar2") == 0
+            system("$binDir/nix-store --restore $v < $tmpNar2") == 0
                 or die "cannot unpack $tmpNar2 into `$v'";
         }
     }
@@ -327,7 +339,7 @@ while (scalar @path > 0) {
         } else {
             # Unpack the archive into the target path.
             print "  unpacking archive...\n";
-            system("@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'") == 0
+            system("@bunzip2@ < '$narFilePath' | $binDir/nix-store --restore '$v'") == 0
                 or die "cannot unpack `$narFilePath' into `$v'";
         }
     }

scripts/nix-build.in

@@ -2,6 +2,8 @@
 
 use strict;
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 my $addDrvLink = 0;
 my $addOutLink = 1;
@@ -9,6 +11,8 @@ my $addOutLink = 1;
 my $outLink;
 my $drvLink;
 
+my $dryRun = 0;
+
 my @instArgs = ();
 my @buildArgs = ();
 my @exprs = ();
@@ -89,6 +93,11 @@ EOF
         push @buildArgs, ($arg, $ARGV[$n]);
     }
 
+    elsif ($arg eq "--dry-run") {
+        push @buildArgs, "--dry-run";
+        $dryRun = 1;
+    }
+    
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -117,7 +126,7 @@ foreach my $expr (@exprs) {
     # Instantiate.
     my @drvPaths;
     # !!! would prefer the perl 5.8.0 pipe open feature here.
-    my $pid = open(DRVPATHS, "-|") || exec "@bindir@/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
+    my $pid = open(DRVPATHS, "-|") || exec "$binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
     while (<DRVPATHS>) {chomp; push @drvPaths, $_;}
     close DRVPATHS or exit 1;
 
@@ -128,11 +137,13 @@ foreach my $expr (@exprs) {
 
     # Build.
     my @outPaths;
-    $pid = open(OUTPATHS, "-|") || exec "@bindir@/nix-store", "--add-root", $outLink, "--indirect", "-rv",
+    $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-rv",
         @buildArgs, @drvPaths;
     while (<OUTPATHS>) {chomp; push @outPaths, $_;}
     close OUTPATHS or exit 1;
 
+    next if $dryRun;
+
     foreach my $outPath (@outPaths) {
         my $target = readlink $outPath or die "cannot read symlink `$outPath'";
         print "$target\n";

scripts/nix-collect-garbage.in

@@ -4,6 +4,8 @@ use strict;
 
 my $profilesDir = "@localstatedir@/nix/profiles";
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 # Process the command line arguments.
 my @args = ();
@@ -34,7 +36,7 @@ sub removeOldGenerations {
         $name = $dir . "/" . $name;
         if (-l $name && (readlink($name) =~ /link/)) {
             print STDERR "removing old generations of profile $name\n";
-            system("@bindir@/nix-env", "-p", $name, "--delete-generations", "old");
+            system("$binDir/nix-env", "-p", $name, "--delete-generations", "old");
         }
         elsif (! -l $name && -d $name) {
             removeOldGenerations $name;
@@ -48,4 +50,4 @@ removeOldGenerations $profilesDir if $removeOld;
 
 
 # Run the actual garbage collector.
-exec "@bindir@/nix-store", "--gc", @args;
+exec "$binDir/nix-store", "--gc", @args;

scripts/nix-copy-closure.in

@@ -1,7 +1,6 @@
 #! @perl@ -w
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 
 if (scalar @ARGV < 1) {

scripts/nix-http-export.cgi.in

@@ -0,0 +1,50 @@
+#! /bin/sh
+
+export HOME=/tmp
+export NIX_REMOTE=daemon
+
+TMP_DIR="${TMP_DIR:-/tmp/nix-export}"
+
+@coreutils@/mkdir -p "$TMP_DIR" || true
+@coreutils@/chmod a+r "$TMP_DIR"
+
+needed_path="?$QUERY_STRING"
+needed_path="${needed_path#*[?&]needed_path=}"
+needed_path="${needed_path%%&*}"
+#needed_path="$(echo $needed_path  | ./unhttp)"
+needed_path="$(echo $needed_path  | sed -e 's/%2B/+/g; s/%3D/=/g')"
+
+echo needed_path: "$needed_path" >&2
+
+NIX_STORE="${NIX_STORE_DIR:-/nix/store}"
+
+echo NIX_STORE: "${NIX_STORE}" >&2
+
+full_path="${NIX_STORE}"/"$needed_path"
+
+if [ "$needed_path" != "${needed_path%.drv}" ]; then
+	echo "Status: 403 You should create the derivation file yourself"
+	echo "Content-Type: text/plain"
+	echo
+	echo "Refusing to disclose derivation contents"
+	exit
+fi
+
+if @bindir@/nix-store --check-validity "$full_path"; then
+	if ! [ -e nix-export/"$needed_path".nar.gz ]; then
+		@bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz
+		@coreutils@/ln -fs  "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz 
+	fi;
+	echo "Status: 301 Moved"
+	echo "Location: nix-export/"$needed_path".nar.gz"
+	echo
+else 
+	echo "Status: 404 No such path found"
+	echo "Content-Type: text/plain"
+	echo
+	echo "Path not found:"
+	echo "$needed_path"
+	echo "checked:"
+	echo "$full_path"
+fi
+

scripts/nix-install-package.in

@@ -3,6 +3,8 @@
 use strict;
 use File::Temp qw(tempdir);
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 sub usageError {
     print STDERR <<EOF;
@@ -59,7 +61,7 @@ if ($interactive && !defined $ENV{"NIX_HAVE_TERMINAL"}) {
     $ENV{"NIX_HAVE_TERMINAL"} = "1";
     $ENV{"LD_LIBRARY_PATH"} = "";
     foreach my $term ("xterm", "konsole", "gnome-terminal", "xterm") {
-        exec($term, "-e", "@bindir@/nix-install-package", @ARGV);
+        exec($term, "-e", "$binDir/nix-install-package", @ARGV);
     }
     die "cannot execute `xterm'";
 }
@@ -122,12 +124,12 @@ if ($interactive) {
 
 
 print "\nPulling manifests...\n";
-system("@bindir@/nix-pull", $manifestURL) == 0
+system("$binDir/nix-pull", $manifestURL) == 0
     or barf "nix-pull failed: $?";
 
 
 print "\nInstalling package...\n";
-system("@bindir@/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0
+system("$binDir/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0
     or barf "nix-env failed: $?";
 
 

scripts/nix-pack-closure.in

@@ -1,75 +0,0 @@
-#! @perl@ -w
-
-# This tool computes the closure of a path (using "nix-store --query
-# --requisites") and puts the contents of each path in the closure in
-# a big NAR archive that can be installed on another Nix installation
-# using "nix-unpack-closure".
-
-# TODO: make this program "streamy", i.e., don't use a temporary
-# directory.
-
-use strict;
-use File::Temp qw(tempdir);
-
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
-
-my $tmpDir = tempdir("nix-pack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-mkdir "$tmpDir/contents", 0755 or die;
-mkdir "$tmpDir/references", 0755 or die;
-mkdir "$tmpDir/derivers", 0755 or die;
-
-open TOPLEVEL, ">$tmpDir/top-level" or die;
-
-
-my %storePaths;
-
-
-while (@ARGV) {
-    my $storePath = shift @ARGV;
-
-    # $storePath might be a symlink to the store, so resolve it.
-    $storePath = (`$binDir/nix-store --query --resolve '$storePath'`
-        or die "cannot resolve `$storePath'");
-    chomp $storePath;
-    print TOPLEVEL $storePath, "\n";
-
-    # Get the closure of this path.
-    my $pid = open(READ,
-        "$binDir/nix-store --query --requisites '$storePath'|") or die;
-    
-    while (<READ>) {
-        chomp;
-        die "bad: $_" unless /^\//;
-        $storePaths{$_} = "";
-    }
-
-    close READ or die "nix-store failed: $?";
-}
-
-
-close TOPLEVEL or die;
-
-
-foreach my $storePath (sort(keys %storePaths)) {
-    print STDERR "packing `$storePath'...\n";
-
-    $storePath =~ /\/([^\/]+)$/;
-    my $name = $1;
-
-    system("$binDir/nix-store --dump '$storePath' > $tmpDir/contents/$name") == 0
-        or die "nix-store --dump failed on `$storePath': $?";
-
-    system("$binDir/nix-store --query --references '$storePath' > $tmpDir/references/$name") == 0
-        or die "nix-store --query --references failed on `$storePath': $?";
-
-    system("$binDir/nix-store --query --deriver '$storePath' > $tmpDir/derivers/$name") == 0
-        or die "nix-store --query --deriver failed on `$storePath': $?";
-}
-
-
-# Write a NAR archive of everything to standard output.
-system("nix-store --dump '$tmpDir'") == 0
-    or die "nix-store --dump failed";

scripts/nix-prefetch-url.in

@@ -37,6 +37,7 @@ fi
 
 
 mkTempDir() {
+    if test -n "$tmpPath"; then return; fi
     local i=0
     while true; do
         if test -z "$TMPDIR"; then TMPDIR=/tmp; fi
@@ -64,6 +65,27 @@ doDownload() {
 }
 
 
+# Hack to support the mirror:// scheme from Nixpkgs.
+if test "${url:0:9}" = "mirror://"; then
+    if test -z "$NIXPKGS_ALL"; then
+        echo "Resolving mirror:// URLs requires Nixpkgs.  Please point \$NIXPKGS_ALL at a Nixpkgs tree." >&2
+        exit 1
+    fi
+
+    mkTempDir
+    nix-build "$NIXPKGS_ALL" -A resolveMirrorURLs --argstr url "$url" -o $tmpPath/urls > /dev/null
+
+    expanded=($(cat $tmpPath/urls))
+    if test "${#expanded[*]}" = 0; then
+        echo "$0: cannot resolve $url." >&2
+        exit 1
+    fi
+
+    echo "$url expands to ${expanded[*]} (using ${expanded[0]})" >&2
+    url="${expanded[0]}"
+fi
+
+
 # If we don't know the hash or a file with that hash doesn't exist,
 # download the file and add it to the store.
 if test -z "$finalPath"; then
@@ -78,7 +100,7 @@ if test -z "$finalPath"; then
     # garbage-collected independently.
     if test -n "$NIX_DOWNLOAD_CACHE"; then
         echo -n "$url" > $tmpPath/url
-        urlHash=$(nix-hash --type sha256 --base32 --flat $tmpPath/url)
+        urlHash=$(@bindir@/nix-hash --type sha256 --base32 --flat $tmpPath/url)
         echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url"
         cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType"
         cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp"

scripts/nix-profile.sh.in

@@ -7,5 +7,12 @@ if test -n "$HOME"; then
         @coreutils@/ln -s "$_NIX_DEF_LINK" "$NIX_LINK"
     fi
 
-    export PATH=$NIX_LINK/bin:@prefix@/bin:$PATH
+    export PATH=$NIX_LINK/bin:$PATH
+fi
+
+# This is a quick hack to make fontconfig-based packages in Nixpkgs
+# work out of the box on non-NixOS systems.  Of course, we should
+# really fix fontconfig...
+if test -z "$FONTCONFIG_FILE" -a -e /etc/fonts/fonts.conf; then
+    export FONTCONFIG_FILE=/etc/fonts/fonts.conf
 fi

scripts/nix-pull.in

@@ -7,8 +7,7 @@ use readmanifest;
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
 $libexecDir = "@libexecdir@" unless defined $libexecDir;

scripts/nix-push.in

@@ -16,8 +16,7 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 my $dataDir = $ENV{"NIX_DATA_DIR"};
 $dataDir = "@datadir@" unless defined $dataDir;
@@ -107,7 +106,7 @@ foreach my $storePath (@storePaths) {
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
         "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
-        "{storePath = builtins.toPath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
+        "{storePath = builtins.storePath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
     
     print NIX $nixexpr;
 }
@@ -224,13 +223,14 @@ writeManifest $manifest, \%narFiles, \%patches;
 sub copyFile {
     my $src = shift;
     my $dst = shift;
-    system("@coreutils@/cp", $src, "$dst.tmp") == 0 or die "cannot copy file";
-    rename("$dst.tmp", "$dst") or die "cannot rename file";
+    my $tmp = "$dst.tmp.$$";
+    system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file";
+    rename($tmp, $dst) or die "cannot rename file: $!";
 }
 
 
-# Upload the archives.
-print STDERR "uploading archives...\n";
+# Upload/copy the archives.
+print STDERR "uploading/copying archives...\n";
 
 sub archiveExists {
     my $name = shift;
@@ -244,9 +244,12 @@ foreach my $narArchive (@narArchives) {
     my $basename = $1;
 
     if ($localCopy) {
+        # Since nix-push creates $dst atomically, if it exists we
+        # don't have to copy again.
+        my $dst = "$localArchivesDir/$basename";
         if (! -f "$localArchivesDir/$basename") {
             print STDERR "  $narArchive\n";
-            copyFile $narArchive, "$localArchivesDir/$basename";
+            copyFile $narArchive, $dst;
         }
     }
     else {

scripts/nix-reduce-build.in

@@ -3,10 +3,30 @@
 WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX);
 cd "$WORKING_DIRECTORY";
 
-if test -z "$1" ; then
-	echo 'nix-reduce-build (paths or Nix expressions) -- (logins at remote computers)' >&2
+if test -z "$1" || test "a--help" = "a$1" ; then
+	echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2
 	echo As in: >&2
-	echo nix-reduce-build /etc/nixos/nixos -- user@somewhere.nowhere.example.org >&2
+	echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2
+	echo nix-reduce-build /etc/nixos/nixos -- \\
+	echo "   " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2
+	echo "  store path name will be added into the end of the URL" >&2
+	echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2
+	echo "  that should be a directory where gzipped 'nix-store --export' ">&2
+	echo "  files are located (they should have .nar.gz extension)"  >&2
+	echo "        Or all together: " >&2
+	echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\
+	"    ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2
+	echo "        Also supports best-effort local builds of failing expression set:" >&2
+	echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2
+	echo "  nix-daemon:// builds using daemon"
+	echo "  nix-self:// builds directly using nix-store from current installation" >&2
+	echo "  nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2;
+	echo "derivations with specified output hash (sha256, sha1 or md5)." >&2
+	echo "  nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2;
+	echo "maximum amount of paths" >&2;
+	echo "  nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2;
+	echo "maximum amount of paths" >&2;
+	echo "        If no package sources are specified, required paths are listed." >&2;
 	exit;
 fi;
 
@@ -19,11 +39,14 @@ echo Will work on $(cat initial | wc -l) targets. >&2
 
 while read ; do
 	case "$REPLY" in 
-		${NIX_STORE_PATH:-/nix/store}/*)
+		${NIX_STORE_DIR:-/nix/store}/*)
 			echo "$REPLY" >> paths; >&2
 			;;
 		*)
-			nix-instantiate "$REPLY" >> paths;
+			(
+				IFS=: ;
+				nix-instantiate $REPLY >> paths;
+			);
 			;;
 	esac;
 done < initial;
@@ -51,14 +74,94 @@ echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2
 cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
 echo We need $(cat needed-paths | wc -l) paths. >&2
 
+egrep '[.]drv$' derivers-closure > critical-derivers;
+
 if test -z "$1" ; then
 	cat needed-paths;	
 fi;
 
+refresh_critical_derivers() {
+    echo "Finding needed derivers..." >&2;
+    cat critical-derivers | while read; do
+        if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then
+            echo "$REPLY";
+        fi;
+    done > new-critical-derivers;
+    mv new-critical-derivers critical-derivers;
+    echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2
+}
+
+build_here() {
+    cat critical-derivers | while read; do 
+        echo "Realising $REPLY using nix-daemon" >&2
+        @bindir@/nix-store -r "${REPLY}"
+    done;
+}
+
+try_to_substitute(){
+    cat needed-paths | while read ; do 
+        echo "Building $REPLY using nix-daemon" >&2
+        @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}"
+    done;
+}
+
 for i in "$@"; do 
-	cat needed-paths | while read; do 
-		nix-copy-closure --from "$i" --gzip "$REPLY" </dev/null || true; 
-	done;
+	sshHost="${i#ssh://}";
+	httpHost="${i#http://}";
+	httpsHost="${i#https://}";
+	filePath="${i#file:/}";
+	if [ "$i" != "$sshHost" ]; then
+		cat needed-paths | while read; do 
+			echo "Getting $REPLY and its closure over ssh" >&2
+			nix-copy-closure --from "$sshHost" --gzip "$REPLY" </dev/null || true; 
+		done;
+	elif [ "$i" != "$httpHost" ] || [ "$i" != "$httpsHost" ]; then
+		cat needed-paths | while read; do
+			echo "Getting $REPLY over http/https" >&2
+			curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import;
+		done;
+	elif [ "$i" != "$filePath" ] ; then
+		cat needed-paths | while read; do 
+			echo "Installing $REPLY from file" >&2
+			gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import;
+		done;
+	elif [ "$i" = "nix-daemon://" ] ; then
+		NIX_REMOTE=daemon try_to_substitute;
+		refresh_critical_derivers;
+		NIX_REMOTE=daemon build_here;
+	elif [ "$i" = "nix-self://" ] ; then
+		NIX_REMOTE= try_to_substitute;
+		refresh_critical_derivers;
+		NIX_REMOTE= build_here;
+	elif [ "$i" = "nix-daemon-fixed://" ] ; then
+		refresh_critical_derivers;
+
+		cat critical-derivers | while read; do 
+			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
+				echo "Realising $REPLY using nix-daemon" >&2
+				NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}"
+			fi;
+		done;
+	elif [ "$i" = "nix-self-fixed://" ] ; then
+		refresh_critical_derivers;
+
+		cat critical-derivers | while read; do 
+			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
+				echo "Realising $REPLY using direct Nix build" >&2
+				NIX_REMOTE= @bindir@/nix-store -r "${REPLY}"
+			fi;
+		done;
+	elif [ "$i" = "nix-daemon-substitute://" ] ; then
+		NIX_REMOTE=daemon try_to_substitute;
+	elif [ "$i" = "nix-self-substitute://" ] ; then
+		NIX_REMOTE= try_to_substitute;
+	elif [ "$i" = "nix-daemon-build://" ] ; then
+		refresh_critical_derivers;
+		NIX_REMOTE=daemon build_here;
+	elif [ "$i" = "nix-self-build://" ] ; then
+		refresh_critical_derivers;
+		NIX_REMOTE= build_here;
+	fi;
 	mv needed-paths wanted-paths;
 	cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
 	echo We still need $(cat needed-paths | wc -l) paths. >&2

scripts/nix-unpack-closure.in

@@ -1,88 +0,0 @@
-#! @perl@ -w
-
-# This tool unpacks the closures created by "nix-pack-closure" and
-# adds them to the Nix store.
-
-# TODO: make this program "streamy", i.e., don't use a temporary
-# directory.
-
-use strict;
-use File::Temp qw(tempdir);
-
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
-
-my $tmpDir = tempdir("nix-unpack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-
-# Unpack the NAR archive on standard input.
-system("nix-store --restore '$tmpDir/unpacked'") == 0
-    or die "nix-store --restore failed";
-
-
-open VALID, ">$tmpDir/validity" or die;
-
-
-# For each path in the closure that is not yet valid, add it to the
-# store.  TODO: use proper locking.  Or even better, let nix-store do
-# this.
-opendir(DIR, "$tmpDir/unpacked/contents") or die "cannot open directory: $!";
-
-foreach my $name (sort(readdir DIR)) {
-    next if $name eq "." or $name eq "..";
-
-    my $storePath = "@storedir@/$name"; # !!!
-
-    # !!! this really isn't a good validity check!
-    system "$binDir/nix-store --check-validity '$storePath' 2> /dev/null";
-    if ($? != 0) {
-        print STDERR "unpacking `$storePath'...\n";
-
-        # !!! race
-        system("@coreutils@/rm -rf '$storePath'") == 0
-            or die "cannot remove `$storePath': $?";
-
-        system("$binDir/nix-store --restore '$storePath' < '$tmpDir/unpacked/contents/$name'") == 0
-            or die "nix-store --dump failed on `$storePath': $?";
-        
-        print VALID "$storePath\n";
-
-        open DRV, "<$tmpDir/unpacked/derivers/$name" or die;
-        my $deriver = <DRV>;
-        chomp $deriver;
-        $deriver = "" if $deriver eq "unknown-deriver";
-        close DRV;
-
-        my @refs;
-        open REFS, "<$tmpDir/unpacked/references/$name" or die;
-        while (<REFS>) {
-            chomp;
-            push @refs, $_;
-        }
-        close REFS;
-
-        print VALID "$deriver\n";
-        
-        print VALID (scalar @refs), "\n";
-        foreach my $ref (@refs) {
-            print VALID "$ref\n";
-        }
-    }
-}
-
-closedir(DIR) or die;
-
-
-# Register the invalid paths as valid.
-system("nix-store --register-validity <'$tmpDir/validity'") == 0
-    or die "nix-store --register-validity failed";
-
-
-# Show the top-level paths so that something useful can be done with
-# them, e.g., passing them to `nix-env -i'.
-if (-e "$tmpDir/unpacked/top-level") {
-    open TOPLEVEL, "<$tmpDir/unpacked/top-level" or die;
-    while (<TOPLEVEL>) { print "$_"; }
-    close TOPLEVEL;
-}

src/libexpr/Makefile.am

@@ -2,11 +2,13 @@ pkglib_LTLIBRARIES = libexpr.la
 
 libexpr_la_SOURCES = \
  nixexpr.cc eval.cc primops.cc lexer-tab.cc parser-tab.cc \
- get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc
+ get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc \
+ names.cc
 
 pkginclude_HEADERS = \
  nixexpr.hh eval.hh parser.hh lexer-tab.hh parser-tab.hh \
- get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh
+ get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh \
+ names.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
  ../boost/format/libformat.la

src/libexpr/eval.cc

@@ -48,6 +48,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
     throw EvalError(format(s) % s2);
 }
 
+LocalNoInlineNoReturn(void throwTypeError(const char * s))
+{
+    throw TypeError(s);
+}
+
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s2))
 {
     throw TypeError(format(s) % s2);
@@ -69,77 +74,91 @@ LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2,
 }
 
 
+/* Pattern-match `pat' against `arg'.  The result is a set of
+   substitutions (`subs') and a set of recursive substitutions
+   (`subsRecursive').  The latter can refer to the variables bound by
+   both `subs' and `subsRecursive'. */
+static void patternMatch(EvalState & state,
+    Pattern pat, Expr arg, ATermMap & subs, ATermMap & subsRecursive)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    
+    if (matchVarPat(pat, name)) 
+        subs.set(name, arg);
+
+    else if (matchAttrsPat(pat, formals, ellipsis)) {
+
+        arg = evalExpr(state, arg);
+
+        /* Get the actual arguments. */
+        ATermMap attrs;
+        queryAllAttrs(arg, attrs);
+        unsigned int nrAttrs = attrs.size();
+
+        /* For each formal argument, get the actual argument.  If
+           there is no matching actual argument but the formal
+           argument has a default, use the default. */
+        unsigned int attrsUsed = 0;
+        for (ATermIterator i(formals); i; ++i) {
+            Expr name, def;
+            DefaultValue def2;
+            if (!matchFormal(*i, name, def2)) abort(); /* can't happen */
+
+            Expr value = attrs[name];
+
+            if (value == 0) {
+                if (!matchDefaultValue(def2, def)) def = 0;
+                if (def == 0) throw TypeError(format("the argument named `%1%' required by the function is missing")
+                    % aterm2String(name));
+                subsRecursive.set(name, def);
+            } else {
+                attrsUsed++;
+                attrs.remove(name);
+                subs.set(name, value);
+            }
+
+        }
+
+        /* Check that each actual argument is listed as a formal
+           argument (unless the attribute match specifies a `...'). */
+        if (ellipsis == eFalse && attrsUsed != nrAttrs)
+            throw TypeError(format("the function does not expect an argument named `%1%'")
+                % aterm2String(attrs.begin()->key));
+    }
+
+    else if (matchAtPat(pat, pat1, pat2)) {
+        patternMatch(state, pat1, arg, subs, subsRecursive);
+        patternMatch(state, pat2, arg, subs, subsRecursive);
+    }
+
+    else abort();
+}
+
+
 /* Substitute an argument set into the body of a function. */
 static Expr substArgs(EvalState & state,
-    Expr body, ATermList formals, Expr arg)
+    Expr body, Pattern pat, Expr arg)
 {
-    unsigned int nrFormals = ATgetLength(formals);
-    ATermMap subs(nrFormals);
-
-    /* Get the actual arguments and put them in the substitution. */
-    ATermMap args;
-    queryAllAttrs(arg, args);
-    for (ATermMap::const_iterator i = args.begin(); i != args.end(); ++i)
-        subs.set(i->key, i->value);
+    ATermMap subs(16), subsRecursive(16);
     
-    /* Get the formal arguments. */
-    ATermVector defsUsed;
-    ATermList recAttrs = ATempty;
-    for (ATermIterator i(formals); i; ++i) {
-        Expr name, def;
-        ValidValues valids2;
-        DefaultValue def2;
-        if (!matchFormal(*i, name, valids2, def2)) abort(); /* can't happen */
+    patternMatch(state, pat, arg, subs, subsRecursive);
 
-        Expr value = subs[name];
-        
-        if (value == 0) {
-            if (!matchDefaultValue(def2, def)) def = 0;
-            if (def == 0) throw TypeError(format("the argument named `%1%' required by the function is missing")
-                % aterm2String(name));
-            value = def;
-            defsUsed.push_back(name);
-            recAttrs = ATinsert(recAttrs, makeBind(name, def, makeNoPos()));
-        }
-
-        ATermList valids;
-        if (matchValidValues(valids2, valids)) {
-            value = evalExpr(state, value);
-            bool found = false;
-            for (ATermIterator j(valids); j; ++j) {
-                Expr v = evalExpr(state, *j);
-                if (value == v) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found) throw TypeError(format("the argument named `%1%' has an illegal value")
-                % aterm2String(name));            
-        }
-    }
-
-    /* Make a recursive attribute set out of the (argument-name,
-       value) tuples.  This is so that we can support default
-       parameters that refer to each other, e.g.  ({x, y ? x + x}: y)
-       {x = "foo";} evaluates to "foofoo". */
-    if (defsUsed.size() != 0) {
-        for (ATermMap::const_iterator i = args.begin(); i != args.end(); ++i)
+    /* If we used any default values, make a recursive attribute set
+       out of the (argument-name, value) tuples.  This is so that we
+       can support default values that refer to each other, e.g.  ({x,
+       y ? x + x}: y) {x = "foo";} evaluates to "foofoo". */
+    if (subsRecursive.size() != 0) {
+        ATermList recAttrs = ATempty;
+        foreach (ATermMap::const_iterator, i, subs)
+            recAttrs = ATinsert(recAttrs, makeBind(i->key, i->value, makeNoPos()));
+        foreach (ATermMap::const_iterator, i, subsRecursive)
             recAttrs = ATinsert(recAttrs, makeBind(i->key, i->value, makeNoPos()));
         Expr rec = makeRec(recAttrs, ATempty);
-        for (ATermVector::iterator i = defsUsed.begin(); i != defsUsed.end(); ++i)
-            subs.set(*i, makeSelect(rec, *i));
-    }
-    
-    if (subs.size() != nrFormals) {
-        /* One or more actual arguments were not declared as formal
-           arguments.  Find out which. */
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            subs.remove(name);
-        }
-        throw TypeError(format("the function does not expect an argument named `%1%'")
-            % aterm2String(subs.begin()->key));
+        foreach (ATermMap::const_iterator, i, subsRecursive)
+            subs.set(i->key, makeSelect(rec, i->key));
     }
 
     return substitute(Substitution(0, &subs), body);
@@ -151,11 +170,12 @@ static Expr substArgs(EvalState & state,
    to attributes substituted with selection expressions on the
    original set.  E.g., e = `rec {x = f x y; y = x;}' becomes `{x = f
    (e.x) (e.y); y = e.x;}'. */
-LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
+LocalNoInline(ATerm expandRec(EvalState & state, ATerm e, ATermList rbnds, ATermList nrbnds))
 {
     ATerm name;
     Expr e2;
     Pos pos;
+    Expr eOverrides = 0;
 
     /* Create the substitution list. */
     ATermMap subs(ATgetLength(rbnds) + ATgetLength(nrbnds));
@@ -165,9 +185,26 @@ LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
     }
     for (ATermIterator i(nrbnds); i; ++i) {
         if (!matchBind(*i, name, e2, pos)) abort(); /* can't happen */
+        if (name == sOverrides) eOverrides = e2;
         subs.set(name, e2);
     }
 
+    /* If the rec contains an attribute called `__overrides', then
+       evaluate it, and add the attributes in that set to the rec.
+       This allows overriding of recursive attributes, which is
+       otherwise not possible.  (You can use the // operator to
+       replace an attribute, but other attributes in the rec will
+       still reference the original value, because that value has been
+       substituted into the bodies of the other attributes.  Hence we
+       need __overrides.) */
+    ATermMap overrides;
+    if (eOverrides) {
+        eOverrides = evalExpr(state, eOverrides);
+        queryAllAttrs(eOverrides, overrides, false);
+        foreach (ATermMap::const_iterator, i, overrides)
+            subs.set(i->key, i->value);
+    }
+
     Substitution subs_(0, &subs);
 
     /* Create the non-recursive set. */
@@ -177,6 +214,10 @@ LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
         as.set(name, makeAttrRHS(substitute(subs_, e2), pos));
     }
 
+    if (eOverrides)
+        foreach (ATermMap::const_iterator, i, overrides)
+            as.set(i->key, makeAttrRHS(i->value, makeNoPos()));
+
     /* Copy the non-recursive bindings.  !!! inefficient */
     for (ATermIterator i(nrbnds); i; ++i) {
         if (!matchBind(*i, name, e2, pos)) abort(); /* can't happen */
@@ -305,9 +346,11 @@ string coerceToString(EvalState & state, Expr e, PathSet & context,
     }
         
     ATermList es;
-    if (matchAttrs(e, es))
-        return coerceToString(state, makeSelect(e, toATerm("outPath")),
-            context, coerceMore, copyToStore);
+    if (matchAttrs(e, es)) {
+        Expr e2 = queryAttr(e, "outPath");
+        if (!e2) throwTypeError("cannot coerce an attribute set (except a derivation) to a string");
+        return coerceToString(state, e2, context, coerceMore, copyToStore);
+    }
 
     if (coerceMore) {
 
@@ -379,15 +422,18 @@ Path coerceToPath(EvalState & state, Expr e, PathSet & context)
 
 Expr autoCallFunction(Expr e, const ATermMap & args)
 {
-    ATermList formals;
+    Pattern pat;
     ATerm body, pos;
+    ATermList formals;
+    ATermBool ellipsis;
     
-    if (matchFunction(e, formals, body, pos)) {
+    /* !!! this should be more general */
+    if (matchFunction(e, pat, body, pos) && matchAttrsPat(pat, formals, ellipsis)) {
         ATermMap actualArgs(ATgetLength(formals));
         
         for (ATermIterator i(formals); i; ++i) {
-            Expr name, def, value; ATerm values, def2;
-            if (!matchFormal(*i, name, values, def2)) abort();
+            Expr name, def, value; ATerm def2;
+            if (!matchFormal(*i, name, def2)) abort();
             if ((value = args.get(name)))
                 actualArgs.set(name, makeAttrRHS(value, makeNoPos()));
             else if (!matchDefaultValue(def2, def))
@@ -427,8 +473,8 @@ LocalNoInline(Expr evalVar(EvalState & state, ATerm name))
 
 LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
 {
-    ATermList formals;
-    ATerm pos, name;
+    Pattern pat;
+    ATerm pos;
     Expr body;
         
     /* Evaluate the left-hand side. */
@@ -454,10 +500,9 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
             return makePrimOp(arity, funBlob, args);
     }
 
-    else if (matchFunction(fun, formals, body, pos)) {
-        arg = evalExpr(state, arg);
+    else if (matchFunction(fun, pat, body, pos)) {
         try {
-            return evalExpr(state, substArgs(state, body, formals, arg));
+            return evalExpr(state, substArgs(state, body, pat, arg));
         } catch (Error & e) {
             addErrorPrefix(e, "while evaluating the function at %1%:\n",
                 showPos(pos));
@@ -465,18 +510,6 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
         }
     }
         
-    else if (matchFunction1(fun, name, body, pos)) {
-        try {
-            ATermMap subs(1);
-            subs.set(name, arg);
-            return evalExpr(state, substitute(Substitution(0, &subs), body));
-        } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating the function at %1%:\n",
-                showPos(pos));
-            throw;
-        }
-    }
-
     else throwTypeError(
         "attempt to call something which is neither a function nor a primop (built-in operation) but %1%",
         showType(fun));
@@ -633,7 +666,6 @@ Expr evalExpr2(EvalState & state, Expr e)
         sym == symInt ||
         sym == symBool ||
         sym == symFunction ||
-        sym == symFunction1 ||
         sym == symAttrs ||
         sym == symList ||
         sym == symPrimOp)
@@ -657,7 +689,7 @@ Expr evalExpr2(EvalState & state, Expr e)
     /* Mutually recursive sets. */
     ATermList rbnds, nrbnds;
     if (matchRec(e, rbnds, nrbnds))
-        return expandRec(e, rbnds, nrbnds);
+        return expandRec(state, e, rbnds, nrbnds);
 
     /* Conditionals. */
     if (matchIf(e, e1, e2, e3))
@@ -793,29 +825,6 @@ static Expr strictEvalExpr_(EvalState & state, Expr e, ATermMap & nfs)
         return makeList(ATreverse(es2));
     }
     
-    ATermList formals;
-    ATerm body, pos;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermList formals2 = ATempty;
-        
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ValidValues valids; ATerm dummy;
-            if (!matchFormal(*i, name, valids, dummy)) abort();
-
-            ATermList valids2;
-            if (matchValidValues(valids, valids2)) {
-                ATermList valids3 = ATempty;
-                for (ATermIterator j(valids2); j; ++j)
-                    valids3 = ATinsert(valids3, strictEvalExpr(state, *j, nfs));
-                valids = makeValidValues(ATreverse(valids3));
-            }
-
-            formals2 = ATinsert(formals2, makeFormal(name, valids, dummy));
-        }
-        
-        return makeFunction(ATreverse(formals2), body, pos);
-    }
-    
     return e;
 }
 

src/libexpr/expr-to-xml.cc

@@ -4,6 +4,8 @@
 #include "aterm.hh"
 #include "util.hh"
 
+#include <cstdlib>
+
 
 namespace nix {
 
@@ -38,6 +40,31 @@ static void showAttrs(const ATermMap & attrs, XMLWriter & doc,
 }
 
 
+static void printPatternAsXML(Pattern pat, XMLWriter & doc)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    if (matchVarPat(pat, name))
+        doc.writeEmptyElement("varpat", singletonAttrs("name", aterm2String(name)));
+    else if (matchAttrsPat(pat, formals, ellipsis)) {
+        XMLOpenElement _(doc, "attrspat");
+        for (ATermIterator i(formals); i; ++i) {
+            Expr name; ATerm dummy;
+            if (!matchFormal(*i, name, dummy)) abort();
+            doc.writeEmptyElement("attr", singletonAttrs("name", aterm2String(name)));
+        }
+        if (ellipsis == eTrue) doc.writeEmptyElement("ellipsis");
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        XMLOpenElement _(doc, "at");
+        printPatternAsXML(pat1, doc);
+        printPatternAsXML(pat2, doc);
+    }
+}
+
+
 static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
     ExprSet & drvsSeen)
 {
@@ -45,8 +72,8 @@ static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
     string s;
     ATerm s2;
     int i;
-    ATermList as, es, formals;
-    ATerm body, pos;
+    ATermList as, es;
+    ATerm pat, body, pos;
 
     checkInterrupt();
 
@@ -107,22 +134,9 @@ static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
             printTermAsXML(*i, doc, context, drvsSeen);
     }
 
-    else if (matchFunction(e, formals, body, pos)) {
+    else if (matchFunction(e, pat, body, pos)) {
         XMLOpenElement _(doc, "function");
-        
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ValidValues valids; ATerm dummy;
-            if (!matchFormal(*i, name, valids, dummy)) abort();
-            XMLOpenElement _(doc, "arg", singletonAttrs("name", aterm2String(name)));
-
-            ATermList valids2;
-            if (matchValidValues(valids, valids2)) {
-                for (ATermIterator j(valids2); j; ++j) {
-                    XMLOpenElement _(doc, "value");
-                    printTermAsXML(*j, doc, context, drvsSeen);
-                }
-            }
-        }
+        printPatternAsXML(pat, doc);
     }
 
     else

src/libexpr/get-drvs.cc

@@ -102,7 +102,7 @@ static bool getDerivation(EvalState & state, Expr e,
 
         boost::shared_ptr<ATermMap> attrs(new ATermMap());
         queryAllAttrs(e, *attrs, false);
-    
+        
         Expr a = attrs->get(toATerm("type"));
         if (!a || evalStringNoCtx(state, a) != "derivation") return true;
 
@@ -173,24 +173,30 @@ static void getDerivations(EvalState & state, Expr e,
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        Expr e2 = drvMap.get(toATerm("_combineChannels"));
-        bool combineChannels = e2 && evalBool(state, e2);
-        
-        for (ATermMap::const_iterator i = drvMap.begin(); i != drvMap.end(); ++i) {
-            startNest(nest, lvlDebug,
-                format("evaluating attribute `%1%'") % aterm2String(i->key));
-            string pathPrefix2 = addToPath(pathPrefix, aterm2String(i->key));
-            if (combineChannels) {
-                if (((string) aterm2String(i->key)) != "_combineChannels")
-                    getDerivations(state, i->value, pathPrefix2, autoArgs, drvs, doneExprs);
-            }
-            else if (getDerivation(state, i->value, pathPrefix2, drvs, doneExprs)) {
+        bool combineChannels = drvMap.get(toATerm("_combineChannels"));
+
+        /* Consider the attributes in sorted order to get more
+           deterministic behaviour in nix-env operations (e.g. when
+           there are names clashes between derivations, the derivation
+           bound to the attribute with the "lower" name should take
+           precedence). */
+        typedef std::map<string, Expr> AttrsSorted;
+        AttrsSorted attrsSorted;
+        foreach (ATermMap::const_iterator, i, drvMap)
+            attrsSorted[aterm2String(i->key)] = i->value;
+
+        foreach (AttrsSorted::iterator, i, attrsSorted) {
+            startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first);
+            string pathPrefix2 = addToPath(pathPrefix, i->first);
+            if (combineChannels)
+                getDerivations(state, i->second, pathPrefix2, autoArgs, drvs, doneExprs);
+            else if (getDerivation(state, i->second, pathPrefix2, drvs, doneExprs)) {
                 /* If the value of this attribute is itself an
                    attribute set, should we recurse into it?  => Only
                    if it has a `recurseForDerivations = true'
                    attribute. */
                 ATermList es;
-                Expr e = evalExpr(state, i->value);
+                Expr e = evalExpr(state, i->second), e2;
                 if (matchAttrs(e, es)) {
                     ATermMap attrs(ATgetLength(es));
                     queryAllAttrs(e, attrs, false);

src/libexpr/lexer.l

@@ -95,6 +95,7 @@ let         { return LET; }
 in          { return IN; }
 rec         { return REC; }
 inherit     { return INHERIT; }
+\.\.\.      { return ELLIPSIS; }
 
 \=\=        { return EQ; }
 \!\=        { return NEQ; }

src/libexpr/names.cc

@@ -92,25 +92,6 @@ int compareVersions(const string & v1, const string & v2)
 }
 
 
-static void testCompareVersions()
-{
-#define TEST(v1, v2, n) assert( \
-    compareVersions(v1, v2) == n && compareVersions(v2, v1) == -n)
-    TEST("1.0", "2.3", -1);
-    TEST("2.1", "2.3", -1);
-    TEST("2.3", "2.3", 0);
-    TEST("2.5", "2.3", 1);
-    TEST("3.1", "2.3", 1);
-    TEST("2.3.1", "2.3", 1);
-    TEST("2.3.1", "2.3a", 1);
-    TEST("2.3pre1", "2.3", -1);
-    TEST("2.3pre3", "2.3pre12", -1);
-    TEST("2.3a", "2.3c", -1);
-    TEST("2.3pre1", "2.3c", -1);
-    TEST("2.3pre1", "2.3q", -1);
-}
-
-
 DrvNames drvNamesFromArgs(const Strings & opArgs)
 {
     DrvNames result;

src/libexpr/nix.sdf

@@ -1,10 +1,13 @@
+%% Note: this SDF grammar is no longer used in the Nix expression
+%% parser and may not be up to date.
+
 definition
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Top level syntax.
 
-module Nix
+module Main
 imports Nix-Exprs Nix-Layout
 
 
@@ -38,6 +41,7 @@ exports
     "with" Expr ";" Expr -> Expr {cons("With")}
 
     "rec" "{" Bind* "}" -> Expr {cons("Rec")}
+    "let" Bind* "in" Expr -> Expr {cons("Let")}
     "let" "{" Bind* "}" -> Expr {cons("LetRec")}
     "{" Bind* "}" -> Expr {cons("Attrs")}
 
@@ -102,7 +106,8 @@ exports
 
     [0-9]+ -> Int
 
-    "\"" ~[\n\"]* "\"" -> Str
+    "\"" (~[\"\\] | ("\\" ~[]) )* "\"" -> Str
+    "''" (~[\"\\] | ("\\" ~[]) )* "''" -> Str
 
     [a-zA-Z0-9\.\_\-\+]* ("/"[a-zA-Z0-9\.\_\-\+]+)+ -> Path
 

src/libexpr/nixexpr-ast.def

@@ -3,8 +3,7 @@ init initNixExprHelpers
 Pos | string int int | Pos |
 NoPos | | Pos |
 
-Function | ATermList Expr Pos | Expr |
-Function1 | string Expr Pos | Expr |
+Function | Pattern Expr Pos | Expr |
 Assert | Expr Expr Pos | Expr |
 With | Expr Expr Pos | Expr |
 If | Expr Expr Expr | Expr |
@@ -67,7 +66,7 @@ PrimOp | int ATermBlob ATermList | Expr |
 Attrs | ATermList | Expr |
 Closed | Expr | Expr |
 Rec | ATermList ATermList | Expr |
-Bool | ATerm | Expr |
+Bool | ATermBool | Expr |
 Null | | Expr |
 
 Bind | string Expr Pos | ATerm |
@@ -76,16 +75,17 @@ Inherit | Expr ATermList Pos | ATerm |
 
 Scope | | Expr |
 
-Formal | string ValidValues DefaultValue | ATerm |
+VarPat | string | Pattern |
+AttrsPat | ATermList ATermBool | Pattern | # bool = `...'
+AtPat | Pattern Pattern | Pattern |
 
-ValidValues | ATermList | ValidValues |
-UnrestrictedValues | | ValidValues |
+Formal | string DefaultValue | ATerm |
 
 DefaultValue | Expr | DefaultValue |
 NoDefaultValue | | DefaultValue |
 
-True | | ATerm |
-False | | ATerm |
+True | | ATermBool |
+False | | ATermBool |
 
 PrimOpDef | int ATermBlob | ATerm |
 
@@ -93,3 +93,4 @@ AttrRHS | Expr Pos | ATerm |
 
 eTrue = makeBool(makeTrue())
 eFalse = makeBool(makeFalse())
+sOverrides = toATerm("__overrides")

src/libexpr/nixexpr.cc

@@ -6,6 +6,8 @@
 #include "nixexpr-ast.hh"
 #include "nixexpr-ast.cc"
 
+#include <cstdlib>
+
 
 namespace nix {
     
@@ -108,6 +110,31 @@ Expr makeAttrs(const ATermMap & attrs)
 }
 
 
+static void varsBoundByPattern(ATermMap & map, Pattern pat)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    /* Use makeRemoved() so that it can be used directly in
+       substitute(). */
+    if (matchVarPat(pat, name))
+        map.set(name, makeRemoved());
+    else if (matchAttrsPat(pat, formals, ellipsis)) { 
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm d1;
+            if (!matchFormal(*i, name, d1)) abort();
+            map.set(name, makeRemoved());
+        }
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        varsBoundByPattern(map, pat1);
+        varsBoundByPattern(map, pat2);
+    }
+    else abort();
+}
+
+
 Expr substitute(const Substitution & subs, Expr e)
 {
     checkInterrupt();
@@ -133,27 +160,17 @@ Expr substitute(const Substitution & subs, Expr e)
 
     /* In case of a function, filter out all variables bound by this
        function. */
-    ATermList formals;
+    Pattern pat;
     ATerm body;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermMap map(ATgetLength(formals));
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            map.set(name, makeRemoved());
-        }
+    if (matchFunction(e, pat, body, pos)) {
+        ATermMap map(16);
+        varsBoundByPattern(map, pat);
         Substitution subs2(&subs, &map);
         return makeFunction(
-            (ATermList) substitute(subs2, (ATerm) formals),
+            (Pattern) substitute(subs2, (Expr) pat),
             substitute(subs2, body), pos);
     }
 
-    if (matchFunction1(e, name, body, pos)) {
-        ATermMap map(1);
-        map.set(name, makeRemoved());
-        return makeFunction1(name, substitute(Substitution(&subs, &map), body), pos);
-    }
-        
     /* Idem for a mutually recursive attribute set. */
     ATermList rbnds, nrbnds;
     if (matchRec(e, rbnds, nrbnds)) {
@@ -211,9 +228,9 @@ static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
     done.insert(e);
     
     ATerm name, pos, value;
-    ATermList formals;
     ATerm with, body;
     ATermList rbnds, nrbnds;
+    Pattern pat;
 
     /* Closed terms don't have free variables, so we don't have to
        check by definition. */
@@ -225,28 +242,11 @@ static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
                 % aterm2String(name));
     }
 
-    else if (matchFunction(e, formals, body, pos)) {
+    else if (matchFunction(e, pat, body, pos)) {
         ATermMap defs2(defs);
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            defs2.set(name, (ATerm) ATempty);
-        }
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm valids, deflt;
-            set<Expr> done2;
-            if (!matchFormal(*i, name, valids, deflt)) abort();
-            checkVarDefs2(done, defs, valids);
-            checkVarDefs2(done2, defs2, deflt);
-        }
-        set<Expr> done2;
-        checkVarDefs2(done2, defs2, body);
-    }
-        
-    else if (matchFunction1(e, name, body, pos)) {
-        ATermMap defs2(defs);
-        defs2.set(name, (ATerm) ATempty);
+        varsBoundByPattern(defs2, pat);
         set<Expr> done2;
+        checkVarDefs2(done2, defs2, pat);
         checkVarDefs2(done2, defs2, body);
     }
         
@@ -360,17 +360,17 @@ Expr makeStr(const string & s, const PathSet & context)
 
 string showType(Expr e)
 {
-    ATerm t1, t2, t3;
+    ATerm t1, t2;
     ATermList l1;
     ATermBlob b1;
     int i1;
+    Pattern p1;
     if (matchStr(e, t1, l1)) return "a string";
     if (matchPath(e, t1)) return "a path";
     if (matchNull(e)) return "null";
     if (matchInt(e, i1)) return "an integer";
     if (matchBool(e, t1)) return "a boolean";
-    if (matchFunction(e, l1, t1, t2)) return "a function";
-    if (matchFunction1(e, t1, t2, t3)) return "a function";
+    if (matchFunction(e, p1, t1, t2)) return "a function";
     if (matchAttrs(e, l1)) return "an attribute set";
     if (matchList(e, l1)) return "a list";
     if (matchPrimOp(e, i1, b1, l1)) return "a partially applied built-in function";

src/libexpr/nixexpr.hh

@@ -21,11 +21,10 @@ MakeError(TypeError, EvalError)
    property of the ATerm library allows us to implement caching of
    normals forms efficiently. */
 typedef ATerm Expr;
-
 typedef ATerm DefaultValue;
-typedef ATerm ValidValues;
-
 typedef ATerm Pos;
+typedef ATerm Pattern;
+typedef ATerm ATermBool;
 
 
 /* A STL vector of ATerms.  Should be used with great care since it's

src/libexpr/parser.y

@@ -208,15 +208,22 @@ static void freeAndUnprotect(void * p)
 %union {
   ATerm t;
   ATermList ts;
+  struct {
+    ATermList formals;
+    bool ellipsis;
+  } formals;
 }
 
 %type <t> start expr expr_function expr_if expr_op
 %type <t> expr_app expr_select expr_simple bind inheritsrc formal
-%type <ts> binds ids expr_list formals string_parts ind_string_parts
+%type <t> pattern pattern2
+%type <ts> binds ids expr_list string_parts ind_string_parts
+%type <formals> formals
 %token <t> ID INT STR IND_STR PATH URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
+%token ELLIPSIS
 
 %nonassoc IMPL
 %left OR
@@ -236,10 +243,8 @@ start: expr { data->result = $1; };
 expr: expr_function;
 
 expr_function
-  : '{' formals '}' ':' expr_function
-    { $$ = makeFunction($2, $5, CUR_POS); }
-  | ID ':' expr_function
-    { $$ = makeFunction1($1, $3, CUR_POS); }
+  : pattern ':' expr_function
+    { $$ = makeFunction($1, $3, CUR_POS); }
   | ASSERT expr ';' expr_function
     { $$ = makeAssert($2, $4, CUR_POS); }
   | WITH expr ';' expr_function
@@ -320,6 +325,16 @@ ind_string_parts
   | { $$ = ATempty; }
   ;
 
+pattern
+  : pattern2 '@' pattern { $$ = makeAtPat($1, $3); }
+  | pattern2
+  ;
+
+pattern2
+  : ID { $$ = makeVarPat($1); }
+  | '{' formals '}' { $$ = makeAttrsPat($2.formals, $2.ellipsis ? eTrue : eFalse); }
+  ;
+
 binds
   : binds bind { $$ = ATinsert($1, $2); }
   | { $$ = ATempty; }
@@ -348,15 +363,19 @@ expr_list
   ;
 
 formals
-  : formal ',' formals { $$ = ATinsert($3, $1); } /* idem - right recursive */
-  | formal { $$ = ATinsert(ATempty, $1); }
-  | { $$ = ATempty; }
+  : formal ',' formals /* idem - right recursive */
+    { $$.formals = ATinsert($3.formals, $1); $$.ellipsis = $3.ellipsis; }
+  | formal
+    { $$.formals = ATinsert(ATempty, $1); $$.ellipsis = false; }
+  |
+    { $$.formals = ATempty; $$.ellipsis = false; }
+  | ELLIPSIS
+    { $$.formals = ATempty; $$.ellipsis = true; }
   ;
 
 formal
-  : ID { $$ = makeFormal($1, makeUnrestrictedValues(), makeNoDefaultValue()); }
-  | ID ':' '[' expr_list ']' { $$ = makeFormal($1, makeValidValues($4), makeNoDefaultValue()); }
-  | ID '?' expr { $$ = makeFormal($1, makeUnrestrictedValues(), makeDefaultValue($3)); }
+  : ID { $$ = makeFormal($1, makeNoDefaultValue()); }
+  | ID '?' expr { $$ = makeFormal($1, makeDefaultValue($3)); }
   ;
   
 %%
@@ -388,22 +407,43 @@ static void checkAttrs(ATermMap & names, ATermList bnds)
 }
 
 
-static void checkAttrSets(ATerm e)
+static void checkPatternVars(ATerm pos, ATermMap & map, Pattern pat)
 {
+    ATerm name;
     ATermList formals;
-    ATerm body, pos;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermMap names(ATgetLength(formals));
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    if (matchVarPat(pat, name)) {
+        if (map.get(name))
+            throw EvalError(format("duplicate formal function argument `%1%' at %2%")
+                % aterm2String(name) % showPos(pos));
+        map.set(name, name);
+    }
+    else if (matchAttrsPat(pat, formals, ellipsis)) { 
         for (ATermIterator i(formals); i; ++i) {
-            ATerm name;
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            if (names.get(name))
+            ATerm d1;
+            if (!matchFormal(*i, name, d1)) abort();
+            if (map.get(name))
                 throw EvalError(format("duplicate formal function argument `%1%' at %2%")
                     % aterm2String(name) % showPos(pos));
-            names.set(name, name);
+            map.set(name, name);
         }
     }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        checkPatternVars(pos, map, pat1);
+        checkPatternVars(pos, map, pat2);
+    }
+    else abort();
+}
+
+
+static void checkAttrSets(ATerm e)
+{
+    ATerm pat, body, pos;
+    if (matchFunction(e, pat, body, pos)) {
+        ATermMap map(16);
+        checkPatternVars(pos, map, pat);
+    }
 
     ATermList bnds;
     if (matchAttrs(e, bnds)) {

src/libexpr/primops.cc

@@ -7,6 +7,7 @@
 #include "expr-to-xml.hh"
 #include "nixexpr-ast.hh"
 #include "parser.hh"
+#include "names.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -119,67 +120,13 @@ static Expr prim_isNull(EvalState & state, const ATermVector & args)
 static Expr prim_isFunction(EvalState & state, const ATermVector & args)
 {
     Expr e = evalExpr(state, args[0]);
-    ATermList formals;
-    ATerm name, body, pos;
-    return makeBool(
-        matchFunction(e, formals, body, pos) ||
-        matchFunction1(e, name, body, pos));
+    Pattern pat;
+    ATerm body, pos;
+    return makeBool(matchFunction(e, pat, body, pos));
 }
 
 
-static Path findDependency(Path dir, string dep)
-{
-    if (dep[0] == '/') throw EvalError(
-        format("illegal absolute dependency `%1%'") % dep);
-
-    Path p = canonPath(dir + "/" + dep);
-
-    if (pathExists(p))
-        return p;
-    else
-        return "";
-}
-
-
-/* Make path `p' relative to directory `pivot'.  E.g.,
-   relativise("/a/b/c", "a/b/x/y") => "../x/y".  Both input paths
-   should be in absolute canonical form. */
-static string relativise(Path pivot, Path p)
-{
-    assert(pivot.size() > 0 && pivot[0] == '/');
-    assert(p.size() > 0 && p[0] == '/');
-        
-    if (pivot == p) return ".";
-
-    /* `p' is in `pivot'? */
-    Path pivot2 = pivot + "/";
-    if (p.substr(0, pivot2.size()) == pivot2) {
-        return p.substr(pivot2.size());
-    }
-
-    /* Otherwise, `p' is in a parent of `pivot'.  Find up till which
-       path component `p' and `pivot' match, and add an appropriate
-       number of `..' components. */
-    string::size_type i = 1;
-    while (1) {
-        string::size_type j = pivot.find('/', i);
-        if (j == string::npos) break;
-        j++;
-        if (pivot.substr(0, j) != p.substr(0, j)) break;
-        i = j;
-    }
-
-    string prefix;
-    unsigned int slashes = count(pivot.begin() + i, pivot.end(), '/') + 1;
-    while (slashes--) {
-        prefix += "../";
-    }
-
-    return prefix + p.substr(i);
-}
-
-
-static Expr prim_dependencyClosure(EvalState & state, const ATermVector & args)
+static Expr prim_genericClosure(EvalState & state, const ATermVector & args)
 {
     startNest(nest, lvlDebug, "finding dependencies");
 
@@ -190,87 +137,40 @@ static Expr prim_dependencyClosure(EvalState & state, const ATermVector & args)
     if (!startSet) throw EvalError("attribute `startSet' required");
     ATermList startSet2 = evalList(state, startSet);
 
-    Path pivot;
-    PathSet workSet;
-    for (ATermIterator i(startSet2); i; ++i) {
-        PathSet context; /* !!! what to do? */
-        Path p = coerceToPath(state, *i, context);
-        workSet.insert(p);
-        pivot = dirOf(p);
-    }
+    set<Expr> workSet; // !!! gc roots
+    for (ATermIterator i(startSet2); i; ++i) workSet.insert(*i);
 
-    /* Get the search path. */
-    PathSet searchPath;
-    Expr e = queryAttr(attrs, "searchPath");
-    if (e) {
-        ATermList list = evalList(state, e);
-        for (ATermIterator i(list); i; ++i) {
-            PathSet context; /* !!! what to do? */
-            Path p = coerceToPath(state, *i, context);
-            searchPath.insert(p);
-        }
-    }
-
-    Expr scanner = queryAttr(attrs, "scanner");
-    if (!scanner) throw EvalError("attribute `scanner' required");
+    /* Get the operator. */
+    Expr op = queryAttr(attrs, "operator");
+    if (!op) throw EvalError("attribute `operator' required");
     
-    /* Construct the dependency closure by querying the dependency of
-       each path in `workSet', adding the dependencies to
-       `workSet'. */
-    PathSet doneSet;
+    /* Construct the closure by applying the operator to element of
+       `workSet', adding the result to `workSet', continuing until
+       no new elements are found. */
+    ATermList res = ATempty;
+    set<Expr> doneKeys; // !!! gc roots
     while (!workSet.empty()) {
-	Path path = *(workSet.begin());
-	workSet.erase(path);
+	Expr e = *(workSet.begin());
+	workSet.erase(e);
 
-	if (doneSet.find(path) != doneSet.end()) continue;
-        doneSet.insert(path);
+        e = strictEvalExpr(state, e);
 
-        try {
-            
-            /* Call the `scanner' function with `path' as argument. */
-            debug(format("finding dependencies in `%1%'") % path);
-            ATermList deps = evalList(state, makeCall(scanner, makeStr(path)));
+        Expr key = queryAttr(e, "key");
+        if (!key) throw EvalError("attribute `key' required");
 
-            /* Try to find the dependencies relative to the `path'. */
-            for (ATermIterator i(deps); i; ++i) {
-                string s = evalStringNoCtx(state, *i);
-                
-                Path dep = findDependency(dirOf(path), s);
+	if (doneKeys.find(key) != doneKeys.end()) continue;
+        doneKeys.insert(key);
+        res = ATinsert(res, e);
+        
+        /* Call the `operator' function with `e' as argument. */
+        ATermList res = evalList(state, makeCall(op, e));
 
-                if (dep == "") {
-                    for (PathSet::iterator j = searchPath.begin();
-                         j != searchPath.end(); ++j)
-                    {
-                        dep = findDependency(*j, s);
-                        if (dep != "") break;
-                    }
-                }
-                
-                if (dep == "")
-                    debug(format("did NOT find dependency `%1%'") % s);
-                else {
-                    debug(format("found dependency `%1%'") % dep);
-                    workSet.insert(dep);
-                }
-            }
-
-        } catch (Error & e) {
-            e.addPrefix(format("while finding dependencies in `%1%':\n")
-                % path);
-            throw;
-        }
+        /* Try to find the dependencies relative to the `path'. */
+        for (ATermIterator i(res); i; ++i)
+            workSet.insert(evalExpr(state, *i));
     }
 
-    /* Return a list of the dependencies we've just found. */
-    ATermList deps = ATempty;
-    for (PathSet::iterator i = doneSet.begin(); i != doneSet.end(); ++i) {
-        deps = ATinsert(deps, makeStr(relativise(pivot, *i)));
-        deps = ATinsert(deps, makeStr(*i));
-    }
-
-    debug(format("dependency list is `%1%'") % makeList(deps));
-    
-    return makeList(deps);
+    return makeList(res);
 }
 
 
@@ -310,15 +210,6 @@ static Expr prim_trace(EvalState & state, const ATermVector & args)
 }
 
 
-static Expr prim_relativise(EvalState & state, const ATermVector & args)
-{
-    PathSet context; /* !!! what to do? */
-    Path pivot = coerceToPath(state, args[0], context);
-    Path path = coerceToPath(state, args[1], context);
-    return makeStr(relativise(pivot, path));
-}
-
-
 /*************************************************************
  * Derivations
  *************************************************************/
@@ -591,6 +482,27 @@ static Expr prim_toPath(EvalState & state, const ATermVector & args)
 }
 
 
+/* Allow a valid store path to be used in an expression.  This is
+   useful in some generated expressions such as in nix-push, which
+   generates a call to a function with an already existing store path
+   as argument.  You don't want to use `toPath' here because it copies
+   the path to the Nix store, which yields a copy like
+   /nix/store/newhash-oldhash-oldname.  In the past, `toPath' had
+   special case behaviour for store paths, but that created weird
+   corner cases. */
+static Expr prim_storePath(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    Path path = canonPath(coerceToPath(state, args[0], context));
+    if (!isInStore(path))
+        throw EvalError(format("path `%1%' is not in the Nix store") % path);
+    if (!store->isValidPath(path))
+        throw EvalError(format("store path `%1%' is not valid") % path);
+    context.insert(toStorePath(path));
+    return makeStr(path, context);
+}
+
+
 static Expr prim_pathExists(EvalState & state, const ATermVector & args)
 {
     PathSet context;
@@ -817,7 +729,7 @@ static Expr prim_removeAttrs(EvalState & state, const ATermVector & args)
 }
 
 
-/* Determine whether the argument is a list. */
+/* Determine whether the argument is an attribute set. */
 static Expr prim_isAttrs(EvalState & state, const ATermVector & args)
 {
     ATermList list;
@@ -873,6 +785,14 @@ static Expr prim_map(EvalState & state, const ATermVector & args)
 }
 
 
+/* Return the length of a list.  This is an O(1) time operation. */
+static Expr prim_length(EvalState & state, const ATermVector & args)
+{
+    ATermList list = evalList(state, args[0]);
+    return makeInt(ATgetLength(list));
+}
+
+
 /*************************************************************
  * Integer arithmetic
  *************************************************************/
@@ -894,6 +814,23 @@ static Expr prim_sub(EvalState & state, const ATermVector & args)
 }
 
 
+static Expr prim_mul(EvalState & state, const ATermVector & args)
+{
+    int i1 = evalInt(state, args[0]);
+    int i2 = evalInt(state, args[1]);
+    return makeInt(i1 * i2);
+}
+
+
+static Expr prim_div(EvalState & state, const ATermVector & args)
+{
+    int i1 = evalInt(state, args[0]);
+    int i2 = evalInt(state, args[1]);
+    if (i2 == 0) throw EvalError("division by zero");
+    return makeInt(i1 / i2);
+}
+
+
 static Expr prim_lessThan(EvalState & state, const ATermVector & args)
 {
     int i1 = evalInt(state, args[0]);
@@ -950,23 +887,54 @@ static Expr prim_unsafeDiscardStringContext(EvalState & state, const ATermVector
     return makeStr(s, PathSet());
 }
 
-/* Expression serialization/deserialization */ 
 
-static Expr prim_ExprToString ( EvalState & state, const ATermVector & args)
+/* Expression serialization/deserialization */
+
+
+static Expr prim_exprToString(EvalState & state, const ATermVector & args)
 {
-	return makeStr ( atPrint ( evalExpr ( state, args [ 0 ] ) ) );
+    /* !!! this disregards context */
+    return makeStr(atPrint(evalExpr(state, args[0])));
 }
 
-static Expr prim_StringToExpr ( EvalState & state, const ATermVector & args)
+
+static Expr prim_stringToExpr(EvalState & state, const ATermVector & args)
 {
-	string s;
-	PathSet l;
-	if (! matchStr ( evalExpr ( state, args[0] ), s, l )) {
-		throw EvalError("__stringToExpr needs string argument!");
-	}
-	return ATreadFromString(s.c_str());
+    /* !!! this can introduce arbitrary garbage terms in the
+       evaluator! */;
+    string s;
+    PathSet l;
+    if (!matchStr(evalExpr(state, args[0]), s, l))
+        throw EvalError("stringToExpr needs string argument!");
+    return ATreadFromString(s.c_str());
 }
 
+
+/*************************************************************
+ * Versions
+ *************************************************************/
+
+
+static Expr prim_parseDrvName(EvalState & state, const ATermVector & args)
+{
+    string name = evalStringNoCtx(state, args[0]);
+    DrvName parsed(name);
+    ATermMap attrs(2);
+    attrs.set(toATerm("name"), makeAttrRHS(makeStr(parsed.name), makeNoPos()));
+    attrs.set(toATerm("version"), makeAttrRHS(makeStr(parsed.version), makeNoPos()));
+    return makeAttrs(attrs);
+}
+
+
+static Expr prim_compareVersions(EvalState & state, const ATermVector & args)
+{
+    string version1 = evalStringNoCtx(state, args[0]);
+    string version2 = evalStringNoCtx(state, args[1]);
+    int d = compareVersions(version1, version2);
+    return makeInt(d);
+}
+
+
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -987,17 +955,15 @@ void EvalState::addPrimOps()
     addPrimOp("import", 1, prim_import);
     addPrimOp("isNull", 1, prim_isNull);
     addPrimOp("__isFunction", 1, prim_isFunction);
-    addPrimOp("dependencyClosure", 1, prim_dependencyClosure);
+    addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("throw", 1, prim_throw);
     addPrimOp("__getEnv", 1, prim_getEnv);
     addPrimOp("__trace", 2, prim_trace);
     
     // Expr <-> String
-    addPrimOp("__exprToString", 1, prim_ExprToString);
-    addPrimOp("__stringToExpr", 1, prim_StringToExpr);
-
-    addPrimOp("relativise", 2, prim_relativise);
+    addPrimOp("__exprToString", 1, prim_exprToString);
+    addPrimOp("__stringToExpr", 1, prim_stringToExpr);
 
     // Derivations
     addPrimOp("derivation!", 1, prim_derivationStrict);
@@ -1005,6 +971,7 @@ void EvalState::addPrimOps()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
+    addPrimOp("__storePath", 1, prim_storePath);
     addPrimOp("__pathExists", 1, prim_pathExists);
     addPrimOp("baseNameOf", 1, prim_baseNameOf);
     addPrimOp("dirOf", 1, prim_dirOf);
@@ -1028,10 +995,13 @@ void EvalState::addPrimOps()
     addPrimOp("__head", 1, prim_head);
     addPrimOp("__tail", 1, prim_tail);
     addPrimOp("map", 2, prim_map);
+    addPrimOp("__length", 1, prim_length);
 
     // Integer arithmetic
     addPrimOp("__add", 2, prim_add);
     addPrimOp("__sub", 2, prim_sub);
+    addPrimOp("__mul", 2, prim_mul);
+    addPrimOp("__div", 2, prim_div);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation
@@ -1039,7 +1009,10 @@ void EvalState::addPrimOps()
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    
+
+    // Versions
+    addPrimOp("__parseDrvName", 1, prim_parseDrvName);
+    addPrimOp("__compareVersions", 2, prim_compareVersions);
 }
 
 

src/libmain/shared.cc

@@ -4,6 +4,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "misc.hh"
 
 #include <iostream>
 #include <cctype>
@@ -49,6 +50,34 @@ void printGCWarning()
 }
 
 
+void printMissing(const PathSet & paths)
+{
+    unsigned long long downloadSize;
+    PathSet willBuild, willSubstitute, unknown;
+    queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize);
+
+    if (!willBuild.empty()) {
+        printMsg(lvlInfo, format("the following derivations will be built:"));
+        foreach (PathSet::iterator, i, willBuild)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+
+    if (!willSubstitute.empty()) {
+        printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") %
+            (downloadSize / (1024.0 * 1024.0)));
+        foreach (PathSet::iterator, i, willSubstitute)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+
+    if (!unknown.empty()) {
+        printMsg(lvlInfo, format("don't know how to build the following paths%1%:")
+            % (readOnlyMode ? " (may be caused by read-only store access)" : ""));
+        foreach (PathSet::iterator, i, unknown)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+}
+
+
 static void setLogType(string lt)
 {
     if (lt == "pretty") logType = ltPretty;
@@ -58,27 +87,18 @@ static void setLogType(string lt)
 }
 
 
-static unsigned int getIntArg(const string & opt,
+unsigned long long getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
     if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
-    int n;
+    long long n;
     if (!string2Int(*i, n) || n < 0)
         throw UsageError(format("`%1%' requires a non-negative integer") % opt);
     return n;
 }
 
 
-struct RemoveTempRoots 
-{
-    ~RemoveTempRoots()
-    {
-        removeTempRoots();
-    }
-};
-
-
 void initDerivationsHelpers();
 
 
@@ -99,6 +119,12 @@ static void closeStore()
 }
 
 
+RemoveTempRoots::~RemoveTempRoots()
+{
+    removeTempRoots();
+}
+
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
@@ -112,11 +138,13 @@ static void initAndRun(int argc, char * * argv)
     nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
     nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
     nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
+    nixChrootsDir = canonPath(getEnv("NIX_CHROOTS_DIR", nixStateDir + "/chroots"));
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
-    if (subs == "default")
-        substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl");
-    else
+    if (subs == "default") {
+        substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+        substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+    } else
         substituters = tokenizeString(subs, ":");
 
     /* Get some settings from the configuration file. */
@@ -192,6 +220,8 @@ static void initAndRun(int argc, char * * argv)
             ; /* !!! obsolete - remove eventually */
         else if (arg == "--no-build-output" || arg == "-Q")
             buildVerbosity = lvlVomit;
+        else if (arg == "--print-build-trace")
+            printBuildTrace = true;
         else if (arg == "--help") {
             printHelp();
             return;
@@ -219,7 +249,7 @@ static void initAndRun(int argc, char * * argv)
 
     /* Automatically clean up the temporary roots file when we
        exit. */
-    RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
+    RemoveTempRoots removeTempRoots __attribute__((unused));
 
     /* Make sure that the database gets closed properly, even if
        terminate() is called (which happens sometimes due to bugs in

src/libmain/shared.hh

@@ -26,6 +26,11 @@ namespace nix {
 Path makeRootName(const Path & gcRoot, int & counter);
 void printGCWarning();
 
+void printMissing(const PathSet & paths);
+
+unsigned long long getIntArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end);
+
 /* Whether we're running setuid. */
 extern bool setuidMode;
 
@@ -33,6 +38,11 @@ extern volatile ::sig_atomic_t blockInt;
 
 MakeError(UsageError, nix::Error);
 
+struct RemoveTempRoots 
+{
+    ~RemoveTempRoots();    
+};
+
 }
 
 

src/libstore/build.cc

@@ -184,11 +184,6 @@ private:
     /* Goals waiting for a build slot. */
     WeakGoals wantingToBuild;
 
-    /* Goals waiting for info from substituters (using --query-info),
-       and the info they're (collectively) waiting for. */
-    WeakGoals waitingForInfo;
-    map<Path, PathSet> requestedInfo;
-
     /* Child processes currently running. */
     Children children;
 
@@ -243,11 +238,6 @@ public:
        call is made to childTerminate(..., true).  */
     void waitForChildTermination(GoalPtr goal);
 
-    /* Put `goal' to sleep until the top-level loop has run `sub' to
-       get info about `storePath' (with --query-info).  We combine
-       substituter invocations to reduce overhead. */
-    void waitForInfo(GoalPtr goal, Path sub, Path storePath);
-
     /* Wait for any goal to finish.  Pretty indiscriminate way to
        wait for some resource that some other goal is holding. */
     void waitForAnyGoal(GoalPtr goal);
@@ -257,12 +247,6 @@ public:
 
     /* Wait for input to become available. */
     void waitForInput();
-
-private:
-
-    /* Process the pending paths in requestedInfo and wake up the
-       goals in waitingForInfo. */
-    void getInfo();
     
 };
 
@@ -377,6 +361,15 @@ const char * * strings2CharPtrs(const Strings & ss)
 }
 
 
+/* Restore default handling of SIGPIPE, otherwise some programs will
+   randomly say "Broken pipe". */
+static void restoreSIGPIPE() 
+{
+    struct sigaction act, oact;
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    if (sigaction(SIGPIPE, &act, &oact)) throw SysError("resetting SIGPIPE");
+}
 
 
 //////////////////////////////////////////////////////////////////////
@@ -530,6 +523,8 @@ static void runSetuidHelper(const string & command,
             args.push_back(arg.c_str());
             args.push_back(0);
 
+            restoreSIGPIPE();
+            
             execve(program.c_str(), (char * *) &args[0], 0);
             throw SysError(format("executing `%1%'") % program);
         }
@@ -578,17 +573,17 @@ void getOwnership(const Path & path)
 
 
 void deletePathWrapped(const Path & path,
-    unsigned long long & bytesFreed)
+    unsigned long long & bytesFreed, unsigned long long & blocksFreed)
 {
     try {
         /* First try to delete it ourselves. */
-        deletePath(path, bytesFreed);
+        deletePath(path, bytesFreed, blocksFreed);
     } catch (SysError & e) {
         /* If this failed due to a permission error, then try it with
            the setuid helper. */
         if (haveBuildUsers() && !amPrivileged()) {
             getOwnership(path);
-            deletePath(path, bytesFreed);
+            deletePath(path, bytesFreed, blocksFreed);
         } else
             throw;
     }
@@ -597,8 +592,8 @@ void deletePathWrapped(const Path & path,
 
 void deletePathWrapped(const Path & path)
 {
-    unsigned long long dummy;
-    deletePathWrapped(path, dummy);
+    unsigned long long dummy1, dummy2;
+    deletePathWrapped(path, dummy1, dummy2);
 }
 
 
@@ -731,12 +726,15 @@ private:
     /* Whether we're currently doing a chroot build. */
     bool useChroot;
 
-    /* A RAII object to delete the chroot directory. */
-    boost::shared_ptr<AutoDelete> autoDelChroot;
+    /* RAII objects to delete the chroot directory and its /tmp
+       directory.  Don't change the order: autoDelChrootTmp has to be
+       deleted before autoDelChrootRoot. */
+    boost::shared_ptr<AutoDelete> autoDelChrootRoot;
+    boost::shared_ptr<AutoDelete> autoDelChrootTmp;
     
     /* In chroot builds, the list of bind mounts currently active.
        The destructor of BindMount will cause the binds to be
-       unmounted. */
+       unmounted.  Keep this *below* autoDelChroot* just to be safe! */
     list<boost::shared_ptr<BindMount> > bindMounts;
 
     typedef void (DerivationGoal::*GoalState)();
@@ -791,7 +789,7 @@ private:
     void computeClosure();
 
     /* Open a log file and a pipe to it. */
-    void openLogFile();
+    Path openLogFile();
 
     /* Common initialisation to be performed in child processes (i.e.,
        both in builders and in build hooks). */
@@ -912,7 +910,7 @@ void DerivationGoal::haveDerivation()
 
     /* If they are all valid, then we're done. */
     if (invalidOutputs.size() == 0) {
-        if(! forceInputs) {
+        if(!forceInputs) {
 	    amDone(ecSuccess);
 	    return;
 	}
@@ -1094,6 +1092,10 @@ void DerivationGoal::tryToBuild()
 
     } catch (BuildError & e) {
         printMsg(lvlError, e.msg());
+        if (printBuildTrace) {
+            printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
+                % drvPath % drv.outputs["out"].path % 0 % e.msg());
+        }
         amDone(ecFailed);
         return;
     }
@@ -1184,9 +1186,13 @@ void DerivationGoal::buildDone()
         /* Compute the FS closure of the outputs and register them as
            being valid. */
         computeClosure();
-        
+
     } catch (BuildError & e) {
         printMsg(lvlError, e.msg());
+        if (printBuildTrace) {
+            printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
+                % drvPath % drv.outputs["out"].path % status % e.msg());
+        }
         amDone(ecFailed);
         return;
     }
@@ -1194,6 +1200,11 @@ void DerivationGoal::buildDone()
     /* Release the build user, if applicable. */
     buildUser.release();
 
+    if (printBuildTrace) {
+        printMsg(lvlError, format("@ build-succeeded %1% %2%")
+            % drvPath % drv.outputs["out"].path);
+    }
+    
     amDone(ecSuccess);
 }
 
@@ -1263,7 +1274,7 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
     tmpDir = createTempDir();
     
     /* Create the log file and pipe. */
-    openLogFile();
+    Path logFile = openLogFile();
 
     /* Create the communication pipes. */
     toHook.create();
@@ -1382,6 +1393,11 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
         /* Tell the hook to proceed. */ 
         writeLine(toHook.writeSide, "okay");
 
+        if (printBuildTrace) {
+            printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
+                % drvPath % drv.outputs["out"].path % drv.platform % logFile);
+        }
+        
         return rpAccept;
     }
 
@@ -1556,7 +1572,7 @@ void DerivationGoal::startBuilder()
 
     /* Create a temporary directory where the build will take
        place. */
-    tmpDir = createTempDir();
+    tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false);
 
     /* For convenience, set an environment pointing to the top build
        directory. */
@@ -1565,6 +1581,12 @@ void DerivationGoal::startBuilder()
     /* Also set TMPDIR and variants to point to this directory. */
     env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir;
 
+    /* Explicitly set PWD to prevent problems with chroot builds.  In
+       particular, dietlibc cannot figure out the cwd because the
+       inode of the current directory doesn't appear in .. (because
+       getdents returns the inode of the mount point). */
+    env["PWD"] = tmpDir;
+
     /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
        derivation, tell the builder, so that for instance `fetchurl'
        can skip checking the output.  On older Nixes, this environment
@@ -1626,6 +1648,7 @@ void DerivationGoal::startBuilder()
     }
 
     // The same for derivations
+    // !!! urgh, cut&paste duplication
     s = drv.env["exportBuildReferencesGraph"];
     ss = tokenizeString(s);
     if (ss.size() % 2 != 0)
@@ -1701,39 +1724,63 @@ void DerivationGoal::startBuilder()
     }
 
 
-    /* Are we doing a chroot build? */
+    /* Are we doing a chroot build?  Note that fixed-output
+       derivations are never done in a chroot, mainly so that
+       functions like fetchurl (which needs a proper /etc/resolv.conf)
+       work properly.  Purity checking for fixed-output derivations
+       is somewhat pointless anyway. */
     useChroot = queryBoolSetting("build-use-chroot", false);
-    Path tmpRootDir;
+    Path chrootRootDir;
+
+    if (fixedOutput) useChroot = false;
 
     if (useChroot) {
 #if CHROOT_ENABLED
         /* Create a temporary directory in which we set up the chroot
            environment using bind-mounts.
 
-           !!! Big danger here: since we're doing this in /tmp, there
-           is a risk that the admin does something like "rm -rf
-           /tmp/chroot-nix-*" to clean up aborted builds, and if some
-           of the bind-mounts are still active, then "rm -rf" will
-           happily recurse into those mount points (thereby deleting,
-           say, /nix/store).  Ideally, tmpRootDir should be created in
-           some special location (maybe in /nix/var/nix) where Nix
-           takes care of unmounting / deleting old chroots
-           automatically. */
-        tmpRootDir = createTempDir("", "chroot-nix");
+           !!! Bind mounts are potentially dangerous: if the user
+           cleans up his system by doing "rm -rf
+           /nix/var/nix/chroots/*", this will recurse into /nix/store
+           via the bind mounts (and potentially other parts of the
+           filesystem, depending on the setting of the
+           `build-chroot-dirs' option). */
+        chrootRootDir = createTempDir(nixChrootsDir, "chroot-nix");
 
         /* Clean up the chroot directory automatically, but don't
            recurse; that would be very very bad if the unmount of a
            bind-mount fails. Instead BindMount::unbind() unmounts and
            deletes exactly those directories that it created to
            produce the mount point, so that after all the BindMount
-           destructors have run, tmpRootDir should be empty. */
-        autoDelChroot = boost::shared_ptr<AutoDelete>(new AutoDelete(tmpRootDir, false));
+           destructors have run, chrootRootDir should be empty. */
+        autoDelChrootRoot = boost::shared_ptr<AutoDelete>(new AutoDelete(chrootRootDir, false));
         
-        printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % tmpRootDir);
+        printMsg(lvlChatty, format("setting up chroot environment in `%1%'") % chrootRootDir);
 
+        /* Create a writable /tmp in the chroot.  Many builders need
+           this.  (Of course they should really respect $TMPDIR
+           instead.) */
+        Path chrootTmpDir = chrootRootDir + "/tmp";
+        createDirs(chrootTmpDir);
+
+        if (chmod(chrootTmpDir.c_str(), 01777) == -1)
+            throw SysError("creating /tmp in the chroot");
+
+        /* When deleting this, do recurse (the builder might have left
+           crap there).  As long as nothing important is bind-mounted
+           under /tmp it's okay (and the bind-mounts are unmounted
+           before autoDelChrootTmp's destructor runs, anyway).  */
+        autoDelChrootTmp = boost::shared_ptr<AutoDelete>(new AutoDelete(chrootTmpDir));
+
+        /* Bind-mount a user-configurable set of directories from the
+           host file system.  The `/dev/pts' directory must be mounted
+           separately so that newly-created pseudo-terminals show
+           up. */
         Paths defaultDirs;
         defaultDirs.push_back("/dev");
+        defaultDirs.push_back("/dev/pts");
         defaultDirs.push_back("/proc");
+        
         Paths dirsInChroot = querySetting("build-chroot-dirs", defaultDirs);
 
         dirsInChroot.push_front(nixStore);
@@ -1743,7 +1790,7 @@ void DerivationGoal::startBuilder()
            unmounted in LIFO order.  (!!! Does the C++ standard
            guarantee that list elements are destroyed in order?) */
         for (Paths::iterator i = dirsInChroot.begin(); i != dirsInChroot.end(); ++i)
-            bindMounts.push_front(boost::shared_ptr<BindMount>(new BindMount(*i, tmpRootDir + *i)));
+            bindMounts.push_front(boost::shared_ptr<BindMount>(new BindMount(*i, chrootRootDir + *i)));
         
 #else
         throw Error("chroot builds are not supported on this platform");
@@ -1756,7 +1803,7 @@ void DerivationGoal::startBuilder()
         drv.builder);
 
     /* Create the log file and pipe. */
-    openLogFile();
+    Path logFile = openLogFile();
     
     /* Fork a child to build the package.  Note that while we
        currently use forks to run and wait for the children, it
@@ -1782,8 +1829,8 @@ void DerivationGoal::startBuilder()
                chroot.  (Actually the order doesn't matter, since due
                to the bind mount tmpDir and tmpRootDit/tmpDir are the
                same directories.) */
-            if (useChroot && chroot(tmpRootDir.c_str()) == -1)
-                throw SysError(format("cannot change root directory to `%1%'") % tmpRootDir);
+            if (useChroot && chroot(chrootRootDir.c_str()) == -1)
+                throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir);
 #endif
             
             initChild();
@@ -1842,6 +1889,8 @@ void DerivationGoal::startBuilder()
                 args.push_back(i->c_str());
             args.push_back(0);
 
+            restoreSIGPIPE();
+
             /* Execute the program.  This should not return. */
             execve(program.c_str(), (char * *) &args[0], (char * *) envArr);
 
@@ -1860,6 +1909,11 @@ void DerivationGoal::startBuilder()
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(), pid,
         singleton<set<int> >(logPipe.readSide), true);
+
+    if (printBuildTrace) {
+        printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
+            % drvPath % drv.outputs["out"].path % drv.platform % logFile);
+    }
 }
 
 
@@ -2005,7 +2059,7 @@ void DerivationGoal::computeClosure()
 string drvsLogDir = "drvs";
 
 
-void DerivationGoal::openLogFile()
+Path DerivationGoal::openLogFile()
 {
     /* Create a log file. */
     Path dir = (format("%1%/%2%") % nixLogDir % drvsLogDir).str();
@@ -2019,6 +2073,8 @@ void DerivationGoal::openLogFile()
 
     /* Create a pipe to get the output of the child. */
     logPipe.create();
+
+    return logFileName;
 }
 
 
@@ -2042,12 +2098,12 @@ void DerivationGoal::initChild()
     }
 
     /* Close all other file descriptors. */
-    int maxFD = 0;
-    maxFD = sysconf(_SC_OPEN_MAX);
-    for (int fd = 0; fd < maxFD; ++fd)
-        if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO
-            && (!inHook || (fd != 3 && fd != 4)))
-            close(fd); /* ignore result */
+    set<int> exceptions;
+    if (inHook) {
+        exceptions.insert(3);
+        exceptions.insert(4);
+    }
+    closeMostFDs(exceptions);
 }
 
 
@@ -2118,10 +2174,8 @@ private:
     /* The current substituter. */
     Path sub;
 
-    /* Path info returned by the substituter's --query-info operation. */
-    bool infoOkay;
-    PathSet references;
-    Path deriver;
+    /* Path info returned by the substituter's query info operation. */
+    SubstitutablePathInfo info;
 
     /* Pipe for the substitute's standard output/error. */
     Pipe logPipe;
@@ -2206,7 +2260,7 @@ void SubstitutionGoal::init()
     }
 
     subs = substituters;
-
+    
     tryNext();
 }
 
@@ -2228,25 +2282,14 @@ void SubstitutionGoal::tryNext()
     sub = subs.front();
     subs.pop_front();
 
-    infoOkay = false;
-    state = &SubstitutionGoal::gotInfo;
-    worker.waitForInfo(shared_from_this(), sub, storePath);
-}
-
-
-void SubstitutionGoal::gotInfo()
-{
-    trace("got info");
-
-    if (!infoOkay) {
+    if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
         tryNext();
         return;
     }
-    
+
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    for (PathSet::iterator i = references.begin();
-         i != references.end(); ++i)
+    foreach (PathSet::iterator, i, info.references)
         if (*i != storePath) /* ignore self-references */
             addWaitee(worker.makeSubstitutionGoal(*i));
 
@@ -2268,8 +2311,7 @@ void SubstitutionGoal::referencesValid()
         return;
     }
 
-    for (PathSet::iterator i = references.begin();
-         i != references.end(); ++i)
+    foreach (PathSet::iterator, i, info.references)
         if (*i != storePath) /* ignore self-references */
             assert(worker.store.isValidPath(*i));
 
@@ -2363,6 +2405,11 @@ void SubstitutionGoal::tryToRun()
         pid, singleton<set<int> >(logPipe.readSide), true);
 
     state = &SubstitutionGoal::finished;
+
+    if (printBuildTrace) {
+        printMsg(lvlError, format("@ substituter-started %1% %2%")
+            % storePath % sub);
+    }
 }
 
 
@@ -2402,6 +2449,11 @@ void SubstitutionGoal::finished()
             format("substitution of path `%1%' using substituter `%2%' failed: %3%")
             % storePath % sub % e.msg());
         
+        if (printBuildTrace) {
+            printMsg(lvlError, format("@ substituter-failed %1% %2% %3%")
+                % storePath % status % e.msg());
+        }
+        
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
         worker.wakeUp(shared_from_this());
@@ -2413,13 +2465,17 @@ void SubstitutionGoal::finished()
     Hash contentHash = hashPath(htSHA256, storePath);
 
     worker.store.registerValidPath(storePath, contentHash,
-        references, deriver);
+        info.references, info.deriver);
 
     outputLock->setDeletion(true);
     
     printMsg(lvlChatty,
         format("substitution of path `%1%' succeeded") % storePath);
 
+    if (printBuildTrace) {
+        printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath);
+    }
+    
     amDone(ecSuccess);
 }
 
@@ -2608,14 +2664,6 @@ void Worker::waitForChildTermination(GoalPtr goal)
 }
 
 
-void Worker::waitForInfo(GoalPtr goal, Path sub, Path storePath)
-{
-    debug("wait for info");
-    requestedInfo[sub].insert(storePath);
-    waitingForInfo.insert(goal);
-}
-
-
 void Worker::waitForAnyGoal(GoalPtr goal)
 {
     debug("wait for any goal");
@@ -2623,68 +2671,6 @@ void Worker::waitForAnyGoal(GoalPtr goal)
 }
 
 
-void Worker::getInfo()
-{
-    for (map<Path, PathSet>::iterator i = requestedInfo.begin();
-         i != requestedInfo.end(); ++i)
-    {
-        Path sub = i->first;
-        PathSet paths = i->second;
-
-        while (!paths.empty()) {
-
-            /* Run the substituter for at most 100 paths at a time to
-               prevent command line overflows. */
-            PathSet paths2;
-            while (!paths.empty() && paths2.size() < 100) {
-                paths2.insert(*paths.begin());
-                paths.erase(paths.begin());
-            }
-
-            /* Ask the substituter for the references and deriver of
-               the paths. */
-            debug(format("running `%1%' to get info about `%2%'") % sub % showPaths(paths2));
-            Strings args;
-            args.push_back("--query-info");
-            args.insert(args.end(), paths2.begin(), paths2.end());
-            string res = runProgram(sub, false, args);
-            std::istringstream str(res);
-
-            while (true) {
-                ValidPathInfo info = decodeValidPathInfo(str);
-                if (info.path == "") break;
-
-                /* !!! inefficient */
-                for (WeakGoals::iterator k = waitingForInfo.begin();
-                     k != waitingForInfo.end(); ++k)
-                {
-                    GoalPtr goal = k->lock();
-                    if (goal) {
-                        SubstitutionGoal * goal2 = dynamic_cast<SubstitutionGoal *>(goal.get());
-                        if (goal2->storePath == info.path) {
-                            goal2->references = info.references;
-                            goal2->deriver = info.deriver;
-                            goal2->infoOkay = true;
-                            wakeUp(goal);
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    for (WeakGoals::iterator k = waitingForInfo.begin();
-         k != waitingForInfo.end(); ++k)
-    {
-        GoalPtr goal = k->lock();
-        if (goal) wakeUp(goal);
-    }
-    
-    requestedInfo.clear();
-    waitingForInfo.clear(); // !!! have we done them all?
-}
-
-
 void Worker::run(const Goals & _topGoals)
 {
     for (Goals::iterator i = _topGoals.begin();
@@ -2711,8 +2697,6 @@ void Worker::run(const Goals & _topGoals)
 
         if (topGoals.empty()) break;
 
-        getInfo();
-
         /* Wait for input. */
         if (!children.empty())
             waitForInput();

src/libstore/derivations.cc

@@ -2,6 +2,7 @@
 #include "store-api.hh"
 #include "aterm.hh"
 #include "globals.hh"
+#include "util.hh"
 
 #include "derivations-ast.hh"
 #include "derivations-ast.cc"
@@ -163,9 +164,7 @@ ATerm unparseDerivation(const Derivation & drv)
 
 bool isDerivation(const string & fileName)
 {
-    return
-        fileName.size() >= drvExtension.size() &&
-        string(fileName, fileName.size() - drvExtension.size()) == drvExtension;
+    return hasSuffix(fileName, drvExtension);
 }
 
  

src/libstore/gc.cc

@@ -5,6 +5,9 @@
 
 #include <boost/shared_ptr.hpp>
 
+#include <functional>
+#include <queue>
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <errno.h>
@@ -123,7 +126,11 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
         createSymlink(gcRoot, storePath, false);
     }
 
-    /* Check that the root can be found by the garbage collector. */
+    /* Check that the root can be found by the garbage collector.
+       !!! This can be very slow on machines that have many roots.
+       Instead of reading all the roots, it would be more efficient to
+       check if the root is in a directory in or linked from the
+       gcroots directory. */
     if (queryBoolSetting("gc-check-reachability", true)) {
         Roots roots = store->findRoots();
         if (roots.find(gcRoot) == roots.end())
@@ -435,12 +442,143 @@ Paths topoSortPaths(const PathSet & paths)
 }
 
 
-void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
-    bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
+static time_t lastFileAccessTime(const Path & path)
 {
-    result.clear();
-    bytesFreed = 0;
+    checkInterrupt();
+    
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1)
+        throw SysError(format("statting `%1%'") % path);
 
+    if (S_ISDIR(st.st_mode)) {
+        time_t last = 0;
+	Strings names = readDirectory(path);
+	for (Strings::iterator i = names.begin(); i != names.end(); ++i) {
+            time_t t = lastFileAccessTime(path + "/" + *i);
+            if (t > last) last = t;
+        }
+        return last;
+    }
+
+    else if (S_ISLNK(st.st_mode)) return 0;
+
+    else return st.st_atime;
+}
+
+
+struct GCLimitReached { };
+
+
+void LocalStore::gcPath(const GCOptions & options, GCResults & results, 
+    const Path & path)
+{
+    results.paths.insert(path);
+
+    if (!pathExists(path)) return;
+                
+#ifndef __CYGWIN__
+    AutoCloseFD fdLock;
+        
+    /* Only delete a lock file if we can acquire a write lock on it.
+       That means that it's either stale, or the process that created
+       it hasn't locked it yet.  In the latter case the other process
+       will detect that we deleted the lock, and retry (see
+       pathlocks.cc). */
+    if (path.size() >= 5 && string(path, path.size() - 5) == ".lock") {
+        fdLock = openLockFile(path, false);
+        if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
+            debug(format("skipping active lock `%1%'") % path);
+            return;
+        }
+    }
+#endif
+
+    /* Okay, it's safe to delete. */
+    unsigned long long bytesFreed, blocksFreed;
+    deleteFromStore(path, bytesFreed, blocksFreed);
+    results.bytesFreed += bytesFreed;
+    results.blocksFreed += blocksFreed;
+
+    if (results.bytesFreed > options.maxFreed) {
+        printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % options.maxFreed);
+        throw GCLimitReached();
+    }
+
+    if (options.maxLinks) {
+        struct stat st;
+        if (stat(nixStore.c_str(), &st) == -1)
+            throw SysError(format("statting `%1%'") % nixStore);
+        if (st.st_nlink < options.maxLinks) {
+            printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % options.maxLinks);
+            throw GCLimitReached();
+        }
+    }
+
+#ifndef __CYGWIN__
+    if (fdLock != -1)
+        /* Write token to stale (deleted) lock file. */
+        writeFull(fdLock, (const unsigned char *) "d", 1);
+#endif
+}
+
+
+void LocalStore::gcPathRecursive(const GCOptions & options,
+    GCResults & results, PathSet & done, const Path & path)
+{
+    if (done.find(path) != done.end()) return;
+    done.insert(path);
+
+    startNest(nest, lvlDebug, format("looking at `%1%'") % path);
+        
+    /* Delete all the referrers first.  They must be garbage too,
+       since if they were live, then the current path would also be
+       live.  Note that deleteFromStore() below still makes sure that
+       the referrer set has become empty, just in case.  (However that
+       doesn't guard against deleting top-level paths that are only
+       reachable from GC roots.) */
+    PathSet referrers;
+    if (isValidPath(path))
+        queryReferrers(path, referrers);
+    foreach (PathSet::iterator, i, referrers)
+        if (*i != path) gcPathRecursive(options, results, done, *i);
+
+    printMsg(lvlInfo, format("deleting `%1%'") % path);
+            
+    gcPath(options, results, path);
+}
+
+
+struct CachingAtimeComparator : public std::binary_function<Path, Path, bool> 
+{
+    std::map<Path, time_t> cache;
+
+    time_t lookup(const Path & p)
+    {
+        std::map<Path, time_t>::iterator i = cache.find(p);
+        if (i != cache.end()) return i->second;
+        debug(format("computing atime of `%1%'") % p);
+        cache[p] = lastFileAccessTime(p);
+        assert(cache.find(p) != cache.end());
+        return cache[p];
+    }
+        
+    bool operator () (const Path & p1, const Path & p2)
+    {
+        return lookup(p2) < lookup(p1);
+    }
+};
+
+
+string showTime(const string & format, time_t t)
+{
+    char s[128];
+    strftime(s, sizeof s, format.c_str(), localtime(&t));
+    return string(s);
+}
+
+
+void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
     bool gcKeepOutputs =
         queryBoolSetting("gc-keep-outputs", false);
     bool gcKeepDerivations =
@@ -455,7 +593,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);
+    printMsg(lvlError, format("finding garbage collector roots..."));
+    Roots rootMap = options.ignoreLiveness ? Roots() : nix::findRoots(true);
 
     PathSet roots;
     for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i)
@@ -465,16 +604,17 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
-    if (!ignoreLiveness)
+    if (!options.ignoreLiveness)
         addAdditionalRoots(roots);
 
-    if (action == gcReturnRoots) {
-        result = roots;
+    if (options.action == GCOptions::gcReturnRoots) {
+        results.paths = roots;
         return;
     }
 
     /* Determine the live paths which is just the closure of the
        roots under the `references' relation. */
+    printMsg(lvlError, format("computing live paths..."));
     PathSet livePaths;
     for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
         computeFSClosure(canonPath(*i), livePaths);
@@ -486,8 +626,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
             /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
-            Path deriver = store->queryDeriver(*i);
-            if (deriver != "" && store->isValidPath(deriver))
+            Path deriver = queryDeriver(*i);
+            if (deriver != "" && isValidPath(deriver))
                 computeFSClosure(deriver, livePaths);
         }
     }
@@ -507,13 +647,13 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
 		if (gcLevel >= gcKeepOutputsThreshold)    
 		    for (DerivationOutputs::iterator j = drv.outputs.begin();
                          j != drv.outputs.end(); ++j)
-			if (store->isValidPath(j->second.path))
+			if (isValidPath(j->second.path))
 			    computeFSClosure(j->second.path, livePaths);
             }
     }
 
-    if (action == gcReturnLive) {
-        result = livePaths;
+    if (options.action == GCOptions::gcReturnLive) {
+        results.paths = livePaths;
         return;
     }
 
@@ -532,108 +672,132 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
        means that it has already been closed). */
     PathSet tempRootsClosed;
     for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
-        if (store->isValidPath(*i))
+        if (isValidPath(*i))
             computeFSClosure(*i, tempRootsClosed);
         else
             tempRootsClosed.insert(*i);
 
-    /* For testing - see tests/gc-concurrent.sh. */
-    if (getenv("NIX_DEBUG_GC_WAIT"))
-        sleep(2);
-    
     /* After this point the set of roots or temporary roots cannot
        increase, since we hold locks on everything.  So everything
        that is not currently in in `livePaths' or `tempRootsClosed'
        can be deleted. */
     
     /* Read the Nix store directory to find all currently existing
-       paths. */
-    PathSet storePathSet;
-    if (action != gcDeleteSpecific) {
+       paths and filter out all live paths. */
+    printMsg(lvlError, format("reading the Nix store..."));
+    PathSet storePaths;
+    
+    if (options.action != GCOptions::gcDeleteSpecific) {
         Paths entries = readDirectory(nixStore);
-        for (Paths::iterator i = entries.begin(); i != entries.end(); ++i)
-            storePathSet.insert(canonPath(nixStore + "/" + *i));
-    } else {
-        for (PathSet::iterator i = pathsToDelete.begin();
-             i != pathsToDelete.end(); ++i)
-        {
-            assertStorePath(*i);
-            storePathSet.insert(*i);
+        foreach (Paths::iterator, i, entries) {
+            Path path = canonPath(nixStore + "/" + *i);
+            if (livePaths.find(path) == livePaths.end() &&
+                tempRootsClosed.find(path) == tempRootsClosed.end())
+                storePaths.insert(path);
         }
     }
 
-    /* Topologically sort them under the `referrers' relation.  That
-       is, a < b iff a is in referrers(b).  This gives us the order in
-       which things can be deleted safely. */
-    /* !!! when we have multiple output paths per derivation, this
-       will not work anymore because we get cycles. */
-    Paths storePaths = topoSortPaths(storePathSet);
-
-    /* Try to delete store paths in the topologically sorted order. */
-    for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
-
-        debug(format("considering deletion of `%1%'") % *i);
-        
-        if (livePaths.find(*i) != livePaths.end()) {
-            if (action == gcDeleteSpecific)
+    else {
+        foreach (PathSet::iterator, i, options.pathsToDelete) {
+            assertStorePath(*i);
+            storePaths.insert(*i);
+            if (livePaths.find(*i) != livePaths.end())
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
-            debug(format("live path `%1%'") % *i);
-            continue;
+            if (tempRootsClosed.find(*i) != tempRootsClosed.end())
+                throw Error(format("cannot delete path `%1%' since it is temporarily in use") % *i);
+        }
+    }
+
+    if (options.action == GCOptions::gcReturnDead) {
+        results.paths.insert(storePaths.begin(), storePaths.end());
+        return;
+    }
+
+    /* Delete all dead store paths (or until one of the stop
+       conditions is reached). */
+
+    PathSet done;
+    try {
+
+        if (!options.useAtime) {
+            /* Delete the paths, respecting the partial ordering
+               determined by the references graph. */
+            printMsg(lvlError, format("deleting garbage..."));
+            foreach (PathSet::iterator, i, storePaths)
+                gcPathRecursive(options, results, done, *i);
         }
 
-        if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
-            debug(format("temporary root `%1%'") % *i);
-            continue;
-        }
+        else {
 
-        debug(format("dead path `%1%'") % *i);
-        result.insert(*i);
+            /* Delete in order of ascending last access time, still
+               maintaining the partial ordering of the reference
+               graph.  Note that we can't use a topological sort for
+               this because that takes time O(V+E), and in this case
+               E=O(V^2) (i.e. the graph is dense because of the edges
+               due to the atime ordering).  So instead we put all
+               deletable paths in a priority queue (ordered by atime),
+               and after deleting a path, add additional paths that
+               have become deletable to the priority queue. */
 
-        /* If just returning the set of dead paths, we also return the
-           space that would be freed if we deleted them. */
-        if (action == gcReturnDead)
-            bytesFreed += computePathSize(*i);
+            CachingAtimeComparator atimeComp;
 
-        if (action == gcDeleteDead || action == gcDeleteSpecific) {
+            /* Create a priority queue that orders paths by ascending
+               atime.  This is why C++ needs type inferencing... */
+            std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> > prioQueue =
+                std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> >(binary_function_ref_adapter<CachingAtimeComparator>(&atimeComp));
 
-#ifndef __CYGWIN__
-            AutoCloseFD fdLock;
+           /* Initially put the paths that are invalid or have no
+              referrers into the priority queue. */
+            printMsg(lvlError, format("finding deletable paths..."));
+            foreach (PathSet::iterator, i, storePaths) {
+                checkInterrupt();
+                /* We can safely delete a path if it's invalid or
+                   it has no referrers.  Note that all the invalid
+                   paths will be deleted in the first round. */
+                if (isValidPath(*i)) {
+                    if (queryReferrersNoSelf(*i).empty()) prioQueue.push(*i);
+                } else prioQueue.push(*i);
+            }
 
-            /* Only delete a lock file if we can acquire a write lock
-               on it.  That means that it's either stale, or the
-               process that created it hasn't locked it yet.  In the
-               latter case the other process will detect that we
-               deleted the lock, and retry (see pathlocks.cc). */
-            if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
-                fdLock = openLockFile(*i, false);
-                if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
-                    debug(format("skipping active lock `%1%'") % *i);
+            debug(format("%1% initially deletable paths") % prioQueue.size());
+
+            /* Now delete everything in the order of the priority
+               queue until nothing is left. */
+            printMsg(lvlError, format("deleting garbage..."));
+            while (!prioQueue.empty()) {
+                checkInterrupt();
+                Path path = prioQueue.top(); prioQueue.pop();
+
+                if (options.maxAtime != (time_t) -1 &&
+                    atimeComp.lookup(path) > options.maxAtime)
                     continue;
+                
+                printMsg(lvlInfo, format("deleting `%1%' (last accessed %2%)") % path % showTime("%F %H:%M:%S", atimeComp.lookup(path)));
+
+                PathSet references;
+                if (isValidPath(path)) references = queryReferencesNoSelf(path);
+
+                gcPath(options, results, path);
+
+                /* For each reference of the current path, see if the
+                   reference has now become deletable (i.e. is in the
+                   set of dead paths and has no referrers left).  If
+                   so add it to the priority queue. */
+                foreach (PathSet::iterator, i, references) {
+                    if (storePaths.find(*i) != storePaths.end() &&
+                        queryReferrersNoSelf(*i).empty())
+                    {
+                        debug(format("path `%1%' has become deletable") % *i);
+                        prioQueue.push(*i);
+                    }
                 }
             }
-#endif
-
-            if (!pathExists(*i)) continue;
-                
-            printMsg(lvlInfo, format("deleting `%1%'") % *i);
             
-            /* Okay, it's safe to delete. */
-            try {
-                unsigned long long freed;
-                deleteFromStore(*i, freed);
-                bytesFreed += freed;
-            } catch (PathInUse & e) {
-                printMsg(lvlError, format("warning: %1%") % e.msg());
-            }
-
-#ifndef __CYGWIN__
-            if (fdLock != -1)
-                /* Write token to stale (deleted) lock file. */
-                writeFull(fdLock, (const unsigned char *) "d", 1);
-#endif
         }
+        
+    } catch (GCLimitReached & e) {
     }
 }
 
- 
+
 }

src/libstore/globals.cc

@@ -16,6 +16,7 @@ string nixDBPath = "/UNINIT";
 string nixConfDir = "/UNINIT";
 string nixLibexecDir = "/UNINIT";
 string nixBinDir = "/UNINIT";
+string nixChrootsDir = "/UNINIT";
 
 bool keepFailed = false;
 bool keepGoing = false;
@@ -27,6 +28,7 @@ string thisSystem = "unset";
 unsigned int maxSilentTime = 0;
 Paths substituters;
 bool useBuildHook = true;
+bool printBuildTrace = false;
 
 
 static bool settingsRead = false;
@@ -116,5 +118,12 @@ unsigned int queryIntSetting(const string & name, unsigned int def)
     return n;
 }
 
+
+void reloadSettings()
+{
+    settingsRead = false;
+    settings.clear();
+}
+
  
 }

src/libstore/globals.hh

@@ -35,6 +35,12 @@ extern string nixLibexecDir;
 /* nixBinDir is the directory where the main programs are stored. */
 extern string nixBinDir;
 
+/* nixChrootsDir is the directory where we create chroot environments
+   (when chroot builds are enabled).  We don't put these under /tmp to
+   prevent "rm -rf /tmp" from recursing into /nix/store via the bind
+   mounts in the chroots. */
+extern string nixChrootsDir;
+
 
 /* Misc. global flags. */
 
@@ -76,6 +82,22 @@ extern Paths substituters;
    users want to disable this from the command-line. */
 extern bool useBuildHook;
 
+/* Whether buildDerivations() should print out lines on stderr in a
+   fixed format to allow its progress to be monitored.  Each line
+   starts with a "@".  The following are defined:
+
+   @ build-started <drvpath> <outpath> <system> <logfile>
+   @ build-failed <drvpath> <outpath> <exitcode> <error text>
+   @ build-succeeded <drvpath> <outpath>
+   @ substituter-started <outpath> <substituter>
+   @ substituter-failed <outpath> <exitcode> <error text>
+   @ substituter-succeeded <outpath>
+
+   Best combined with --no-build-output, otherwise stderr might
+   conceivably contain lines in this format printed by the builders.
+*/
+extern bool printBuildTrace;
+
 
 Strings querySetting(const string & name, const Strings & def);
 
@@ -85,6 +107,8 @@ bool queryBoolSetting(const string & name, bool def);
 
 unsigned int queryIntSetting(const string & name, unsigned int def);
 
+void reloadSettings();
+
     
 }
 

src/libstore/local-store.cc

@@ -15,6 +15,7 @@
 #include <unistd.h>
 #include <utime.h>
 #include <fcntl.h>
+#include <errno.h>
 
 
 namespace nix {
@@ -48,8 +49,14 @@ LocalStore::LocalStore()
 
     checkStoreNotSymlink();
 
-    Path globalLockPath = nixDBPath + "/big-lock";
-    globalLock = openLockFile(globalLockPath.c_str(), true);
+    try {
+        Path globalLockPath = nixDBPath + "/big-lock";
+        globalLock = openLockFile(globalLockPath.c_str(), true);
+    } catch (SysError & e) {
+        if (e.errNo != EACCES) throw;
+        readOnlyMode = true;
+        return;
+    }
     
     if (!lockFile(globalLock, ltRead, false)) {
         printMsg(lvlError, "waiting for the big Nix store lock...");
@@ -59,9 +66,6 @@ LocalStore::LocalStore()
     createDirs(nixDBPath + "/info");
     createDirs(nixDBPath + "/referrer");
 
-    //printMsg(lvlTalkative, "cannot access Nix database; continuing anyway");
-    //readOnlyMode = true;
-
     int curSchema = getSchema();
     if (curSchema > nixSchemaVersion)
         throw Error(format("current Nix store schema is version %1%, but I only support %2%")
@@ -79,6 +83,13 @@ LocalStore::~LocalStore()
 {
     try {
         flushDelayedUpdates();
+
+        foreach (RunningSubstituters::iterator, i, runningSubstituters) {
+            i->second.toBuf.reset();
+            i->second.to.reset();
+            i->second.pid.wait(true);
+        }
+                
     } catch (...) {
         ignoreException();
     }
@@ -363,8 +374,6 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
     std::map<Path, ValidPathInfo>::iterator lookup = pathInfoCache.find(path);
     if (lookup != pathInfoCache.end()) return lookup->second;
     
-    //printMsg(lvlError, "queryPathInfo: " + path);
-    
     /* Read the info file. */
     Path infoFile = infoFileFor(path);
     if (!pathExists(infoFile))
@@ -399,6 +408,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
 
 bool LocalStore::isValidPath(const Path & path)
 {
+    /* Files in the info directory starting with a `.' are temporary
+       files. */
+    if (baseNameOf(path).at(0) == '.') return false;
     return pathExists(infoFileFor(path));
 }
 
@@ -407,8 +419,8 @@ PathSet LocalStore::queryValidPaths()
 {
     PathSet paths;
     Strings entries = readDirectory(nixDBPath + "/info");
-    for (Strings::iterator i = entries.begin(); i != entries.end(); ++i) 
-        paths.insert(nixStore + "/" + *i);
+    for (Strings::iterator i = entries.begin(); i != entries.end(); ++i)
+        if (i->at(0) != '.') paths.insert(nixStore + "/" + *i);
     return paths;
 }
 
@@ -463,33 +475,109 @@ Path LocalStore::queryDeriver(const Path & path)
 }
 
 
-PathSet LocalStore::querySubstitutablePaths()
+void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
 {
-    if (!substitutablePathsLoaded) {
-        for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) {
-            debug(format("running `%1%' to find out substitutable paths") % *i);
-            Strings args;
-            args.push_back("--query-paths");
-            Strings ss = tokenizeString(runProgram(*i, false, args), "\n");
-            for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) {
-                if (!isStorePath(*j))
-                    throw Error(format("`%1%' returned a bad substitutable path `%2%'")
-                        % *i % *j);
-                substitutablePaths.insert(*j);
-            }
+    if (run.pid != -1) return;
+    
+    debug(format("starting substituter program `%1%'") % substituter);
+
+    Pipe toPipe, fromPipe;
+            
+    toPipe.create();
+    fromPipe.create();
+
+    run.pid = fork();
+            
+    switch (run.pid) {
+
+    case -1:
+        throw SysError("unable to fork");
+
+    case 0: /* child */
+        try {
+            fromPipe.readSide.close();
+            toPipe.writeSide.close();
+            if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
+                throw SysError("dupping stdin");
+            if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
+                throw SysError("dupping stdout");
+            closeMostFDs(set<int>());
+            execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
+            throw SysError(format("executing `%1%'") % substituter);
+        } catch (std::exception & e) {
+            std::cerr << "error: " << e.what() << std::endl;
         }
-        substitutablePathsLoaded = true;
+        quickExit(1);
     }
 
-    return substitutablePaths;
+    /* Parent. */
+    
+    toPipe.readSide.close();
+    fromPipe.writeSide.close();
+
+    run.toBuf = boost::shared_ptr<stdio_filebuf>(new stdio_filebuf(toPipe.writeSide.borrow(), std::ios_base::out));
+    run.to = boost::shared_ptr<std::ostream>(new std::ostream(&*run.toBuf));
+
+    run.fromBuf = boost::shared_ptr<stdio_filebuf>(new stdio_filebuf(fromPipe.readSide.borrow(), std::ios_base::in));
+    run.from = boost::shared_ptr<std::istream>(new std::istream(&*run.fromBuf));
+}
+
+
+template<class T> T getIntLine(std::istream & str)
+{
+    string s;
+    T res;
+    getline(str, s);
+    if (!str || !string2Int(s, res)) throw Error("integer expected from stream");
+    return res;
 }
 
 
 bool LocalStore::hasSubstitutes(const Path & path)
 {
-    if (!substitutablePathsLoaded)
-        querySubstitutablePaths(); 
-    return substitutablePaths.find(path) != substitutablePaths.end();
+    foreach (Paths::iterator, i, substituters) {
+        RunningSubstituter & run(runningSubstituters[*i]);
+        startSubstituter(*i, run);
+
+        *run.to << "have\n" << path << "\n" << std::flush;
+
+        if (getIntLine<int>(*run.from)) return true;
+    }
+
+    return false;
+}
+
+
+bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
+    const Path & path, SubstitutablePathInfo & info)
+{
+    RunningSubstituter & run(runningSubstituters[substituter]);
+    startSubstituter(substituter, run);
+
+    *run.to << "info\n" << path << "\n" << std::flush;
+        
+    if (!getIntLine<int>(*run.from)) return false;
+    
+    getline(*run.from, info.deriver);
+    if (info.deriver != "") assertStorePath(info.deriver);
+    int nrRefs = getIntLine<int>(*run.from);
+    while (nrRefs--) {
+        Path p; getline(*run.from, p);
+        assertStorePath(p);
+        info.references.insert(p);
+    }
+    info.downloadSize = getIntLine<long long>(*run.from);
+    
+    return true;
+}
+
+
+bool LocalStore::querySubstitutablePathInfo(const Path & path,
+    SubstitutablePathInfo & info)
+{
+    foreach (Paths::iterator, i, substituters)
+        if (querySubstitutablePathInfo(*i, path, info)) return true;
+    return false;
 }
 
 
@@ -851,7 +939,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 }
 
 
-void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed)
+void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed,
+    unsigned long long & blocksFreed)
 {
     bytesFreed = 0;
 
@@ -871,7 +960,7 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr
         invalidatePath(path);
     }
 
-    deletePathWrapped(path, bytesFreed);
+    deletePathWrapped(path, bytesFreed, blocksFreed);
 }
 
 

src/libstore/local-store.hh

@@ -3,6 +3,8 @@
 
 #include <string>
 
+#include <ext/stdio_filebuf.h>
+
 #include "store-api.hh"
 #include "util.hh"
 
@@ -25,19 +27,35 @@ struct OptimiseStats
     unsigned long sameContents;
     unsigned long filesLinked;
     unsigned long long bytesFreed;
+    unsigned long long blocksFreed;
     OptimiseStats()
     {
         totalFiles = sameContents = filesLinked = 0;
-        bytesFreed = 0;
+        bytesFreed = blocksFreed = 0;
     }
 };
 
 
+typedef __gnu_cxx::stdio_filebuf<char> stdio_filebuf;
+
+
+struct RunningSubstituter
+{
+    Pid pid;
+    boost::shared_ptr<stdio_filebuf> toBuf, fromBuf;
+    boost::shared_ptr<std::ostream> to;
+    boost::shared_ptr<std::istream> from;
+};
+
+
 class LocalStore : public StoreAPI
 {
 private:
     bool substitutablePathsLoaded;
     PathSet substitutablePaths;
+
+    typedef std::map<Path, RunningSubstituter> RunningSubstituters;
+    RunningSubstituters runningSubstituters;
     
 public:
 
@@ -64,6 +82,12 @@ public:
     PathSet querySubstitutablePaths();
     
     bool hasSubstitutes(const Path & path);
+
+    bool querySubstitutablePathInfo(const Path & path,
+        SubstitutablePathInfo & info);
+    
+    bool querySubstitutablePathInfo(const Path & substituter,
+        const Path & path, SubstitutablePathInfo & info);
     
     Path addToStore(const Path & srcPath, bool fixed = false,
         bool recursive = false, string hashAlgo = "",
@@ -89,11 +113,11 @@ public:
 
     Roots findRoots();
 
-    void collectGarbage(GCAction action, const PathSet & pathsToDelete,
-        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
+    void collectGarbage(const GCOptions & options, GCResults & results);
 
     /* Delete a path from the Nix store. */
-    void deleteFromStore(const Path & path, unsigned long long & bytesFreed);
+    void deleteFromStore(const Path & path, unsigned long long & bytesFreed,
+        unsigned long long & blocksFreed);
     
     /* Optimise the disk space usage of the Nix store by hard-linking
        files with the same contents. */
@@ -143,6 +167,14 @@ private:
     
     void upgradeStore12();
 
+    void gcPath(const GCOptions & options, GCResults & results,
+        const Path & path);
+
+    void gcPathRecursive(const GCOptions & options,
+        GCResults & results, PathSet & done, const Path & path);
+
+    void startSubstituter(const Path & substituter,
+        RunningSubstituter & runningSubstituter);
 };
 
 
@@ -175,7 +207,7 @@ void getOwnership(const Path & path);
 /* Like deletePath(), but changes the ownership of `path' using the
    setuid wrapper if necessary (and possible). */
 void deletePathWrapped(const Path & path,
-    unsigned long long & bytesFreed);
+    unsigned long long & bytesFreed, unsigned long long & blocksFreed);
 
 void deletePathWrapped(const Path & path);
  

src/libstore/misc.cc

@@ -1,5 +1,6 @@
 #include "misc.hh"
 #include "store-api.hh"
+#include "local-store.hh"
 
 #include <aterm2.h>
 
@@ -45,8 +46,11 @@ Path findOutput(const Derivation & drv, string id)
 
 
 void queryMissing(const PathSet & targets,
-    PathSet & willBuild, PathSet & willSubstitute)
+    PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    unsigned long long & downloadSize)
 {
+    downloadSize = 0;
+    
     PathSet todo(targets.begin(), targets.end()), done;
 
     while (!todo.empty()) {
@@ -56,7 +60,10 @@ void queryMissing(const PathSet & targets,
         done.insert(p);
 
         if (isDerivation(p)) {
-            if (!store->isValidPath(p)) continue;
+            if (!store->isValidPath(p)) {
+                unknown.insert(p);
+                continue;
+            }
             Derivation drv = derivationFromPath(p);
 
             bool mustBuild = false;
@@ -79,10 +86,13 @@ void queryMissing(const PathSet & targets,
 
         else {
             if (store->isValidPath(p)) continue;
-            if (store->hasSubstitutes(p))
+            SubstitutablePathInfo info;
+            if (store->querySubstitutablePathInfo(p, info)) {
                 willSubstitute.insert(p);
-            // XXX call the substituters
-            // store->queryReferences(p, todo);
+                downloadSize += info.downloadSize;
+                todo.insert(info.references.begin(), info.references.end());
+            } else
+                unknown.insert(p);
         }
     }
 }

src/libstore/misc.hh

@@ -29,7 +29,8 @@ Path findOutput(const Derivation & drv, string id);
    derivations that will be built, and the set of output paths that
    will be substituted. */
 void queryMissing(const PathSet & targets,
-    PathSet & willBuild, PathSet & willSubstitute);
+    PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    unsigned long long & downloadSize);
 
 
 }

src/libstore/optimise-store.cc

@@ -4,6 +4,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <errno.h>
 
 
 namespace nix {
@@ -22,6 +23,21 @@ static void makeWritable(const Path & path)
 }
 
 
+struct MakeReadOnly
+{
+    Path path;
+    MakeReadOnly(const Path & path) : path(path) { }
+    ~MakeReadOnly()
+    {
+        try {
+            if (path != "") canonicalisePathMetaData(path, false);
+        } catch (...) {
+            ignoreException();
+        }
+    }
+};
+
+
 static void hashAndLink(bool dryRun, HashToPath & hashToPath,
     OptimiseStats & stats, const Path & path)
 {
@@ -82,25 +98,35 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath,
                mess with  its permissions). */
             bool mustToggle = !isStorePath(path);
             if (mustToggle) makeWritable(dirOf(path));
+            
+            /* When we're done, make the directory read-only again and
+               reset its timestamp back to 0. */
+            MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
         
-            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1)
+            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) {
+                if (errno == EMLINK) {
+                    /* Too many links to the same file (>= 32000 on
+                       most file systems).  This is likely to happen
+                       with empty files.  Just start over, creating
+                       links to the current file. */
+                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
+                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
+                    return;
+                }
                 throw SysError(format("cannot link `%1%' to `%2%'")
                     % tempLink % prevPath.first);
+            }
 
             /* Atomically replace the old file with the new hard link. */
             if (rename(tempLink.c_str(), path.c_str()) == -1)
                 throw SysError(format("cannot rename `%1%' to `%2%'")
                     % tempLink % path);
-
-            /* Make the directory read-only again and reset its
-               timestamp back to 0. */
-            if (mustToggle) canonicalisePathMetaData(dirOf(path), false);
-            
         } else
             printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
         
         stats.filesLinked++;
         stats.bytesFreed += st.st_size;
+        stats.blocksFreed += st.st_blocks;
     }
 
     if (S_ISDIR(st.st_mode)) {

src/libstore/pathlocks.cc

@@ -2,6 +2,7 @@
 #include "util.hh"
 
 #include <cerrno>
+#include <cstdlib>
 
 #include <sys/types.h>
 #include <sys/stat.h>

src/libstore/pathlocks.hh

@@ -17,7 +17,7 @@ int openLockFile(const Path & path, bool create);
 void deleteLockFilePreClose(const Path & path, int fd);
 void deleteLockFilePostClose(const Path & path);
 
-typedef enum LockType { ltRead, ltWrite, ltNone };
+enum LockType { ltRead, ltWrite, ltNone };
 
 bool lockFile(int fd, LockType lockType, bool wait);
 

src/libstore/references.cc

@@ -3,6 +3,8 @@
 #include "util.hh"
 
 #include <cerrno>
+#include <cstring>
+#include <cstdlib>
 #include <map>
 
 #include <sys/types.h>

src/libstore/remote-store.cc

@@ -137,14 +137,26 @@ void RemoteStore::connectToDaemon()
 
     string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
 
+    /* Urgh, sockaddr_un allows path names of only 108 characters.  So
+       chdir to the socket directory so that we can pass a relative
+       path name.  !!! this is probably a bad idea in multi-threaded
+       applications... */
+    AutoCloseFD fdPrevDir = open(".", O_RDONLY);
+    if (fdPrevDir == -1) throw SysError("couldn't open current directory");
+    chdir(dirOf(socketPath).c_str()); 
+    Path socketPathRel = "./" + baseNameOf(socketPath);
+    
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
-    if (socketPath.size() >= sizeof(addr.sun_path))
-        throw Error(format("socket path `%1%' is too long") % socketPath);
-    strcpy(addr.sun_path, socketPath.c_str());
+    if (socketPathRel.size() >= sizeof(addr.sun_path))
+        throw Error(format("socket path `%1%' is too long") % socketPathRel);
+    strcpy(addr.sun_path, socketPathRel.c_str());
     
     if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1)
         throw SysError(format("cannot connect to daemon at `%1%'") % socketPath);
+
+    if (fchdir(fdPrevDir) == -1)
+        throw SysError("couldn't change back to previous directory");
 }
 
 
@@ -171,6 +183,11 @@ void RemoteStore::setOptions()
     writeInt(maxSilentTime, to);
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
         writeInt(useBuildHook, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) {
+        writeInt(buildVerbosity, to);
+        writeInt(logType, to);
+        writeInt(printBuildTrace, to);
+    }
     processStderr();
 }
 
@@ -201,6 +218,23 @@ bool RemoteStore::hasSubstitutes(const Path & path)
 }
 
 
+bool RemoteStore::querySubstitutablePathInfo(const Path & path,
+    SubstitutablePathInfo & info)
+{
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false;
+    writeInt(wopQuerySubstitutablePathInfo, to);
+    writeString(path, to);
+    processStderr();
+    unsigned int reply = readInt(from);
+    if (reply == 0) return false;
+    info.deriver = readString(from);
+    if (info.deriver != "") assertStorePath(info.deriver);
+    info.references = readStorePaths(from);
+    info.downloadSize = readLongLong(from);
+    return true;
+}
+
+
 Hash RemoteStore::queryPathHash(const Path & path)
 {
     writeInt(wopQueryPathHash, to);
@@ -244,12 +278,6 @@ Path RemoteStore::queryDeriver(const Path & path)
 }
 
 
-PathSet RemoteStore::querySubstitutablePaths()
-{
-    throw Error("not implemented");
-}
-
-
 Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
     bool recursive, string hashAlgo, PathFilter & filter)
 {
@@ -360,24 +388,20 @@ Roots RemoteStore::findRoots()
 }
 
 
-void RemoteStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
-    bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
+void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    result.clear();
-    bytesFreed = 0;
     writeInt(wopCollectGarbage, to);
-    writeInt(action, to);
-    writeStringSet(pathsToDelete, to);
-    writeInt(ignoreLiveness, to);
+    writeInt(options.action, to);
+    writeStringSet(options.pathsToDelete, to);
+    writeInt(options.ignoreLiveness, to);
+    writeLongLong(options.maxFreed, to);
+    writeInt(options.maxLinks, to);
     
     processStderr();
     
-    result = readStringSet(from);
-
-    /* Ugh - NAR integers are 64 bits, but read/writeInt() aren't. */
-    unsigned int lo = readInt(from);
-    unsigned int hi = readInt(from);
-    bytesFreed = (((unsigned long long) hi) << 32) | lo;
+    results.paths = readStringSet(from);
+    results.bytesFreed = readLongLong(from);
+    results.blocksFreed = readLongLong(from);
 }
 
 

src/libstore/remote-store.hh

@@ -37,10 +37,11 @@ public:
 
     Path queryDeriver(const Path & path);
     
-    PathSet querySubstitutablePaths();
-    
     bool hasSubstitutes(const Path & path);
     
+    bool querySubstitutablePathInfo(const Path & path,
+        SubstitutablePathInfo & info);
+    
     Path addToStore(const Path & srcPath, bool fixed = false,
         bool recursive = false, string hashAlgo = "",
         PathFilter & filter = defaultPathFilter);
@@ -65,8 +66,7 @@ public:
     
     Roots findRoots();
 
-    void collectGarbage(GCAction action, const PathSet & pathsToDelete,
-        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
+    void collectGarbage(const GCOptions & options, GCResults & results);
     
 private:
     AutoCloseFD fdSocket;

src/libstore/store-api.cc

@@ -2,14 +2,26 @@
 #include "globals.hh"
 #include "util.hh"
 
+#include <limits.h>
+
+
+/* Needed for some ancient environments. */
+#ifndef ULLONG_MAX
+#define ULLONG_MAX 18446744073709551615
+#endif
+
 
 namespace nix {
 
 
-bool StoreAPI::hasSubstitutes(const Path & path)
+GCOptions::GCOptions()
 {
-    PathSet paths = querySubstitutablePaths();
-    return paths.find(path) != paths.end();
+    action = gcDeleteDead;
+    ignoreLiveness = false;
+    maxFreed = ULLONG_MAX;
+    maxLinks = 0;
+    useAtime = false;
+    maxAtime = (time_t) -1;
 }
 
 

src/libstore/store-api.hh

@@ -16,14 +16,100 @@ namespace nix {
 typedef std::map<Path, Path> Roots;
 
 
-/* Garbage collector operation. */
-typedef enum {
-    gcReturnRoots,
-    gcReturnLive,
-    gcReturnDead,
-    gcDeleteDead,
-    gcDeleteSpecific,
-} GCAction;
+
+
+struct GCOptions
+{
+    /* Garbage collector operation:
+
+       - `gcReturnRoots': find and return the set of roots for the
+         garbage collector.  These are the store paths symlinked to in
+         the `gcroots' directory.
+
+       - `gcReturnLive': return the set of paths reachable from
+         (i.e. in the closure of) the roots.
+
+       - `gcReturnDead': return the set of paths not reachable from
+         the roots.
+
+       - `gcDeleteDead': actually delete the latter set.
+
+       - `gcDeleteSpecific': delete the paths listed in
+          `pathsToDelete', insofar as they are not reachable.
+    */
+    typedef enum {
+        gcReturnRoots,
+        gcReturnLive,
+        gcReturnDead,
+        gcDeleteDead,
+        gcDeleteSpecific,
+    } GCAction;
+
+    GCAction action;
+
+    /* If `ignoreLiveness' is set, then reachability from the roots is
+       ignored (dangerous!).  However, the paths must still be
+       unreferenced *within* the store (i.e., there can be no other
+       store paths that depend on them). */
+    bool ignoreLiveness;
+
+    /* For `gcDeleteSpecific', the paths to delete. */
+    PathSet pathsToDelete;
+
+    /* Stop after at least `maxFreed' bytes have been freed. */
+    unsigned long long maxFreed;
+
+    /* Stop after the number of hard links to the Nix store directory
+       has dropped below `maxLinks'. */
+    unsigned int maxLinks;
+
+    /* Delete paths in order of ascending last access time.  I.e.,
+       prefer deleting unrecently used paths.  Useful in conjunction
+       with `maxFreed' and `maxLinks' (or manual interruption).  The
+       access time of a path is defined as the highest atime of any
+       non-directory, non-symlink file under that path.  Directories
+       and symlinks are ignored because their atimes are frequently
+       mass-updated, e.g. by `locate'.  Note that optimiseStore()
+       somewhat reduces the usefulness of this option: it hard-links
+       regular files and symlink together, giving them a "shared"
+       atime. */
+    bool useAtime;
+
+    /* Do not delete paths newer than `maxAtime'.  -1 means no age
+       limit. */
+    time_t maxAtime;
+
+    GCOptions();
+};
+
+
+struct GCResults 
+{
+    /* Depending on the action, the GC roots, or the paths that would
+       be or have been deleted. */
+    PathSet paths;
+
+    /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
+       number of bytes that would be or was freed. */
+    unsigned long long bytesFreed;
+
+    /* The number of file system blocks that would be or was freed. */
+    unsigned long long blocksFreed;
+
+    GCResults()
+    {
+        bytesFreed = 0;
+        blocksFreed = 0;
+    }
+};
+
+
+struct SubstitutablePathInfo
+{
+    Path deriver;
+    PathSet references;
+    unsigned long long downloadSize; /* 0 = unknown or inapplicable */
+};
 
 
 class StoreAPI 
@@ -46,21 +132,40 @@ public:
     virtual void queryReferences(const Path & path,
         PathSet & references) = 0;
 
+    /* Like queryReferences, but with self-references filtered out. */
+    PathSet queryReferencesNoSelf(const Path & path)
+    {
+        PathSet res;
+        queryReferences(path, res);
+        res.erase(path);
+        return res;
+    }
+
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
     virtual void queryReferrers(const Path & path,
         PathSet & referrers) = 0;
 
+    /* Like queryReferrers, but with self-references filtered out. */
+    PathSet queryReferrersNoSelf(const Path & path)
+    {
+        PathSet res;
+        queryReferrers(path, res);
+        res.erase(path);
+        return res;
+    }
+
     /* Query the deriver of a store path.  Return the empty string if
        no deriver has been set. */
     virtual Path queryDeriver(const Path & path) = 0;
 
-    /* Query the set of substitutable paths. */
-    virtual PathSet querySubstitutablePaths() = 0;
+    /* Query whether a path has substitutes. */
+    virtual bool hasSubstitutes(const Path & path) = 0;
 
-    /* More efficient variant if we just want to know if a path has
-       substitutes. */
-    virtual bool hasSubstitutes(const Path & path);
+    /* Query the references, deriver and download size of a
+       substitutable path. */
+    virtual bool querySubstitutablePathInfo(const Path & path,
+        SubstitutablePathInfo & info) = 0;
     
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
@@ -137,33 +242,8 @@ public:
        outside of the Nix store that point to `storePath'.  */
     virtual Roots findRoots() = 0;
 
-    /* Depending on `action', this function does the following:
-
-       - `gcReturnRoots': find and return the set of roots for the
-         garbage collector.  These are the store paths symlinked to in
-         the `gcroots' directory.
-
-       - `gcReturnLive': return the set of paths reachable from
-         (i.e. in the closure of) the roots.
-
-       - `gcReturnDead': return the set of paths not reachable from
-         the roots.
-
-       - `gcDeleteDead': actually delete the latter set.
-
-       - `gcDeleteSpecific': delete the paths listed in
-         `pathsToDelete', insofar as they are not reachable.
-
-       If `ignoreLiveness' is set, then reachability from the roots is
-       ignored (dangerous!).  However, the paths must still be
-       unreferenced *within* the store (i.e., there can be no other
-       store paths that depend on them).
-
-       For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
-       number of bytes that would be or was freed is returned in
-       `bytesFreed'. */
-    virtual void collectGarbage(GCAction action, const PathSet & pathsToDelete,
-        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed) = 0;
+    /* Perform a garbage collection. */
+    virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
 };
 
 

src/libstore/worker-protocol.hh

@@ -8,31 +8,32 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x102
+#define PROTOCOL_VERSION 0x104
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
 typedef enum {
     wopQuit = 0,
-    wopIsValidPath,
+    wopIsValidPath = 1,
     wopHasSubstitutes = 3,
-    wopQueryPathHash,
-    wopQueryReferences,
-    wopQueryReferrers,
-    wopAddToStore,
-    wopAddTextToStore,
-    wopBuildDerivations,
-    wopEnsurePath,
-    wopAddTempRoot,
-    wopAddIndirectRoot,
-    wopSyncWithGC,
-    wopFindRoots,
-    wopCollectGarbage,
-    wopExportPath,
-    wopImportPath,
-    wopQueryDeriver,
-    wopSetOptions,
+    wopQueryPathHash = 4,
+    wopQueryReferences = 5,
+    wopQueryReferrers = 6,
+    wopAddToStore = 7,
+    wopAddTextToStore = 8,
+    wopBuildDerivations = 9,
+    wopEnsurePath = 10,
+    wopAddTempRoot = 11,
+    wopAddIndirectRoot = 12,
+    wopSyncWithGC = 13,
+    wopFindRoots = 14,
+    wopExportPath = 16,
+    wopImportPath = 17,
+    wopQueryDeriver = 18,
+    wopSetOptions = 19,
+    wopCollectGarbage = 20,
+    wopQuerySubstitutablePathInfo = 21,
 } WorkerOp;
 
 

src/libutil/aterm.cc

@@ -1,5 +1,7 @@
 #include "aterm.hh"
 
+#include <cstring>
+
 using std::string;
 
 

src/libutil/hash.cc

@@ -1,6 +1,7 @@
 #include "config.h"
 
 #include <iostream>
+#include <cstring>
 
 #ifdef HAVE_OPENSSL
 #include <openssl/md5.h>

src/libutil/hash.hh

@@ -83,7 +83,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize);
 HashType parseHashType(const string & s);
 
 
-typedef union Ctx;
+union Ctx;
 
 class HashSink : public Sink
 {

src/libutil/serialise.cc

@@ -1,6 +1,8 @@
 #include "serialise.hh"
 #include "util.hh"
 
+#include <cstring>
+
 
 namespace nix {
 
@@ -39,6 +41,21 @@ void writeInt(unsigned int n, Sink & sink)
 }
 
 
+void writeLongLong(unsigned long long n, Sink & sink)
+{
+    unsigned char buf[8];
+    buf[0] = n & 0xff;
+    buf[1] = (n >> 8) & 0xff;
+    buf[2] = (n >> 16) & 0xff;
+    buf[3] = (n >> 24) & 0xff;
+    buf[4] = (n >> 32) & 0xff;
+    buf[5] = (n >> 40) & 0xff;
+    buf[6] = (n >> 48) & 0xff;
+    buf[7] = (n >> 56) & 0xff;
+    sink(buf, sizeof(buf));
+}
+
+
 void writeString(const string & s, Sink & sink)
 {
     unsigned int len = s.length();
@@ -82,6 +99,22 @@ unsigned int readInt(Source & source)
 }
 
 
+unsigned long long readLongLong(Source & source)
+{
+    unsigned char buf[8];
+    source(buf, sizeof(buf));
+    return
+        ((unsigned long long) buf[0]) |
+        ((unsigned long long) buf[1] << 8) |
+        ((unsigned long long) buf[2] << 16) |
+        ((unsigned long long) buf[3] << 24) |
+        ((unsigned long long) buf[4] << 32) |
+        ((unsigned long long) buf[5] << 40) |
+        ((unsigned long long) buf[6] << 48) |
+        ((unsigned long long) buf[7] << 56);
+}
+
+
 string readString(Source & source)
 {
     unsigned int len = readInt(source);

src/libutil/serialise.hh

@@ -95,11 +95,13 @@ struct StringSource : Source
 
 void writePadding(unsigned int len, Sink & sink);
 void writeInt(unsigned int n, Sink & sink);
+void writeLongLong(unsigned long long n, Sink & sink);
 void writeString(const string & s, Sink & sink);
 void writeStringSet(const StringSet & ss, Sink & sink);
 
 void readPadding(unsigned int len, Source & source);
 unsigned int readInt(Source & source);
+unsigned long long readLongLong(Source & source);
 string readString(Source & source);
 StringSet readStringSet(Source & source);
 

src/libutil/util.cc

@@ -7,6 +7,7 @@
 #include <iostream>
 #include <cerrno>
 #include <cstdio>
+#include <cstdlib>
 #include <sstream>
 #include <cstring>
 
@@ -228,30 +229,38 @@ void writeFile(const Path & path, const string & s)
 }
 
 
-unsigned long long computePathSize(const Path & path)
+static void _computePathSize(const Path & path,
+    unsigned long long & bytes, unsigned long long & blocks)
 {
-    unsigned long long size = 0;
-    
     checkInterrupt();
 
     struct stat st;
     if (lstat(path.c_str(), &st))
 	throw SysError(format("getting attributes of path `%1%'") % path);
 
-    size += st.st_size;
+    bytes += st.st_size;
+    blocks += st.st_blocks;
 
     if (S_ISDIR(st.st_mode)) {
 	Strings names = readDirectory(path);
 
 	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
-            size += computePathSize(path + "/" + *i);
+            _computePathSize(path + "/" + *i, bytes, blocks);
     }
-
-    return size;
 }
 
 
-static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+void computePathSize(const Path & path,
+    unsigned long long & bytes, unsigned long long & blocks)
+{
+    bytes = 0;
+    blocks = 0;
+    _computePathSize(path, bytes, blocks);
+}
+
+
+static void _deletePath(const Path & path, unsigned long long & bytesFreed,
+    unsigned long long & blocksFreed)
 {
     checkInterrupt();
 
@@ -262,6 +271,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 	throw SysError(format("getting attributes of path `%1%'") % path);
 
     bytesFreed += st.st_size;
+    blocksFreed += st.st_blocks;
 
     if (S_ISDIR(st.st_mode)) {
 	Strings names = readDirectory(path);
@@ -273,7 +283,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 	}
 
 	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
-            _deletePath(path + "/" + *i, bytesFreed);
+            _deletePath(path + "/" + *i, bytesFreed, blocksFreed);
     }
 
     if (remove(path.c_str()) == -1)
@@ -283,17 +293,19 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 
 void deletePath(const Path & path)
 {
-    unsigned long long dummy;
-    deletePath(path, dummy);
+    unsigned long long dummy1, dummy2;
+    deletePath(path, dummy1, dummy2);
 }
 
 
-void deletePath(const Path & path, unsigned long long & bytesFreed)
+void deletePath(const Path & path, unsigned long long & bytesFreed,
+    unsigned long long & blocksFreed)
 {
     startNest(nest, lvlDebug,
         format("recursively deleting path `%1%'") % path);
     bytesFreed = 0;
-    _deletePath(path, bytesFreed);
+    blocksFreed = 0;
+    _deletePath(path, bytesFreed, blocksFreed);
 }
 
 
@@ -318,19 +330,27 @@ void makePathReadOnly(const Path & path)
 }
 
 
-static Path tempName(const Path & tmpRoot, const Path & prefix)
+static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
+    int & counter)
 {
-    static int counter = 0;
-    Path tmpRoot2 = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true);
-    return (format("%1%/%2%-%3%-%4%") % tmpRoot2 % prefix % getpid() % counter++).str();
+    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true);
+    if (includePid)
+        return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
+    else
+        return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
 }
 
 
-Path createTempDir(const Path & tmpRoot, const Path & prefix)
+Path createTempDir(const Path & tmpRoot, const Path & prefix,
+    bool includePid, bool useGlobalCounter)
 {
+    static int globalCounter = 0;
+    int localCounter = 0;
+    int & counter(useGlobalCounter ? globalCounter : localCounter);
+    
     while (1) {
         checkInterrupt();
-	Path tmpDir = tempName(tmpRoot, prefix);
+	Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
 	if (mkdir(tmpDir.c_str(), 0777) == 0) {
 	    /* Explicitly set the group of the directory.  This is to
 	       work around around problems caused by BSD's group
@@ -522,13 +542,14 @@ AutoDelete::AutoDelete(const string & p, bool recursive) : path(p)
 AutoDelete::~AutoDelete()
 {
     try {
-        if (del)
+        if (del) {
             if (recursive)
                 deletePath(path);
             else {
                 if (remove(path.c_str()) == -1)
                     throw SysError(format("cannot unlink `%1%'") % path);
             }
+        }
     } catch (...) {
         ignoreException();
     }
@@ -558,7 +579,14 @@ AutoCloseFD::AutoCloseFD(int fd)
 
 AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd)
 {
-    abort();
+    /* Copying a AutoCloseFD isn't allowed (who should get to close
+       it?).  But as a edge case, allow copying of closed
+       AutoCloseFDs.  This is necessary due to tiresome reasons
+       involving copy constructor use on default object values in STL
+       containers (like when you do `map[value]' where value isn't in
+       the map yet). */
+    this->fd = fd.fd;
+    if (this->fd != -1) abort();
 }
 
 
@@ -811,7 +839,7 @@ string runProgram(Path program, bool searchPath, const Strings & args)
             pipe.readSide.close();
 
             if (dup2(pipe.writeSide, STDOUT_FILENO) == -1)
-                throw SysError("dupping from-hook write side");
+                throw SysError("dupping stdout");
 
             std::vector<const char *> cargs; /* careful with c_str()! */
             cargs.push_back(program.c_str());
@@ -847,6 +875,17 @@ string runProgram(Path program, bool searchPath, const Strings & args)
 }
 
 
+void closeMostFDs(const set<int> & exceptions)
+{
+    int maxFD = 0;
+    maxFD = sysconf(_SC_OPEN_MAX);
+    for (int fd = 0; fd < maxFD; ++fd)
+        if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO
+            && exceptions.find(fd) == exceptions.end())
+            close(fd); /* ignore result */
+}
+
+
 void quickExit(int status)
 {
 #ifdef __CYGWIN__
@@ -999,6 +1038,20 @@ bool string2Int(const string & s, int & n)
 }
 
 
+bool string2Int(const string & s, long long & n)
+{
+    std::istringstream str(s);
+    str >> n;
+    return str && str.get() == EOF;
+}
+
+
+bool hasSuffix(const string & s, const string & suffix)
+{
+    return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix;
+}
+
+
 void ignoreException()
 {
     try {

src/libutil/util.hh

@@ -61,20 +61,23 @@ string readFile(const Path & path);
 void writeFile(const Path & path, const string & s);
 
 /* Compute the sum of the sizes of all files in `path'. */
-unsigned long long computePathSize(const Path & path);
+void computePathSize(const Path & path,
+    unsigned long long & bytes, unsigned long long & blocks);
 
 /* Delete a path; i.e., in the case of a directory, it is deleted
    recursively.  Don't use this at home, kids.  The second variant
-   returns the number of bytes freed. */
+   returns the number of bytes and blocks freed. */
 void deletePath(const Path & path);
 
-void deletePath(const Path & path, unsigned long long & bytesFreed);
+void deletePath(const Path & path, unsigned long long & bytesFreed,
+    unsigned long long & blocksFreed);
 
 /* Make a path read-only recursively. */
 void makePathReadOnly(const Path & path);
 
 /* Create a temporary directory. */
-Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix");
+Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
+    bool includePid = true, bool useGlobalCounter = true);
 
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
@@ -242,6 +245,10 @@ void killUser(uid_t uid);
 string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings());
 
+/* Close all file descriptors except stdin, stdout, stderr, and those
+   listed in the given set.  Good practice in child processes. */
+void closeMostFDs(const set<int> & exceptions);
+
 /* Wrapper around _exit() on Unix and ExitProcess() on Windows.  (On
    Cygwin, _exit() doesn't seem to do the right thing.) */
 void quickExit(int status);
@@ -284,6 +291,11 @@ bool statusOk(int status);
 /* Parse a string into an integer. */
 string int2String(int n);
 bool string2Int(const string & s, int & n);
+bool string2Int(const string & s, long long & n);
+
+
+/* Return true iff `s' ends in `suffix'. */
+bool hasSuffix(const string & s, const string & suffix);
 
 
 /* Exception handling in destructors: print an error message, then
@@ -291,6 +303,29 @@ bool string2Int(const string & s, int & n);
 void ignoreException();
 
 
+/* STL functions such as sort() pass a binary function object around
+   by value, so it gets cloned a lot.  This is bad if the function
+   object has state or is simply large.  This adapter wraps the
+   function object to simulate passing by reference. */
+template<class F>
+struct binary_function_ref_adapter
+{
+    F * p;
+
+    binary_function_ref_adapter(F * _p)
+    {
+        p = _p;
+    }
+    
+    typename F::result_type operator () (
+        const typename F::first_argument_type & x,
+        const typename F::second_argument_type & y)
+    {
+        return (*p)(x, y);
+    }
+};
+
+
 }
 
 

src/nix-env/Makefile.am

@@ -1,7 +1,6 @@
 bin_PROGRAMS = nix-env
 
-nix_env_SOURCES = nix-env.cc names.cc names.hh \
- profiles.cc profiles.hh help.txt
+nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt
 nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \
  ../libstore/libstore.la ../libutil/libutil.la \
  ../boost/format/libformat.la ${bdb_lib} ${aterm_lib}

src/nix-env/nix-env.cc

@@ -115,18 +115,29 @@ static void getAllExprs(EvalState & state,
     const Path & path, ATermMap & attrs)
 {
     Strings names = readDirectory(path);
+    StringSet namesSorted(names.begin(), names.end());
 
-    for (Strings::iterator i = names.begin(); i != names.end(); ++i) {
+    foreach (StringSet::iterator, i, namesSorted) {
         Path path2 = path + "/" + *i;
         
         struct stat st;
         if (stat(path2.c_str(), &st) == -1)
             continue; // ignore dangling symlinks in ~/.nix-defexpr
         
-        if (isNixExpr(path2))
-            attrs.set(toATerm(*i), makeAttrRHS(
+        if (isNixExpr(path2)) {
+            /* Strip off the `.nix' filename suffix (if applicable),
+               otherwise the attribute cannot be selected with the
+               `-A' option.  Useful if you want to stick a Nix
+               expression directly in ~/.nix-defexpr. */
+            string attrName = *i;
+            if (hasSuffix(attrName, ".nix"))
+                attrName = string(attrName, 0, attrName.size() - 4);
+            attrs.set(toATerm(attrName), makeAttrRHS(
                     parseExprFromFile(state, absPath(path2)), makeNoPos()));
+        }
         else
+            /* `path2' is a directory (with no default.nix in it);
+               recurse into it. */
             getAllExprs(state, path2, attrs);
     }
 }
@@ -143,7 +154,7 @@ static Expr loadSourceExpr(EvalState & state, const Path & path)
        for a user to have a ~/.nix-defexpr directory that includes
        some system-wide directory). */
     ATermMap attrs;
-    attrs.set(toATerm("_combineChannels"), makeAttrRHS(eTrue, makeNoPos()));
+    attrs.set(toATerm("_combineChannels"), makeAttrRHS(makeList(ATempty), makeNoPos()));
     getAllExprs(state, path, attrs);
     return makeAttrs(attrs);
 }
@@ -215,8 +226,34 @@ static DrvInfos queryInstalled(EvalState & state, const Path & userEnv)
 }
 
 
-static void createUserEnv(EvalState & state, const DrvInfos & elems,
-    const Path & profile, bool keepDerivations)
+/* Ensure exclusive access to a profile.  Any command that modifies
+   the profile first acquires this lock. */
+static void lockProfile(PathLocks & lock, const Path & profile)
+{
+    lock.lockPaths(singleton<PathSet>(profile),
+        (format("waiting for lock on profile `%1%'") % profile).str());
+    lock.setDeletion(true);
+}
+
+
+/* Optimistic locking is used by long-running operations like `nix-env
+   -i'.  Instead of acquiring the exclusive lock for the entire
+   duration of the operation, we just perform the operation
+   optimistically (without an exclusive lock), and check at the end
+   whether the profile changed while we were busy (i.e., the symlink
+   target changed).  If so, the operation is restarted.  Restarting is
+   generally cheap, since the build results are still in the Nix
+   store.  Most of the time, only the user environment has to be
+   rebuilt. */
+static string optimisticLockProfile(const Path & profile)
+{
+    return pathExists(profile) ? readLink(profile) : "";
+}
+
+
+static bool createUserEnv(EvalState & state, const DrvInfos & elems,
+    const Path & profile, bool keepDerivations,
+    const string & lockToken)
 {
     /* Build the components in the user environment, if they don't
        exist already. */
@@ -247,8 +284,14 @@ static void createUserEnv(EvalState & state, const DrvInfos & elems,
            output path, and optionally the derivation path, as well as
            the meta attributes. */
         Path drvPath = keepDerivations ? i->queryDrvPath(state) : "";
+
+        MetaInfo meta = i->queryMetaInfo(state);
+        ATermList metaList = ATempty;
+        foreach (MetaInfo::iterator, j, meta)
+            metaList = ATinsert(metaList,
+                makeBind(toATerm(j->first), makeStr(j->second), makeNoPos()));
         
-        ATermList as = ATmakeList4(
+        ATermList as = ATmakeList5(
             makeBind(toATerm("type"),
                 makeStr("derivation"), makeNoPos()),
             makeBind(toATerm("name"),
@@ -256,17 +299,13 @@ static void createUserEnv(EvalState & state, const DrvInfos & elems,
             makeBind(toATerm("system"),
                 makeStr(i->system), makeNoPos()),
             makeBind(toATerm("outPath"),
-                makeStr(i->queryOutPath(state)), makeNoPos()));
+                makeStr(i->queryOutPath(state)), makeNoPos()), 
+            makeBind(toATerm("meta"), makeAttrs(metaList), makeNoPos()));
         
         if (drvPath != "") as = ATinsert(as, 
             makeBind(toATerm("drvPath"),
                 makeStr(drvPath), makeNoPos()));
         
-        if (i->attrs->get(toATerm("meta"))) as = ATinsert(as, 
-            makeBind(toATerm("meta"),
-                strictEvalExpr(state, i->attrs->get(toATerm("meta"))),
-                makeNoPos()));
-
         manifest = ATinsert(manifest, makeAttrs(as));
         
         inputs = ATinsert(inputs, makeStr(i->queryOutPath(state)));
@@ -305,9 +344,20 @@ static void createUserEnv(EvalState & state, const DrvInfos & elems,
     store->buildDerivations(singleton<PathSet>(topLevelDrv.queryDrvPath(state)));
 
     /* Switch the current user environment to the output path. */
+    PathLocks lock;
+    lockProfile(lock, profile);
+
+    Path lockTokenCur = optimisticLockProfile(profile);
+    if (lockToken != lockTokenCur) {
+        printMsg(lvlError, format("profile `%1%' changed while we were busy; restarting") % profile);
+        return false;
+    }
+    
     debug(format("switching to new user environment"));
     Path generation = createGeneration(profile, topLevelDrv.queryOutPath(state));
     switchLink(profile, generation);
+
+    return true;
 }
 
 
@@ -531,8 +581,8 @@ static void queryInstSources(EvalState & state,
 
 static void printMissing(EvalState & state, const DrvInfos & elems)
 {
-    PathSet targets, willBuild, willSubstitute;
-    for (DrvInfos::const_iterator i = elems.begin(); i != elems.end(); ++i) {
+    PathSet targets;
+    foreach (DrvInfos::const_iterator, i, elems) {
         Path drvPath = i->queryDrvPath(state);
         if (drvPath != "")
             targets.insert(drvPath);
@@ -540,27 +590,7 @@ static void printMissing(EvalState & state, const DrvInfos & elems)
             targets.insert(i->queryOutPath(state));
     }
 
-    queryMissing(targets, willBuild, willSubstitute);
-
-    if (!willBuild.empty()) {
-        printMsg(lvlInfo, format("the following derivations will be built:"));
-        for (PathSet::iterator i = willBuild.begin(); i != willBuild.end(); ++i)
-            printMsg(lvlInfo, format("  %1%") % *i);
-    }
-
-    if (!willSubstitute.empty()) {
-        printMsg(lvlInfo, format("the following paths will be substituted:"));
-        for (PathSet::iterator i = willSubstitute.begin(); i != willSubstitute.end(); ++i)
-            printMsg(lvlInfo, format("  %1%") % *i);
-    }
-}
-
-
-static void lockProfile(PathLocks & lock, const Path & profile)
-{
-    lock.lockPaths(singleton<PathSet>(profile),
-        (format("waiting for lock on profile `%1%'") % profile).str());
-    lock.setDeletion(true);
+    printMissing(targets);
 }
 
 
@@ -586,36 +616,35 @@ static void installDerivations(Globals & globals,
 
     /* Add in the already installed derivations, unless they have the
        same name as a to-be-installed element. */
-    PathLocks lock;
-    lockProfile(lock, profile);
-    DrvInfos installedElems = queryInstalled(globals.state, profile);
 
-    DrvInfos allElems(newElems);
-    for (DrvInfos::iterator i = installedElems.begin();
-         i != installedElems.end(); ++i)
-    {
-        DrvName drvName(i->name);
-        MetaInfo meta = i->queryMetaInfo(globals.state);
-        if (!globals.preserveInstalled &&
-            newNames.find(drvName.name) != newNames.end() &&
-            meta["keep"] != "true")
-            printMsg(lvlInfo,
-                format("replacing old `%1%'") % i->name);
-        else
-            allElems.push_back(*i);
-    }
-
-    for (DrvInfos::iterator i = newElems.begin(); i != newElems.end(); ++i)
-        printMsg(lvlInfo,
-            format("installing `%1%'") % i->name);
+    while (true) {
+        string lockToken = optimisticLockProfile(profile);
         
-    if (globals.dryRun) {
-        printMissing(globals.state, newElems);
-        return;
-    }
+        DrvInfos installedElems = queryInstalled(globals.state, profile);
+        
+        DrvInfos allElems(newElems);
+        foreach (DrvInfos::iterator, i, installedElems) {
+            DrvName drvName(i->name);
+            MetaInfo meta = i->queryMetaInfo(globals.state);
+            if (!globals.preserveInstalled &&
+                newNames.find(drvName.name) != newNames.end() &&
+                meta["keep"] != "true")
+                printMsg(lvlInfo,
+                    format("replacing old `%1%'") % i->name);
+            else
+                allElems.push_back(*i);
+        }
 
-    createUserEnv(globals.state, allElems,
-        profile, globals.keepDerivations);
+        foreach (DrvInfos::iterator, i, newElems)
+            printMsg(lvlInfo, format("installing `%1%'") % i->name);
+    
+        printMissing(globals.state, newElems);
+    
+        if (globals.dryRun) return;
+
+        if (createUserEnv(globals.state, allElems,
+                profile, globals.keepDerivations, lockToken)) break;
+    }
 }
 
 
@@ -647,78 +676,75 @@ static void upgradeDerivations(Globals & globals,
        for a derivation in the input Nix expression that has the same
        name and a higher version number. */
 
-    /* Load the currently installed derivations. */
-    PathLocks lock;
-    lockProfile(lock, globals.profile);
-    DrvInfos installedElems = queryInstalled(globals.state, globals.profile);
+    while (true) {
+        string lockToken = optimisticLockProfile(globals.profile);
+        
+        DrvInfos installedElems = queryInstalled(globals.state, globals.profile);
 
-    /* Fetch all derivations from the input file. */
-    DrvInfos availElems;
-    queryInstSources(globals.state, globals.instSource, args, availElems, false);
+        /* Fetch all derivations from the input file. */
+        DrvInfos availElems;
+        queryInstSources(globals.state, globals.instSource, args, availElems, false);
 
-    /* Go through all installed derivations. */
-    DrvInfos newElems;
-    for (DrvInfos::iterator i = installedElems.begin();
-         i != installedElems.end(); ++i)
-    {
-        DrvName drvName(i->name);
+        /* Go through all installed derivations. */
+        DrvInfos newElems;
+        foreach (DrvInfos::iterator, i, installedElems) {
+            DrvName drvName(i->name);
 
-        MetaInfo meta = i->queryMetaInfo(globals.state);
-        if (meta["keep"] == "true") {
-            newElems.push_back(*i);
-            continue;
-        }
+            MetaInfo meta = i->queryMetaInfo(globals.state);
+            if (meta["keep"] == "true") {
+                newElems.push_back(*i);
+                continue;
+            }
 
-        /* Find the derivation in the input Nix expression with the
-           same name that satisfies the version constraints specified
-           by upgradeType.  If there are multiple matches, take the
-           one with the highest priority.  If there are still multiple
-           matches, take the one with the highest version. */
-        DrvInfos::iterator bestElem = availElems.end();
-        DrvName bestName;
-        for (DrvInfos::iterator j = availElems.begin();
-             j != availElems.end(); ++j)
-        {
-            DrvName newName(j->name);
-            if (newName.name == drvName.name) {
-                int d = comparePriorities(globals.state, *i, *j);
-                if (d == 0) d = compareVersions(drvName.version, newName.version);
-                if (upgradeType == utLt && d < 0 ||
-                    upgradeType == utLeq && d <= 0 ||
-                    upgradeType == utEq && d == 0 ||
-                    upgradeType == utAlways)
-                {
-                    int d2 = -1;
-                    if (bestElem != availElems.end()) {
-                        d2 = comparePriorities(globals.state, *bestElem, *j);
-                        if (d2 == 0) d2 = compareVersions(bestName.version, newName.version);
-                    }
-                    if (d2 < 0) {
-                        bestElem = j;
-                        bestName = newName;
+            /* Find the derivation in the input Nix expression with
+               the same name that satisfies the version constraints
+               specified by upgradeType.  If there are multiple
+               matches, take the one with the highest priority.  If
+               there are still multiple matches, take the one with the
+               highest version. */
+            DrvInfos::iterator bestElem = availElems.end();
+            DrvName bestName;
+            foreach (DrvInfos::iterator, j, availElems) {
+                DrvName newName(j->name);
+                if (newName.name == drvName.name) {
+                    int d = comparePriorities(globals.state, *i, *j);
+                    if (d == 0) d = compareVersions(drvName.version, newName.version);
+                    if (upgradeType == utLt && d < 0 ||
+                        upgradeType == utLeq && d <= 0 ||
+                        upgradeType == utEq && d == 0 ||
+                        upgradeType == utAlways)
+                    {
+                        int d2 = -1;
+                        if (bestElem != availElems.end()) {
+                            d2 = comparePriorities(globals.state, *bestElem, *j);
+                            if (d2 == 0) d2 = compareVersions(bestName.version, newName.version);
+                        }
+                        if (d2 < 0) {
+                            bestElem = j;
+                            bestName = newName;
+                        }
                     }
                 }
             }
-        }
 
-        if (bestElem != availElems.end() &&
-            i->queryOutPath(globals.state) !=
+            if (bestElem != availElems.end() &&
+                i->queryOutPath(globals.state) !=
                 bestElem->queryOutPath(globals.state))
-        {
-            printMsg(lvlInfo,
-                format("upgrading `%1%' to `%2%'")
-                % i->name % bestElem->name);
-            newElems.push_back(*bestElem);
-        } else newElems.push_back(*i);
-    }
+            {
+                printMsg(lvlInfo,
+                    format("upgrading `%1%' to `%2%'")
+                    % i->name % bestElem->name);
+                newElems.push_back(*bestElem);
+            } else newElems.push_back(*i);
+        }
     
-    if (globals.dryRun) {
         printMissing(globals.state, newElems);
-        return;
-    }
+    
+        if (globals.dryRun) return;
 
-    createUserEnv(globals.state, newElems,
-        globals.profile, globals.keepDerivations);
+        if (createUserEnv(globals.state, newElems,
+                globals.profile, globals.keepDerivations, lockToken)) break;
+    }
 }
 
 
@@ -762,29 +788,27 @@ static void opSetFlag(Globals & globals,
     string flagValue = *arg++;
     DrvNames selectors = drvNamesFromArgs(Strings(arg, opArgs.end()));
 
-    /* Load the currently installed derivations. */
-    PathLocks lock;
-    lockProfile(lock, globals.profile);
-    DrvInfos installedElems = queryInstalled(globals.state, globals.profile);
+    while (true) {
+        string lockToken = optimisticLockProfile(globals.profile);
 
-    /* Update all matching derivations. */
-    for (DrvInfos::iterator i = installedElems.begin();
-         i != installedElems.end(); ++i)
-    {
-        DrvName drvName(i->name);
-        for (DrvNames::iterator j = selectors.begin();
-             j != selectors.end(); ++j)
-            if (j->matches(drvName)) {
-                printMsg(lvlInfo,
-                    format("setting flag on `%1%'") % i->name);
-                setMetaFlag(globals.state, *i, flagName, flagValue);
-                break;
-            }
+        DrvInfos installedElems = queryInstalled(globals.state, globals.profile);
+
+        /* Update all matching derivations. */
+        foreach (DrvInfos::iterator, i, installedElems) {
+            DrvName drvName(i->name);
+            foreach (DrvNames::iterator, j, selectors)
+                if (j->matches(drvName)) {
+                    printMsg(lvlInfo,
+                        format("setting flag on `%1%'") % i->name);
+                    setMetaFlag(globals.state, *i, flagName, flagValue);
+                    break;
+                }
+        }
+
+        /* Write the new user environment. */
+        if (createUserEnv(globals.state, installedElems,
+                globals.profile, globals.keepDerivations, lockToken)) break;
     }
-
-    /* Write the new user environment. */
-    createUserEnv(globals.state, installedElems,
-        globals.profile, globals.keepDerivations);
 }
 
 
@@ -805,10 +829,17 @@ static void opSet(Globals & globals,
     
     DrvInfo & drv(elems.front());
 
-    if (drv.queryDrvPath(globals.state) != "")
-        store->buildDerivations(singleton<PathSet>(drv.queryDrvPath(globals.state)));
-    else
+    if (drv.queryDrvPath(globals.state) != "") {
+        PathSet paths = singleton<PathSet>(drv.queryDrvPath(globals.state));
+        printMissing(paths);
+        if (globals.dryRun) return;
+        store->buildDerivations(paths);
+    }
+    else {
+        printMissing(singleton<PathSet>(drv.queryOutPath(globals.state)));
+        if (globals.dryRun) return;
         store->ensurePath(drv.queryOutPath(globals.state));
+    }
 
     debug(format("switching to new user environment"));
     Path generation = createGeneration(globals.profile, drv.queryOutPath(globals.state));
@@ -819,33 +850,33 @@ static void opSet(Globals & globals,
 static void uninstallDerivations(Globals & globals, Strings & selectors,
     Path & profile)
 {
-    PathLocks lock;
-    lockProfile(lock, profile);
-    DrvInfos installedElems = queryInstalled(globals.state, profile);
-    DrvInfos newElems;
+    while (true) {
+        string lockToken = optimisticLockProfile(profile);
 
-    for (DrvInfos::iterator i = installedElems.begin();
-         i != installedElems.end(); ++i)
-    {
-        DrvName drvName(i->name);
-        bool found = false;
-        for (Strings::iterator j = selectors.begin(); j != selectors.end(); ++j)
-            /* !!! the repeated calls to followLinksToStorePath() are
-               expensive, should pre-compute them. */
-            if ((isPath(*j) && i->queryOutPath(globals.state) == followLinksToStorePath(*j))
-                || DrvName(*j).matches(drvName))
-            {
-                printMsg(lvlInfo, format("uninstalling `%1%'") % i->name);
-                found = true;
-                break;
-            }
-        if (!found) newElems.push_back(*i);
+        DrvInfos installedElems = queryInstalled(globals.state, profile);
+        DrvInfos newElems;
+
+        foreach (DrvInfos::iterator, i, installedElems) {
+            DrvName drvName(i->name);
+            bool found = false;
+            foreach (Strings::iterator, j, selectors)
+                /* !!! the repeated calls to followLinksToStorePath()
+                   are expensive, should pre-compute them. */
+                if ((isPath(*j) && i->queryOutPath(globals.state) == followLinksToStorePath(*j))
+                    || DrvName(*j).matches(drvName))
+                {
+                    printMsg(lvlInfo, format("uninstalling `%1%'") % i->name);
+                    found = true;
+                    break;
+                }
+            if (!found) newElems.push_back(*i);
+        }
+
+        if (globals.dryRun) return;
+
+        if (createUserEnv(globals.state, newElems,
+                profile, globals.keepDerivations, lockToken)) break;
     }
-
-    if (globals.dryRun) return;
-
-    createUserEnv(globals.state, newElems,
-        profile, globals.keepDerivations);
 }
 
 
@@ -1376,10 +1407,12 @@ void run(Strings args)
             globals.instSource.systemFilter = needArg(i, args, arg);
         else {
             remaining.push_back(arg);
-            if (arg[0] == '-') 
+            if (arg[0] == '-') {
                 opFlags.push_back(arg);
-            else
-                opArgs.push_back(arg);
+                if (arg == "--from-profile") { /* !!! hack */
+                    if (i != args.end()) opFlags.push_back(*i++);
+                }
+            } else opArgs.push_back(arg);
         }
 
         if (oldOp && oldOp != op)

src/nix-log2xml/log2xml.cc

@@ -2,6 +2,7 @@
 #include <iostream>
 #include <cstdio>
 #include <string>
+#include <cstring>
 
 using namespace std;
 

src/nix-store/help.txt

@@ -4,7 +4,7 @@ Usage: nix-store [OPTIONS...] [ARGUMENTS...]
 
 Operations:
 
-  --realise / -r: ensure path validity; if a derivation, ensure that
+  --realise / -r: ensure path validity; if a derivation, ensure the
       validity of the outputs
   --add / -A: copy a path to the Nix store
   --delete: safely delete paths from the Nix store

src/nix-store/nix-store.cc

@@ -77,23 +77,26 @@ static Path realisePath(const Path & path)
 /* Realise the given paths. */
 static void opRealise(Strings opFlags, Strings opArgs)
 {
-    if (!opFlags.empty()) throw UsageError("unknown flag");
+    bool dryRun = false;
+    
+    foreach (Strings::iterator, i, opFlags)
+        if (*i == "--dry-run") dryRun = true;
+        else throw UsageError(format("unknown flag `%1%'") % *i);
 
-    for (Strings::iterator i = opArgs.begin();
-         i != opArgs.end(); ++i)
+    foreach (Strings::iterator, i, opArgs)
         *i = followLinksToStorePath(*i);
             
-    if (opArgs.size() > 1) {
-        PathSet drvPaths;
-        for (Strings::iterator i = opArgs.begin();
-             i != opArgs.end(); ++i)
-            if (isDerivation(*i))
-                drvPaths.insert(*i);
-        store->buildDerivations(drvPaths);
-    }
+    printMissing(PathSet(opArgs.begin(), opArgs.end()));
+    
+    if (dryRun) return;
+    
+    /* Build all derivations at the same time to exploit parallelism. */
+    PathSet drvPaths;
+    foreach (Strings::iterator, i, opArgs)
+        if (isDerivation(*i)) drvPaths.insert(*i);
+    store->buildDerivations(drvPaths);
 
-    for (Strings::iterator i = opArgs.begin();
-         i != opArgs.end(); ++i)
+    foreach (Strings::iterator, i,opArgs)
         cout << format("%1%\n") % realisePath(*i);
 }
 
@@ -414,9 +417,9 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
     PathSet validPaths = store->queryValidPaths();
-    /* !!! this isn't streamy; makeValidityRegistration() builds a
-       potentially gigantic string. */
-    cout << makeValidityRegistration(validPaths, true, true);
+    foreach (PathSet::iterator, i, validPaths) {
+        cout << makeValidityRegistration(singleton<PathSet>(*i), true, true);
+    }
 }
 
 
@@ -489,53 +492,59 @@ static void opCheckValidity(Strings opFlags, Strings opArgs)
 }
 
 
-static string showBytes(unsigned long long bytes)
+static string showBytes(unsigned long long bytes, unsigned long long blocks)
 {
-    return (format("%d bytes (%.2f MiB)")
-        % bytes % (bytes / (1024.0 * 1024.0))).str();
+    return (format("%d bytes (%.2f MiB, %d blocks)")
+        % bytes % (bytes / (1024.0 * 1024.0)) % blocks).str();
 }
 
 
 struct PrintFreed 
 {
-    bool show, dryRun;
-    unsigned long long bytesFreed;
-    PrintFreed(bool show, bool dryRun)
-        : show(show), dryRun(dryRun), bytesFreed(0) { }
+    bool show;
+    const GCResults & results;
+    PrintFreed(bool show, const GCResults & results)
+        : show(show), results(results) { }
     ~PrintFreed() 
     {
         if (show)
-            cout << format(
-                (dryRun
-                    ? "%1% would be freed\n"
-                    : "%1% freed\n"))
-                % showBytes(bytesFreed);
+            cout << format("%1% store paths deleted, %2% freed\n")
+                % results.paths.size()
+                % showBytes(results.bytesFreed, results.blocksFreed);
     }
 };
 
 
 static void opGC(Strings opFlags, Strings opArgs)
 {
-    GCAction action = gcDeleteDead;
+    GCOptions options;
+    options.action = GCOptions::gcDeleteDead;
+    
+    GCResults results;
     
     /* Do what? */
-    for (Strings::iterator i = opFlags.begin();
-         i != opFlags.end(); ++i)
-        if (*i == "--print-roots") action = gcReturnRoots;
-        else if (*i == "--print-live") action = gcReturnLive;
-        else if (*i == "--print-dead") action = gcReturnDead;
-        else if (*i == "--delete") action = gcDeleteDead;
+    foreach (Strings::iterator, i, opFlags)
+        if (*i == "--print-roots") options.action = GCOptions::gcReturnRoots;
+        else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
+        else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
+        else if (*i == "--delete") options.action = GCOptions::gcDeleteDead;
+        else if (*i == "--max-freed") options.maxFreed = getIntArg(*i, i, opFlags.end());
+        else if (*i == "--max-links") options.maxLinks = getIntArg(*i, i, opFlags.end());
+        else if (*i == "--use-atime") options.useAtime = true;
+        else if (*i == "--max-atime") {
+            options.useAtime = true;
+            options.maxAtime = getIntArg(*i, i, opFlags.end());
+        }
         else throw UsageError(format("bad sub-operation `%1%' in GC") % *i);
 
-    PathSet result;
-    PrintFreed freed(action == gcDeleteDead || action == gcReturnDead,
-        action == gcReturnDead);
-    store->collectGarbage(action, PathSet(), false, result, freed.bytesFreed);
+    if (!opArgs.empty()) throw UsageError("no arguments expected");
+    
+    PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
+    store->collectGarbage(options, results);
 
-    if (action != gcDeleteDead) {
-        for (PathSet::iterator i = result.begin(); i != result.end(); ++i)
+    if (options.action != GCOptions::gcDeleteDead)
+        foreach (PathSet::iterator, i, results.paths)
             cout << *i << std::endl;
-    }
 }
 
 
@@ -544,22 +553,19 @@ static void opGC(Strings opFlags, Strings opArgs)
    roots). */
 static void opDelete(Strings opFlags, Strings opArgs)
 {
-    bool ignoreLiveness = false;
+    GCOptions options;
+    options.action = GCOptions::gcDeleteSpecific;
     
-    for (Strings::iterator i = opFlags.begin();
-         i != opFlags.end(); ++i)
-        if (*i == "--ignore-liveness") ignoreLiveness = true;
+    foreach (Strings::iterator, i, opFlags)
+        if (*i == "--ignore-liveness") options.ignoreLiveness = true;
         else throw UsageError(format("unknown flag `%1%'") % *i);
 
-    PathSet pathsToDelete;
-    for (Strings::iterator i = opArgs.begin();
-         i != opArgs.end(); ++i)
-        pathsToDelete.insert(followLinksToStorePath(*i));
+    foreach (Strings::iterator, i, opArgs)
+        options.pathsToDelete.insert(followLinksToStorePath(*i));
     
-    PathSet dummy;
-    PrintFreed freed(true, false);
-    store->collectGarbage(gcDeleteSpecific, pathsToDelete, ignoreLiveness,
-        dummy, freed.bytesFreed);
+    GCResults results;
+    PrintFreed freed(true, results);
+    store->collectGarbage(options, results);
 }
 
 
@@ -653,7 +659,7 @@ static void showOptimiseStats(OptimiseStats & stats)
 {
     printMsg(lvlError,
         format("%1% freed by hard-linking %2% files; there are %3% files with equal contents out of %4% files in total")
-        % showBytes(stats.bytesFreed)
+        % showBytes(stats.bytesFreed, stats.blocksFreed)
         % stats.filesLinked
         % stats.sameContents
         % stats.totalFiles);
@@ -743,8 +749,12 @@ void run(Strings args)
         }
         else if (arg == "--indirect")
             indirectRoot = true;
-        else if (arg[0] == '-')
+        else if (arg[0] == '-') {            
             opFlags.push_back(arg);
+            if (arg == "--max-freed" || arg == "--max-links" || arg == "--max-atime") { /* !!! hack */
+                if (i != args.end()) opFlags.push_back(*i++);
+            }
+        }
         else
             opArgs.push_back(arg);
 

src/nix-worker/nix-worker.cc

@@ -395,23 +395,24 @@ static void performOp(unsigned int clientVersion,
     }
 
     case wopCollectGarbage: {
-        GCAction action = (GCAction) readInt(from);
-        PathSet pathsToDelete = readStorePaths(from);
-        bool ignoreLiveness = readInt(from);
+        GCOptions options;
+        options.action = (GCOptions::GCAction) readInt(from);
+        options.pathsToDelete = readStorePaths(from);
+        options.ignoreLiveness = readInt(from);
+        options.maxFreed = readLongLong(from);
+        options.maxLinks = readInt(from);
 
-        PathSet result;
-        unsigned long long bytesFreed;
+        GCResults results;
         
         startWork();
-        if (ignoreLiveness)
+        if (options.ignoreLiveness)
             throw Error("you are not allowed to ignore liveness");
-        store->collectGarbage(action, pathsToDelete, ignoreLiveness,
-            result, bytesFreed);
+        store->collectGarbage(options, results);
         stopWork();
         
-        writeStringSet(result, to);
-        writeInt(bytesFreed & 0xffffffff, to);
-        writeInt(bytesFreed >> 32, to);
+        writeStringSet(results.paths, to);
+        writeLongLong(results.bytesFreed, to);
+        writeLongLong(results.blocksFreed, to);
         
         break;
     }
@@ -425,11 +426,30 @@ static void performOp(unsigned int clientVersion,
         maxSilentTime = readInt(from);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
             useBuildHook = readInt(from) != 0;
+        if (GET_PROTOCOL_MINOR(clientVersion) >= 4) {
+            buildVerbosity = (Verbosity) readInt(from);
+            logType = (LogType) readInt(from);
+            printBuildTrace = readInt(from) != 0;
+        }
         startWork();
         stopWork();
         break;
     }
-            
+
+    case wopQuerySubstitutablePathInfo: {
+        Path path = absPath(readString(from));
+        startWork();
+        SubstitutablePathInfo info;
+        bool res = store->querySubstitutablePathInfo(path, info);
+        stopWork();
+        writeInt(res ? 1 : 0, to);
+        if (res) {
+            writeString(info.deriver, to);
+            writeStringSet(info.references, to);
+            writeLongLong(info.downloadSize, to);
+        }
+        break;
+    }
             
     default:
         throw Error(format("invalid operation %1%") % op);
@@ -439,6 +459,8 @@ static void performOp(unsigned int clientVersion,
 
 static void processConnection()
 {
+    RemoveTempRoots removeTempRoots __attribute__((unused));
+
     canSendStderr = false;
     myPid = getpid();    
     writeToStderr = tunnelStderr;
@@ -512,7 +534,7 @@ static void processConnection()
 static void sigChldHandler(int sigNo)
 {
     /* Reap all dead children. */
-    while (waitpid(-1, 0, WNOHANG) == 0) ;
+    while (waitpid(-1, 0, WNOHANG) > 0) ;
 }
 
 
@@ -542,11 +564,17 @@ static void daemonLoop()
 
     createDirs(dirOf(socketPath));
 
+    /* Urgh, sockaddr_un allows path names of only 108 characters.  So
+       chdir to the socket directory so that we can pass a relative
+       path name. */
+    chdir(dirOf(socketPath).c_str());
+    Path socketPathRel = "./" + baseNameOf(socketPath);
+    
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
-    if (socketPath.size() >= sizeof(addr.sun_path))
-        throw Error(format("socket path `%1%' is too long") % socketPath);
-    strcpy(addr.sun_path, socketPath.c_str());
+    if (socketPathRel.size() >= sizeof(addr.sun_path))
+        throw Error(format("socket path `%1%' is too long") % socketPathRel);
+    strcpy(addr.sun_path, socketPathRel.c_str());
 
     unlink(socketPath.c_str());
 
@@ -559,6 +587,8 @@ static void daemonLoop()
     if (res == -1)
         throw SysError(format("cannot bind to socket `%1%'") % socketPath);
 
+    chdir("/"); /* back to the root */
+
     if (listen(fdSocket, 5) == -1)
         throw SysError(format("cannot listen on socket `%1%'") % socketPath);
 
@@ -597,13 +627,17 @@ static void daemonLoop()
 
             case 0:
                 try { /* child */
-                    
+
                     /* Background the worker. */
                     if (setsid() == -1)
                         throw SysError(format("creating a new session"));
 
                     /* Restore normal handling of SIGCHLD. */
                     setSigChldAction(false);
+
+                    /* Since the daemon can be long-running, the
+                       settings may have changed.  So force a reload. */
+                    reloadSettings();
                     
                     /* Handle the connection. */
                     from.fd = remote;

substitute.mk

@@ -9,7 +9,7 @@
 	 -e "s^@libexecdir\@^$(libexecdir)^g" \
 	 -e "s^@storedir\@^$(storedir)^g" \
 	 -e "s^@system\@^$(system)^g" \
-	 -e "s^@shell\@^$(shell)^g" \
+	 -e "s^@shell\@^$(bash)^g" \
 	 -e "s^@curl\@^$(curl)^g" \
 	 -e "s^@bzip2\@^$(bzip2_bin)/bzip2^g" \
 	 -e "s^@bunzip2\@^$(bzip2_bin)/bunzip2^g" \
@@ -17,6 +17,7 @@
 	 -e "s^@perl\@^$(perl)^g" \
 	 -e "s^@coreutils\@^$(coreutils)^g" \
 	 -e "s^@tar\@^$(tar)^g" \
+	 -e "s^@gzip\@^$(gzip)^g" \
 	 -e "s^@tr\@^$(tr)^g" \
 	 -e "s^@dot\@^$(dot)^g" \
 	 -e "s^@xmllint\@^$(xmllint)^g" \

tests/gc-concurrent.builder.sh

@@ -1,8 +1,11 @@
 mkdir $out
 echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
 
-sleep 5
-mkdir $out || true
+sleep 10
+
+# $out should not have been GC'ed while we were sleeping, but just in
+# case...
+mkdir -p $out
 
 # Check that the GC hasn't deleted the lock on our output.
 test -e "$out.lock"

tests/gc-concurrent.sh

@@ -1,5 +1,7 @@
 source common.sh
 
+$NIX_BIN_DIR/nix-collect-garbage -vvvvv
+
 drvPath1=$($nixinstantiate gc-concurrent.nix)
 outPath1=$($nixstore -q $drvPath1)
 
@@ -13,15 +15,13 @@ ln -s $drvPath2 "$NIX_STATE_DIR"/gcroots/foo
 $nixstore -rvv "$drvPath1" &
 pid1=$!
 
-# Start build #2 in the background after 3 seconds.
-(sleep 3 && $nixstore -rvv "$drvPath2") &
+# Start build #2 in the background after 6 seconds.
+(sleep 6 && $nixstore -rvv "$drvPath2") &
 pid2=$!
 
-# Run the garbage collector while the build is running.  Note: the GC
-# sleeps for *another* 2 seconds after acquiring the GC lock.  This
-# checks whether build #1
-sleep 2
-NIX_DEBUG_GC_WAIT=1 $NIX_BIN_DIR/nix-collect-garbage -vvvvv
+# Run the garbage collector while the build is running.
+sleep 4
+$NIX_BIN_DIR/nix-collect-garbage -vvvvv
 
 # Wait for build #1/#2 to finish.
 echo waiting for pid $pid1 to finish...
@@ -30,11 +30,13 @@ echo waiting for pid $pid2 to finish...
 wait $pid2
 
 # Check that the root of build #1 and its dependencies haven't been
-# deleted.
+# deleted.  The should not be deleted by the GC because they were
+# being built during the GC.
 cat $outPath1/foobar
 cat $outPath1/input-2/bar
 
-# Build #2 should have failed because its derivation got garbage collected.
+# Check that build #2 has succeeded.  It should succeed because the
+# derivation is a GC root.
 cat $outPath2/foobar
 
 rm "$NIX_STATE_DIR"/gcroots/foo

tests/gc-concurrent2.builder.sh

@@ -4,4 +4,4 @@ echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar
 # Check that the GC hasn't deleted the lock on our output.
 test -e "$out.lock"
 
-sleep 3
+sleep 6

tests/init.sh

@@ -29,9 +29,10 @@ ln -s $TOP/scripts/nix-build $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-install-package $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-push $NIX_BIN_DIR/
 ln -s $TOP/scripts/nix-pull $NIX_BIN_DIR/
-ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/
-ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/
 mkdir $NIX_BIN_DIR/nix
+ln -s $bzip2_bin_test/bzip2 $NIX_BIN_DIR/nix/
+ln -s $bzip2_bin_test/bunzip2 $NIX_BIN_DIR/nix/
+ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/
 ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/
 ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/
 
@@ -54,6 +55,7 @@ cp -pr $TOP/corepkgs $NIX_DATA_DIR/nix/
 for i in \
     $NIX_DATA_DIR/nix/corepkgs/nar/nar.sh \
     $NIX_BIN_DIR/nix/download-using-manifests.pl \
+    $NIX_BIN_DIR/nix/copy-from-other-stores.pl \
     $NIX_BIN_DIR/nix-prefetch-url \
     $NIX_BIN_DIR/nix-collect-garbage \
     $NIX_BIN_DIR/nix-build \
@@ -63,7 +65,8 @@ for i in \
     $NIX_BIN_DIR/nix/readmanifest.pm \
     ; do
     sed < $i > $i.tmp \
-        -e "s^$REAL_BIN_DIR^$NIX_BIN_DIR^" \
+        -e "s^$REAL_BIN_DIR/nix-store^$NIX_BIN_DIR/nix-store^" \
+        -e "s^$REAL_BIN_DIR/nix-hash^$NIX_BIN_DIR/nix-hash^" \
         -e "s^$REAL_LIBEXEC_DIR^$NIX_LIBEXEC_DIR^" \
         -e "s^$REAL_LOCALSTATE_DIR^$NIX_LOCALSTATE_DIR^" \
         -e "s^$REAL_DATA_DIR^$NIX_DATA_DIR^" \
@@ -91,6 +94,10 @@ exec $NIX_BIN_DIR/nix/download-using-manifests.pl.real "\$@"
 EOF
 chmod +x $NIX_BIN_DIR/nix/download-using-manifests.pl
 
+mkdir -p $NIX_BIN_DIR/nix/substituters
+mv $NIX_BIN_DIR/nix/copy-from-other-stores.pl $NIX_BIN_DIR/nix/substituters/copy-from-other-stores.pl
+mv $NIX_BIN_DIR/nix/download-using-manifests.pl $NIX_BIN_DIR/nix/substituters/download-using-manifests.pl
+
 # Initialise the database.
 $nixstore --init
 

tests/lang/eval-fail-bad-value.nix

@@ -1,7 +0,0 @@
-let {
-
-  f = {x, y : ["baz" "bat"]}: x + y;
-
-  body = f {x = "foo"; y = "bar";};
-
-}

tests/lang/eval-okay-closure.exp.xml

@@ -0,0 +1,343 @@
+<?xml version='1.0' encoding='utf-8'?>
+<expr>
+  <list>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-13" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-12" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-11" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-9" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-8" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-7" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-5" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-4" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="-3" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="-1" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="0" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="1" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="2" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="4" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="5" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="6" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="8" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="9" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="10" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="13" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="14" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="15" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="17" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="18" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="19" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="22" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="23" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="26" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="27" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="28" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="31" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="32" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="35" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="36" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="40" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="41" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="44" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="45" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="49" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="53" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="54" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="58" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="62" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="foo">
+        <bool value="true" />
+      </attr>
+      <attr name="key">
+        <int value="67" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="71" />
+      </attr>
+    </attrs>
+    <attrs>
+      <attr name="key">
+        <int value="80" />
+      </attr>
+    </attrs>
+  </list>
+</expr>

tests/lang/eval-okay-closure.nix

@@ -0,0 +1,13 @@
+let
+
+  closure = builtins.genericClosure {
+    startSet = [{key = 80;}];
+    operator = {key, foo ? false}:
+      if builtins.lessThan key 0
+      then []
+      else [{key = builtins.sub key 9;} {key = builtins.sub key 13; foo = true;}];
+  };
+
+  sort = (import ./lib.nix).sortBy (a: b: builtins.lessThan a.key b.key);
+
+in sort closure

tests/lang/eval-okay-good-value.exp

@@ -1 +0,0 @@
-Str("foobar",[])

tests/lang/eval-okay-good-value.nix

@@ -1,7 +0,0 @@
-let {
-
-  f = {x, y : ["baz" "bar" "bat"]}: x + y;
-
-  body = f {x = "foo"; y = "bar";};
-
-}

tests/lang/eval-okay-patterns.exp

@@ -0,0 +1 @@
+Str("abcxyzDDDDEFghijk",[])

tests/lang/eval-okay-patterns.nix

@@ -0,0 +1,19 @@
+let
+
+  f = args@{x, y, z}: x + args.y + z;
+
+  g = {x, y, z}@args: f args;
+
+  h = {x ? "d", y ? x, z ? args.x}@args: x + y + z;
+
+  i = args@args2: args.x + args2.y;
+
+  j = {x, y, z, ...}: x + y + z;
+
+in
+  f {x = "a"; y = "b"; z = "c";} +
+  g {x = "x"; y = "y"; z = "z";} +
+  h {x = "D";} +
+  h {x = "D"; y = "E"; z = "F";} +
+  i {x = "g"; y = "h";} +
+  j {x = "i"; y = "j"; z = "k"; bla = "bla"; foo = "bar";}

tests/lang/eval-okay-versions.exp

@@ -0,0 +1 @@
+Bool(True)