* Tagged Nix 0.14.

processes.

Makefile.am

@@ -29,11 +29,9 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
-	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
-	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
-	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
+	-$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
 

configure.ac

@@ -55,7 +55,7 @@ test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 # except that it requires the ATerm "lib" directory to be in $PATH, as
 # Windows doesn't have anything like an RPATH embedded in executable.
 # Since this is kind of annoying, we use static libraries for now.
-    
+
 AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
   [produce statically linked binaries]),
   static_nix=$enableval, static_nix=no)
@@ -63,19 +63,25 @@ AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
 if test "$sys_name" = cygwin; then
    static_nix=yes
 fi
-  
+
 if test "$static_nix" = yes; then
     AC_DISABLE_SHARED
     AC_ENABLE_STATIC
 fi
 
-  
+
 # Windows-specific stuff.
 if test "$sys_name" = "cygwin"; then
     # We cannot delete open files.
     AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.])
 fi
 
+# Solaris-specific stuff.
+if test "$sys_name" = "sunos"; then
+    # Solaris requires -lsocket -lnsl for network functions
+    ADDITIONAL_NETWORK_LIBS="-lsocket -lnsl"
+    AC_SUBST(ADDITIONAL_NETWORK_LIBS)
+fi
 
 AC_PROG_CC
 AC_PROG_CXX
@@ -88,8 +94,7 @@ AC_PROG_LIBTOOL
 
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
-CFLAGS="-D_FILE_OFFSET_BITS=64 $CFLAGS"
-CXXFLAGS="-D_FILE_OFFSET_BITS=64 $CXXFLAGS"
+AC_SYS_LARGEFILE
 
 
 # Check for pubsetbuf.
@@ -140,7 +145,6 @@ NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
-AC_PATH_PROG(jing, jing, false) # needed because xmllint --relaxng seems broken
 AC_PATH_PROG(w3m, w3m, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
@@ -189,31 +193,6 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
   storedir=$withval, storedir='/nix/store')
 AC_SUBST(storedir)
 
-AC_ARG_ENABLE(old-db-compat, AC_HELP_STRING([--disable-old-db-compat],
-  [disable support for converting from old Berkeley DB-based Nix stores]),
-  old_db_compat=$enableval, old_db_compat=yes)
-AM_CONDITIONAL(OLD_DB_COMPAT, test "$old_db_compat" = "yes")
-
-AC_ARG_WITH(bdb, AC_HELP_STRING([--with-bdb=PATH],
-  [prefix of Berkeley DB (for Nix <= 0.11 compatibility)]),
-  bdb=$withval, bdb=)
-AM_CONDITIONAL(HAVE_BDB, test -n "$bdb")
-if test -z "$bdb"; then
-  bdb_lib='-L${top_builddir}/externals/inst-bdb/lib -ldb_cxx'
-  bdb_include='-I${top_builddir}/externals/inst-bdb/include'
-else
-  bdb_lib="-L$bdb/lib -ldb_cxx"
-  bdb_include="-I$bdb/include"
-fi
-if test "$old_db_compat" = "no"; then
-  bdb_lib=
-  bdb_include=
-else
-  AC_DEFINE(OLD_DB_COMPAT, 1, [Whether to support converting from old Berkeley DB-based Nix stores.])
-fi
-AC_SUBST(bdb_lib)
-AC_SUBST(bdb_include)
-
 AC_ARG_WITH(aterm, AC_HELP_STRING([--with-aterm=PATH],
   [prefix of CWI ATerm library]),
   aterm=$withval, aterm=)
@@ -260,7 +239,7 @@ else
   bzip2_include="-I$bzip2/include"
   bzip2_bin="$bzip2/bin"
   bzip2_bin_test="$bzip2/bin"
-fi   
+fi
 AC_SUBST(bzip2_lib)
 AC_SUBST(bzip2_include)
 AC_SUBST(bzip2_bin)
@@ -283,9 +262,10 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
 # Nice to have, but not essential.
 AC_CHECK_FUNCS([strsignal])
 AC_CHECK_FUNCS([posix_fallocate])
+AC_CHECK_FUNCS([fdatasync])
 
 
-# This is needed if ATerm, Berkeley DB or bzip2 are static libraries,
+# This is needed if ATerm or bzip2 are static libraries,
 # and the Nix libraries are dynamic.
 if test "$(uname)" = "Darwin"; then
     LDFLAGS="-all_load $LDFLAGS"

corepkgs/channels/unpack.sh.in

@@ -1,5 +1,8 @@
 #! @shell@ -e
 
+# Cygwin compatibility hack: bunzip2 expects cygwin.dll in $PATH.
+export PATH=@coreutils@
+
 @coreutils@/mkdir $out
 @coreutils@/mkdir $out/tmp
 cd $out/tmp

doc/manual/Makefile.am

@@ -6,7 +6,12 @@ XSLTPROC = $(xsltproc) $(xmlflags) \
  --param xref.with.number.and.title 1 \
  --param toc.section.depth 3 \
  --param admon.style \'\' \
- --param callout.graphics.extension \'.gif\'
+ --param callout.graphics.extension \'.gif\' \
+ --param contrib.inline.enabled 0
+
+dblatex_opts = \
+ -P doc.collab.show=0 \
+ -P latex.output.revhistory=0
 
 # Note: we use GIF for now, since the PNGs shipped with Docbook aren't
 # transparent.
@@ -29,13 +34,9 @@ MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  conf-file.xml release-notes.xml \
  style.css images
 
+# Note: RelaxNG validation requires xmllint >= 2.7.4.
 manual.is-valid: $(MANUAL_SRCS) version.txt
-#	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --relaxng $(docbookrng)/docbook.rng -
-	if test "$(jing)" != "false"; then \
-		$(XMLLINT) --xinclude $< | $(jing) $(docbookrng)/docbook.rng /dev/fd/0; \
-	else \
-		echo "Not validating."; \
-	fi
+	$(XMLLINT) --noout --nonet --xinclude --noxincludenode --relaxng $(docbookrng)/docbook.rng $<
 	touch $@
 
 version.txt:
@@ -50,7 +51,7 @@ manual.html: $(MANUAL_SRCS) manual.is-valid images
 
 manual.pdf: $(MANUAL_SRCS) manual.is-valid images
 	if test "$(dblatex)" != ""; then \
-		$(dblatex) manual.xml; \
+		$(dblatex) $(dblatex_opts) manual.xml; \
 	else \
 		echo "Please install dblatex and rerun configure."; \
 		exit 1; \

doc/manual/builtins.xml

@@ -324,6 +324,16 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.intersectAttrs</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return an attribute set consisting of the
+    attributes in the set <replaceable>e2</replaceable> that also
+    exist in the set <replaceable>e1</replaceable>.</para></listitem>
+
+  </varlistentry>
+        
+    
   <varlistentry><term><function>builtins.isAttrs</function>
   <replaceable>e</replaceable></term>
 

doc/manual/conf-file.xml

@@ -233,7 +233,17 @@ build-use-chroot = /dev /proc /bin</programlisting>
     <filename>configure</filename> at build time.</para></listitem>
 
   </varlistentry>
-  
+
+
+  <varlistentry><term><literal>fsync-metadata</literal></term>
+
+    <listitem><para>If set to <literal>true</literal>, changes to the
+    Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
+    synchronously flushed to disk.  This improves robustness in case
+    of system crashes, but reduces performance.  The default is
+    <literal>false</literal>.</para></listitem>
+
+  </varlistentry>
     
 </variablelist>
 

doc/manual/installation.xml

@@ -107,13 +107,6 @@ you can use <command>configure</command>'s
 <option>--with-aterm</option> and <option>--with-bzip2</option>
 options to point to their respective locations.</para>
 
-<para>If you want to be able to upgrade Nix stores from before version
-0.12pre12020, you need Sleepycat's Berkeley DB version version 4.5.
-(Other versions may not have compatible database formats.).  Berkeley
-DB 4.5 is included in the Nix source distribution.  If you do not need
-this ability, you can build Nix with the
-<option>--disable-old-db-compat</option> configure option.</para>
-
 </section>
 
 
@@ -140,16 +133,25 @@ $ ./bootstrap</screen>
 <para>The installation path can be specified by passing the
 <option>--prefix=<replaceable>prefix</replaceable></option> to
 <command>configure</command>.  The default installation directory is
-<filename>/nix</filename>.  You can change this to any location you
-like.  You must have write permission to the
+<filename>/usr/local</filename>.  You can change this to any location
+you like.  You must have write permission to the
 <replaceable>prefix</replaceable> path.</para>
 
-<warning><para>It is best <emphasis>not</emphasis> to change the
-installation prefix from its default, since doing so makes it
-impossible to use pre-built binaries from the standard Nixpkgs
-channels.</para></warning>
+<para>Nix keeps its <emphasis>store</emphasis> (the place where
+packages are stored) in <filename>/nix/store</filename> by default.
+This can be changed using
+<option>--with-store-dir=<replaceable>path</replaceable></option>.</para>
 
-<para>If you want to rebuilt the documentation, pass the full path to
+<warning><para>It is best <emphasis>not</emphasis> to change the Nix
+store from its default, since doing so makes it impossible to use
+pre-built binaries from the standard Nixpkgs channels — that is, all
+packages will need to be built from source.</para></warning>
+
+<para>Nix keeps state (such as its database and log files) in
+<filename>/nix/var</filename> by default.  This can be changed using
+<option>--localstatedir=<replaceable>path</replaceable></option>.</para>
+
+<para>If you want to rebuild the documentation, pass the full path to
 the DocBook RELAX NG schemas and to the DocBook XSL stylesheets using
 the
 <option>--with-docbook-rng=<replaceable>path</replaceable></option>
@@ -160,27 +162,26 @@ options.</para>
 </section>
 
 
-<section><title>Installing from RPMs</title>
+<section><title>Installing a binary distribution</title>
 
-<para>RPM packages of Nix can be downloaded from <link
-xlink:href="http://nixos.org/" />.  These RPMs should work for most
-fairly recent releases of SuSE and Red Hat Linux.  They have been
-known to work work on SuSE Linux 8.1 and 9.0, and Red Hat 9.0.  In
-fact, it should work on any RPM-based Linux distribution based on
-<literal>glibc</literal> 2.3 or later.</para>
-
-<para>Once downloaded, the RPMs can be installed or upgraded using
-<command>rpm -U</command>.  For example,
+<para>RPM and Deb packages of Nix for a number of different versions
+of Fedora, openSUSE, Debian and Ubuntu can be downloaded from <link
+xlink:href="http://nixos.org/" />.  Once downloaded, the RPMs can be
+installed or upgraded using <command>rpm -U</command>.  For example,
 
 <screen>
-$ rpm -U nix-0.5pre664-1.i386.rpm</screen>
+$ rpm -U nix-0.13pre18104-1.i386.rpm</screen>
+
+Likewise, for a Deb package:
+
+<screen>
+$ dpkg -i nix_0.13pre18104-1_amd64.deb</screen>
 
 </para>
 
-<para>The RPMs install into the directory <filename>/nix</filename>.
-Nix can be uninstalled using <command>rpm -e nix</command>.  After
-this it will be necessary to manually remove the Nix store and other
-auxiliary data:
+<para>Nix can be uninstalled using <command>rpm -e nix</command> or
+<command>dpkg -r nix</command>.  After this you should manually remove
+the Nix store and other auxiliary data, if desired:
 
 <screen>
 $ rm -rf /nix/store
@@ -191,6 +192,7 @@ $ rm -rf /nix/var</screen>
 </section>
 
 
+<!-- TODO: should be updated
 <section><title>Upgrading Nix through Nix</title>
 
 <para>You can install the latest stable version of Nix through Nix
@@ -203,6 +205,7 @@ installation</link> by clicking on the package links at <link
 xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para>
 
 </section>
+-->
 
 
 <section><title>Security</title>

doc/manual/introduction.xml

@@ -320,7 +320,7 @@ overview of NixOS is given in the HotOS XI paper <citetitle
 xlink:href="http://www.st.ewi.tudelft.nl/~dolstra/pubs/hotos-final.pdf">Purely
 Functional System Configuration Management</citetitle>.  The Nix
 homepage has <link
-xlink:href="http://nix.cs.uu.nl/docs/papers.html">an up-to-date list
+xlink:href="http://nixos.org/docs/papers.html">an up-to-date list
 of Nix-related papers</link>.</para>
 
 <para>Nix is the subject of Eelco Dolstra’s PhD thesis <citetitle

doc/manual/manual.xml

@@ -17,6 +17,7 @@
         <orgname>Delft University of Technology</orgname>
         <orgdiv>Department of Software Technology</orgdiv>
       </affiliation>
+      <contrib>Author</contrib>
     </author>
 
     <copyright>
@@ -25,10 +26,11 @@
       <year>2006</year>
       <year>2007</year>
       <year>2008</year>
+      <year>2009</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>November 2008</date>
+    <date>September 2009</date>
     
   </info>
 
@@ -49,64 +51,25 @@
     
     <section>
       <title>Main commands</title>
-      <section xml:id="sec-nix-env">
-        <title>nix-env</title>
-        <xi:include href="nix-env.xml" />
-      </section>
-      <section xml:id="sec-nix-instantiate">
-        <title>nix-instantiate</title>
-        <xi:include href="nix-instantiate.xml" />
-      </section>
-      <section xml:id="sec-nix-store">
-        <title>nix-store</title>
-        <xi:include href="nix-store.xml" />
-      </section>
+      <xi:include href="nix-env.xml" />
+      <xi:include href="nix-instantiate.xml" />
+      <xi:include href="nix-store.xml" />
     </section>
     
     <section>
       <title>Utilities</title>
-      <section xml:id="sec-nix-build">
-        <title>nix-build</title>
-        <xi:include href="nix-build.xml" />
-      </section>
-      <section xml:id="sec-nix-channel">
-        <title>nix-channel</title>
-        <xi:include href="nix-channel.xml" />
-      </section>
-      <section xml:id="sec-nix-collect-garbage">
-        <title>nix-collect-garbage</title>
-        <xi:include href="nix-collect-garbage.xml" />
-      </section>
-      <section xml:id="sec-nix-copy-closure">
-        <title>nix-copy-closure</title>
-        <xi:include href="nix-copy-closure.xml" />
-      </section>
-      <section xml:id="sec-nix-hash">
-        <title>nix-hash</title>
-        <xi:include href="nix-hash.xml" />
-      </section>
-      <section xml:id="sec-nix-install-package">
-        <title>nix-install-package</title>
-        <xi:include href="nix-install-package.xml" />
-      </section>
-      <section xml:id="sec-nix-prefetch-url">
-        <title>nix-prefetch-url</title>
-        <xi:include href="nix-prefetch-url.xml" />
-      </section>
-      <section xml:id="sec-nix-pull">
-        <title>nix-pull</title>
-        <xi:include href="nix-pull.xml" />
-      </section>
-      <section xml:id="sec-nix-push">
-        <title>nix-push</title>
-        <xi:include href="nix-push.xml" />
-      </section>
-      <section xml:id="sec-nix-worker">
-        <title>nix-worker</title>
-        <xi:include href="nix-worker.xml" />
-      </section>
+      <xi:include href="nix-build.xml" />
+      <xi:include href="nix-channel.xml" />
+      <xi:include href="nix-collect-garbage.xml" />
+      <xi:include href="nix-copy-closure.xml" />
+      <xi:include href="nix-hash.xml" />
+      <xi:include href="nix-install-package.xml" />
+      <xi:include href="nix-prefetch-url.xml" />
+      <xi:include href="nix-pull.xml" />
+      <xi:include href="nix-push.xml" />
+      <xi:include href="nix-worker.xml" />
     </section>
-    
+
   </appendix>
 
   <xi:include href="troubleshooting.xml" />

doc/manual/nix-build.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-build">
 
 <refmeta>
   <refentrytitle>nix-build</refentrytitle>

doc/manual/nix-channel.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-channel">
   
 <refmeta>
   <refentrytitle>nix-channel</refentrytitle>

doc/manual/nix-collect-garbage.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-collect-garbage">
   
 <refmeta>
   <refentrytitle>nix-collect-garbage</refentrytitle>

doc/manual/nix-copy-closure.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-copy-closure">
 
 <refmeta>
   <refentrytitle>nix-copy-closure</refentrytitle>

doc/manual/nix-env.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-env">
   
 <refmeta>
   <refentrytitle>nix-env</refentrytitle>

doc/manual/nix-hash.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-hash">
   
 <refmeta>
   <refentrytitle>nix-hash</refentrytitle>

doc/manual/nix-install-package.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-install-package">
   
 <refmeta>
   <refentrytitle>nix-install-package</refentrytitle>

doc/manual/nix-instantiate.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-instantiate">
   
 <refmeta>
   <refentrytitle>nix-instantiate</refentrytitle>

doc/manual/nix-lang-ref.xml

@@ -178,100 +178,5 @@
     </productionset>
     
   </sect1>
-  
-
-
-  <sect1>
-    <title>Semantics</title>
-
-
-    
-    <sect2>
-      <title>Built-in functions</title>
-
-      <para>
-        The Nix language provides the following built-in function
-        (<quote>primops</quote>):
-      </para>
-
-      <variablelist>
-
-        <varlistentry>
-          <term><function>import</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield a path value.  The Nix expression
-              stored at this path in the file system is then read,
-              parsed, and evaluated.  Returns the result of the
-              evaluation of the Nix expression just read.
-            </para>
-
-            <para>
-              Example: <literal>import ./foo.nix</literal> evaluates
-              the expression stored in <filename>foo.nix</filename>
-              (in the directory containing the expression in which the
-              <function>import</function> occurs).
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>derivation</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield an attribute set.  [...]
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>baseNameOf</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield a string value, and returns a string
-              representing its <emphasis>base name</emphasis>.  This
-              is the substring following the last path separator
-              (<literal>/</literal>).
-            </para>
-
-            <para>
-              Example: <literal>baseNameOf "/foo/bar"</literal>
-              returns <literal>"bar"</literal>, and
-              <literal>baseNameOf "/foo/bar/"</literal> returns
-              <literal>""</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>toString</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>
-              and coerces it into a string, if possible.  Only
-              strings, paths, and URIs can be so coerced.
-            </para>
-
-            <para>
-              Example: <literal>toString
-              http://www.cs.uu.nl/</literal> returns
-              <literal>"http://www.cs.uu.nl/"</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-            
-      </variablelist>
-
-    </sect2>
-
-  </sect1>
-  
 
 </appendix>

doc/manual/nix-prefetch-url.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-prefetch-url">
   
 <refmeta>
   <refentrytitle>nix-prefetch-url</refentrytitle>

doc/manual/nix-pull.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-pull">
 
 <refmeta>
   <refentrytitle>nix-pull</refentrytitle>

doc/manual/nix-push.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-push">
 
 <refmeta>
   <refentrytitle>nix-push</refentrytitle>

doc/manual/nix-store.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-store">
 
 <refmeta>
   <refentrytitle>nix-store</refentrytitle>
@@ -212,8 +213,6 @@ linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
   <arg><option>--max-links</option> <replaceable>nrlinks</replaceable></arg>
-  <arg><option>--max-atime</option> <replaceable>atime</replaceable></arg>
-  <arg><option>--use-atime</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -291,42 +290,6 @@ options control what gets deleted and in what order:
     
   </varlistentry>
 
-  <varlistentry><term><option>--max-atime</option> <replaceable>atime</replaceable></term>
-  
-    <listitem><para>Only delete a store path if its last-accessed time
-    is less than <replaceable>atime</replaceable>.  This allows you to
-    garbage-collect only packages that haven’t been used recently.
-    The time is expressed as the number of seconds in the Unix epoch,
-    i.e., since 1970-01-01 00:00:00 UTC.  An easy way to convert to
-    this format is <literal>date +%s -d "<replaceable>date
-    specification</replaceable>"</literal>.</para>
-
-    <para>For directories, the last-accessed time is the highest
-    last-accessed time of any regular file in the directory (or in any
-    of its subdirectories).  That is, the <literal>atime</literal>
-    field maintained by the filesystem is ignored for directories.
-    This is because operations such as rebuilding the
-    <command>locate</command> database tend to update the
-    <literal>atime</literal> values of all directories, so they’re not
-    a useful indicator of whether a package was recently used.</para>
-
-    <para>Note that <command>nix-store --optimise</command> reads all
-    regular files in the Nix store, and so causes all last-accessed
-    times to be set to the present time.  This makes
-    <option>--max-atime</option> ineffective (for a while at
-    least).</para></listitem>
-    
-  </varlistentry>
-
-  <varlistentry><term><option>--use-atime</option></term>
-
-    <listitem><para>Delete store paths in order of ascending
-    last-accessed time.  This is useful in conjunction with the other
-    options to delete only the least recently used
-    packages.</para></listitem>
-
-  </varlistentry>
-  
 </variablelist>
 
 </para>
@@ -357,13 +320,6 @@ deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
 
 </para>
 
-<para>To delete unreachable paths not accessed in the last two months:
-
-<screen>
-$ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
-
-</para>
-
 <para>To delete at least 100 MiBs of unreachable paths:
 
 <screen>
@@ -448,6 +404,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
     <arg choice='plain'><option>--tree</option></arg>
     <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
     <arg choice='plain'><option>--hash</option></arg>
+    <arg choice='plain'><option>--roots</option></arg>
   </group>
   <arg><option>--use-output</option></arg>
   <arg><option>-u</option></arg>
@@ -630,12 +587,20 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--hash</option></term>
   
     <listitem><para>Prints the SHA-256 hash of the contents of the
-    store path <replaceable>paths</replaceable>.  Since the hash is
+    store paths <replaceable>paths</replaceable>.  Since the hash is
     stored in the Nix database, this is a fast
     operation.</para></listitem>
 
   </varlistentry>
 
+  <varlistentry><term><option>--roots</option></term>
+  
+    <listitem><para>Prints the garbage collector roots that point,
+    directly or indirectly, at the store paths
+    <replaceable>paths</replaceable>.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 </refsection>
@@ -713,6 +678,18 @@ $ gv graph.ps</screen>
 
 </para>
 
+<para>Show every garbage collector root that points to a store path
+that depends on <command>svn</command>:
+
+<screen>
+$ nix-store -q --roots $(which svn)
+/nix/var/nix/profiles/default-81-link
+/nix/var/nix/profiles/default-82-link
+/nix/var/nix/profiles/per-user/eelco/profile-97-link
+</screen>
+
+</para>
+
 </refsection>
 
 

doc/manual/nix-worker.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-worker">
 
 <refmeta>
   <refentrytitle>nix-worker</refentrytitle>

doc/manual/package-management.xml

@@ -331,7 +331,7 @@ default profile, respectively.  If the profile doesn’t exist, it will
 be created automatically.  You should be careful about storing a
 profile in another location than the <filename>profiles</filename>
 directory, since otherwise it might not be used as a root of the
-garbage collector (see section <xref linkend='sec-garbage-collection'
+garbage collector (see <xref linkend='sec-garbage-collection'
 />).</para>
 
 <para>All <command>nix-env</command> operations work on the profile
@@ -507,19 +507,16 @@ click on it, and it will be installed with all the necessary
 dependencies.</para>
 
 <para>For instance, you can go to <link
-xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/" /> —
-or to any older release of Nix Packages — and click on any link for
-the individual packages for your platform (say, <link
-xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.10pre6622/pkgs/subversion-1.4.0-i686-linux.nixpkg'><literal>subversion-1.4.0</literal>
-for <literal>i686-linux</literal></link>).  The first time you do
-this, your browser will ask what to do with
-<literal>application/nix-package</literal> files.  You should open
-them with <filename>/nix/bin/nix-install-package</filename>.  This
-will open a window that asks you to confirm that you want to install
-the package.  When you answer <literal>Y</literal>, the package and
-all its dependencies will be installed.  This is a binary deployment
-mechanism — you get packages pre-compiled for the selected platform
-type.</para>
+xlink:href="http://hydra.nixos.org/jobset/nixpkgs/trunk/channel/latest"
+/> and click on any link for the individual packages for your
+platform.  The first time you do this, your browser will ask what to
+do with <literal>application/nix-package</literal> files.  You should
+open them with <filename>/nix/bin/nix-install-package</filename>.
+This will open a window that asks you to confirm that you want to
+install the package.  When you answer <literal>Y</literal>, the
+package and all its dependencies will be installed.  This is a binary
+deployment mechanism — you get packages pre-compiled for the selected
+platform type.</para>
 
 <para>You can also install <literal>application/nix-package</literal>
 files from the command line directly.  See <xref

doc/manual/quick-start.xml

@@ -1,5 +1,6 @@
 <chapter xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink">
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xml:id="chap-quick-start">
 
 <title>Quick Start</title>
 
@@ -10,9 +11,9 @@ to the following chapters.</para>
 
 <orderedlist>
 
-<listitem><para>Download a source tarball or RPM from <link
-xlink:href='http://nixos.org/'/>.  Build source
-distributions using the regular sequence:
+<listitem><para>Download a source tarball, RPM or Deb from <link
+xlink:href='http://nixos.org/'/>.  Build source distributions using
+the regular sequence:
         
 <screen>
 $ tar xvfj nix-<replaceable>version</replaceable>.tar.bz2
@@ -20,13 +21,21 @@ $ ./configure
 $ make
 $ make install <lineannotation>(as root)</lineannotation></screen>
 
-This will install Nix in <filename>/nix</filename>.  You shouldn't
-change the prefix if at all possible since that will make it
-impossible to use pre-built binaries from the Nixpkgs channel and
-other channels.  Alternatively, you could grab an RPM if you're on an
-RPM-based system.  You should also add
-<filename>/nix/etc/profile.d/nix.sh</filename> to your
-<filename>~/.bashrc</filename> (or some other login
+This will install the Nix binaries in <filename>/usr/local</filename>
+and keep the Nix store and other state in <filename>/nix</filename>.
+You can change the former by specifying
+<option>--prefix=<replaceable>path</replaceable></option>.  The
+location of the store can be changed using
+<option>--with-store-dir=<replaceable>path</replaceable></option>.
+However, you shouldn't change the store location, if at all possible,
+since that will make it impossible to use pre-built binaries from the
+Nixpkgs channel and other channels.  The location of the state can be
+changed using
+<option>--localstatedir=<replaceable>path</replaceable>.</option></para></listitem>
+
+<listitem><para>You should add
+<filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
+to your <filename>~/.bashrc</filename> (or some other login
 file).</para></listitem>
 
 <listitem><para>Subscribe to the Nix Packages channel.
@@ -99,7 +108,7 @@ numbers).</para></listitem>
 
 <listitem><para>You can also install specific packages directly from
 your web browser.  For instance, you can go to <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/" />
+xlink:href="http://hydra.nixos.org/jobset/nixpkgs/trunk/channel/latest" />
 and click on any link for the individual packages for your platform.
 Associate <literal>application/nix-package</literal> with the program
 <filename>/nix/bin/nix-install-package</filename>.  A window should

doc/manual/release-notes.xml

@@ -8,7 +8,51 @@
 
 <!--==================================================================-->
 
-<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (April NN,
+<section xml:id="ssec-relnotes-0.14"><title>Release 0.14 (February 4, 2010)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The garbage collector now starts deleting garbage much
+    faster than before.  It no longer determines liveness of all paths
+    in the store, but does so on demand.</para>
+  </listitem>
+
+  <listitem>
+    <para>Added a new operation, <command>nix-store --query
+    --roots</command>, that shows the garbage collector roots that
+    directly or indirectly point to the given store paths.</para>
+  </listitem>
+
+  <listitem>
+    <para>Removed support for converting Berkeley DB-based Nix
+    databases to the new schema.</para>
+  </listitem>
+
+  <listitem>
+    <para>Removed the <option>--use-atime</option> and
+    <option>--max-atime</option> garbage collector options.  They were
+    not very useful in practice.</para>
+  </listitem>
+
+  <listitem>
+    <para>On Windows, Nix now requires Cygwin 1.7.x.</para>
+  </listitem>
+
+  <listitem>
+    <para>A few bug fixes.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (November 5,
 2009)</title>
 
 <para>This is primarily a bug fix release.  It has some new
@@ -85,7 +129,25 @@ features:</para>
     add a string to stack traces — useful for debugging),
     <varname>builtins.isBool</varname>,
     <varname>builtins.isString</varname>,
-    <varname>builtins.isInt</varname>.</para>
+    <varname>builtins.isInt</varname>,
+    <varname>builtins.intersectAttrs</varname>.</para>
+  </listitem>
+
+  <listitem>
+    <para>OpenSolaris support (Sander van der Burg).</para>
+  </listitem>
+
+  <listitem>
+    <para>Stack traces are no longer displayed unless the
+    <option>--show-trace</option> option is used.</para>
+  </listitem>
+
+  <listitem>
+    <para>The scoping rules for <literal>inherit
+    (<replaceable>e</replaceable>) ...</literal> in recursive
+    attribute sets have changed.  The expression
+    <replaceable>e</replaceable> can now refer to the attributes
+    defined in the containing set.</para>
   </listitem>
 
 </itemizedlist>

doc/manual/writing-nix-expressions.xml

@@ -1106,7 +1106,7 @@ used in the Nix expression for Subversion.</para>
     incompatibility might occur.</para>
   </callout>
 
-  <callout arearefs='ex-subversion-nix-co-2'>
+  <callout arearefs='ex-subversion-nix-co-3'>
     <para>This assertion says that in order for Subversion to have SSL
     support (so that it can access <literal>https</literal> URLs), an
     OpenSSL library must be passed.  Additionally, it says that

externals/Makefile.am

@@ -1,46 +1,3 @@
-# Berkeley DB
-
-DB = db-4.5.20
-
-if OLD_DB_COMPAT
-
-$(DB).tar.gz:
-	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.5.20 from"
-	@echo "  http://download-east.oracle.com/berkeley-db/db-4.5.20.tar.gz"
-	@echo "and place it in the externals/ directory."
-	false
-
-$(DB): $(DB).tar.gz
-	gunzip < $(srcdir)/$(DB).tar.gz | tar xvf -
-	(cd $(DB) && $(patch) -p1) < $(srcdir)/bdb-cygwin.patch
-
-have-db:
-	$(MAKE) $(DB)
-	touch have-db
-
-if HAVE_BDB
-build-db:
-else
-build-db: have-db
-	(pfx=`pwd` && \
-	cd $(DB)/build_unix && \
-	CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" \
-	 ../dist/configure --prefix=$$pfx/inst-bdb \
-	 --enable-cxx --disable-shared --disable-cryptography \
-         --disable-replication --disable-verify && \
-	$(MAKE) && \
-	$(MAKE) install_include install_lib)
-	touch build-db
-endif
-
-else
-
-build-db:
-
-endif
-
-
 # CWI ATerm
 
 ATERM = aterm-2.4.2-fixes-r2
@@ -107,11 +64,10 @@ install:
 endif
 
 
-all: build-db build-aterm build-bzip2
+all: build-aterm build-bzip2
 
-EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.bz2 $(BZIP2).tar.gz \
-  bdb-cygwin.patch
+EXTRA_DIST = $(ATERM).tar.bz2 $(BZIP2).tar.gz
 
 ext-clean:
-	$(RM) -f have-db build-db have-aterm build-aterm have-bzip2 build-bzip2
-	$(RM) -rf $(DB) $(ATERM) $(BZIP2)
+	$(RM) -f have-aterm build-aterm have-bzip2 build-bzip2
+	$(RM) -rf $(ATERM) $(BZIP2)

externals/bdb-cygwin.patch

@@ -1,22 +0,0 @@
-diff -rc db-4.5.20-orig/os/os_flock.c db-4.5.20/os/os_flock.c
-*** db-4.5.20-orig/os/os_flock.c	2006-10-13 12:36:12.000000000 +0200
---- db-4.5.20/os/os_flock.c	2006-10-13 12:40:11.000000000 +0200
-***************
-*** 30,35 ****
---- 30,44 ----
-  
-  	DB_ASSERT(dbenv, F_ISSET(fhp, DB_FH_OPENED) && fhp->fd != -1);
-  
-+ #ifdef __CYGWIN__
-+ 	/*
-+ 	 * Windows file locking interferes with read/write operations, so we
-+ 	 * map the ranges to an area past the end of the file.
-+ 	 */
-+ 	DB_ASSERT(dbenv, offset < (off_t) 1 << 62);
-+ 	offset += (off_t) 1 << 62;
-+ #endif
-+ 
-  	fl.l_start = offset;
-  	fl.l_len = 1;
-  	fl.l_type = acquire ? F_WRLCK : F_UNLCK;
-Only in db-4.5.20/os: os_flock.c~

release.nix

@@ -18,7 +18,10 @@ let
         src = nix;
         inherit officialRelease;
 
-        buildInputs = [curl bison flex2533 perl libxml2 libxslt w3m bzip2 jing_tools tetex dblatex];
+        buildInputs =
+          [ curl bison flex2533 perl libxml2 libxslt w3m bzip2
+            tetex dblatex nukeReferences
+          ];
 
         configureFlags = ''
           --with-docbook-rng=${docbook5}/xml/rng/docbook
@@ -26,27 +29,16 @@ let
           --with-xml-flags=--nonet
         '';
 
-        # Include the BDB, ATerm and Bzip2 tarballs in the distribution.
+        # Include the ATerm and Bzip2 tarballs in the distribution.
         preConfigure = ''
-          stripHash ${db45.src}
-          # Remove unnecessary stuff from the Berkeley DB tarball.
-          ( mkdir bdb-temp
-            cd bdb-temp
-            tar xfz ${db45.src}
-            cd *
-            rm -rf docs test tcl perl libdb_java java rpc_server build_vxworks \
-              examples_java examples_c examples_cxx dist/tags
-            mkdir test
-            touch test/include.tcl
-            cd ..
-            tar cvfz ../externals/$strippedName *
-          )
-
           stripHash ${aterm242fixes.src}
           cp -pv ${aterm242fixes.src} externals/$strippedName
 
           stripHash ${bzip2.src}
           cp -pv ${bzip2.src} externals/$strippedName
+
+          # TeX needs a writable font cache.
+          export VARTEXFONTS=$TMPDIR/texfonts
         '';
 
         preDist = ''
@@ -54,6 +46,14 @@ let
           
           make -C doc/manual manual.pdf prefix=$out
           cp doc/manual/manual.pdf $out/manual.pdf
+
+          # The PDF containes filenames of included graphics (see
+          # http://www.tug.org/pipermail/pdftex/2007-August/007290.html).
+          # This causes a retained dependency on dblatex, which Hydra
+          # doesn't like (the output of the tarball job is distributed
+          # to Windows and Macs, so there should be no Linux binaries
+          # in the closure).
+          nuke-refs $out/manual.pdf
           
           echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
           echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products
@@ -77,11 +77,12 @@ let
 
         configureFlags = ''
           --disable-init-state
-          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
         '';
       };
 
-      
+
+    /*
     static =
       { tarball ? jobs.tarball {}
       , system ? "i686-linux"
@@ -97,10 +98,11 @@ let
 
         configureFlags = ''
           --disable-init-state
-          --disable-old-db-compat --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
           --enable-static-nix
         '';
       };
+    */
 
       
     coverage =
@@ -121,7 +123,7 @@ let
 
         configureFlags = ''
           --disable-init-state --disable-shared
-          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
         '';
 
         lcovFilter = ["*/boost/*" "*-tab.*"];
@@ -133,22 +135,32 @@ let
       };
 
       
-    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 20;
-    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 50;
-    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 50;
-    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 40;
-    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 40;
+    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 10;
+    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 20;
+    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 20;
+    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 30;
+    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 30;
+    rpm_fedora11i386 = makeRPM_i686 (diskImages: diskImages.fedora11i386) 40;
+    rpm_fedora11x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora11x86_64) 40;
+    rpm_fedora12i386 = makeRPM_i686 (diskImages: diskImages.fedora12i386) 50;
+    rpm_fedora12x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora12x86_64) 50;
     rpm_opensuse103i386 = makeRPM_i686 (diskImages: diskImages.opensuse103i386) 40;
+    rpm_opensuse110i386 = makeRPM_i686 (diskImages: diskImages.opensuse110i386) 50;
+    rpm_opensuse110x86_64 = makeRPM_x86_64 (diskImages: diskImages.opensuse110x86_64) 50;
 
     
     deb_debian40i386 = makeDeb_i686 (diskImages: diskImages.debian40i386) 40;
     deb_debian40x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian40x86_64) 40;
-    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 30;
-    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 30;
-    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 50;
-    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 50;
-    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 40;
-    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 40;
+    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 50;
+    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 50;
+    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 20;
+    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 20;
+    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 30;
+    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 30;
+    deb_ubuntu904i386 = makeDeb_i686 (diskImages: diskImages.ubuntu904i386) 40;
+    deb_ubuntu904x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu904x86_64) 40;
+    deb_ubuntu910i386 = makeDeb_i686 (diskImages: diskImages.ubuntu910i386) 50;
+    deb_ubuntu910x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu910x86_64) 50;
 
 
   };
@@ -190,6 +202,7 @@ let
       memSize = 1024;
       meta = { schedulingPriority = toString prio; };
       configureFlags = "--sysconfdir=/etc";
+      debRequires = ["curl"];
     };
 
 

scripts/Makefile.am

@@ -15,6 +15,7 @@ install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) ssh.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
@@ -31,6 +32,7 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-channel.in \
   readmanifest.pm.in \
   readconfig.pm.in \
+  ssh.pm \
   nix-build.in \
   download-using-manifests.pl.in \
   copy-from-other-stores.pl.in \

scripts/build-remote.pl.in

@@ -1,9 +1,10 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
 
-use strict;
 use Fcntl ':flock';
 use English '-no_match_vars';
 use IO::Handle;
+use ssh qw/sshOpts openSSHConnection/;
+
 
 # General operation:
 #
@@ -22,6 +23,14 @@ use IO::Handle;
 # The nice thing about this scheme is that if we die prematurely, the
 # locks are released automatically.
 
+
+# Make sure that we don't get any SSH passphrase or host key popups -
+# if there is any problem it should fail, not do something
+# interactive.
+$ENV{"DISPLAY"} = "";
+$ENV{"SSH_ASKPASS"} = "";
+
+
 my $loadIncreased = 0;
 
 my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV;
@@ -47,24 +56,22 @@ decline if !defined $conf || ! -e $conf;
 my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
 
 
-# Otherwise find a willing remote machine.
-my @machines;
-my %curJobs;
-
-
 # Read the list of machines.
+my @machines;
 open CONF, "< $conf" or die;
 
 while (<CONF>) {
     chomp;
     s/\#.*$//g;
     next if /^\s*$/;
-    /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die;
+    /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)(\s+([0-9\.]+))?\s*$/ or die;
     push @machines,
         { hostName => $1
-        , systemType => $2
+        , systemTypes => [split(/,/, $2)]
         , sshKeys => $3
         , maxJobs => $4
+        , speedFactor => 1.0 * ($6 || 1)
+        , enabled => 1
         };
 }
 
@@ -77,50 +84,105 @@ open MAINLOCK, ">>$mainLock" or die;
 flock(MAINLOCK, LOCK_EX) or die;
 
 
-# Find a suitable system.
-my $rightType = 0;
-my $machine;
-my $slotLock;
-LOOP: foreach my $cur (@machines) {
-    if ($neededSystem eq $cur->{systemType}
-        || ($neededSystem eq "i686-linux" && $cur->{systemType} eq "x86_64-linux"))
-    {
-        $rightType = 1;
+sub openSlotLock {
+    my ($machine, $slot) = @_;
+    my $slotLockFn = "$currentLoad/" . (join '+', @{$machine->{systemTypes}}) . "-" . $machine->{hostName} . "-$slot";
+    my $slotLock = new IO::Handle;
+    open $slotLock, ">>$slotLockFn" or die;
+    return $slotLock;
+}
+    
 
-        # We have a machine of the right type.  Try to get a lock on
-        # one of the machine's lock files.
-        my $slot = 0;
-        while ($slot < $cur->{maxJobs}) {
-            my $slotLockFn = "$currentLoad/" . $cur->{systemType} . "-" . $cur->{hostName} . "-$slot";
-            $slotLock = new IO::Handle;
-            open $slotLock, ">>$slotLockFn" or die;
-            if (flock($slotLock, LOCK_EX | LOCK_NB)) {
-                utime undef, undef, $slotLock;
-                $machine = $cur;
-                last LOOP;
+my $hostName;
+my $slotLock;
+
+while (1) {
+    
+    # Find all machine that can execute this build, i.e., that support
+    # builds for the given platform and are not at their job limit.
+    my $rightType = 0;
+    my @available = ();
+    LOOP: foreach my $cur (@machines) {
+        if ($cur->{enabled} && grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) {
+            $rightType = 1;
+
+            # We have a machine of the right type.  Determine the load on
+            # the machine.
+            my $slot = 0;
+            my $load = 0;
+            my $free;
+            while ($slot < $cur->{maxJobs}) {
+                my $slotLock = openSlotLock($cur, $slot);
+                if (flock($slotLock, LOCK_EX | LOCK_NB)) {
+                    $free = $slot unless defined $free;
+                    flock($slotLock, LOCK_UN) or die;
+                } else {
+                    $load++;
+                }
+                close $slotLock;
+                $slot++;
             }
-            close $slotLock;
-            $slot++;
+
+            push @available, { machine => $cur, load => $load, free => $free }
+            if $load < $cur->{maxJobs};
         }
     }
-}
 
-close MAINLOCK;
-
-
-# Didn't find one?  Then decline or postpone.
-if (!defined $machine) {
-    # Postpone if we have a machine of the right type, except if the
-    # local system can and wants to do the build.
-    if ($rightType && !$canBuildLocally) {
-        sendReply "postpone";
-        exit 0;
-    } else {
-        decline;
+    if (defined $ENV{NIX_DEBUG_HOOK}) {
+        print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n"
+            foreach @available;
     }
+
+
+    # Didn't find any available machine?  Then decline or postpone.
+    if (scalar @available == 0) {
+        # Postpone if we have a machine of the right type, except if the
+        # local system can and wants to do the build.
+        if ($rightType && !$canBuildLocally) {
+            sendReply "postpone";
+            exit 0;
+        } else {
+            decline;
+        }
+    }
+
+
+    # Prioritise the available machines as follows:
+    # - First by load divided by speed factor, rounded to the nearest
+    #   integer.  This causes fast machines to be preferred over slow
+    #   machines with similar loads.
+    # - Then by speed factor.
+    # - Finally by load.
+    sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); }
+    @available = sort
+        { lf($a) <=> lf($b)
+              || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor}
+              || $a->{load} <=> $b->{load}
+        } @available;
+
+
+    # Select the best available machine and lock a free slot.
+    my $selected = $available[0]; 
+    my $machine = $selected->{machine};
+
+    $slotLock = openSlotLock($machine, $selected->{free});
+    flock($slotLock, LOCK_EX | LOCK_NB) or die;
+    utime undef, undef, $slotLock;
+
+    close MAINLOCK;
+
+
+    # Connect to the selected machine.
+    @sshOpts = ("-i", $machine->{sshKeys}, "-x");
+    $hostName = $machine->{hostName};
+    last if openSSHConnection $hostName;
+    
+    warn "unable to open SSH connection to $hostName, trying other available machines...\n";
+    $machine->{enabled} = 0;
 }
 
-# Yes we did, accept.
+
+# Tell Nix we've accepted the build.
 sendReply "accept";
 my $x = <STDIN>;
 chomp $x;
@@ -130,43 +192,9 @@ if ($x ne "okay") {
 }
 
 
-# Do the actual job.
-my $hostName = $machine->{hostName};
+# Do the actual build.
 print STDERR "building `$drvPath' on `$hostName'\n";
 
-# Make sure that we don't get any SSH passphrase or host key popups -
-# if there is any problem it should fail, not do something
-# interactive.
-$ENV{"DISPLAY"} = "";
-$ENV{"SSH_PASSWORD_FILE="} = "";
-$ENV{"SSH_ASKPASS="} = "";
-
-my $sshOpts = "-i " . $machine->{sshKeys} . " -x";
-
-# Hack to support Cygwin: if we login without a password, we don't
-# have exactly the same rights as when we do.  This causes the
-# Microsoft C compiler to fail with certain flags:
-#
-#   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676
-#
-# So as a workaround, we pass a verbatim password.  ssh tries to makes
-# this very hard; the trick is to make it call SSH_ASKPASS to get the
-# password.  (It only calls this command when there is no controlling
-# terminal, but Nix ensures that is is the case.  When doing this
-# manually, use setsid(1).)
-if ($machine->{sshKeys} =~ /^password:/) {
-    my $passwordFile = $machine->{sshKeys};
-    $passwordFile =~ s/^password://;
-    $sshOpts = "ssh -x";
-    $ENV{"SSH_PASSWORD_FILE"} = $passwordFile;
-    $ENV{"SSH_ASKPASS"} = "/tmp/writepass";
-
-    open WRITEPASS, ">/tmp/writepass" or die;
-    print WRITEPASS "#! /bin/sh\ncat \"\$SSH_PASSWORD_FILE\"";
-    close WRITEPASS;
-    chmod 0755, "/tmp/writepass" or die;
-}
-
 my $inputs = `cat inputs`; die if ($? != 0);
 $inputs =~ s/\n/ /g;
 
@@ -178,7 +206,7 @@ print "copying inputs...\n";
 my $maybeSign = "";
 $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
 
-system("NIX_SSHOPTS=\"$sshOpts\" @bindir@/nix-copy-closure --gzip $hostName $maybeSign $drvPath $inputs") == 0
+system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath $inputs") == 0
     or die "cannot copy inputs to $hostName: $?";
 
 print "building...\n";
@@ -190,10 +218,10 @@ my $buildFlags = "--max-silent-time $maxSilentTime";
 # connection dies.  Without it, the remote process might continue to
 # run indefinitely (that is, until it next tries to write to
 # stdout/stderr).
-if (system("ssh -tt $sshOpts $hostName 'nix-store --realise -K $buildFlags $drvPath > /dev/null'") != 0) {
+if (system("ssh $hostName @sshOpts -tt 'nix-store --realise $buildFlags $drvPath > /dev/null'") != 0) {
     # If we couldn't run ssh or there was an ssh problem (indicated by
     # exit code 255), then we return exit code 1; otherwise we assume
-    # that the builder failed, which we indicated to Nix using exit
+    # that the builder failed, which we indicate to Nix using exit
     # code 100.  It's important to distinguish between the two because
     # the first is a transient failure and the latter is permanent.
     my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100;
@@ -207,6 +235,6 @@ foreach my $output (split '\n', $outputs) {
     my $maybeSignRemote = "";
     $maybeSignRemote = "--sign" if $UID != 0;
     
-    system("ssh $sshOpts $hostName 'nix-store --export $maybeSignRemote $output | gzip' | gunzip | @bindir@/nix-store --import > /dev/null") == 0
+    system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output' | @bindir@/nix-store --import > /dev/null") == 0
 	or die "cannot copy $output from $hostName: $?";
 }

scripts/download-using-manifests.pl.in

@@ -73,7 +73,7 @@ if ($ARGV[0] eq "--query") {
 }
 
 elsif ($ARGV[0] ne "--substitute") {
-    die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n";
+    die;
 }
 
 
@@ -85,8 +85,6 @@ my $targetPath = $ARGV[1];
 my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-chdir $tmpDir or die "cannot change to `$tmpDir': $!";
-
 my $tmpNar = "$tmpDir/nar";
 my $tmpNar2 = "$tmpDir/nar2";
 

scripts/nix-build.in

@@ -12,6 +12,7 @@ my $outLink;
 my $drvLink;
 
 my $dryRun = 0;
+my $verbose = 0;
 
 my @instArgs = ();
 my @buildArgs = ();
@@ -112,6 +113,16 @@ EOF
         $dryRun = 1;
     }
     
+    elsif ($arg eq "--show-trace") {
+        push @instArgs, $arg;
+    }
+    
+    elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") {
+        push @buildArgs, $arg;
+        push @instArgs, $arg;
+        $verbose = 1;
+    }
+    
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -149,7 +160,7 @@ foreach my $expr (@exprs) {
 
     foreach my $drvPath (@drvPaths) {
         my $target = readlink $drvPath or die "cannot read symlink `$drvPath'";
-        print STDERR "store derivation is $target\n";
+        print STDERR "derivation is $target\n" if $verbose;
     }
 
     # Build.

scripts/nix-channel.in

@@ -78,6 +78,9 @@ sub removeChannel {
 sub update {
     readChannels;
 
+    # Create the manifests directory if it doesn't exist.
+    mkdir "$stateDir/manifests", 0755 unless -e "$stateDir/manifests";
+
     # Do we have write permission to the manifests directory?  If not,
     # then just skip pulling the manifest and just download the Nix
     # expressions.  If the user is a non-privileged user in a
@@ -85,11 +88,6 @@ sub update {
     # source.
     if (-W "$stateDir/manifests") {
 
-        # Remove all the old manifests.
-        for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
-            unlink $manifest or die "cannot remove `$manifest': $!";
-        }
-
         # Pull cache manifests.
         foreach my $url (@channels) {
             #print "pulling cache manifest from `$url'\n";

scripts/nix-copy-closure.in

@@ -1,4 +1,6 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
+
+use ssh;
 
 my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
@@ -14,12 +16,11 @@ EOF
 
 # Get the target host.
 my $sshHost;
-my @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
 
 my $sign = 0;
 
-my $compressor = "cat";
-my $decompressor = "cat";
+my $compressor = "";
+my $decompressor = "";
 
 my $toMode = 1;
 
@@ -34,8 +35,8 @@ while (@ARGV) {
         $sign = 1;
     }
     elsif ($arg eq "--gzip") {
-        $compressor = "gzip";
-        $decompressor = "gunzip";
+        $compressor = "| gzip";
+        $decompressor = "gunzip |";
     }
     elsif ($arg eq "--from") {
         $toMode = 0;
@@ -52,6 +53,9 @@ while (@ARGV) {
 }
 
 
+openSSHConnection $sshHost or die "$0: unable to start SSH\n";
+
+
 if ($toMode) { # Copy TO the remote machine.
 
     my @allStorePaths;
@@ -69,11 +73,10 @@ if ($toMode) { # Copy TO the remote machine.
 
 
     # Ask the remote host which paths are invalid.
-    open(READ, "ssh @sshOpts $sshHost nix-store --check-validity --print-invalid @allStorePaths|");
+    open(READ, "ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
     my @missing = ();
     while (<READ>) {
         chomp;
-        print STDERR "target machine needs $_\n";
         push @missing, $_;
     }
     close READ or die;
@@ -81,9 +84,11 @@ if ($toMode) { # Copy TO the remote machine.
 
     # Export the store paths and import them on the remote machine.
     if (scalar @missing > 0) {
+        print STDERR "copying these missing paths:\n";
+        print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("nix-store --export $extraOpts @missing | $compressor | ssh @sshOpts $sshHost '$decompressor | nix-store --import'") == 0
+        system("nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
             or die "copying store paths to remote machine `$sshHost' failed: $?";
     }
 
@@ -114,7 +119,6 @@ else { # Copy FROM the remote machine.
     my @missing = ();
     while (<READ>) {
         chomp;
-        print STDERR "local machine needs $_\n";
         push @missing, $_;
     }
     close READ or die;
@@ -122,10 +126,12 @@ else { # Copy FROM the remote machine.
 
     # Export the store paths on the remote machine and import them on locally.
     if (scalar @missing > 0) {
+        print STDERR "copying these missing paths:\n";
+        print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("ssh @sshOpts $sshHost 'nix-store --export $extraOpts @missing | $compressor' | $decompressor | @bindir@/nix-store --import") == 0
-            or die "copying store paths to remote machine `$sshHost' failed: $?";
+        system("ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0
+            or die "copying store paths from remote machine `$sshHost' failed: $?";
     }
 
 }

scripts/nix-pull.in

@@ -18,6 +18,12 @@ my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "$stateDir/manifests");
 umask 0022;
 
 
+# Create the manifests directory if it doesn't exist.
+if (! -e $manifestDir) {
+    mkdir $manifestDir, 0755 or die "cannot create directory `$manifestDir'";
+}
+
+
 # Process the URLs specified on the command line.
 my %narFiles;
 my %localPaths;
@@ -45,7 +51,7 @@ sub processURL {
 
     # First see if a bzipped manifest is available.
     if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) {
-        print "obtaining list of Nix archives at `$url.bz2'...\n";
+        print "fetching list of Nix archives at `$url.bz2'...\n";
         my $bzipped = downloadFile "$url.bz2";
 
         $manifest = "$tmpDir/MANIFEST";
@@ -86,11 +92,31 @@ sub processURL {
     my $hash = `$binDir/nix-hash --flat '$manifest'`
         or die "cannot hash `$manifest'";
     chomp $hash;
+
+    my $urlFile = "$manifestDir/$baseName-$hash.url";
+    open URL, ">$urlFile" or die "cannot create `$urlFile'";
+    print URL "$url";
+    close URL;
     
     my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest";
-    
-    system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
+
+    unlink $finalPath if -e $finalPath;
+        
+    symlink("$manifest", "$finalPath")
         or die "cannot link `$finalPath to `$manifest'";
+
+    # Delete all old manifests downloaded from this URL.
+    for my $urlFile2 (glob "$manifestDir/*.url") {
+        next if $urlFile eq $urlFile2;
+        open URL, "<$urlFile2" or die;
+        my $url2 = <URL>;
+        chomp $url2;
+        close URL;
+        next unless $url eq $url2;
+        my $base = $urlFile2; $base =~ s/.url$//;
+        unlink "${base}.url";
+        unlink "${base}.nixmanifest";
+    }
 }
 
 while (@ARGV) {

scripts/ssh.pm

@@ -0,0 +1,47 @@
+use strict;
+use File::Temp qw(tempdir);
+
+our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
+
+my $sshStarted = 0;
+my $sshHost;
+
+# Open a master SSH connection to `host', unless there already is a
+# running master connection (as determined by `-O check').
+sub openSSHConnection {
+    my ($host) = @_;
+    die if $sshStarted;
+    $sshHost = $host;
+    return 1 if system("ssh $sshHost @sshOpts -O check 2> /dev/null") == 0;
+
+    my $tmpDir = tempdir("nix-ssh.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+        or die "cannot create a temporary directory";
+    
+    push @sshOpts, "-S", "$tmpDir/control";
+
+    # Start the master.  We can't use the `-f' flag (fork into
+    # background after establishing the connection) because then the
+    # child continues to run if we are killed.  So instead make SSH
+    # print "started" when it has established the connection, and wait
+    # until we see that.
+    open SSH, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die;
+    while (<SSH>) {
+        chomp;
+        last if /started/;
+    }
+    
+    $sshStarted = 1;
+    return 1;
+}
+
+# Tell the master SSH client to exit.
+sub closeSSHConnection {
+    if ($sshStarted) {
+        system("ssh $sshHost @sshOpts -O exit 2> /dev/null") == 0
+            or warn "unable to stop SSH master: $?";
+    }
+}
+
+END { my $saved = $?; closeSSHConnection; $? = $saved; }
+
+return 1;

src/libexpr/Makefile.am

@@ -19,7 +19,7 @@ BUILT_SOURCES = nixexpr-ast.cc nixexpr-ast.hh \
 EXTRA_DIST = lexer.l parser.y nixexpr-ast.def nixexpr-ast.cc
 
 AM_CXXFLAGS = \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} \
+ -I$(srcdir)/.. ${aterm_include} \
  -I$(srcdir)/../libutil -I$(srcdir)/../libstore
 AM_CFLAGS = \
  ${aterm_include}

src/libexpr/primops.cc

@@ -223,6 +223,23 @@ static Expr prim_addErrorContext(EvalState & state, const ATermVector & args)
     }
 }
 
+/* Try evaluating the argument. Success => {success=true; value=something;}, 
+ * else => {success=false; value=false;} */
+static Expr prim_tryEval(EvalState & state, const ATermVector & args)
+{
+    ATermMap res = ATermMap();
+    try {
+        Expr val = evalExpr(state, args[0]);
+        res.set(toATerm("value"), makeAttrRHS(val, makeNoPos()));
+        res.set(toATerm("success"), makeAttrRHS(eTrue, makeNoPos()));
+    } catch (AssertionError & e) {
+        printMsg(lvlDebug, format("tryEval caught an error: %1%: %2%") % e.prefix() % e.msg());
+        res.set(toATerm("value"), makeAttrRHS(eFalse, makeNoPos()));
+        res.set(toATerm("success"), makeAttrRHS(eFalse, makeNoPos()));
+    }
+    return makeAttrs(res);
+}
+
 
 /* Return an environment variable.  Use with care. */
 static Expr prim_getEnv(EvalState & state, const ATermVector & args)
@@ -239,7 +256,12 @@ static Expr prim_getEnv(EvalState & state, const ATermVector & args)
 static Expr prim_trace(EvalState & state, const ATermVector & args)
 {
     Expr e = evalExpr(state, args[0]);
-    printMsg(lvlError, format("trace: %1%") % e);
+    string s;
+    PathSet context;
+    if (matchStr(e, s, context))
+        printMsg(lvlError, format("trace: %1%") % s);
+    else
+        printMsg(lvlError, format("trace: %1%") % e);
     return evalExpr(state, args[1]);
 }
 
@@ -418,10 +440,18 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
                     drv.inputDrvs[*j] = singleton<StringSet>("out");
             }
         }
-        
-        debug(format("derivation uses `%1%'") % path);
+
+        /* See prim_unsafeDiscardOutputDependency. */
+        bool useDrvAsSrc = false;
+        if (path.at(0) == '~') {
+            path = string(path, 1);
+            useDrvAsSrc = true;
+        }
+
         assert(isStorePath(path));
-        if (isDerivation(path))
+
+        debug(format("derivation uses `%1%'") % path);
+        if (!useDrvAsSrc && isDerivation(path))
             drv.inputDrvs[path] = singleton<StringSet>("out");
         else
             drv.inputSrcs.insert(path);
@@ -795,6 +825,70 @@ static Expr prim_isAttrs(EvalState & state, const ATermVector & args)
 }
 
 
+/* Return the right-biased intersection of two attribute sets as1 and
+   as2, i.e. a set that contains every attribute from as2 that is also
+   a member of as1. */
+static Expr prim_intersectAttrs(EvalState & state, const ATermVector & args)
+{
+    ATermMap as1, as2;
+    queryAllAttrs(evalExpr(state, args[0]), as1, true);
+    queryAllAttrs(evalExpr(state, args[1]), as2, true);
+
+    ATermMap res;
+    foreach (ATermMap::const_iterator, i, as2)
+        if (as1[i->key]) res.set(i->key, i->value);
+
+    return makeAttrs(res);
+}
+
+
+static void attrsInPattern(ATermMap & map, Pattern pat)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    if (matchAttrsPat(pat, formals, ellipsis)) { 
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm def;
+            if (!matchFormal(*i, name, def)) abort();
+            map.set(name, makeAttrRHS(makeBool(def != constNoDefaultValue), makeNoPos()));
+        }
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        attrsInPattern(map, pat1);
+        attrsInPattern(map, pat2);
+    }
+}
+
+
+/* Return a set containing the names of the formal arguments expected
+   by the function `f'.  The value of each attribute is a Boolean
+   denoting whether has a default value.  For instance,
+
+      functionArgs ({ x, y ? 123}: ...)
+   => { x = false; y = true; }
+
+   "Formal argument" here refers to the attributes pattern-matched by
+   the function.  Plain lambdas are not included, e.g.
+
+      functionArgs (x: ...)
+   => { }
+*/
+static Expr prim_functionArgs(EvalState & state, const ATermVector & args)
+{
+    Expr f = evalExpr(state, args[0]);
+    ATerm pat, body, pos;
+    if (!matchFunction(f, pat, body, pos))
+        throw TypeError("`functionArgs' required a function");
+    
+    ATermMap as;
+    attrsInPattern(as, pat);
+
+    return makeAttrs(as);
+}
+
+
 /*************************************************************
  * Lists
  *************************************************************/
@@ -946,6 +1040,28 @@ static Expr prim_unsafeDiscardStringContext(EvalState & state, const ATermVector
 }
 
 
+/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
+   builder without causing the derivation to be built (for instance,
+   in the derivation that builds NARs in nix-push, when doing
+   source-only deployment).  This primop marks the string context so
+   that builtins.derivation adds the path to drv.inputSrcs rather than
+   drv.inputDrvs. */
+static Expr prim_unsafeDiscardOutputDependency(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    string s = coerceToString(state, args[0], context);
+
+    PathSet context2;
+    foreach (PathSet::iterator, i, context) {
+        Path p = *i;
+        if (p.at(0) == '=') p = "~" + string(p, 1);
+        context2.insert(p);
+    }
+    
+    return makeStr(s, context2);
+}
+
+
 /* Expression serialization/deserialization */
 
 
@@ -1020,6 +1136,7 @@ void EvalState::addPrimOps()
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("throw", 1, prim_throw);
     addPrimOp("__addErrorContext", 2, prim_addErrorContext);
+    addPrimOp("__tryEval", 1, prim_tryEval);
     addPrimOp("__getEnv", 1, prim_getEnv);
     addPrimOp("__trace", 2, prim_trace);
 
@@ -1052,6 +1169,8 @@ void EvalState::addPrimOps()
     addPrimOp("__isAttrs", 1, prim_isAttrs);
     addPrimOp("removeAttrs", 2, prim_removeAttrs);
     addPrimOp("__listToAttrs", 1, prim_listToAttrs);
+    addPrimOp("__intersectAttrs", 2, prim_intersectAttrs);
+    addPrimOp("__functionArgs", 1, prim_functionArgs);
 
     // Lists
     addPrimOp("__isList", 1, prim_isList);
@@ -1072,6 +1191,7 @@ void EvalState::addPrimOps()
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
+    addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
 
     // Versions
     addPrimOp("__parseDrvName", 1, prim_parseDrvName);

src/libmain/shared.cc

@@ -87,18 +87,6 @@ static void setLogType(string lt)
 }
 
 
-unsigned long long getIntArg(const string & opt,
-    Strings::iterator & i, const Strings::iterator & end)
-{
-    ++i;
-    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
-    long long n;
-    if (!string2Int(*i, n) || n < 0)
-        throw UsageError(format("`%1%' requires a non-negative integer") % opt);
-    return n;
-}
-
-
 void initDerivationsHelpers();
 
 
@@ -155,23 +143,29 @@ static void initAndRun(int argc, char * * argv)
     maxSilentTime = queryIntSetting("build-max-silent-time", 0);
 
     /* Catch SIGINT. */
-    struct sigaction act, oact;
+    struct sigaction act;
     act.sa_handler = sigintHandler;
     sigfillset(&act.sa_mask);
     act.sa_flags = 0;
-    if (sigaction(SIGINT, &act, &oact))
+    if (sigaction(SIGINT, &act, 0))
         throw SysError("installing handler for SIGINT");
-    if (sigaction(SIGTERM, &act, &oact))
+    if (sigaction(SIGTERM, &act, 0))
         throw SysError("installing handler for SIGTERM");
-    if (sigaction(SIGHUP, &act, &oact))
+    if (sigaction(SIGHUP, &act, 0))
         throw SysError("installing handler for SIGHUP");
 
     /* Ignore SIGPIPE. */
     act.sa_handler = SIG_IGN;
     act.sa_flags = 0;
-    if (sigaction(SIGPIPE, &act, &oact))
+    if (sigaction(SIGPIPE, &act, 0))
         throw SysError("ignoring SIGPIPE");
 
+    /* Reset SIGCHLD to its default. */
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    if (sigaction(SIGCHLD, &act, 0))
+        throw SysError("resetting SIGCHLD");
+
     /* There is no privacy in the Nix system ;-)  At least not for
        now.  In particular, store objects should be readable by
        everybody.  This prevents nasty surprises when using a shared
@@ -195,7 +189,7 @@ static void initAndRun(int argc, char * * argv)
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
         if (string(arg, 0, 4) == "-at-") ;
-        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-') {
+        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && !isdigit(arg[1])) {
             for (unsigned int j = 1; j < arg.length(); j++)
                 if (isalpha(arg[j]))
                     remaining.push_back((string) "-" + arg[j]);
@@ -239,11 +233,11 @@ static void initAndRun(int argc, char * * argv)
         else if (arg == "--fallback")
             tryFallback = true;
         else if (arg == "--max-jobs" || arg == "-j")
-            maxBuildJobs = getIntArg(arg, i, args.end());
+            maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end());
         else if (arg == "--readonly-mode")
             readOnlyMode = true;
         else if (arg == "--max-silent-time")
-            maxSilentTime = getIntArg(arg, i, args.end());
+            maxSilentTime = getIntArg<unsigned int>(arg, i, args.end());
         else if (arg == "--no-build-hook")
             useBuildHook = false;
         else if (arg == "--show-trace")

src/libmain/shared.hh

@@ -22,22 +22,30 @@ extern std::string programId;
 
 namespace nix {
 
+MakeError(UsageError, nix::Error);
+
 /* Ugh.  No better place to put this. */
 Path makeRootName(const Path & gcRoot, int & counter);
 void printGCWarning();
 
 void printMissing(const PathSet & paths);
 
-unsigned long long getIntArg(const string & opt,
-    Strings::iterator & i, const Strings::iterator & end);
+template<class N> N getIntArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end)
+{
+    ++i;
+    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
+    N n;
+    if (!string2Int(*i, n))
+        throw UsageError(format("`%1%' requires an integer argument") % opt);
+    return n;
+}
 
 /* Whether we're running setuid. */
 extern bool setuidMode;
 
 extern volatile ::sig_atomic_t blockInt;
 
-MakeError(UsageError, nix::Error);
-
 struct RemoveTempRoots 
 {
     ~RemoveTempRoots();    

src/libstore/Makefile.am

@@ -2,22 +2,22 @@ pkglib_LTLIBRARIES = libstore.la
 
 libstore_la_SOURCES = \
   store-api.cc local-store.cc remote-store.cc derivations.cc build.cc misc.cc \
-  globals.cc db.cc references.cc pathlocks.cc gc.cc upgrade-schema.cc \
+  globals.cc references.cc pathlocks.cc gc.cc \
   optimise-store.cc
 
 pkginclude_HEADERS = \
   store-api.hh local-store.hh remote-store.hh derivations.hh misc.hh \
-  globals.hh db.hh references.hh pathlocks.hh \
+  globals.hh references.hh pathlocks.hh \
   worker-protocol.hh
 
-libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la
+libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
 
 BUILT_SOURCES = derivations-ast.cc derivations-ast.hh
 
 EXTRA_DIST = derivations-ast.def derivations-ast.cc
 
 AM_CXXFLAGS = -Wall \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} -I$(srcdir)/../libutil
+ -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil
 
 derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
 	$(perl) $(srcdir)/../aterm-helper.pl derivations-ast.hh derivations-ast.cc < $(srcdir)/derivations-ast.def

src/libstore/build.cc

@@ -1,9 +1,12 @@
+#include "config.h"
+
 #include "references.hh"
 #include "pathlocks.hh"
 #include "misc.hh"
 #include "globals.hh"
 #include "local-store.hh"
 #include "util.hh"
+#include "archive.hh"
 
 #include <map>
 #include <iostream>
@@ -15,19 +18,19 @@
 
 #include <time.h>
 #include <sys/time.h>
+#include <sys/wait.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
+#include <stdio.h>
 
 #include <pwd.h>
 #include <grp.h>
 
 
 /* Includes required for chroot support. */
-#include "config.h"
-
 #if HAVE_SYS_PARAM_H
 #include <sys/param.h>
 #endif
@@ -159,6 +162,7 @@ struct Child
 {
     WeakGoalPtr goal;
     set<int> fds;
+    bool monitorForSilence;
     bool inBuildSlot;
     time_t lastOutput; /* time we last got output on stdout/stderr */
 };
@@ -231,7 +235,7 @@ public:
     /* Registers a running child process.  `inBuildSlot' means that
        the process counts towards the jobs limit. */
     void childStarted(GoalPtr goal, pid_t pid,
-        const set<int> & fds, bool inBuildSlot);
+        const set<int> & fds, bool inBuildSlot, bool monitorForSilence);
 
     /* Unregisters a running child process.  `wakeSleepers' should be
        false if there is no sense in waking up goals that are sleeping
@@ -466,6 +470,8 @@ void UserLock::acquire()
         if (!pw)
             throw Error(format("the user `%1%' in the group `%2%' does not exist")
                 % *i % buildUsersGroup);
+
+        createDirs(nixStateDir + "/userpool");
         
         fnUserLock = (format("%1%/userpool/%2%") % nixStateDir % pw->pw_uid).str();
 
@@ -791,6 +797,9 @@ void DerivationGoal::init()
 {
     trace("init");
 
+    if (readOnlyMode)
+        throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
+
     /* The first thing to do is to make sure that the derivation
        exists.  If it doesn't, it may be created through a
        substitute. */
@@ -1254,7 +1263,7 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
     pid.setKillSignal(SIGTERM);
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(),
-        pid, singleton<set<int> >(logPipe.readSide), false);
+        pid, singleton<set<int> >(logPipe.readSide), false, false);
 
     toHook.readSide.close();
 
@@ -1310,18 +1319,18 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
         string s;
         foreach (PathSet::iterator, i, allInputs) s += *i + "\n";
         
-        writeStringToFile(inputListFN, s);
+        writeFile(inputListFN, s);
 
         /* The `outputs' file lists all outputs that have to be copied
            from the remote system. */
         s = "";
         foreach (DerivationOutputs::iterator, i, drv.outputs)
             s += i->second.path + "\n";
-        writeStringToFile(outputListFN, s);
+        writeFile(outputListFN, s);
 
         /* The `references' file has exactly the format accepted by
            `nix-store --register-validity'. */
-        writeStringToFile(referencesFN,
+        writeFile(referencesFN,
             makeValidityRegistration(allInputs, true, false));
 
         /* Tell the hook to proceed. */
@@ -1485,7 +1494,7 @@ void DerivationGoal::startBuilder()
         }
 
         /* Write closure info to `fileName'. */
-        writeStringToFile(tmpDir + "/" + fileName,
+        writeFile(tmpDir + "/" + fileName,
             makeValidityRegistration(paths, false, false));
     }
 
@@ -1562,7 +1571,7 @@ void DerivationGoal::startBuilder()
            support Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
 
-        writeStringToFile(chrootRootDir + "/etc/passwd",
+        writeFile(chrootRootDir + "/etc/passwd",
             (format(
                 "nixbld:x:%1%:65534:Nix build user:/:/noshell\n"
                 "nobody:x:65534:65534:Nobody:/:/noshell\n")
@@ -1600,8 +1609,18 @@ void DerivationGoal::startBuilder()
                 dirsInChroot.insert(*i);
             else {
                 Path p = chrootRootDir + *i;
-                if (link(i->c_str(), p.c_str()) == -1)
-                    throw SysError(format("linking `%1%' to `%2%'") % p % *i);
+                if (link(i->c_str(), p.c_str()) == -1) {
+                    /* Hard-linking fails if we exceed the maximum
+                       link count on a file (e.g. 32000 of ext3),
+                       which is quite possible after a `nix-store
+                       --optimise'.  Make a copy instead. */
+                    if (errno != EMLINK)
+                        throw SysError(format("linking `%1%' to `%2%'") % p % *i);
+                    StringSink sink;
+                    dumpPath(*i, sink);
+                    StringSource source(sink.s);
+                    restorePath(p, source);
+                }
             }
         }
         
@@ -1749,7 +1768,7 @@ void DerivationGoal::startBuilder()
     pid.setSeparatePG(true);
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(), pid,
-        singleton<set<int> >(logPipe.readSide), true);
+        singleton<set<int> >(logPipe.readSide), true, true);
 
     if (printBuildTrace) {
         printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
@@ -2107,6 +2126,9 @@ void SubstitutionGoal::init()
         return;
     }
 
+    if (readOnlyMode)
+        throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
+
     subs = substituters;
     
     tryNext();
@@ -2253,7 +2275,7 @@ void SubstitutionGoal::tryToRun()
     pid.setKillSignal(SIGTERM);
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(),
-        pid, singleton<set<int> >(logPipe.readSide), true);
+        pid, singleton<set<int> >(logPipe.readSide), true, true);
 
     state = &SubstitutionGoal::finished;
 
@@ -2453,13 +2475,15 @@ unsigned Worker::getNrLocalBuilds()
 
 
 void Worker::childStarted(GoalPtr goal,
-    pid_t pid, const set<int> & fds, bool inBuildSlot)
+    pid_t pid, const set<int> & fds, bool inBuildSlot,
+    bool monitorForSilence)
 {
     Child child;
     child.goal = goal;
     child.fds = fds;
     child.lastOutput = time(0);
     child.inBuildSlot = inBuildSlot;
+    child.monitorForSilence = monitorForSilence;
     children[pid] = child;
     if (inBuildSlot) nrLocalBuilds++;
 }
@@ -2580,12 +2604,16 @@ void Worker::waitForInput()
     if (maxSilentTime != 0) {
         time_t oldest = 0;
         foreach (Children::iterator, i, children) {
-            oldest = oldest == 0 || i->second.lastOutput < oldest
-                ? i->second.lastOutput : oldest;
+            if (i->second.monitorForSilence) {
+                oldest = oldest == 0 || i->second.lastOutput < oldest
+                    ? i->second.lastOutput : oldest;
+            }
+        }
+        if (oldest) {
+            useTimeout = true;
+            timeout.tv_sec = std::max((time_t) 0, oldest + maxSilentTime - before);
+            printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
         }
-        useTimeout = true;
-        timeout.tv_sec = std::max((time_t) 0, oldest + maxSilentTime - before);
-        printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
     }
 
     /* If we are polling goals that are waiting for a lock, then wake
@@ -2660,6 +2688,7 @@ void Worker::waitForInput()
         }
 
         if (maxSilentTime != 0 &&
+            j->second.monitorForSilence &&
             after - j->second.lastOutput >= (time_t) maxSilentTime)
         {
             printMsg(lvlError,

src/libstore/db.cc

@@ -1,474 +0,0 @@
-#include "config.h"
-
-#ifdef OLD_DB_COMPAT
-
-#include "db.hh"
-#include "util.hh"
-#include "pathlocks.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-
-#include <memory>
-
-#include <db_cxx.h>
-
-
-namespace nix {
-
-
-/* Wrapper class to ensure proper destruction. */
-class DestroyDbc 
-{
-    Dbc * dbc;
-public:
-    DestroyDbc(Dbc * _dbc) : dbc(_dbc) { }
-    ~DestroyDbc() { dbc->close(); /* close() frees dbc */ }
-};
-
-
-class DestroyDbEnv
-{
-    DbEnv * dbenv;
-public:
-    DestroyDbEnv(DbEnv * _dbenv) : dbenv(_dbenv) { }
-    ~DestroyDbEnv() {
-        if (dbenv) {
-            if (dbenv->get_DB_ENV()) dbenv->close(0);
-            delete dbenv;
-        }
-    }
-    void release() { dbenv = 0; };
-};
-
-
-static void rethrow(DbException & e)
-{
-    throw Error(e.what());
-}
-
-
-Transaction::Transaction()
-    : txn(0)
-{
-}
-
-
-Transaction::Transaction(Database & db)
-    : txn(0)
-{
-    begin(db);
-}
-
-
-Transaction::~Transaction()
-{
-    if (txn) abort();
-}
-
-
-void Transaction::begin(Database & db)
-{
-    assert(txn == 0);
-    db.requireEnv();
-    try {
-        db.env->txn_begin(0, &txn, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::commit()
-{
-    if (!txn) throw Error("commit called on null transaction");
-    debug(format("committing transaction %1%") % (void *) txn);
-    DbTxn * txn2 = txn;
-    txn = 0;
-    try {
-        txn2->commit(0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::abort()
-{
-    if (!txn) throw Error("abort called on null transaction");
-    debug(format("aborting transaction %1%") % (void *) txn);
-    DbTxn * txn2 = txn;
-    txn = 0;
-    try {
-        txn2->abort();
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::moveTo(Transaction & t)
-{
-    if (t.txn) throw Error("target txn already exists");
-    t.txn = txn;
-    txn = 0;
-}
-
-
-void Database::requireEnv()
-{
-    checkInterrupt();
-    if (!env) throw Error("database environment is not open "
-        "(maybe you don't have sufficient permission?)");
-}
-
-
-Db * Database::getDb(TableId table)
-{
-    if (table == 0)
-        throw Error("database table is not open "
-            "(maybe you don't have sufficient permission?)");
-    std::map<TableId, Db *>::iterator i = tables.find(table);
-    if (i == tables.end())
-        throw Error("unknown table id");
-    return i->second;
-}
-
-
-Database::Database()
-    : env(0)
-    , nextId(1)
-{
-}
-
-
-Database::~Database()
-{
-    close();
-}
-
-
-void openEnv(DbEnv * & env, const string & path, u_int32_t flags)
-{
-    try {
-        createDirs(path);
-    } catch (SysError & e) {
-        if (e.errNo == EPERM || e.errNo == EACCES)
-            throw DbNoPermission(format("cannot create the Nix database in `%1%'") % path);
-        else
-            throw;
-    }
-        
-    try {
-        env->open(path.c_str(),
-            DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN |
-            DB_CREATE | flags,
-            0666);
-    } catch (DbException & e) {
-        printMsg(lvlError, format("environment open failed: %1%") % e.what());
-        throw;
-    }
-}
-
-
-static int my_fsync(int fd)
-{
-    return 0;
-}
-
-
-static void errorPrinter(const DbEnv * env, const char * errpfx, const char * msg)
-{
-    printMsg(lvlError, format("Berkeley DB error: %1%") % msg);
-}
-
-
-static void messagePrinter(const DbEnv * env, const char * msg)
-{
-    printMsg(lvlError, format("Berkeley DB message: %1%") % msg);
-}
-
-
-void Database::open2(const string & path, bool removeOldEnv)
-{
-    if (env) throw Error(format("environment already open"));
-
-    debug(format("opening database environment"));
-
-
-    /* Create the database environment object. */
-    DbEnv * env = new DbEnv(0);
-    DestroyDbEnv deleteEnv(env);
-
-    env->set_errcall(errorPrinter);
-    env->set_msgcall(messagePrinter);
-    if (getEnv("NIX_DEBUG_DB_REGISTER") == "1")
-        env->set_verbose(DB_VERB_REGISTER, 1);
-    env->set_verbose(DB_VERB_RECOVERY, 1);
-    
-    /* Smaller log files. */
-    env->set_lg_bsize(32 * 1024); /* default */
-    env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */
-    
-    /* Write the log, but don't sync.  This protects transactions
-       against application crashes, but if the system crashes, some
-       transactions may be undone.  An acceptable risk, I think. */
-    env->set_flags(DB_TXN_WRITE_NOSYNC | DB_LOG_AUTOREMOVE, 1);
-    
-    /* Increase the locking limits.  If you ever get `Dbc::get: Cannot
-       allocate memory' or similar, especially while running
-       `nix-store --verify', just increase the following number, then
-       run db_recover on the database to remove the existing DB
-       environment (since changes only take effect on new
-       environments). */
-    env->set_lk_max_locks(10000);
-    env->set_lk_max_lockers(10000);
-    env->set_lk_max_objects(10000);
-    env->set_lk_detect(DB_LOCK_DEFAULT);
-    
-    /* Dangerous, probably, but from the docs it *seems* that BDB
-       shouldn't sync when DB_TXN_WRITE_NOSYNC is used, but it still
-       fsync()s sometimes. */
-    db_env_set_func_fsync(my_fsync);
-
-    
-    if (removeOldEnv) {
-        printMsg(lvlError, "removing old Berkeley DB database environment...");
-        env->remove(path.c_str(), DB_FORCE);
-        return;
-    }
-
-    openEnv(env, path, DB_REGISTER | DB_RECOVER);
-
-    deleteEnv.release();
-    this->env = env;
-}
-
-
-void Database::open(const string & path)
-{
-    try {
-
-        open2(path, false);
-        
-    } catch (DbException e) {
-        
-        if (e.get_errno() == DB_VERSION_MISMATCH) {
-            /* Remove the environment while we are holding the global
-               lock.  If things go wrong there, we bail out.
-               !!! argh, we abolished the global lock :-( */
-            open2(path, true);
-
-            /* Try again. */
-            open2(path, false);
-
-            /* Force a checkpoint, as per the BDB docs. */
-            env->txn_checkpoint(DB_FORCE, 0, 0);
-
-            printMsg(lvlError, "database succesfully upgraded to new version");
-        }
-
-#if 0        
-        else if (e.get_errno() == DB_RUNRECOVERY) {
-            /* If recovery is needed, do it. */
-            printMsg(lvlError, "running recovery...");
-            open2(path, false, true);
-        }
-#endif        
-        
-        else
-            rethrow(e);
-    }
-}
-
-
-void Database::close()
-{
-    if (!env) return;
-
-    /* Close the database environment. */
-    debug(format("closing database environment"));
-
-    try {
-
-        for (std::map<TableId, Db *>::iterator i = tables.begin();
-             i != tables.end(); )
-        {
-            std::map<TableId, Db *>::iterator j = i;
-            ++j;
-            closeTable(i->first);
-            i = j;
-        }
-
-        /* Do a checkpoint every 128 kilobytes, or every 5 minutes. */
-        env->txn_checkpoint(128, 5, 0);
-        
-        env->close(0);
-
-    } catch (DbException e) { rethrow(e); }
-
-    delete env;
-
-    env = 0;
-}
-
-
-TableId Database::openTable(const string & tableName, bool sorted)
-{
-    requireEnv();
-    TableId table = nextId++;
-
-    try {
-
-        Db * db = new Db(env, 0);
-
-        try {
-            db->open(0, tableName.c_str(), 0, 
-                sorted ? DB_BTREE : DB_HASH,
-                DB_CREATE | DB_AUTO_COMMIT, 0666);
-        } catch (...) {
-            delete db;
-            throw;
-        }
-
-        tables[table] = db;
-
-    } catch (DbException e) { rethrow(e); }
-
-    return table;
-}
-
-
-void Database::closeTable(TableId table)
-{
-    try {
-        Db * db = getDb(table);
-        db->close(DB_NOSYNC);
-        delete db;
-        tables.erase(table);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::deleteTable(const string & table)
-{
-    try {
-        env->dbremove(0, table.c_str(), 0, DB_AUTO_COMMIT);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-bool Database::queryString(const Transaction & txn, TableId table, 
-    const string & key, string & data)
-{
-    checkInterrupt();
-
-    try {
-        Db * db = getDb(table);
-
-        Dbt kt((void *) key.c_str(), key.length());
-        Dbt dt;
-
-        int err = db->get(txn.txn, &kt, &dt, 0);
-        if (err) return false;
-
-        if (!dt.get_data())
-            data = "";
-        else
-            data = string((char *) dt.get_data(), dt.get_size());
-    
-    } catch (DbException e) { rethrow(e); }
-
-    return true;
-}
-
-
-bool Database::queryStrings(const Transaction & txn, TableId table, 
-    const string & key, Strings & data)
-{
-    string d;
-    if (!queryString(txn, table, key, d))
-        return false;
-    data = unpackStrings(d);
-    return true;
-}
-
-
-void Database::setString(const Transaction & txn, TableId table,
-    const string & key, const string & data)
-{
-    checkInterrupt();
-    try {
-        Db * db = getDb(table);
-        Dbt kt((void *) key.c_str(), key.length());
-        Dbt dt((void *) data.c_str(), data.length());
-        db->put(txn.txn, &kt, &dt, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::setStrings(const Transaction & txn, TableId table,
-    const string & key, const Strings & data, bool deleteEmpty)
-{
-    if (deleteEmpty && data.size() == 0)
-        delPair(txn, table, key);
-    else
-        setString(txn, table, key, packStrings(data));
-}
-
-
-void Database::delPair(const Transaction & txn, TableId table,
-    const string & key)
-{
-    checkInterrupt();
-    try {
-        Db * db = getDb(table);
-        Dbt kt((void *) key.c_str(), key.length());
-        db->del(txn.txn, &kt, 0);
-        /* Non-existence of a pair with the given key is not an
-           error. */
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::enumTable(const Transaction & txn, TableId table,
-    Strings & keys, const string & keyPrefix)
-{
-    try {
-        Db * db = getDb(table);
-
-        Dbc * dbc;
-        db->cursor(txn.txn, &dbc, 0);
-        DestroyDbc destroyDbc(dbc);
-
-        Dbt kt, dt;
-        u_int32_t flags = DB_NEXT;
-
-        if (!keyPrefix.empty()) {
-            flags = DB_SET_RANGE;
-            kt = Dbt((void *) keyPrefix.c_str(), keyPrefix.size());
-        }
-
-        while (dbc->get(&kt, &dt, flags) != DB_NOTFOUND) {
-            checkInterrupt();
-            string data((char *) kt.get_data(), kt.get_size());
-            if (!keyPrefix.empty() &&
-                string(data, 0, keyPrefix.size()) != keyPrefix)
-                break;
-            keys.push_back(data);
-            flags = DB_NEXT;
-        }
-
-    } catch (DbException e) { rethrow(e); }
-}
-
- 
-void Database::clearTable(const Transaction & txn, TableId table)
-{
-    try {
-        Db * db = getDb(table);
-        u_int32_t count;
-        db->truncate(txn.txn, &count, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-}
-
-#endif

src/libstore/db.hh

@@ -1,107 +0,0 @@
-#ifndef __DB_H
-#define __DB_H
-
-#include "types.hh"
-
-#include <map>
-
-
-/* Defined externally. */
-class DbTxn;
-class DbEnv;
-class Db;
-
-
-namespace nix {
-
-
-class Database;
-
-
-class Transaction
-{
-    friend class Database;
-
-private:
-    DbTxn * txn;
-    
-public:
-    Transaction();
-    Transaction(Database & _db);
-    ~Transaction();
-
-    void begin(Database & db);
-    void abort();
-    void commit();
-
-    void moveTo(Transaction & t);
-};
-
-
-#define noTxn Transaction()
-
-
-typedef unsigned int TableId; /* table handles */
-
-
-class Database
-{
-    friend class Transaction;
-
-private:
-    DbEnv * env;
-
-    TableId nextId;
-    std::map<TableId, Db *> tables;
-
-    void requireEnv();
-
-    Db * getDb(TableId table);
-
-    void open2(const string & path, bool removeOldEnv);
-    
-public:
-    Database();
-    ~Database();
-    
-    void open(const string & path);
-    void close();
-
-    TableId openTable(const string & table, bool sorted = false);
-    void closeTable(TableId table);
-    void deleteTable(const string & table);
-
-    bool queryString(const Transaction & txn, TableId table, 
-        const string & key, string & data);
-
-    bool queryStrings(const Transaction & txn, TableId table, 
-        const string & key, Strings & data);
-
-    void setString(const Transaction & txn, TableId table,
-        const string & key, const string & data);
-
-    void setStrings(const Transaction & txn, TableId table,
-        const string & key, const Strings & data,
-        bool deleteEmpty = true);
-
-    void delPair(const Transaction & txn, TableId table,
-        const string & key);
-
-    void enumTable(const Transaction & txn, TableId table,
-        Strings & keys, const string & keyPrefix = "");
-
-    void clearTable(const Transaction & txn, TableId table);
-};
-
-
-class DbNoPermission : public Error
-{
-public:
-    DbNoPermission(const format & f) : Error(f) { };
-};
-
- 
-}
-
-
-#endif /* !__DB_H */

src/libstore/derivations.hh

@@ -1,7 +1,7 @@
 #ifndef __DERIVATIONS_H
 #define __DERIVATIONS_H
 
-typedef struct _ATerm * ATerm;
+#include <aterm1.h>
 
 #include "hash.hh"
 

src/libstore/gc.cc

@@ -7,6 +7,7 @@
 
 #include <functional>
 #include <queue>
+#include <algorithm>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -14,11 +15,6 @@
 #include <fcntl.h>
 #include <unistd.h>
 
-#ifdef __CYGWIN__
-#include <windows.h>
-#include <sys/cygwin.h>
-#endif
-
 
 namespace nix {
 
@@ -178,15 +174,6 @@ void LocalStore::addTempRoot(const Path & path)
 
             fdGCLock.close();
       
-	    /* Note that on Cygwin a lot of the following complexity
-	       is unnecessary, since we cannot delete open lock
-	       files.  If we have the lock file open, then it's valid;
-	       if we can delete it, then it wasn't in use any more. 
-
-	       Also note that on Cygwin we cannot "upgrade" a lock
-	       from a read lock to a write lock. */
-
-#ifndef __CYGWIN__
             debug(format("acquiring read lock on `%1%'") % fnTempRoots);
             lockFile(fdTempRoots, ltRead, true);
 
@@ -200,10 +187,6 @@ void LocalStore::addTempRoot(const Path & path)
             /* The garbage collector deleted this file before we could
                get a lock.  (It won't delete the file after we get a
                lock.)  Try again. */
-
-#else
-            break;
-#endif
         }
 
     }
@@ -216,14 +199,9 @@ void LocalStore::addTempRoot(const Path & path)
     string s = path + '\0';
     writeFull(fdTempRoots, (const unsigned char *) s.c_str(), s.size());
 
-#ifndef __CYGWIN__
     /* Downgrade to a read lock. */
     debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
     lockFile(fdTempRoots, ltRead, true);
-#else
-    debug(format("releasing write lock on `%1%'") % fnTempRoots);
-    lockFile(fdTempRoots, ltNone, true);
-#endif
 }
 
 
@@ -251,19 +229,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
         Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
 
         debug(format("reading temporary root file `%1%'") % path);
-
-#ifdef __CYGWIN__
-	/* On Cygwin we just try to delete the lock file. */
-	char win32Path[MAX_PATH];
-	cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
-	if (DeleteFile(win32Path)) {
-            printMsg(lvlError, format("removed stale temporary roots file `%1%'")
-                % path);
-            continue;
-        } else
-            debug(format("delete of `%1%' failed: %2%") % path % GetLastError());
-#endif
-
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
         if (*fd == -1) {
             /* It's okay if the file has disappeared. */
@@ -275,7 +240,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
         //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
         //if (*fd == -1) continue;
 
-#ifndef __CYGWIN__
         /* Try to acquire a write lock without blocking.  This can
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
@@ -286,7 +250,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
             writeFull(*fd, (const unsigned char *) "d", 1);
             continue;
         }
-#endif
 
         /* Acquire a read lock.  This will prevent the owning process
            from upgrading to a write lock, therefore it will block in
@@ -401,7 +364,7 @@ static void addAdditionalRoots(PathSet & roots)
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
             if (roots.find(path) == roots.end() && store->isValidPath(path)) {
-                debug(format("found additional root `%1%'") % path);
+                debug(format("got additional root `%1%'") % path);
                 roots.insert(path);
             }
         }
@@ -439,147 +402,218 @@ Paths topoSortPaths(const PathSet & paths)
 }
 
 
-static time_t lastFileAccessTime(const Path & path)
-{
-    checkInterrupt();
-    
-    struct stat st;
-    if (lstat(path.c_str(), &st) == -1)
-        throw SysError(format("statting `%1%'") % path);
-
-    if (S_ISDIR(st.st_mode)) {
-        time_t last = 0;
-	Strings names = readDirectory(path);
-	foreach (Strings::iterator, i, names) {
-            time_t t = lastFileAccessTime(path + "/" + *i);
-            if (t > last) last = t;
-        }
-        return last;
-    }
-
-    else if (S_ISLNK(st.st_mode)) return 0;
-
-    else return st.st_atime;
-}
-
-
 struct GCLimitReached { };
 
 
-void LocalStore::gcPath(const GCOptions & options, GCResults & results, 
-    const Path & path)
+struct LocalStore::GCState
 {
-    results.paths.insert(path);
+    GCOptions options;
+    GCResults & results;
+    PathSet roots;
+    PathSet tempRoots;
+    PathSet deleted;
+    PathSet live;
+    PathSet busy;
+    bool gcKeepOutputs;
+    bool gcKeepDerivations;
 
-    if (!pathExists(path)) return;
-                
-    /* Okay, it's safe to delete. */
-    unsigned long long bytesFreed, blocksFreed;
-    deleteFromStore(path, bytesFreed, blocksFreed);
-    results.bytesFreed += bytesFreed;
-    results.blocksFreed += blocksFreed;
+    bool drvsIndexed;
+    typedef std::multimap<string, Path> DrvsByName;
+    DrvsByName drvsByName; // derivation paths hashed by name attribute
 
-    if (options.maxFreed && results.bytesFreed > options.maxFreed) {
-        printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % options.maxFreed);
-        throw GCLimitReached();
-    }
-
-    if (options.maxLinks) {
-        struct stat st;
-        if (stat(nixStore.c_str(), &st) == -1)
-            throw SysError(format("statting `%1%'") % nixStore);
-        if (st.st_nlink < options.maxLinks) {
-            printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % options.maxLinks);
-            throw GCLimitReached();
-        }
-    }
-}
-
-
-void LocalStore::gcPathRecursive(const GCOptions & options,
-    GCResults & results, PathSet & done, const Path & path)
-{
-    if (done.find(path) != done.end()) return;
-    done.insert(path);
-
-    startNest(nest, lvlDebug, format("looking at `%1%'") % path);
-        
-    /* Delete all the referrers first.  They must be garbage too,
-       since if they were live, then the current path would also be
-       live.  Note that deleteFromStore() below still makes sure that
-       the referrer set has become empty, just in case.  (However that
-       doesn't guard against deleting top-level paths that are only
-       reachable from GC roots.) */
-    PathSet referrers;
-    if (isValidPath(path))
-        queryReferrers(path, referrers);
-    foreach (PathSet::iterator, i, referrers)
-        if (*i != path) gcPathRecursive(options, results, done, *i);
-
-    printMsg(lvlInfo, format("deleting `%1%'") % path);
-            
-    gcPath(options, results, path);
-}
-
-
-struct CachingAtimeComparator : public std::binary_function<Path, Path, bool> 
-{
-    std::map<Path, time_t> cache;
-
-    time_t lookup(const Path & p)
+    GCState(GCResults & results_) : results(results_), drvsIndexed(false)
     {
-        std::map<Path, time_t>::iterator i = cache.find(p);
-        if (i != cache.end()) return i->second;
-        debug(format("computing atime of `%1%'") % p);
-        cache[p] = lastFileAccessTime(p);
-        assert(cache.find(p) != cache.end());
-        return cache[p];
-    }
-        
-    bool operator () (const Path & p1, const Path & p2)
-    {
-        return lookup(p2) < lookup(p1);
     }
 };
 
 
-static string showTime(const string & format, time_t t)
+static bool doDelete(GCOptions::GCAction action)
 {
-    char s[128];
-    strftime(s, sizeof s, format.c_str(), localtime(&t));
-    return string(s);
+    return action == GCOptions::gcDeleteDead
+        || action == GCOptions::gcDeleteSpecific;
 }
 
 
-static bool isLive(const Path & path, const PathSet & livePaths,
-    const PathSet & tempRoots, const PathSet & tempRootsClosed)
+bool LocalStore::isActiveTempFile(const GCState & state,
+    const Path & path, const string & suffix)
 {
-    if (livePaths.find(path) != livePaths.end() ||
-        tempRootsClosed.find(path) != tempRootsClosed.end()) return true;
+    return hasSuffix(path, suffix)
+        && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
+}
 
-    /* A lock file belonging to a path that we're building right
-       now isn't garbage. */
-    if (hasSuffix(path, ".lock") && tempRoots.find(string(path, 0, path.size() - 5)) != tempRoots.end())
-        return true;
 
-    /* Don't delete .chroot directories for derivations that are
-       currently being built. */
-    if (hasSuffix(path, ".chroot") && tempRoots.find(string(path, 0, path.size() - 7)) != tempRoots.end())
-        return true;
+/* Return all the derivations in the Nix store that have `path' as an
+   output.  This function assumes that derivations have the same name
+   as their outputs. */
+PathSet LocalStore::findDerivers(GCState & state, const Path & path)
+{
+    PathSet derivers;
 
+    Path deriver = queryDeriver(path);
+    if (deriver != "") derivers.insert(deriver);
+
+    if (!state.drvsIndexed) {
+        Paths entries = readDirectory(nixStore);
+        foreach (Paths::iterator, i, entries)
+            if (isDerivation(*i))
+                state.drvsByName.insert(std::pair<string, Path>(
+                        getNameOfStorePath(*i), nixStore + "/" + *i));
+        state.drvsIndexed = true;
+    }
+    
+    string name = getNameOfStorePath(path);
+
+    // Urgh, I should have used Haskell...
+    std::pair<GCState::DrvsByName::iterator, GCState::DrvsByName::iterator> range =
+        state.drvsByName.equal_range(name);
+
+    for (GCState::DrvsByName::iterator i = range.first; i != range.second; ++i)
+        if (isValidPath(i->second)) {
+            Derivation drv = derivationFromPath(i->second);
+            foreach (DerivationOutputs::iterator, j, drv.outputs)
+                if (j->second.path == path) derivers.insert(i->second);
+        }
+
+    return derivers;
+}
+
+    
+bool LocalStore::tryToDelete(GCState & state, const Path & path)
+{
+    if (!pathExists(path)) return true;
+    if (state.deleted.find(path) != state.deleted.end()) return true;
+    if (state.live.find(path) != state.live.end()) return false;
+
+    startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path);
+
+    if (state.roots.find(path) != state.roots.end()) {
+        printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path);
+        goto isLive;
+    }
+
+    if (isValidPath(path)) {
+
+        /* Recursively try to delete the referrers of this path.  If
+           any referrer can't be deleted, then this path can't be
+           deleted either. */
+        PathSet referrers;
+        queryReferrers(path, referrers);
+        foreach (PathSet::iterator, i, referrers)
+            if (*i != path && !tryToDelete(state, *i)) {
+                printMsg(lvlDebug, format("cannot delete `%1%' because it has live referrers") % path);
+                goto isLive;
+            }
+
+        /* If gc-keep-derivations is set and this is a derivation,
+           then don't delete the derivation if any of the outputs are
+           live. */
+        if (state.gcKeepDerivations && isDerivation(path)) {
+            Derivation drv = derivationFromPath(path);
+            foreach (DerivationOutputs::iterator, i, drv.outputs)
+                if (!tryToDelete(state, i->second.path)) {
+                    printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output is alive") % path);
+                    goto isLive;
+                }
+        }
+
+        /* If gc-keep-derivations and gc-keep-outputs are both set,
+           it's possible that the path has already been deleted (due
+           to the recursion below), so bail out. */
+        if (!pathExists(path)) return true;
+
+        /* If gc-keep-outputs is set, then don't delete this path if
+           its deriver is not garbage.  !!! Nix does not reliably
+           store derivers, so we have to look at all derivations to
+           determine which of them derive `path'.  Since this makes
+           the garbage collector very slow to start on large Nix
+           stores, here we just look for all derivations that have the
+           same name as `path' (where the name is the part of the
+           filename after the hash, i.e. the `name' attribute of the
+           derivation).  This is somewhat hacky: currently, the
+           deriver of a path always has the same name as the output,
+           but this might change in the future. */
+        if (state.gcKeepOutputs) {
+            PathSet derivers = findDerivers(state, path);
+            foreach (PathSet::iterator, deriver, derivers) {
+                /* Break an infinite recursion if gc-keep-derivations
+                   and gc-keep-outputs are both set by tentatively
+                   assuming that this path is garbage.  This is a safe
+                   assumption because at this point, the only thing
+                   that can prevent it from being garbage is the
+                   deriver.  Since tryToDelete() works "upwards"
+                   through the dependency graph, it won't encouter
+                   this path except in the call to tryToDelete() in
+                   the gc-keep-derivation branch. */
+                state.deleted.insert(path);
+                if (!tryToDelete(state, *deriver)) {
+                    state.deleted.erase(path);
+                    printMsg(lvlDebug, format("cannot delete `%1%' because its deriver `%2%' is alive") % path % *deriver);
+                    goto isLive;
+                }
+            }
+        }
+    }
+
+    else {
+
+        /* A lock file belonging to a path that we're building right
+           now isn't garbage. */
+        if (isActiveTempFile(state, path, ".lock")) return false;
+
+        /* Don't delete .chroot directories for derivations that are
+           currently being built. */
+        if (isActiveTempFile(state, path, ".chroot")) return false;
+
+    }
+
+    /* The path is garbage, so delete it. */
+    if (doDelete(state.options.action)) {
+        printMsg(lvlInfo, format("deleting `%1%'") % path);
+
+        unsigned long long bytesFreed, blocksFreed;
+        deleteFromStore(path, bytesFreed, blocksFreed);
+        state.results.bytesFreed += bytesFreed;
+        state.results.blocksFreed += blocksFreed;
+
+        if (state.options.maxFreed && state.results.bytesFreed > state.options.maxFreed) {
+            printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % state.options.maxFreed);
+            throw GCLimitReached();
+        }
+
+        if (state.options.maxLinks) {
+            struct stat st;
+            if (stat(nixStore.c_str(), &st) == -1)
+                throw SysError(format("statting `%1%'") % nixStore);
+            if (st.st_nlink < state.options.maxLinks) {
+                printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % state.options.maxLinks);
+                throw GCLimitReached();
+            }
+        }
+        
+    } else
+        printMsg(lvlTalkative, format("would delete `%1%'") % path);
+    
+    state.deleted.insert(path);
+    if (state.options.action != GCOptions::gcReturnLive)
+        state.results.paths.insert(path);
+    return true;
+
+ isLive:
+    state.live.insert(path);
+    if (state.options.action == GCOptions::gcReturnLive)
+        state.results.paths.insert(path);
     return false;
 }
 
 
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    bool gcKeepOutputs =
-        queryBoolSetting("gc-keep-outputs", false);
-    bool gcKeepDerivations =
-        queryBoolSetting("gc-keep-derivations", true);
-    int gcKeepOutputsThreshold = 
-        queryIntSetting ("gc-keep-outputs-threshold", defaultGcLevel);
-
+    GCState state(results);
+    state.options = options;    
+    
+    state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false);
+    state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true);
+    
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
        b) Processes from creating new temporary root files. */
@@ -590,198 +624,60 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printMsg(lvlError, format("finding garbage collector roots..."));
     Roots rootMap = options.ignoreLiveness ? Roots() : nix::findRoots(true);
 
-    PathSet roots;
-    foreach (Roots::iterator, i, rootMap) roots.insert(i->second);
+    foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second);
 
     /* Add additional roots returned by the program specified by the
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
     if (!options.ignoreLiveness)
-        addAdditionalRoots(roots);
-
-    if (options.action == GCOptions::gcReturnRoots) {
-        results.paths = roots;
-        return;
-    }
-
-    /* Determine the live paths which is just the closure of the
-       roots under the `references' relation. */
-    printMsg(lvlError, format("computing live paths..."));
-    PathSet livePaths;
-    foreach (PathSet::const_iterator, i, roots)
-        computeFSClosure(canonPath(*i), livePaths);
-
-    if (gcKeepDerivations) {
-        foreach (PathSet::iterator, i, livePaths) {
-            /* Note that the deriver need not be valid (e.g., if we
-               previously ran the collector with `gcKeepDerivations'
-               turned off). */
-            Path deriver = queryDeriver(*i);
-            if (deriver != "" && isValidPath(deriver))
-                computeFSClosure(deriver, livePaths);
-        }
-    }
-
-    if (gcKeepOutputs) {
-        /* Hmz, identical to storePathRequisites in nix-store. */
-        foreach (PathSet::iterator, i, livePaths)
-            if (isDerivation(*i)) {
-                Derivation drv = derivationFromPath(*i);
-
-		string gcLevelStr = drv.env["__gcLevel"];
-		int gcLevel;
-		if (!string2Int(gcLevelStr, gcLevel))
-		    gcLevel = defaultGcLevel;
-		
-		if (gcLevel >= gcKeepOutputsThreshold)    
-		    foreach (DerivationOutputs::iterator, j, drv.outputs)
-			if (isValidPath(j->second.path))
-			    computeFSClosure(j->second.path, livePaths);
-            }
-    }
-
-    if (options.action == GCOptions::gcReturnLive) {
-        results.paths = livePaths;
-        return;
-    }
+        addAdditionalRoots(state.roots);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
-    PathSet tempRoots;
     FDs fds;
-    readTempRoots(tempRoots, fds);
-
-    /* Close the temporary roots.  Note that we *cannot* do this in
-       readTempRoots(), because there we may not have all locks yet,
-       meaning that an invalid path can become valid (and thus add to
-       the references graph) after we have added it to the closure
-       (and computeFSClosure() assumes that the presence of a path
-       means that it has already been closed). */
-    PathSet tempRootsClosed;
-    foreach (PathSet::iterator, i, tempRoots)
-        if (isValidPath(*i))
-            computeFSClosure(*i, tempRootsClosed);
-        else
-            tempRootsClosed.insert(*i);
+    readTempRoots(state.tempRoots, fds);
+    state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
        increase, since we hold locks on everything.  So everything
-       that is not currently in in `livePaths' or `tempRootsClosed'
-       can be deleted. */
-    
-    /* Read the Nix store directory to find all currently existing
-       paths and filter out all live paths. */
-    printMsg(lvlError, format("reading the Nix store..."));
-    PathSet storePaths;
-    
-    if (options.action != GCOptions::gcDeleteSpecific) {
-        Paths entries = readDirectory(nixStore);
-        foreach (Paths::iterator, i, entries) {
-            Path path = canonPath(nixStore + "/" + *i);
-            if (!isLive(path, livePaths, tempRoots, tempRootsClosed)) storePaths.insert(path);
-        }
-    }
+       that is not reachable from `roots'. */
+
+    /* Now either delete all garbage paths, or just the specified
+       paths (for gcDeleteSpecific). */
+
+    if (options.action == GCOptions::gcDeleteSpecific) {
 
-    else {
         foreach (PathSet::iterator, i, options.pathsToDelete) {
             assertStorePath(*i);
-            storePaths.insert(*i);
-            if (isLive(*i, livePaths, tempRoots, tempRootsClosed))
+            if (!tryToDelete(state, *i))
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
         }
-    }
-
-    if (options.action == GCOptions::gcReturnDead) {
-        results.paths.insert(storePaths.begin(), storePaths.end());
-        return;
-    }
-
-    /* Delete all dead store paths (or until one of the stop
-       conditions is reached). */
-
-    PathSet done;
-    try {
-
-        if (!options.useAtime) {
-            /* Delete the paths, respecting the partial ordering
-               determined by the references graph. */
-            printMsg(lvlError, format("deleting garbage..."));
-            foreach (PathSet::iterator, i, storePaths)
-                gcPathRecursive(options, results, done, *i);
-        }
-
-        else {
-
-            /* Delete in order of ascending last access time, still
-               maintaining the partial ordering of the reference
-               graph.  Note that we can't use a topological sort for
-               this because that takes time O(V+E), and in this case
-               E=O(V^2) (i.e. the graph is dense because of the edges
-               due to the atime ordering).  So instead we put all
-               deletable paths in a priority queue (ordered by atime),
-               and after deleting a path, add additional paths that
-               have become deletable to the priority queue. */
-
-            CachingAtimeComparator atimeComp;
-
-            /* Create a priority queue that orders paths by ascending
-               atime.  This is why C++ needs type inferencing... */
-            std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> > prioQueue =
-                std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> >(binary_function_ref_adapter<CachingAtimeComparator>(&atimeComp));
-
-           /* Initially put the paths that are invalid or have no
-              referrers into the priority queue. */
-            printMsg(lvlError, format("finding deletable paths..."));
-            foreach (PathSet::iterator, i, storePaths) {
-                checkInterrupt();
-                /* We can safely delete a path if it's invalid or
-                   it has no referrers.  Note that all the invalid
-                   paths will be deleted in the first round. */
-                if (isValidPath(*i)) {
-                    if (queryReferrersNoSelf(*i).empty()) prioQueue.push(*i);
-                } else prioQueue.push(*i);
-            }
-
-            debug(format("%1% initially deletable paths") % prioQueue.size());
-
-            /* Now delete everything in the order of the priority
-               queue until nothing is left. */
-            printMsg(lvlError, format("deleting garbage..."));
-            while (!prioQueue.empty()) {
-                checkInterrupt();
-                Path path = prioQueue.top(); prioQueue.pop();
-
-                if (options.maxAtime != (time_t) -1 &&
-                    atimeComp.lookup(path) > options.maxAtime)
-                    continue;
-                
-                printMsg(lvlInfo, format("deleting `%1%' (last accessed %2%)") % path % showTime("%F %H:%M:%S", atimeComp.lookup(path)));
-
-                PathSet references;
-                if (isValidPath(path)) references = queryReferencesNoSelf(path);
-
-                gcPath(options, results, path);
-
-                /* For each reference of the current path, see if the
-                   reference has now become deletable (i.e. is in the
-                   set of dead paths and has no referrers left).  If
-                   so add it to the priority queue. */
-                foreach (PathSet::iterator, i, references) {
-                    if (storePaths.find(*i) != storePaths.end() &&
-                        queryReferrersNoSelf(*i).empty())
-                    {
-                        debug(format("path `%1%' has become deletable") % *i);
-                        prioQueue.push(*i);
-                    }
-                }
-            }
-            
-        }
         
-    } catch (GCLimitReached & e) {
-    }
+    } else {
+        
+        printMsg(lvlError, format("reading the Nix store..."));
+        Paths entries = readDirectory(nixStore);
+
+        /* Randomise the order in which we delete entries to make the
+           collector less biased towards deleting paths that come
+           alphabetically first (e.g. /nix/store/000...).  This
+           matters when using --max-freed etc. */
+        vector<Path> entries_(entries.begin(), entries.end());
+        random_shuffle(entries_.begin(), entries_.end());
+
+        if (doDelete(state.options.action))
+            printMsg(lvlError, format("deleting garbage..."));
+        else
+            printMsg(lvlError, format("determining live/dead paths..."));
+    
+        try {
+            foreach (vector<Path>::iterator, i, entries_)
+                tryToDelete(state, canonPath(nixStore + "/" + *i));
+        } catch (GCLimitReached & e) {
+        }
+    }        
 }
 
 

src/libstore/local-store.cc

@@ -16,6 +16,7 @@
 #include <utime.h>
 #include <fcntl.h>
 #include <errno.h>
+#include <stdio.h>
 
 
 namespace nix {
@@ -47,8 +48,21 @@ LocalStore::LocalStore()
     
     if (readOnlyMode) return;
 
+    /* Create missing state directories if they don't already exist. */
     createDirs(nixStore);
-    
+    createDirs(nixDBPath + "/info");
+    createDirs(nixDBPath + "/referrer");
+    createDirs(nixDBPath + "/failed");
+    Path profilesDir = nixStateDir + "/profiles";
+    createDirs(nixStateDir + "/profiles");
+    createDirs(nixStateDir + "/temproots");
+    Path gcRootsDir = nixStateDir + "/gcroots";
+    if (!pathExists(gcRootsDir)) {
+        createDirs(gcRootsDir);
+        if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1)
+            throw SysError(format("creating symlink to `%1%'") % profilesDir);
+    }
+  
     checkStoreNotSymlink();
 
     try {
@@ -64,11 +78,7 @@ LocalStore::LocalStore()
         printMsg(lvlError, "waiting for the big Nix store lock...");
         lockFile(globalLock, ltRead, true);
     }
-
-    createDirs(nixDBPath + "/info");
-    createDirs(nixDBPath + "/referrer");
-    createDirs(nixDBPath + "/failed");
-
+    
     int curSchema = getSchema();
     if (curSchema > nixSchemaVersion)
         throw Error(format("current Nix store schema is version %1%, but I only support %2%")
@@ -79,6 +89,8 @@ LocalStore::LocalStore()
     }
     if (curSchema == 1) throw Error("your Nix store is no longer supported");
     if (curSchema < nixSchemaVersion) upgradeStore12();
+
+    doFsync = queryBoolSetting("fsync-metadata", false);
 }
 
 
@@ -212,7 +224,7 @@ static Path tmpFileForAtomicUpdate(const Path & path)
 }
 
 
-static void appendReferrer(const Path & from, const Path & to, bool lock)
+void LocalStore::appendReferrer(const Path & from, const Path & to, bool lock)
 {
     Path referrersFile = referrersFileFor(from);
     
@@ -227,6 +239,8 @@ static void appendReferrer(const Path & from, const Path & to, bool lock)
     
     string s = " " + to;
     writeFull(fd, (const unsigned char *) s.c_str(), s.size());
+
+    if (doFsync) fdatasync(fd);
 }
 
 
@@ -257,6 +271,8 @@ void LocalStore::rewriteReferrers(const Path & path, bool purge, PathSet referre
     
     writeFull(fd, (const unsigned char *) s.c_str(), s.size());
 
+    if (doFsync) fdatasync(fd);
+    
     fd.close(); /* for Windows; can't rename open file */
 
     if (rename(tmpFile.c_str(), referrersFile.c_str()) == -1)
@@ -337,7 +353,7 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi
 
     /* Atomically rewrite the info file. */
     Path tmpFile = tmpFileForAtomicUpdate(infoFile);
-    writeFile(tmpFile, s);
+    writeFile(tmpFile, s, doFsync);
     if (rename(tmpFile.c_str(), infoFile.c_str()) == -1)
         throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile);
 
@@ -380,6 +396,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
 
     assertStorePath(path);
 
+    if (!isValidPath(path))
+        throw Error(format("path `%1%' is not valid") % path);
+
     std::map<Path, ValidPathInfo>::iterator lookup = pathInfoCache.find(path);
     if (lookup != pathInfoCache.end()) return lookup->second;
     
@@ -394,7 +413,11 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
 
     foreach (Strings::iterator, i, lines) {
         string::size_type p = i->find(':');
-        if (p == string::npos) continue; /* bad line */
+        if (p == string::npos) {
+            if (!ignoreErrors)
+                throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i);
+            continue; /* bad line */
+        }
         string name(*i, 0, p);
         string value(*i, p + 2);
         if (name == "References") {
@@ -425,7 +448,22 @@ bool LocalStore::isValidPath(const Path & path)
     /* Files in the info directory starting with a `.' are temporary
        files. */
     if (baseNameOf(path).at(0) == '.') return false;
-    return pathExists(infoFileFor(path));
+
+    /* A path is valid if its info file exists and has a non-zero
+       size.  (The non-zero size restriction is to be robust to
+       certain kinds of filesystem corruption, particularly with
+       ext4.) */
+    Path infoFile = infoFileFor(path);
+
+    struct stat st;
+    if (lstat(infoFile.c_str(), &st)) {
+        if (errno == ENOENT) return false;
+        throw SysError(format("getting status of `%1%'") % infoFile);
+    }
+
+    if (st.st_size == 0) return false;
+    
+    return true;
 }
 
 
@@ -509,6 +547,13 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
 
     case 0: /* child */
         try {
+            /* Hack to let "make check" succeed on Darwin.  The
+               libtool wrapper script sets DYLD_LIBRARY_PATH to our
+               libutil (among others), but Perl also depends on a
+               library named libutil.  As a result, substituters
+               written in Perl (i.e. all of them) fail. */
+            unsetenv("DYLD_LIBRARY_PATH");
+            
             fromPipe.readSide.close();
             toPipe.writeSide.close();
             if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
@@ -698,7 +743,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
                 StringSource source(dump);
                 restorePath(dstPath, source);
             } else
-                writeStringToFile(dstPath, dump);
+                writeFile(dstPath, dump);
 
             canonicalisePathMetaData(dstPath);
 
@@ -753,7 +798,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
             if (pathExists(dstPath)) deletePathWrapped(dstPath);
 
-            writeStringToFile(dstPath, s);
+            writeFile(dstPath, s);
 
             canonicalisePathMetaData(dstPath);
             
@@ -832,7 +877,7 @@ void LocalStore::exportPath(const Path & path, bool sign,
         Path tmpDir = createTempDir();
         AutoDelete delTmp(tmpDir);
         Path hashFile = tmpDir + "/hash";
-        writeStringToFile(hashFile, printHash(hash));
+        writeFile(hashFile, printHash(hash));
 
         Path secretKey = nixConfDir + "/signing-key.sec";
         checkSecrecy(secretKey);
@@ -908,7 +953,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 
         if (requireSignature) {
             Path sigFile = tmpDir + "/sig";
-            writeStringToFile(sigFile, signature);
+            writeFile(sigFile, signature);
 
             Strings args;
             args.push_back("rsautl");
@@ -1011,8 +1056,18 @@ void LocalStore::verifyStore(bool checkContents)
         } else if (!pathExists(*i)) {
             printMsg(lvlError, format("path `%1%' disappeared") % *i);
             invalidatePath(*i);
-        } else
-            validPaths.insert(*i);
+        } else {
+            Path infoFile = infoFileFor(*i);
+            struct stat st;
+            if (lstat(infoFile.c_str(), &st))
+                throw SysError(format("getting status of `%1%'") % infoFile);
+            if (st.st_size == 0) {
+                printMsg(lvlError, format("removing corrupt info file `%1%'") % infoFile);
+                if (unlink(infoFile.c_str()) == -1)
+                    throw SysError(format("unlinking `%1%'") % infoFile);
+            }
+            else validPaths.insert(*i);
+        }
     }
 
 
@@ -1116,4 +1171,16 @@ void LocalStore::verifyStore(bool checkContents)
 }
 
 
+/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12).  The
+   old schema uses Berkeley DB, the new one stores store path
+   meta-information in files. */
+void LocalStore::upgradeStore12()
+{
+    throw Error(
+        "Your Nix store has a database in Berkeley DB format,\n"
+        "which is no longer supported. To convert to the new format,\n"
+        "please upgrade Nix to version 0.12 first.");
+}
+
+
 }

src/libstore/local-store.hh

@@ -158,12 +158,17 @@ private:
     /* Store paths for which the referrers file must be purged. */
     PathSet delayedUpdates;
 
+    /* Whether to do an fsync() after writing Nix metadata. */
+    bool doFsync;
+
     int getSchema();
 
     void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false);
 
     ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false);
 
+    void appendReferrer(const Path & from, const Path & to, bool lock);
+    
     void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
 
     void flushDelayedUpdates();
@@ -174,20 +179,20 @@ private:
     
     void upgradeStore12();
 
-    void gcPath(const GCOptions & options, GCResults & results,
-        const Path & path);
-
-    void gcPathRecursive(const GCOptions & options,
-        GCResults & results, PathSet & done, const Path & path);
+    struct GCState;
 
+    bool tryToDelete(GCState & state, const Path & path);
+    
+    PathSet findDerivers(GCState & state, const Path & path);
+    
+    bool isActiveTempFile(const GCState & state,
+        const Path & path, const string & suffix);
+        
     void startSubstituter(const Path & substituter,
         RunningSubstituter & runningSubstituter);
 };
 
 
-/* Copy a path recursively. */
-void copyPath(const Path & src, const Path & dst);
-
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
    - the last modification date on each file is set to 1 (i.e.,

src/libstore/misc.cc

@@ -19,7 +19,7 @@ Derivation derivationFromPath(const Path & drvPath)
 
 
 void computeFSClosure(const Path & storePath,
-    PathSet & paths, bool flipDirection)
+    PathSet & paths, bool flipDirection, bool includeOutputs)
 {
     if (paths.find(storePath) != paths.end()) return;
     paths.insert(storePath);
@@ -30,8 +30,15 @@ void computeFSClosure(const Path & storePath,
     else
         store->queryReferences(storePath, references);
 
+    if (includeOutputs && isDerivation(storePath)) {
+        Derivation drv = derivationFromPath(storePath);
+        foreach (DerivationOutputs::iterator, i, drv.outputs)
+            if (store->isValidPath(i->second.path))
+                computeFSClosure(i->second.path, paths, flipDirection, true);
+    }
+
     foreach (PathSet::iterator, i, references)
-        computeFSClosure(*i, paths, flipDirection);
+        computeFSClosure(*i, paths, flipDirection, includeOutputs);
 }
 
 

src/libstore/misc.hh

@@ -19,7 +19,8 @@ Derivation derivationFromPath(const Path & drvPath);
    `referrers' relation instead of the `references' relation is
    returned. */
 void computeFSClosure(const Path & storePath,
-    PathSet & paths, bool flipDirection = false);
+    PathSet & paths, bool flipDirection = false,
+    bool includeOutputs = false);
 
 /* Return the path corresponding to the output identifier `id' in the
    given derivation. */

src/libstore/optimise-store.cc

@@ -5,6 +5,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <errno.h>
+#include <stdio.h>
 
 
 namespace nix {

src/libstore/pathlocks.cc

@@ -8,11 +8,6 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
-#ifdef __CYGWIN__
-#include <windows.h>
-#include <sys/cygwin.h>
-#endif
-
 
 namespace nix {
 
@@ -21,72 +16,24 @@ int openLockFile(const Path & path, bool create)
 {
     AutoCloseFD fd;
 
-#ifdef __CYGWIN__
-    /* On Cygwin we have to open the lock file without "DELETE"
-       sharing mode; otherwise Windows will allow open lock files to
-       be deleted (which is almost but not quite what Unix does). */
-    char win32Path[MAX_PATH + 1];
-    cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
-
-    SECURITY_ATTRIBUTES sa; /* required, otherwise inexplicably bad shit happens */
-    sa.nLength = sizeof sa;
-    sa.lpSecurityDescriptor = 0;
-    sa.bInheritHandle = TRUE;
-    HANDLE h = CreateFile(win32Path, GENERIC_READ | GENERIC_WRITE,
-        FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, 
-        (create ? OPEN_ALWAYS : OPEN_EXISTING), 
-        FILE_ATTRIBUTE_NORMAL, 0);
-    if (h == INVALID_HANDLE_VALUE) {
-        if (create || GetLastError() != ERROR_FILE_NOT_FOUND)
-            throw Error(format("opening lock file `%1%'") % path);
-        fd = -1;
-    }
-    else
-        fd = cygwin_attach_handle_to_fd((char *) path.c_str(), -1, h, 1, O_RDWR);
-#else        
     fd = open(path.c_str(), O_RDWR | (create ? O_CREAT : 0), 0666);
     if (fd == -1 && (create || errno != ENOENT))
         throw SysError(format("opening lock file `%1%'") % path);
-#endif
 
     return fd.borrow();
 }
 
 
-void deleteLockFilePreClose(const Path & path, int fd)
+void deleteLockFile(const Path & path, int fd)
 {
-#ifndef __CYGWIN__
     /* Get rid of the lock file.  Have to be careful not to introduce
-       races. */
-    /* On Unix, write a (meaningless) token to the file to indicate to
+       races.  Write a (meaningless) token to the file to indicate to
        other processes waiting on this lock that the lock is stale
        (deleted). */
     unlink(path.c_str());
     writeFull(fd, (const unsigned char *) "d", 1);
     /* Note that the result of unlink() is ignored; removing the lock
        file is an optimisation, not a necessity. */
-#endif
-}
-
-
-void deleteLockFilePostClose(const Path & path)
-{
-#ifdef __CYGWIN__
-    /* On Windows, just try to delete the lock file.  This will fail
-       if anybody still has the file open.  We cannot use unlink()
-       here, because Cygwin emulates Unix semantics of allowing an
-       open file to be deleted (but fakes it - the file isn't actually
-       deleted until later, so a file with the same name cannot be
-       created in the meantime). */
-    char win32Path[MAX_PATH + 1];
-    cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
-    if (DeleteFile(win32Path))
-        debug(format("delete of `%1%' succeeded") % path.c_str());
-    else
-        /* Not an error: probably means that the lock is still opened
-           by someone else. */
-        debug(format("delete of `%1%' failed: %2%") % path.c_str() % GetLastError());
-#endif
 }
 
 
@@ -220,15 +167,13 @@ PathLocks::~PathLocks()
 void PathLocks::unlock()
 {
     foreach (list<FDPair>::iterator, i, fds) {
-        if (deletePaths) deleteLockFilePreClose(i->second, i->first);
+        if (deletePaths) deleteLockFile(i->second, i->first);
 
         lockedPaths.erase(i->second);
         if (close(i->first) == -1)
             printMsg(lvlError,
                 format("error (ignored): cannot close lock file on `%1%'") % i->second);
 
-	if (deletePaths) deleteLockFilePostClose(i->second);
-
         debug(format("lock released on `%1%'") % i->second);
     }
 

src/libstore/pathlocks.hh

@@ -12,10 +12,8 @@ namespace nix {
    because it doesn't exist.  Any other error throws an exception. */
 int openLockFile(const Path & path, bool create);
 
-/* Delete an open lock file.  Both must be called to be fully portable
-   between Unix and Windows. */
-void deleteLockFilePreClose(const Path & path, int fd);
-void deleteLockFilePostClose(const Path & path);
+/* Delete an open lock file. */
+void deleteLockFile(const Path & path, int fd);
 
 enum LockType { ltRead, ltWrite, ltNone };
 

src/libstore/remote-store.cc

@@ -426,8 +426,9 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     writeLongLong(options.maxFreed, to);
     writeInt(options.maxLinks, to);
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) {
-        writeInt(options.useAtime, to);
-        writeInt(options.maxAtime, to);
+        /* removed options */
+        writeInt(0, to);
+        writeInt(0, to);
     }
     
     processStderr();

src/libstore/store-api.cc

@@ -1,6 +1,7 @@
 #include "store-api.hh"
 #include "globals.hh"
 #include "util.hh"
+#include "derivations.hh"
 
 #include <limits.h>
 
@@ -14,8 +15,6 @@ GCOptions::GCOptions()
     ignoreLiveness = false;
     maxFreed = 0;
     maxLinks = 0;
-    useAtime = false;
-    maxAtime = (time_t) -1;
 }
 
 
@@ -54,6 +53,18 @@ Path toStorePath(const Path & path)
 }
 
 
+string getNameOfStorePath(const Path & path)
+{
+    Path::size_type slash = path.rfind('/');
+    string p = slash == Path::npos ? path : string(path, slash + 1);
+    Path::size_type dash = p.find('-');
+    assert(dash != Path::npos);
+    string p2 = string(p, dash + 1);
+    if (isDerivation(p2)) p2 = string(p2, 0, p2.size() - 4);
+    return p2;
+}
+
+
 Path followLinksToStore(const Path & _path)
 {
     Path path = absPath(_path);

src/libstore/store-api.hh

@@ -22,10 +22,6 @@ struct GCOptions
 {
     /* Garbage collector operation:
 
-       - `gcReturnRoots': find and return the set of roots for the
-         garbage collector.  These are the store paths symlinked to in
-         the `gcroots' directory.
-
        - `gcReturnLive': return the set of paths reachable from
          (i.e. in the closure of) the roots.
 
@@ -38,7 +34,6 @@ struct GCOptions
           `pathsToDelete', insofar as they are not reachable.
     */
     typedef enum {
-        gcReturnRoots,
         gcReturnLive,
         gcReturnDead,
         gcDeleteDead,
@@ -64,22 +59,6 @@ struct GCOptions
        has dropped below `maxLinks'. */
     unsigned int maxLinks;
 
-    /* Delete paths in order of ascending last access time.  I.e.,
-       prefer deleting unrecently used paths.  Useful in conjunction
-       with `maxFreed' and `maxLinks' (or manual interruption).  The
-       access time of a path is defined as the highest atime of any
-       non-directory, non-symlink file under that path.  Directories
-       and symlinks are ignored because their atimes are frequently
-       mass-updated, e.g. by `locate'.  Note that optimiseStore()
-       somewhat reduces the usefulness of this option: it hard-links
-       regular files and symlink together, giving them a "shared"
-       atime. */
-    bool useAtime;
-
-    /* Do not delete paths newer than `maxAtime'.  -1 means no age
-       limit. */
-    time_t maxAtime;
-
     GCOptions();
 };
 
@@ -264,6 +243,12 @@ void checkStoreName(const string & name);
 Path toStorePath(const Path & path);
 
 
+/* Get the "name" part of a store path, that is, the part after the
+   hash and the dash, and with any ".drv" suffix removed
+   (e.g. /nix/store/<hash>-foo-1.2.3.drv => foo-1.2.3). */
+string getNameOfStorePath(const Path & path);
+
+
 /* Follow symlinks until we end up with a path in the Nix store. */
 Path followLinksToStore(const Path & path);
 

src/libstore/upgrade-schema.cc

@@ -1,108 +0,0 @@
-#include "db.hh"
-#include "hash.hh"
-#include "util.hh"
-#include "local-store.hh"
-#include "globals.hh"
-#include "pathlocks.hh"
-#include "config.h"
-
-#include <iostream>
-
-
-namespace nix {
-
-
-Hash parseHashField(const Path & path, const string & s);
-
-
-/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12).  The
-   old schema uses Berkeley DB, the new one stores store path
-   meta-information in files. */
-void LocalStore::upgradeStore12()
-{
-#if OLD_DB_COMPAT
-    
-#ifdef __CYGWIN__
-    /* Cygwin can't upgrade a read lock to a write lock... */
-    lockFile(globalLock, ltNone, true);
-#endif
-
-    if (!lockFile(globalLock, ltWrite, false)) {
-        printMsg(lvlError, "waiting for exclusive access to the Nix store...");
-        lockFile(globalLock, ltWrite, true);
-    }
-
-    printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
-
-    if (getSchema() >= nixSchemaVersion) return; /* somebody else beat us to it */
-
-    /* Open the old Nix database and tables. */
-    Database nixDB;
-    nixDB.open(nixDBPath);
-    
-    /* dbValidPaths :: Path -> ()
-
-       The existence of a key $p$ indicates that path $p$ is valid
-       (that is, produced by a succesful build). */
-    TableId dbValidPaths = nixDB.openTable("validpaths");
-
-    /* dbReferences :: Path -> [Path]
-
-       This table lists the outgoing file system references for each
-       output path that has been built by a Nix derivation.  These are
-       found by scanning the path for the hash components of input
-       paths. */
-    TableId dbReferences = nixDB.openTable("references");
-
-    /* dbReferrers :: Path -> Path
-
-       This table is just the reverse mapping of dbReferences.  This
-       table can have duplicate keys, each corresponding value
-       denoting a single referrer. */
-    // Not needed for conversion: it's just the inverse of
-    // references.
-    // TableId dbReferrers = nixDB.openTable("referrers");
-
-    /* dbDerivers :: Path -> [Path]
-
-       This table lists the derivation used to build a path.  There
-       can only be multiple such paths for fixed-output derivations
-       (i.e., derivations specifying an expected hash). */
-    TableId dbDerivers = nixDB.openTable("derivers");
-
-    Paths paths;
-    nixDB.enumTable(noTxn, dbValidPaths, paths);
-    
-    foreach (Paths::iterator, i, paths) {
-        ValidPathInfo info;
-        info.path = *i;
-        
-        Paths references;
-        nixDB.queryStrings(noTxn, dbReferences, *i, references);
-        info.references.insert(references.begin(), references.end());
-        
-        string s;
-        nixDB.queryString(noTxn, dbValidPaths, *i, s);
-        info.hash = parseHashField(*i, s);
-        
-        nixDB.queryString(noTxn, dbDerivers, *i, info.deriver);
-        
-        registerValidPath(info, true);
-        std::cerr << ".";
-    }
-
-    std::cerr << std::endl;
-
-    writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
-
-    lockFile(globalLock, ltRead, true);
-
-#else
-    throw Error(
-        "Your Nix store has a database in Berkeley DB format. To convert\n"
-        "to the new format, please compile Nix with Berkeley DB support.");
-#endif
-}
-
-
-}

src/libutil/archive.cc

@@ -1,3 +1,5 @@
+#include "config.h"
+
 #include <cerrno>
 #include <algorithm>
 #include <vector>
@@ -12,8 +14,6 @@
 #include "archive.hh"
 #include "util.hh"
 
-#include "config.h"
-
 
 namespace nix {
 
@@ -302,7 +302,12 @@ struct RestoreSink : ParseSink
 #if HAVE_POSIX_FALLOCATE
         if (len) {
             errno = posix_fallocate(fd, 0, len);
-            if (errno) throw SysError(format("preallocating file of %1% bytes") % len);
+            /* Note that EINVAL may indicate that the underlying
+               filesystem doesn't support preallocation (e.g. on
+               OpenSolaris).  Since preallocation is just an
+               optimisation, ignore it. */
+            if (errno && errno != EINVAL)
+                throw SysError(format("preallocating file of %1% bytes") % len);
         }
 #endif
     }

src/libutil/aterm-map.hh

@@ -1,8 +1,7 @@
 #ifndef __ATERM_MAP_H
 #define __ATERM_MAP_H
 
-typedef struct _ATerm * ATerm;
-
+#include <aterm1.h>
 #include <assert.h>
 
 

src/libutil/util.cc

@@ -1,9 +1,5 @@
 #include "config.h"
 
-#ifdef __CYGWIN__
-#include <windows.h>
-#endif
-
 #include <iostream>
 #include <cerrno>
 #include <cstdio>
@@ -15,6 +11,7 @@
 #include <sys/wait.h>
 #include <sys/types.h>
 #include <fcntl.h>
+#include <unistd.h>
 
 #include "util.hh"
 
@@ -220,12 +217,13 @@ string readFile(const Path & path)
 }
 
 
-void writeFile(const Path & path, const string & s)
+void writeFile(const Path & path, const string & s, bool doFsync)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
     if (fd == -1)
         throw SysError(format("opening file `%1%'") % path);
     writeFull(fd, (unsigned char *) s.c_str(), s.size());
+    if (doFsync) fdatasync(fd);
 }
 
 
@@ -297,8 +295,10 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed,
     if (lstat(path.c_str(), &st))
 	throw SysError(format("getting attributes of path `%1%'") % path);
 
-    bytesFreed += st.st_size;
-    blocksFreed += st.st_blocks;
+    if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) {
+        bytesFreed += st.st_size;
+        blocksFreed += st.st_blocks;
+    }
 
     if (S_ISDIR(st.st_mode)) {
 	Strings names = readDirectory(path);
@@ -411,16 +411,6 @@ Paths createDirs(const Path & path)
 }
 
 
-void writeStringToFile(const Path & path, const string & s)
-{
-    AutoCloseFD fd(open(path.c_str(),
-        O_CREAT | O_EXCL | O_WRONLY, 0666));
-    if (fd == -1)
-        throw SysError(format("creating file `%1%'") % path);
-    writeFull(fd, (unsigned char *) s.c_str(), s.size());
-}
-
-
 LogType logType = ltPretty;
 Verbosity verbosity = lvlInfo;
 
@@ -911,15 +901,7 @@ void closeMostFDs(const set<int> & exceptions)
 
 void quickExit(int status)
 {
-#ifdef __CYGWIN__
-    /* Hack for Cygwin: _exit() doesn't seem to work quite right,
-       since some Berkeley DB code appears to be called when a child
-       exits through _exit() (e.g., because execve() failed).  So call
-       the Windows API directly. */
-    ExitProcess(status);
-#else
     _exit(status);
-#endif
 }
 
 
@@ -1053,22 +1035,6 @@ string int2String(int n)
 }
 
 
-bool string2Int(const string & s, int & n)
-{
-    std::istringstream str(s);
-    str >> n;
-    return str && str.get() == EOF;
-}
-
-
-bool string2Int(const string & s, long long & n)
-{
-    std::istringstream str(s);
-    str >> n;
-    return str && str.get() == EOF;
-}
-
-
 bool hasSuffix(const string & s, const string & suffix)
 {
     return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix;

src/libutil/util.hh

@@ -1,6 +1,8 @@
 #ifndef __UTIL_H
 #define __UTIL_H
 
+#include "config.h"
+
 #include "types.hh"
 
 #include <sys/types.h>
@@ -8,6 +10,12 @@
 #include <unistd.h>
 #include <signal.h>
 
+#include <cstdio>
+
+#ifndef HAVE_FDATASYNC
+#define fdatasync fsync
+#endif
+
 
 namespace nix {
 
@@ -58,7 +66,7 @@ string readFile(int fd);
 string readFile(const Path & path);
 
 /* Write a string to a file. */
-void writeFile(const Path & path, const string & s);
+void writeFile(const Path & path, const string & s, bool doFsync = false);
 
 /* Read a line from a file descriptor. */
 string readLine(int fd);
@@ -89,10 +97,6 @@ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);
 
-/* Create a file and write the given text to it.  The file is written
-   in binary mode (i.e., no end-of-line conversions).  The path should
-   not already exist. */
-void writeStringToFile(const Path & path, const string & s);
 
 template<class T, class A>
 T singleton(const A & a)
@@ -295,9 +299,14 @@ bool statusOk(int status);
 
 
 /* Parse a string into an integer. */
+template<class N> bool string2Int(const string & s, N & n)
+{
+    std::istringstream str(s);
+    str >> n;
+    return str && str.get() == EOF;
+}
+
 string int2String(int n);
-bool string2Int(const string & s, int & n);
-bool string2Int(const string & s, long long & n);
 
 
 /* Return true iff `s' ends in `suffix'. */

src/nix-env/Makefile.am

@@ -3,7 +3,7 @@ bin_PROGRAMS = nix-env
 nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt
 nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \
  ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la ${bdb_lib} ${aterm_lib}
+ ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@
 
 nix-env.o: help.txt.hh
 
@@ -11,6 +11,6 @@ nix-env.o: help.txt.hh
 	../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1)
 
 AM_CXXFLAGS = \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} \
+ -I$(srcdir)/.. ${aterm_include} \
  -I$(srcdir)/../libutil -I$(srcdir)/../libstore \
  -I$(srcdir)/../libexpr -I$(srcdir)/../libmain -I../libexpr

src/nix-env/nix-env.cc

@@ -704,52 +704,59 @@ static void upgradeDerivations(Globals & globals,
         foreach (DrvInfos::iterator, i, installedElems) {
             DrvName drvName(i->name);
 
-            MetaInfo meta = i->queryMetaInfo(globals.state);
-            if (keep(meta)) {
-                newElems.push_back(*i);
-                continue;
-            }
+            try {
 
-            /* Find the derivation in the input Nix expression with
-               the same name that satisfies the version constraints
-               specified by upgradeType.  If there are multiple
-               matches, take the one with the highest priority.  If
-               there are still multiple matches, take the one with the
-               highest version. */
-            DrvInfos::iterator bestElem = availElems.end();
-            DrvName bestName;
-            foreach (DrvInfos::iterator, j, availElems) {
-                DrvName newName(j->name);
-                if (newName.name == drvName.name) {
-                    int d = comparePriorities(globals.state, *i, *j);
-                    if (d == 0) d = compareVersions(drvName.version, newName.version);
-                    if ((upgradeType == utLt && d < 0) ||
-                        (upgradeType == utLeq && d <= 0) ||
-                        (upgradeType == utEq && d == 0) ||
-                        upgradeType == utAlways)
-                    {
-                        int d2 = -1;
-                        if (bestElem != availElems.end()) {
-                            d2 = comparePriorities(globals.state, *bestElem, *j);
-                            if (d2 == 0) d2 = compareVersions(bestName.version, newName.version);
-                        }
-                        if (d2 < 0) {
-                            bestElem = j;
-                            bestName = newName;
+                MetaInfo meta = i->queryMetaInfo(globals.state);
+                if (keep(meta)) {
+                    newElems.push_back(*i);
+                    continue;
+                }
+
+                /* Find the derivation in the input Nix expression
+                   with the same name that satisfies the version
+                   constraints specified by upgradeType.  If there are
+                   multiple matches, take the one with the highest
+                   priority.  If there are still multiple matches,
+                   take the one with the highest version. */
+                DrvInfos::iterator bestElem = availElems.end();
+                DrvName bestName;
+                foreach (DrvInfos::iterator, j, availElems) {
+                    DrvName newName(j->name);
+                    if (newName.name == drvName.name) {
+                        int d = comparePriorities(globals.state, *i, *j);
+                        if (d == 0) d = compareVersions(drvName.version, newName.version);
+                        if ((upgradeType == utLt && d < 0) ||
+                            (upgradeType == utLeq && d <= 0) ||
+                            (upgradeType == utEq && d == 0) ||
+                            upgradeType == utAlways)
+                        {
+                            int d2 = -1;
+                            if (bestElem != availElems.end()) {
+                                d2 = comparePriorities(globals.state, *bestElem, *j);
+                                if (d2 == 0) d2 = compareVersions(bestName.version, newName.version);
+                            }
+                            if (d2 < 0) {
+                                bestElem = j;
+                                bestName = newName;
+                            }
                         }
                     }
                 }
-            }
+                
+                if (bestElem != availElems.end() &&
+                    i->queryOutPath(globals.state) !=
+                    bestElem->queryOutPath(globals.state))
+                {
+                    printMsg(lvlInfo,
+                        format("upgrading `%1%' to `%2%'")
+                        % i->name % bestElem->name);
+                    newElems.push_back(*bestElem);
+                } else newElems.push_back(*i);
 
-            if (bestElem != availElems.end() &&
-                i->queryOutPath(globals.state) !=
-                bestElem->queryOutPath(globals.state))
-            {
-                printMsg(lvlInfo,
-                    format("upgrading `%1%' to `%2%'")
-                    % i->name % bestElem->name);
-                newElems.push_back(*bestElem);
-            } else newElems.push_back(*i);
+            } catch (Error & e) {
+                e.addPrefix(format("while trying to find an upgrade for `%1%':\n") % i->name);
+                throw;
+            }
         }
     
         printMissing(globals.state, newElems);

src/nix-hash/Makefile.am

@@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash
 
 nix_hash_SOURCES = nix-hash.cc help.txt
 nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la ${bdb_lib} ${aterm_lib}
+ ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@
 
 nix-hash.o: help.txt.hh
 

src/nix-instantiate/Makefile.am

@@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate
 nix_instantiate_SOURCES = nix-instantiate.cc help.txt
 nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \
  ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la ${bdb_lib} ${aterm_lib}
+ ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@
 
 nix-instantiate.o: help.txt.hh
 
@@ -11,6 +11,6 @@ nix-instantiate.o: help.txt.hh
 	../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1)
 
 AM_CXXFLAGS = \
- ${bdb_include} ${aterm_include} \
+ ${aterm_include} \
  -I$(srcdir)/.. -I$(srcdir)/../libutil -I$(srcdir)/../libstore \
  -I$(srcdir)/../libexpr -I$(srcdir)/../libmain -I../libexpr

src/nix-store/Makefile.am

@@ -2,7 +2,7 @@ bin_PROGRAMS = nix-store
 
 nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt
 nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la ${bdb_lib} ${aterm_lib}
+ ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@
 
 nix-store.o: help.txt.hh
 
@@ -10,5 +10,5 @@ nix-store.o: help.txt.hh
 	../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1)
 
 AM_CXXFLAGS = \
- -I$(srcdir)/.. ${bdb_include} $(aterm_include) -I$(srcdir)/../libutil \
+ -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil \
  -I$(srcdir)/../libstore -I$(srcdir)/../libmain

src/nix-store/help.txt

@@ -24,7 +24,6 @@ Operations:
   --import: import a path from a Nix archive, and register as 
       valid
 
-  --init: initialise the Nix database
   --verify: verify Nix structures
   --optimise: optimise the Nix store by hard-linking identical files
 
@@ -34,12 +33,14 @@ Operations:
 Query flags:
 
   --outputs: query the output paths of a Nix derivation (default)
-  --requisites / -R: print all paths necessary to realise a path
-  --references: print all paths referenced by the given path
-  --referrers: print all paths directly refering to the given path
-  --referrers-closure: print all paths (in)directly refering to the given path
-  --tree: print a tree showing the dependency graph of the given paths
-  --graph: print a dot graph rooted at given paths
+  --requisites / -R: print all paths necessary to realise the path
+  --references: print all paths referenced by the path
+  --referrers: print all paths directly refering to the path
+  --referrers-closure: print all paths (in)directly refering to the path
+  --tree: print a tree showing the dependency graph of the path
+  --graph: print a dot graph rooted at given path
+  --hash: print the SHA-256 hash of the contents of the path
+  --roots: print the garbage collector roots that point to the path
 
 Query switches (not applicable to all queries):
 

src/nix-store/nix-store.cc

@@ -165,41 +165,6 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
 }
 
 
-/* Place in `paths' the set of paths that are required to `realise'
-   the given store path, i.e., all paths necessary for valid
-   deployment of the path.  For a derivation, this is the union of
-   requisites of the inputs, plus the derivation; for other store
-   paths, it is the set of paths in the FS closure of the path.  If
-   `includeOutputs' is true, include the requisites of the output
-   paths of derivations as well.
-
-   Note that this function can be used to implement three different
-   deployment policies:
-
-   - Source deployment (when called on a derivation).
-   - Binary deployment (when called on an output path).
-   - Source/binary deployment (when called on a derivation with
-     `includeOutputs' set to true).
-*/
-static void storePathRequisites(const Path & storePath,
-    bool includeOutputs, PathSet & paths)
-{
-    computeFSClosure(storePath, paths);
-
-    if (includeOutputs) {
-        for (PathSet::iterator i = paths.begin();
-             i != paths.end(); ++i)
-            if (isDerivation(*i)) {
-                Derivation drv = derivationFromPath(*i);
-                for (DerivationOutputs::iterator j = drv.outputs.begin();
-                     j != drv.outputs.end(); ++j)
-                    if (store->isValidPath(j->second.path))
-                        computeFSClosure(j->second.path, paths);
-            }
-    }
-}
-
-
 static Path maybeUseOutput(const Path & storePath, bool useOutput, bool forceRealise)
 {
     if (forceRealise) realisePath(storePath);
@@ -261,14 +226,13 @@ static void opQuery(Strings opFlags, Strings opArgs)
 {
     enum { qOutputs, qRequisites, qReferences, qReferrers
          , qReferrersClosure, qDeriver, qBinding, qHash
-         , qTree, qGraph, qResolve } query = qOutputs;
+         , qTree, qGraph, qResolve, qRoots } query = qOutputs;
     bool useOutput = false;
     bool includeOutputs = false;
     bool forceRealise = false;
     string bindingName;
 
-    for (Strings::iterator i = opFlags.begin();
-         i != opFlags.end(); ++i)
+    foreach (Strings::iterator, i, opFlags)
         if (*i == "--outputs") query = qOutputs;
         else if (*i == "--requisites" || *i == "-R") query = qRequisites;
         else if (*i == "--references") query = qReferences;
@@ -286,6 +250,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         else if (*i == "--tree") query = qTree;
         else if (*i == "--graph") query = qGraph;
         else if (*i == "--resolve") query = qResolve;
+        else if (*i == "--roots") query = qRoots;
         else if (*i == "--use-output" || *i == "-u") useOutput = true;
         else if (*i == "--force-realise" || *i == "-f") forceRealise = true;
         else if (*i == "--include-outputs") includeOutputs = true;
@@ -294,9 +259,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
     switch (query) {
         
         case qOutputs: {
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
-            {
+            foreach (Strings::iterator, i, opArgs) {
                 *i = followLinksToStorePath(*i);
                 if (forceRealise) realisePath(*i);
                 Derivation drv = derivationFromPath(*i);
@@ -310,14 +273,11 @@ static void opQuery(Strings opFlags, Strings opArgs)
         case qReferrers:
         case qReferrersClosure: {
             PathSet paths;
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
-            {
+            foreach (Strings::iterator, i, opArgs) {
                 Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise);
-                if (query == qRequisites)
-                    storePathRequisites(path, includeOutputs, paths);
+                if (query == qRequisites) computeFSClosure(path, paths, false, includeOutputs);
                 else if (query == qReferences) store->queryReferences(path, paths);
-                else if (query == qReferrers) store->queryReferrers(path,  paths);
+                else if (query == qReferrers) store->queryReferrers(path, paths);
                 else if (query == qReferrersClosure) computeFSClosure(path, paths, true);
             }
             Paths sorted = topoSortPaths(paths);
@@ -328,9 +288,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         }
 
         case qDeriver:
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
-            {
+            foreach (Strings::iterator, i, opArgs) {
                 Path deriver = store->queryDeriver(followLinksToStorePath(*i));
                 cout << format("%1%\n") %
                     (deriver == "" ? "unknown-deriver" : deriver);
@@ -338,9 +296,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
 
         case qBinding:
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
-            {
+            foreach (Strings::iterator, i, opArgs) {
                 Path path = useDeriver(followLinksToStorePath(*i));
                 Derivation drv = derivationFromPath(path);
                 StringPairs::iterator j = drv.env.find(bindingName);
@@ -352,9 +308,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
 
         case qHash:
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
-            {
+            foreach (Strings::iterator, i, opArgs) {
                 Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise);
                 Hash hash = store->queryPathHash(path);
                 assert(hash.type == htSHA256);
@@ -364,28 +318,37 @@ static void opQuery(Strings opFlags, Strings opArgs)
 
         case qTree: {
             PathSet done;
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
+            foreach (Strings::iterator, i, opArgs)
                 printTree(followLinksToStorePath(*i), "", "", done);
             break;
         }
             
         case qGraph: {
             PathSet roots;
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
+            foreach (Strings::iterator, i, opArgs)
                 roots.insert(maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise));
 	    printDotGraph(roots);
             break;
         }
 
         case qResolve: {
-            for (Strings::iterator i = opArgs.begin();
-                 i != opArgs.end(); ++i)
+            foreach (Strings::iterator, i, opArgs)
                 cout << format("%1%\n") % followLinksToStorePath(*i);
             break;
         }
             
+        case qRoots: {
+            PathSet referrers;
+            foreach (Strings::iterator, i, opArgs)
+                computeFSClosure(maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise),
+                    referrers, true);
+            Roots roots = store->findRoots();
+            foreach (Roots::iterator, i, roots)
+                if (referrers.find(i->second) != referrers.end())
+                    cout << format("%1%\n") % i->first;
+            break;
+        }
+            
         default:
             abort();
     }
@@ -520,6 +483,7 @@ struct PrintFreed
 
 static void opGC(Strings opFlags, Strings opArgs)
 {
+    bool printRoots = false;
     GCOptions options;
     options.action = GCOptions::gcDeleteDead;
     
@@ -527,30 +491,33 @@ static void opGC(Strings opFlags, Strings opArgs)
     
     /* Do what? */
     foreach (Strings::iterator, i, opFlags)
-        if (*i == "--print-roots") options.action = GCOptions::gcReturnRoots;
+        if (*i == "--print-roots") printRoots = true;
         else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
         else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
         else if (*i == "--delete") options.action = GCOptions::gcDeleteDead;
         else if (*i == "--max-freed") {
-            options.maxFreed = getIntArg(*i, i, opFlags.end());
-            if (options.maxFreed == 0) options.maxFreed = 1;
-        }
-        else if (*i == "--max-links") options.maxLinks = getIntArg(*i, i, opFlags.end());
-        else if (*i == "--use-atime") options.useAtime = true;
-        else if (*i == "--max-atime") {
-            options.useAtime = true;
-            options.maxAtime = getIntArg(*i, i, opFlags.end());
+            long long maxFreed = getIntArg<long long>(*i, i, opFlags.end());
+            options.maxFreed = maxFreed >= 1 ? maxFreed : 1;
         }
+        else if (*i == "--max-links") options.maxLinks = getIntArg<unsigned int>(*i, i, opFlags.end());
         else throw UsageError(format("bad sub-operation `%1%' in GC") % *i);
 
     if (!opArgs.empty()) throw UsageError("no arguments expected");
-    
-    PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
-    store->collectGarbage(options, results);
 
-    if (options.action != GCOptions::gcDeleteDead)
-        foreach (PathSet::iterator, i, results.paths)
-            cout << *i << std::endl;
+    if (printRoots) {
+        Roots roots = store->findRoots();
+        foreach (Roots::iterator, i, roots)
+            cout << i->first << " -> " << i->second << std::endl;
+    }
+
+    else {
+        PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
+        store->collectGarbage(options, results);
+
+        if (options.action != GCOptions::gcDeleteDead)
+            foreach (PathSet::iterator, i, results.paths)
+                cout << *i << std::endl;
+    }
 }
 
 

src/nix-worker/Makefile.am

@@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker
 
 nix_worker_SOURCES = nix-worker.cc help.txt
 nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la ${bdb_lib} ${aterm_lib}
+ ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@
 
 nix-worker.o: help.txt.hh
 
@@ -10,5 +10,5 @@ nix-worker.o: help.txt.hh
 	../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1)
 
 AM_CXXFLAGS = \
- -I$(srcdir)/.. ${bdb_include} $(aterm_include) -I$(srcdir)/../libutil \
+ -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil \
  -I$(srcdir)/../libstore -I$(srcdir)/../libmain

src/nix-worker/nix-worker.cc

@@ -20,9 +20,18 @@
 using namespace nix;
 
 
+/* On platforms that have O_ASYNC, we can detect when a client
+   disconnects and immediately kill any ongoing builds.  On platforms
+   that lack it, we only notice the disconnection the next time we try
+   to write to the client.  So if you have a builder that never
+   generates output on stdout/stderr, the worker will never notice
+   that the client has disconnected until the builder terminates. */
+#ifdef O_ASYNC
+#define HAVE_HUP_NOTIFICATION
 #ifndef SIGPOLL
 #define SIGPOLL SIGIO
 #endif
+#endif
 
 
 static FdSource from(STDIN_FILENO);
@@ -90,7 +99,7 @@ static bool isFarSideClosed(int socket)
 
 
 /* A SIGPOLL signal is received when data is available on the client
-   communication scoket, or when the client has closed its side of the
+   communication socket, or when the client has closed its side of the
    socket.  This handler is enabled at precisely those moments in the
    protocol when we're doing work and the client is supposed to be
    quiet.  Thus, if we get a SIGPOLL signal, it means that the client
@@ -131,12 +140,14 @@ static void sigPollHandler(int sigNo)
 
 static void setSigPollAction(bool enable)
 {
+#ifdef HAVE_HUP_NOTIFICATION
     struct sigaction act, oact;
     act.sa_handler = enable ? sigPollHandler : SIG_IGN;
     sigfillset(&act.sa_mask);
     act.sa_flags = 0;
     if (sigaction(SIGPOLL, &act, &oact))
         throw SysError("setting handler for SIGPOLL");
+#endif
 }
 
 
@@ -453,8 +464,9 @@ static void performOp(unsigned int clientVersion,
         options.maxFreed = readLongLong(from);
         options.maxLinks = readInt(from);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 5) {
-            options.useAtime = readInt(from);
-            options.maxAtime = readInt(from);
+            /* removed options */
+            readInt(from);
+            readInt(from);
         }
 
         GCResults results;
@@ -520,12 +532,14 @@ static void processConnection()
     myPid = getpid();    
     writeToStderr = tunnelStderr;
 
+#ifdef HAVE_HUP_NOTIFICATION
     /* Allow us to receive SIGPOLL for events on the client socket. */
     setSigPollAction(false);
     if (fcntl(from.fd, F_SETOWN, getpid()) == -1)
         throw SysError("F_SETOWN");
-    if (fcntl(from.fd, F_SETFL, fcntl(from.fd, F_GETFL, 0) | FASYNC) == -1)
+    if (fcntl(from.fd, F_SETFL, fcntl(from.fd, F_GETFL, 0) | O_ASYNC) == -1)
         throw SysError("F_SETFL");
+#endif
 
     /* Exchange the greeting. */
     unsigned int magic = readInt(from);
@@ -663,11 +677,12 @@ static void daemonLoop()
             AutoCloseFD remote = accept(fdSocket,
                 (struct sockaddr *) &remoteAddr, &remoteAddrLen);
             checkInterrupt();
-            if (remote == -1)
+            if (remote == -1) {
 		if (errno == EINTR)
 		    continue;
 		else
 		    throw SysError("accepting connection");
+            }
 
             printMsg(lvlInfo, format("accepted connection %1%") % remote);
 

tests/init.sh

@@ -36,12 +36,6 @@ ln -s $TOP/scripts/copy-from-other-stores.pl $NIX_BIN_DIR/nix/
 ln -s $TOP/scripts/download-using-manifests.pl $NIX_BIN_DIR/nix/
 ln -s $TOP/scripts/readmanifest.pm $NIX_BIN_DIR/nix/
 
-mkdir -p "$NIX_STATE_DIR"/manifests
-mkdir -p "$NIX_STATE_DIR"/gcroots
-mkdir -p "$NIX_STATE_DIR"/temproots
-mkdir -p "$NIX_STATE_DIR"/profiles
-ln -s "$NIX_STATE_DIR"/profiles "$NIX_STATE_DIR"/gcroots/
-
 cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 gc-keep-outputs = false
 gc-keep-derivations = false

tests/lang/eval-okay-functionargs.exp.xml

@@ -0,0 +1,15 @@
+<?xml version='1.0' encoding='utf-8'?>
+<expr>
+  <list>
+    <string value="stdenv" />
+    <string value="fetchurl" />
+    <string value="aterm-stdenv" />
+    <string value="aterm-stdenv2" />
+    <string value="libX11" />
+    <string value="libXv" />
+    <string value="mplayer-stdenv2.libXv-libX11" />
+    <string value="mplayer-stdenv2.libXv-libX11_2" />
+    <string value="nix-stdenv-aterm-stdenv" />
+    <string value="nix-stdenv2-aterm2-stdenv2" />
+  </list>
+</expr>

tests/lang/eval-okay-functionargs.nix

@@ -0,0 +1,80 @@
+let
+
+  stdenvFun = { }: { name = "stdenv"; };
+  stdenv2Fun = { }: { name = "stdenv2"; };
+  fetchurlFun = { stdenv }: assert stdenv.name == "stdenv"; { name = "fetchurl"; };
+  atermFun = { stdenv, fetchurl }: { name = "aterm-${stdenv.name}"; };
+  aterm2Fun = { stdenv, fetchurl }: { name = "aterm2-${stdenv.name}"; };
+  nixFun = { stdenv, fetchurl, aterm }: { name = "nix-${stdenv.name}-${aterm.name}"; };
+  
+  mplayerFun =
+    { stdenv, fetchurl, enableX11 ? false, xorg ? null, enableFoo ? true, foo ? null  }:
+    assert stdenv.name == "stdenv2";
+    assert enableX11 -> xorg.libXv.name == "libXv";
+    assert enableFoo -> foo != null;
+    { name = "mplayer-${stdenv.name}.${xorg.libXv.name}-${xorg.libX11.name}"; };
+
+  makeOverridable = f: origArgs: f origArgs //
+    { override = newArgs:
+        makeOverridable f (origArgs // (if builtins.isFunction newArgs then newArgs origArgs else newArgs));
+    };
+    
+  callPackage_ = pkgs: f: args:
+    makeOverridable f ((builtins.intersectAttrs (builtins.functionArgs f) pkgs) // args);
+
+  allPackages =
+    { overrides ? (pkgs: pkgsPrev: { }) }:
+    let
+      callPackage = callPackage_ pkgs;
+      pkgs = pkgsStd // (overrides pkgs pkgsStd);
+      pkgsStd = {
+        inherit pkgs;
+        stdenv = callPackage stdenvFun { };
+        stdenv2 = callPackage stdenv2Fun { };
+        fetchurl = callPackage fetchurlFun { };
+        aterm = callPackage atermFun { };
+        xorg = callPackage xorgFun { };
+        mplayer = callPackage mplayerFun { stdenv = pkgs.stdenv2; enableFoo = false; };
+        nix = callPackage nixFun { };
+      };
+    in pkgs;
+
+  libX11Fun = { stdenv, fetchurl }: { name = "libX11"; };
+  libX11_2Fun = { stdenv, fetchurl }: { name = "libX11_2"; };
+  libXvFun = { stdenv, fetchurl, libX11 }: { name = "libXv"; };
+  
+  xorgFun =
+    { pkgs }:
+    let callPackage = callPackage_ (pkgs // pkgs.xorg); in
+    {
+      libX11 = callPackage libX11Fun { };
+      libXv = callPackage libXvFun { };
+    };
+
+in
+
+let
+
+  pkgs = allPackages { };
+  
+  pkgs2 = allPackages {
+    overrides = pkgs: pkgsPrev: {
+      stdenv = pkgs.stdenv2;
+      nix = pkgsPrev.nix.override { aterm = aterm2Fun { inherit (pkgs) stdenv fetchurl; }; };
+      xorg = pkgsPrev.xorg // { libX11 = libX11_2Fun { inherit (pkgs) stdenv fetchurl; }; };
+    };
+  };
+  
+in
+
+  [ pkgs.stdenv.name
+    pkgs.fetchurl.name
+    pkgs.aterm.name
+    pkgs2.aterm.name
+    pkgs.xorg.libX11.name
+    pkgs.xorg.libXv.name
+    pkgs.mplayer.name
+    pkgs2.mplayer.name
+    pkgs.nix.name
+    pkgs2.nix.name
+  ]

tests/parallel.sh

@@ -27,7 +27,7 @@ clearStore
 
 rm -f $SHARED.cur $SHARED.max
 
-drvPath=$($nixinstantiate parallel.nix --argstr sleepTime 8)
+drvPath=$($nixinstantiate parallel.nix --argstr sleepTime 15)
 
 cmd="$nixstore -j1 -r $drvPath"
 

version

@@ -1 +1 @@
-0.13
+0.14