* Sync with the trunk.

when they are defined

Makefile.am

@@ -2,6 +2,8 @@ SUBDIRS = externals src scripts corepkgs doc misc tests
 EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
   nix.conf.example NEWS version
 
+pkginclude_HEADERS = config.h
+
 include ./substitute.mk
 
 nix.spec: nix.spec.in

aterm-gc.supp

@@ -115,3 +115,35 @@
    fun:*
    fun:AT_collect
 }
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:*
+   fun:mark_phase
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Cond
+   fun:*
+   fun:*
+   fun:mark_phase
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:*
+   fun:mark_phase_young
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Cond
+   fun:*
+   fun:*
+   fun:mark_phase_young
+}

bootstrap.sh

@@ -1,4 +1,5 @@
 #! /bin/sh -e
+rm -f aclocal.m4
 mkdir -p config
 libtoolize --copy
 aclocal

configure.ac

@@ -10,8 +10,8 @@ AC_CANONICAL_HOST
 
 # Construct a Nix system name (like "i686-linux").
 AC_MSG_CHECKING([for the canonical Nix system name])
-cpu_name=$(uname -p | tr 'A-Z ' 'a-z_')
-machine_name=$(uname -m | tr 'A-Z ' 'a-z_')
+cpu_name=$(uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
+machine_name=$(uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
 
 case $machine_name in
     i*86)
@@ -30,7 +30,7 @@ case $machine_name in
         ;;
 esac
 
-sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
+sys_name=$(uname -s | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
 
 case $sys_name in
     cygwin*)
@@ -50,39 +50,24 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
-# Whether to produce a statically linked binary.  On Cygwin, this is
-# the default: dynamically linking against the ATerm DLL does work,
-# except that it requires the ATerm "lib" directory to be in $PATH, as
-# Windows doesn't have anything like an RPATH embedded in executable.
-# Since this is kind of annoying, we use static libraries for now.
-
-AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
-  [produce statically linked binaries]),
-  static_nix=$enableval, static_nix=no)
-
-if test "$sys_name" = cygwin; then
-   static_nix=yes
-fi
-
-if test "$static_nix" = yes; then
+# Windows-specific stuff.  On Cygwin, dynamically linking against the
+# ATerm DLL works, except that it requires the ATerm "lib" directory
+# to be in $PATH, as Windows doesn't have anything like an RPATH
+# embedded in executable.  Since this is kind of annoying, we use
+# static libraries for now.
+if test "$sys_name" = "cygwin"; then
     AC_DISABLE_SHARED
     AC_ENABLE_STATIC
 fi
 
 
-# Windows-specific stuff.
-if test "$sys_name" = "cygwin"; then
-    # We cannot delete open files.
-    AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.])
-fi
-
 # Solaris-specific stuff.
 if test "$sys_name" = "sunos"; then
     # Solaris requires -lsocket -lnsl for network functions
-    ADDITIONAL_NETWORK_LIBS="-lsocket -lnsl"
-    AC_SUBST(ADDITIONAL_NETWORK_LIBS)
+    LIBS="-lsocket -lnsl $LIBS"
 fi
 
+
 AC_PROG_CC
 AC_PROG_CXX
 
@@ -101,6 +86,13 @@ AC_DISABLE_STATIC
 AC_ENABLE_SHARED
 AC_PROG_LIBTOOL
 
+if test "$enable_shared" = yes; then
+   SUB_CONFIGURE_FLAGS="--enable-shared --disable-static"
+else
+   SUB_CONFIGURE_FLAGS="--enable-static --disable-shared"
+fi
+AC_SUBST(SUB_CONFIGURE_FLAGS)
+
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
@@ -136,11 +128,22 @@ AC_CHECK_HEADERS([locale], [], [], [])
 AC_LANG_POP(C++)
 
 
+# Check for <err.h>.
+AC_CHECK_HEADER([err.h], [], [bsddiff_compat_include="-Icompat-include"])
+AC_SUBST([bsddiff_compat_include])
+
+
 # Check whether we have the personality() syscall, which allows us to
 # do i686-linux builds on x86_64-linux machines.
 AC_CHECK_HEADERS([sys/personality.h])
 
 
+# Check for tr1/unordered_set.
+AC_LANG_PUSH(C++)
+AC_CHECK_HEADERS([tr1/unordered_set], [], [], [])
+AC_LANG_POP(C++)
+
+
 AC_DEFUN([NEED_PROG],
 [
 AC_PATH_PROG($1, $2)
@@ -218,6 +221,8 @@ AC_ARG_WITH(bzip2, AC_HELP_STRING([--with-bzip2=PATH],
   [prefix of bzip2]),
   bzip2=$withval, bzip2=)
 AM_CONDITIONAL(HAVE_BZIP2, test -n "$bzip2")
+ATERM_VERSION=2.5
+AC_SUBST(ATERM_VERSION)
 if test -z "$bzip2"; then
   # Headers and libraries will be used from the temporary installation
   # in externals/inst-bzip2.
@@ -238,8 +243,36 @@ AC_SUBST(bzip2_include)
 AC_SUBST(bzip2_bin)
 AC_SUBST(bzip2_bin_test)
 
+AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH],
+  [prefix of SQLite]),
+  sqlite=$withval, sqlite=)
+AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite")
+SQLITE_VERSION=3070500
+AC_SUBST(SQLITE_VERSION)
+if test -z "$sqlite"; then
+  sqlite_lib='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)/libsqlite3.la'
+  sqlite_include='-I${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)'
+  sqlite_bin='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)'
+else
+  sqlite_lib="-L$sqlite/lib -lsqlite3"
+  sqlite_include="-I$sqlite/include"
+  sqlite_bin="$sqlite/bin"
+fi
+AC_SUBST(sqlite_lib)
+AC_SUBST(sqlite_include)
+AC_SUBST(sqlite_bin)
 
-AC_CHECK_LIB(pthread, pthread_mutex_init)
+# Whether to use the Boehm garbage collector.
+AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
+  [enable garbage collection in the Nix expression evaluator (requires Boehm GC)]),
+  gc=$enableval, gc=)
+if test -n "$gc"; then
+  PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
+  boehmgc_lib="-L$boehmgc/lib -lgc"
+  CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
+  AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
+fi
+AC_SUBST(boehmgc_lib)
 
 
 AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
@@ -253,8 +286,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
 
 
 # Nice to have, but not essential.
-AC_CHECK_FUNCS([strsignal])
-AC_CHECK_FUNCS([posix_fallocate])
+AC_CHECK_FUNCS([strsignal posix_fallocate nanosleep])
 
 
 # This is needed if ATerm or bzip2 are static libraries,
@@ -264,14 +296,6 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
-if test "$static_nix" = yes; then
-    # `-all-static' has to be added at the end of configure, because
-    # the C compiler doesn't know about -all-static (it's filtered out
-    # by libtool, but configure doesn't use libtool).
-    LDFLAGS="-all-static $LDFLAGS"
-fi
-
-
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    externals/Makefile

corepkgs/buildenv/default.nix

@@ -11,4 +11,8 @@ derivation {
   paths = derivations;
   active = map (x: if x ? meta && x.meta ? active then x.meta.active else "true") derivations;
   priority = map (x: if x ? meta && x.meta ? priority then x.meta.priority else "5") derivations;
+
+  # Building user environments remotely just causes huge amounts of
+  # network traffic, so don't do that.
+  preferLocalBuild = true;
 }

corepkgs/nar/nar.sh.in

@@ -7,8 +7,6 @@ dst=$out/tmp.nar.bz2
 
 @bzip2@ < tmp > $dst
 
-@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
-
 @bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
 
 @coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2

doc/manual/conf-file.xml

@@ -97,6 +97,25 @@ env-keep-derivations = false
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-build-cores"><term><literal>build-cores</literal></term>
+
+    <listitem><para>Sets the value of the
+    <envar>NIX_BUILD_CORES</envar> environment variable in the
+    invocation of builders.  Builders can use this variable at their
+    discretion to control the maximum amount of parallelism.  For
+    instance, in Nixpkgs, if the derivation attribute
+    <varname>enableParallelBuilding</varname> is set to
+    <literal>true</literal>, the builder passes the
+    <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+    It can be overriden using the <option
+    linkend='opt-cores'>--cores</option> command line switch and
+    defaults to <literal>1</literal>.  The value <literal>0</literal>
+    means that the builder should use all available CPU cores in the
+    system.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-build-max-silent-time"><term><literal>build-max-silent-time</literal></term>
 
     <listitem>
@@ -241,7 +260,7 @@ build-use-chroot = /dev /proc /bin</programlisting>
     Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
     synchronously flushed to disk.  This improves robustness in case
     of system crashes, but reduces performance.  The default is
-    <literal>false</literal>.</para></listitem>
+    <literal>true</literal>.</para></listitem>
 
   </varlistentry>
     

doc/manual/env-common.xml

@@ -271,6 +271,17 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 </varlistentry>
 
+
+<varlistentry><term><envar>GC_INITIAL_HEAP_SIZE</envar></term>
+
+  <listitem><para>If Nix has been configured to use the Boehm garbage
+  collector, this variable sets the initial size of the heap in bytes.
+  It defaults to 384 MiB.  Setting it to a low value reduces memory
+  consumption, but will increase runtime due to the overhead of
+  garbage collection.</para></listitem>
+
+</varlistentry>
+
     
 </variablelist>
 

doc/manual/installation.xml

@@ -105,6 +105,13 @@ this packages.  Alternatively, if you already have it installed, you
 can use <command>configure</command>'s <option>--with-bzip2</option>
 options to point to their respective locations.</para>
 
+<para>Nix can optionally use the <link
+xlink:href="http://www.hpl.hp.com/personal/Hans_Boehm/gc/">Boehm
+garbage collector</link> to reduce the evaluator’s memory consumption.
+To enable it, install <literal>pkgconfig</literal> and the Boehm
+garbage collector, and pass the flag <option>--enable-gc</option> to
+<command>configure</command>.</para>
+
 </section>
 
 

doc/manual/introduction.xml

@@ -113,7 +113,7 @@ $ nix-env --rollback
 
 <simplesect><title>Garbage collection</title>
 
-<para>When you install a package like this…
+<para>When you uninstall a package like this…
 
 <screen>
 $ nix-env --uninstall firefox

doc/manual/manual.xml

@@ -27,10 +27,11 @@
       <year>2007</year>
       <year>2008</year>
       <year>2009</year>
+      <year>2010</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>September 2009</date>
+    <date>August 2010</date>
     
   </info>
 

doc/manual/nix-push.xml

@@ -120,7 +120,7 @@ dependencies used in the build, such as compilers).</para>
 dependencies, we can do:
       
 <screen>
-$ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
+$ nix-push <replaceable>urls</replaceable> $(nix-store -r $(nix-instantiate foo.nix))</screen>
     
 </para>
 

doc/manual/nix-store.xml

@@ -404,6 +404,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
     <arg choice='plain'><option>--tree</option></arg>
     <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
     <arg choice='plain'><option>--hash</option></arg>
+    <arg choice='plain'><option>--size</option></arg>
     <arg choice='plain'><option>--roots</option></arg>
   </group>
   <arg><option>--use-output</option></arg>
@@ -587,9 +588,21 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--hash</option></term>
   
     <listitem><para>Prints the SHA-256 hash of the contents of the
-    store paths <replaceable>paths</replaceable>.  Since the hash is
-    stored in the Nix database, this is a fast
-    operation.</para></listitem>
+    store paths <replaceable>paths</replaceable> (that is, the hash of
+    the output of <command>nix-store --dump</command> on the given
+    paths).  Since the hash is stored in the Nix database, this is a
+    fast operation.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--size</option></term>
+  
+    <listitem><para>Prints the size in bytes of the contents of the
+    store paths <replaceable>paths</replaceable> — to be precise, the
+    size of the output of <command>nix-store --dump</command> on the
+    given paths.  Note that the actual disk space required by the
+    store paths may be higher, especially on filesystems with large
+    cluster sizes.</para></listitem>
 
   </varlistentry>
 

doc/manual/opt-common-syn.xml

@@ -13,6 +13,10 @@
   </group>
   <replaceable>number</replaceable>
 </arg>
+<arg>
+  <arg><option>--cores</option></arg>
+  <replaceable>number</replaceable>
+</arg>
 <arg>
   <arg><option>--max-silent-time</option></arg>
   <replaceable>number</replaceable>

doc/manual/opt-common.xml

@@ -98,7 +98,25 @@
   linkend='conf-build-max-jobs'><literal>build-max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
-  exploit I/O latency.  </para></listitem>
+  exploit I/O latency.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry xml:id="opt-cores"><term><option>--cores</option></term>
+
+  <listitem><para>Sets the value of the <envar>NIX_BUILD_CORES</envar>
+  environment variable in the invocation of builders.  Builders can
+  use this variable at their discretion to control the maximum amount
+  of parallelism.  For instance, in Nixpkgs, if the derivation
+  attribute <varname>enableParallelBuilding</varname> is set to
+  <literal>true</literal>, the builder passes the
+  <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+  It defaults to the value of the <link
+  linkend='conf-build-cores'><literal>build-cores</literal></link>
+  configuration setting, if set, or <literal>1</literal> otherwise.
+  The value <literal>0</literal> means that the builder should use all
+  available CPU cores in the system.</para></listitem>
   
 </varlistentry>
 

doc/manual/quick-start.xml

@@ -60,7 +60,7 @@ available remotely.</para></listitem>
 in the channel:
 
 <screen>
-$ nix-env -qa ’*’ <lineannotation>(mind the quotes!)</lineannotation>
+$ nix-env -qa \*
 docbook-xml-4.2
 firefox-1.0pre-PR-0.10.1
 hello-2.1.1

doc/manual/release-notes.xml

@@ -6,6 +6,80 @@
 
 
 
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-1.0"><title>Release 1.0 (TBA)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix can now optionally use the Boehm garbage collector.
+    This significantly reduces the Nix evaluator’s memory footprint,
+    especially when evaluating large NixOS system configurations.  It
+    can be enabled using the <option>--enable-gc</option> configure
+    option.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.16"><title>Release 0.16 (August 17, 2010)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The Nix expression evaluator is now much faster in most
+    cases: typically, <link
+    xlink:href="http://www.mail-archive.com/nix-dev@cs.uu.nl/msg04113.html">3
+    to 8 times compared to the old implementation</link>.  It also
+    uses less memory.  It no longer depends on the ATerm
+    library.</para>
+  </listitem>
+
+  <listitem>
+    <para>
+      Support for configurable parallelism inside builders.  Build
+      scripts have always had the ability to perform multiple build
+      actions in parallel (for instance, by running <command>make -j
+      2</command>), but this was not desirable because the number of
+      actions to be performed in parallel was not configurable.  Nix
+      now has an option <option>--cores
+      <replaceable>N</replaceable></option> as well as a configuration
+      setting <varname>build-cores =
+      <replaceable>N</replaceable></varname> that causes the
+      environment variable <envar>NIX_BUILD_CORES</envar> to be set to
+      <replaceable>N</replaceable> when the builder is invoked.  The
+      builder can use this at its discretion to perform a parallel
+      build, e.g., by calling <command>make -j
+      <replaceable>N</replaceable></command>.  In Nixpkgs, this can be
+      enabled on a per-package basis by setting the derivation
+      attribute <varname>enableParallelBuilding</varname> to
+      <literal>true</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-store -q</command> now supports XML output
+    through the <option>--xml</option> flag.</para>
+  </listitem>
+
+  <listitem>
+    <para>Several bug fixes.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
 <!--==================================================================-->
 
 <section xml:id="ssec-relnotes-0.15"><title>Release 0.15 (March 17, 2010)</title>

externals/Makefile.am

@@ -12,30 +12,56 @@ $(BZIP2).tar.gz:
 $(BZIP2): $(BZIP2).tar.gz
 	gunzip < $(srcdir)/$(BZIP2).tar.gz | tar xvf -
 
-have-bzip2:
-	$(MAKE) $(BZIP2)
-	touch have-bzip2
-
 if HAVE_BZIP2
 build-bzip2:
 else
-build-bzip2: have-bzip2
-	(pfx=`pwd` && \
-	cd $(BZIP2) && \
-	$(MAKE) && \
-	$(MAKE) install PREFIX=$$pfx/inst-bzip2)
+build-bzip2: $(BZIP2)
+	(cd $(BZIP2) && \
+	$(MAKE) CC="$(CC)" && \
+	$(MAKE) install PREFIX=$(abs_builddir)/inst-bzip2)
 	touch build-bzip2
 
-install:
+install-exec-local:: build-bzip2
 	mkdir -p $(DESTDIR)${bzip2_bin}
 	$(INSTALL_PROGRAM) $(bzip2_bin_test)/bzip2 $(bzip2_bin_test)/bunzip2 $(DESTDIR)${bzip2_bin}
 endif
 
 
-all: build-bzip2
+# SQLite
 
-EXTRA_DIST = $(BZIP2).tar.gz
+SQLITE = sqlite-autoconf-$(SQLITE_VERSION)
+SQLITE_TAR = sqlite-autoconf-$(SQLITE_VERSION).tar.gz
 
-ext-clean:
-	$(RM) -f have-bzip2 build-bzip2
-	$(RM) -rf $(BZIP2)
+$(SQLITE_TAR):
+	@echo "Nix requires the SQLite library to build."
+	@echo "Please download version $(SQLITE_VERSION) from"
+	@echo "  http://www.sqlite.org/$(SQLITE_TAR)"
+	@echo "and place it in the externals/ directory."
+	false
+
+$(SQLITE): $(SQLITE_TAR)
+	gzip -d < $(srcdir)/$(SQLITE_TAR) | tar xvf -
+
+if HAVE_SQLITE
+build-sqlite:
+else
+build-sqlite: $(SQLITE)
+	(cd $(SQLITE) && \
+	CC="$(CC)" CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \
+	$(MAKE) )
+	touch build-sqlite
+
+install-exec-local:: build-sqlite
+	cd $(SQLITE) && $(MAKE) install
+	rm -rf "$(DESTDIR)/$(pkglibdir)/dummy"
+endif
+
+
+all: build-bzip2 build-sqlite
+
+EXTRA_DIST = $(BZIP2).tar.gz $(SQLITE_TAR)
+
+clean:
+	$(RM) -f build-bzip2 build-sqlite
+	$(RM) -rf $(BZIP2) $(SQLITE)
+	$(RM) -rf inst-bzip2

nix.conf.example

@@ -59,6 +59,17 @@
 #build-max-jobs = 1
 
 
+### Option `build-cores'
+#
+# This option defines the number of CPU cores to utilize in parallel
+# within a build job, i.e. by passing an appropriate `-jN' flag to GNU
+# Make. The default is 1, meaning that parallel building within jobs
+# is disabled.  Passing the special value `0' causes Nix to try and
+# auto-detect the number of available cores on the local host.  This
+# setting can be overridden using the `--cores' command line switch.
+#build-cores = 1
+
+
 ### Option `build-max-silent-time'
 #
 # This option defines the maximum number of seconds that a builder can

release.nix

@@ -1,4 +1,7 @@
-{ nixpkgs ? ../nixpkgs }:
+{ nixpkgs ? ../nixpkgs
+, nix ? { outPath = ./.; rev = 1234; }
+, officialRelease ? false
+}:
 
 let
 
@@ -6,10 +9,6 @@ let
 
 
     tarball =
-      { nix ? {outPath = ./.; rev = 1234;}
-      , officialRelease ? false
-      }:
-
       with import nixpkgs {};
 
       releaseTools.sourceTarball {
@@ -19,8 +18,8 @@ let
         inherit officialRelease;
 
         buildInputs =
-          [ curl bison flex2533 perl libxml2 libxslt w3m bzip2
-            tetex dblatex nukeReferences
+          [ curl bison24 flex2535 perl libxml2 libxslt w3m bzip2
+            tetex dblatex nukeReferences pkgconfig
           ];
 
         configureFlags = ''
@@ -34,6 +33,9 @@ let
           stripHash ${bzip2.src}
           cp -pv ${bzip2.src} externals/$strippedName
 
+          stripHash ${sqlite.src}
+          cp -pv ${sqlite.src} externals/$strippedName
+
           # TeX needs a writable font cache.
           export VARTEXFONTS=$TMPDIR/texfonts
         '';
@@ -60,70 +62,43 @@ let
 
 
     build =
-      { tarball ? jobs.tarball {}
-      , system ? "i686-linux"
-      }:
+      { system ? "i686-linux" }:
 
-      with import nixpkgs {inherit system;};
+      with import nixpkgs { inherit system; };
 
       releaseTools.nixBuild {
         name = "nix";
         src = tarball;
 
-        buildInputs = [curl perl bzip2 openssl];
+        buildInputs = [ curl perl bzip2 openssl pkgconfig boehmgc ];
 
         configureFlags = ''
           --disable-init-state
-          --with-bzip2=${bzip2}
+          --with-bzip2=${bzip2} --with-sqlite=${sqlite}
+          --enable-gc
         '';
       };
 
 
-    /*
-    static =
-      { tarball ? jobs.tarball {}
-      , system ? "i686-linux"
-      }:
-
-      with import nixpkgs {inherit system;};
-
-      releaseTools.binaryTarball {
-        name = "nix-static-tarball";
-        src = tarball;
-
-        buildInputs = [curl perl bzip2];
-
-        configureFlags = ''
-          --disable-init-state
-          --with-bzip2=${bzip2}
-          --enable-static-nix
-        '';
-      };
-    */
-
-      
     coverage =
-      { tarball ? jobs.tarball {}
-      }:
-
-      with import nixpkgs {};
+      with import nixpkgs { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
         src = tarball;
 
-        buildInputs = [
-          curl perl bzip2 openssl
-          # These are for "make check" only:
-          graphviz libxml2 libxslt
-        ];
+        buildInputs =
+          [ curl perl bzip2 openssl
+            # These are for "make check" only:
+            graphviz libxml2 libxslt
+          ];
 
         configureFlags = ''
           --disable-init-state --disable-shared
-          --with-bzip2=${bzip2}
+          --with-bzip2=${bzip2} --with-sqlite=${sqlite}
         '';
 
-        lcovFilter = ["*/boost/*" "*-tab.*"];
+        lcovFilter = [ "*/boost/*" "*-tab.*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some
@@ -168,17 +143,15 @@ let
 
   makeRPM = 
     system: diskImageFun: prio:
-    { tarball ? jobs.tarball {}
-    }:
 
-    with import nixpkgs {inherit system;};
+    with import nixpkgs { inherit system; };
 
     releaseTools.rpmBuild rec {
       name = "nix-rpm-${diskImage.name}";
-      src = tarball;
+      src = jobs.tarball;
       diskImage = diskImageFun vmTools.diskImages;
       memSize = 1024;
-      meta = { schedulingPriority = toString prio; };
+      meta.schedulingPriority = prio;
     };
 
 
@@ -187,19 +160,17 @@ let
   
   makeDeb =
     system: diskImageFun: prio:
-    { tarball ? jobs.tarball {}
-    }:
 
-    with import nixpkgs {inherit system;};
+    with import nixpkgs { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";
-      src = tarball;
+      src = jobs.tarball;
       diskImage = diskImageFun vmTools.diskImages;
       memSize = 1024;
-      meta = { schedulingPriority = toString prio; };
+      meta.schedulingPriority = prio;
       configureFlags = "--sysconfdir=/etc";
-      debRequires = ["curl"];
+      debRequires = [ "curl" ];
     };
 
 

scripts/GeneratePatches.pm.in

@@ -0,0 +1,334 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use File::Temp qw(tempdir);
+
+
+# Some patch generations options.
+
+# Max size of NAR archives to generate patches for.
+my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
+$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize;
+
+# If patch is bigger than this fraction of full archive, reject.
+my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
+$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
+
+my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"};
+$timeLimit = 180 if !defined $timeLimit;
+
+my $hashAlgo = "sha256";
+
+
+sub findOutputPaths {
+    my $narFiles = shift;
+
+    my %outPaths;
+    
+    foreach my $p (keys %{$narFiles}) {
+
+        # Ignore derivations.
+        next if ($p =~ /\.drv$/);
+        
+        # Ignore builders (too much ambiguity -- they're all called
+        # `builder.sh').
+        next if ($p =~ /\.sh$/);
+        next if ($p =~ /\.patch$/);
+        
+        # Don't bother including tar files etc.
+        next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/);
+
+        $outPaths{$p} = 1;
+    }
+
+    return %outPaths;
+}
+
+
+sub getNameVersion {
+    my $p = shift;
+    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
+    my $name = $1;
+    my $version = $2;
+    return undef unless defined $name && defined $version;
+    $name =~ s/^-//;
+    $version =~ s/^-//;
+    return ($name, $version);
+}
+
+
+# A quick hack to get a measure of the `distance' between two
+# versions: it's just the position of the first character that differs
+# (or 999 if they are the same).
+sub versionDiff {
+    my $s = shift;
+    my $t = shift;
+    my $i;
+    return 999 if $s eq $t;
+    for ($i = 0; $i < length $s; $i++) {
+        return $i if $i >= length $t or
+            substr($s, $i, 1) ne substr($t, $i, 1);
+    }
+    return $i;
+}
+
+
+sub getNarBz2 {
+    my $narPath = shift;
+    my $narFiles = shift;
+    my $storePath = shift;
+    
+    my $narFileList = $$narFiles{$storePath};
+    die "missing path $storePath" unless defined $narFileList;
+
+    my $narFile = @{$narFileList}[0];
+    die unless defined $narFile;
+
+    $narFile->{url} =~ /\/([^\/]+)$/;
+    die unless defined $1;
+    return "$narPath/$1";
+}
+
+
+sub containsPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $basePath = shift;
+    my $patchList = $$patches{$storePath};
+    return 0 if !defined $patchList;
+    my $found = 0;
+    foreach my $patch (@{$patchList}) {
+        # !!! baseHash might differ
+        return 1 if $patch->{basePath} eq $basePath;
+    }
+    return 0;
+}
+
+
+sub generatePatches {
+    my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_;
+
+    my %srcOutPaths = findOutputPaths $srcNarFiles;
+    my %dstOutPaths = findOutputPaths $dstNarFiles;
+
+    # For each output path in the destination, see if we need to / can
+    # create a patch.
+
+    print STDERR "creating patches...\n";
+
+    foreach my $p (keys %dstOutPaths) {
+
+        # If exactly the same path already exists in the source, skip it.
+        next if defined $srcOutPaths{$p};
+    
+        print "  $p\n";
+
+        # If not, then we should find the paths in the source that are
+        # `most' likely to be present on a system that wants to
+        # install this path.
+
+        (my $name, my $version) = getNameVersion $p;
+        next unless defined $name && defined $version;
+
+        my @closest = ();
+        my $closestVersion;
+        my $minDist = -1; # actually, larger means closer
+
+        # Find all source paths with the same name.
+
+        foreach my $q (keys %srcOutPaths) {
+            (my $name2, my $version2) = getNameVersion $q;
+            next unless defined $name2 && defined $version2;
+
+            if ($name eq $name2) {
+
+                my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system};
+                my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system};
+                if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) {
+                    print "    SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n";
+                    next;
+                }
+
+                # If the sizes differ too much, then skip.  This
+                # disambiguates between, e.g., a real component and a
+                # wrapper component (cf. Firefox in Nixpkgs).
+                my $srcSize = @{$$srcNarFiles{$q}}[0]->{size};
+                my $dstSize = @{$$dstNarFiles{$p}}[0]->{size};
+                my $ratio = $srcSize / $dstSize;
+                $ratio = 1 / $ratio if $ratio < 1;
+                # print "  SIZE $srcSize $dstSize $ratio $q\n";
+
+                if ($ratio >= 3) {
+                    print "    SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n";
+                    next;
+                }
+
+                # If there are multiple matching names, include the
+                # ones with the closest version numbers.
+                my $dist = versionDiff $version, $version2;
+                if ($dist > $minDist) {
+                    $minDist = $dist;
+                    @closest = ($q);
+                    $closestVersion = $version2;
+                } elsif ($dist == $minDist) {
+                    push @closest, $q;
+                }
+            }
+        }
+
+        if (scalar(@closest) == 0) {
+            print "    NO BASE: $p\n";
+            next;
+        }
+
+        foreach my $closest (@closest) {
+
+            # Generate a patch between $closest and $p.
+            print STDERR "  $p <- $closest\n";
+
+            # If the patch already exists, skip it.
+            if (containsPatch($srcPatches, $p, $closest) ||
+                containsPatch($dstPatches, $p, $closest))
+            {
+                print "    skipping, already exists\n";
+                next;
+            }
+
+            my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest;
+            my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p;
+
+            if (! -f $srcNarBz2) {
+                warn "patch source archive $srcNarBz2 is missing\n";
+                next;
+            }
+
+            system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
+                or die "cannot unpack $srcNarBz2";
+
+            if ((stat "$tmpDir/A")[7] >= $maxNarSize) {
+                print "    skipping, source is too large\n";
+                next;
+            }
+        
+            system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0
+                or die "cannot unpack $dstNarBz2";
+
+            if ((stat "$tmpDir/B")[7] >= $maxNarSize) {
+                print "    skipping, destination is too large\n";
+                next;
+            }
+        
+            my $time1 = time();
+            my $res = system("ulimit -t $timeLimit; @libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF");
+            my $time2 = time();
+            if ($res) {
+                warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n";
+                next;
+            }
+
+            my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die;
+            chomp $baseHash;
+
+            my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die;
+            chomp $narHash;
+
+            my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die;
+            chomp $narDiffHash;
+
+            my $narDiffSize = (stat "$tmpDir/DIFF")[7];
+            my $dstNarBz2Size = (stat $dstNarBz2)[7];
+
+            print "    size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n";
+        
+            if ($narDiffSize >= $dstNarBz2Size) {
+                print "    rejecting; patch bigger than full archive\n";
+                next;
+            }
+    
+            if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
+                print "    rejecting; patch too large relative to full archive\n";
+                next;
+            }
+    
+            my $finalName = "$narDiffHash.nar-bsdiff";
+
+            if (-e "$patchesPath/$finalName") {
+                print "    not copying, already exists\n";
+            }
+
+            else {
+                system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0
+                    or die "cannot copy diff";
+                rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName")
+                    or die "cannot rename $patchesPath/$finalName.tmp";
+            }
+        
+            # Add the patch to the manifest.
+            addPatch $dstPatches, $p,
+                { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
+                , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
+                , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
+                };
+        }
+    }
+}
+
+
+# Propagate useful patches from $srcPatches to $dstPatches.  A patch
+# is useful if it produces either paths in the $dstNarFiles or paths
+# that can be used as the base for other useful patches.
+sub propagatePatches {
+    my ($srcPatches, $dstNarFiles, $dstPatches) = @_;
+
+    print STDERR "propagating patches...\n";
+
+    my $changed;
+    do {
+        # !!! we repeat this to reach the transitive closure; inefficient
+        $changed = 0;
+
+        print STDERR "loop\n";
+
+        my %dstBasePaths;
+        foreach my $q (keys %{$dstPatches}) {
+            foreach my $patch (@{$$dstPatches{$q}}) {
+                $dstBasePaths{$patch->{basePath}} = 1;
+            }
+        }
+
+        foreach my $p (keys %{$srcPatches}) {
+            my $patchList = $$srcPatches{$p};
+
+            my $include = 0;
+
+            # Is path $p included in the destination?  If so, include
+            # patches that produce it.
+            $include = 1 if defined $$dstNarFiles{$p};
+
+            # Is path $p a path that serves as a base for paths in the
+            # destination?  If so, include patches that produce it.
+            # !!! check baseHash
+            $include = 1 if defined $dstBasePaths{$p};
+
+            if ($include) {
+                foreach my $patch (@{$patchList}) {
+                    $changed = 1 if addPatch $dstPatches, $p, $patch;
+                }
+            }
+        
+        }
+    
+    } while $changed;
+}
+
+
+# Add all new patches in $srcPatches to $dstPatches.
+sub copyPatches {
+    my ($srcPatches, $dstPatches) = @_;
+    foreach my $p (keys %{$srcPatches}) {
+        addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}};
+    }
+}
+
+
+return 1;

scripts/Makefile.am

@@ -1,23 +1,23 @@
 bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build \
-  nix-copy-closure 
+  nix-copy-closure nix-generate-patches
 
-noinst_SCRIPTS = nix-profile.sh generate-patches.pl \
+noinst_SCRIPTS = nix-profile.sh GeneratePatches.pm \
   find-runtime-roots.pl build-remote.pl nix-reduce-build \
   copy-from-other-stores.pl nix-http-export.cgi
 
-nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl
+nix-pull nix-push: NixManifest.pm NixConfig.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
+install-exec-local: NixManifest.pm GeneratePatches.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
-	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_DATA) ssh.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) NixManifest.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) NixConfig.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) SSH.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) GeneratePatches.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
@@ -30,15 +30,16 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-pull.in nix-push.in nix-profile.sh.in \
   nix-prefetch-url.in nix-install-package.in \
   nix-channel.in \
-  readmanifest.pm.in \
-  readconfig.pm.in \
-  ssh.pm \
+  NixManifest.pm.in \
+  NixConfig.pm.in \
+  SSH.pm \
+  GeneratePatches.pm.in \
   nix-build.in \
   download-using-manifests.pl.in \
   copy-from-other-stores.pl.in \
-  generate-patches.pl.in \
   nix-copy-closure.in \
   find-runtime-roots.pl.in \
   build-remote.pl.in \
   nix-reduce-build.in \
-  nix-http-export.cgi.in 
+  nix-http-export.cgi.in \
+  nix-generate-patches.in

scripts/NixManifest.pm.in

@@ -33,18 +33,8 @@ sub readManifest {
 
     my $manifestVersion = 2;
 
-    my $storePath;
-    my $url;
-    my $hash;
-    my $size;
-    my $basePath;
-    my $baseHash;
-    my $patchType;
-    my $narHash;
-    my $references;
-    my $deriver;
-    my $hashAlgo;
-    my $copyFrom;
+    my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType);
+    my ($narHash, $narSize, $references, $deriver, $hashAlgo, $copyFrom, $system);
 
     while (<MANIFEST>) {
         chomp;
@@ -62,9 +52,11 @@ sub readManifest {
                 undef $hash;
                 undef $size;
                 undef $narHash;
+                undef $narSize;
                 undef $basePath;
                 undef $baseHash;
                 undef $patchType;
+                undef $system;
                 $references = "";
                 $deriver = "";
                 $hashAlgo = "md5";
@@ -89,8 +81,10 @@ sub readManifest {
                     if (!$found) {
                         push @{$narFileList},
                             { url => $url, hash => $hash, size => $size
-                            , narHash => $narHash, references => $references
+                            , narHash => $narHash, narSize => $narSize
+                            , references => $references
                             , deriver => $deriver, hashAlgo => $hashAlgo
+                            , system => $system
                             };
                     }
                 
@@ -100,8 +94,8 @@ sub readManifest {
                     addPatch $patches, $storePath,
                         { url => $url, hash => $hash, size => $size
                         , basePath => $basePath, baseHash => $baseHash
-                        , narHash => $narHash, patchType => $patchType
-                        , hashAlgo => $hashAlgo
+                        , narHash => $narHash, narSize => $narSize
+                        , patchType => $patchType, hashAlgo => $hashAlgo
                         };
                 }
 
@@ -132,9 +126,11 @@ sub readManifest {
             elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
             elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
             elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+            elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; }
             elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
             elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
             elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
+            elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; }
 
             # Compatibility;
             elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
@@ -150,7 +146,7 @@ sub readManifest {
 
 
 sub writeManifest {
-    my ($manifest, $narFiles, $patches) = @_;
+    my ($manifest, $narFiles, $patches, $noCompress) = @_;
 
     open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
 
@@ -165,12 +161,14 @@ sub writeManifest {
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $narFile->{url}\n";
             print MANIFEST "  Hash: $narFile->{hash}\n" if defined $narFile->{hash};
-            print MANIFEST "  NarHash: $narFile->{narHash}\n";
             print MANIFEST "  Size: $narFile->{size}\n" if defined $narFile->{size};
+            print MANIFEST "  NarHash: $narFile->{narHash}\n";
+            print MANIFEST "  NarSize: $narFile->{narSize}\n" if $narFile->{narSize};
             print MANIFEST "  References: $narFile->{references}\n"
                 if defined $narFile->{references} && $narFile->{references} ne "";
             print MANIFEST "  Deriver: $narFile->{deriver}\n"
                 if defined $narFile->{deriver} && $narFile->{deriver} ne "";
+            print MANIFEST "  System: $narFile->{system}\n" if defined $narFile->{system};
             print MANIFEST "}\n";
         }
     }
@@ -182,8 +180,9 @@ sub writeManifest {
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $patch->{url}\n";
             print MANIFEST "  Hash: $patch->{hash}\n";
-            print MANIFEST "  NarHash: $patch->{narHash}\n";
             print MANIFEST "  Size: $patch->{size}\n";
+            print MANIFEST "  NarHash: $patch->{narHash}\n";
+            print MANIFEST "  NarSize: $patch->{narSize}\n" if $patch->{narSize};
             print MANIFEST "  BasePath: $patch->{basePath}\n";
             print MANIFEST "  BaseHash: $patch->{baseHash}\n";
             print MANIFEST "  Type: $patch->{patchType}\n";
@@ -199,11 +198,13 @@ sub writeManifest {
 
 
     # Create a bzipped manifest.
-    system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0
-        or die "cannot compress manifest";
+    unless (defined $noCompress) {
+	system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0
+	    or die "cannot compress manifest";
 
-    rename("$manifest.bz2.tmp", "$manifest.bz2")
-        or die "cannot rename $manifest.bz2.tmp: $!";
+	rename("$manifest.bz2.tmp", "$manifest.bz2")
+	    or die "cannot rename $manifest.bz2.tmp: $!";
+    }
 }
 
 

scripts/SSH.pm

@@ -3,6 +3,8 @@ use File::Temp qw(tempdir);
 
 our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
 
+push @sshOpts, "-x";
+
 my $sshStarted = 0;
 my $sshHost;
 
@@ -24,14 +26,17 @@ sub openSSHConnection {
     # child continues to run if we are killed.  So instead make SSH
     # print "started" when it has established the connection, and wait
     # until we see that.
-    open SSH, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die;
-    while (<SSH>) {
+    open SSHPIPE, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die;
+
+    while (<SSHPIPE>) {
         chomp;
-        last if /started/;
+        if ($_ eq "started") {
+            $sshStarted = 1;
+            return 1;
+        }
     }
-    
-    $sshStarted = 1;
-    return 1;
+
+    return 0;
 }
 
 # Tell the master SSH client to exit.

scripts/build-remote.pl.in

@@ -3,7 +3,8 @@
 use Fcntl ':flock';
 use English '-no_match_vars';
 use IO::Handle;
-use ssh qw/sshOpts openSSHConnection/;
+use SSH qw/sshOpts openSSHConnection/;
+no warnings('once');
 
 
 # General operation:
@@ -31,57 +32,22 @@ $ENV{"DISPLAY"} = "";
 $ENV{"SSH_ASKPASS"} = "";
 
 
-my $loadIncreased = 0;
-
-my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV;
-$maxSilentTime = 0 unless defined $maxSilentTime;
-
 sub sendReply {
     my $reply = shift;
     print STDERR "# $reply\n";
 }
 
-sub decline {
-    sendReply "decline";
-    exit 0;
-}
+sub all { $_ || return 0 for @_; 1 }
+
+
+# Initialisation.
+my $loadIncreased = 0;
+
+my ($localSystem, $maxSilentTime, $printBuildTrace) = @ARGV;
+$maxSilentTime = 0 unless defined $maxSilentTime;
 
 my $currentLoad = $ENV{"NIX_CURRENT_LOAD"};
-decline unless defined $currentLoad;
-mkdir $currentLoad, 0777 or die unless -d $currentLoad;
-
 my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
-decline if !defined $conf || ! -e $conf;
-
-my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
-
-
-# Read the list of machines.
-my @machines;
-open CONF, "< $conf" or die;
-
-while (<CONF>) {
-    chomp;
-    s/\#.*$//g;
-    next if /^\s*$/;
-    /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)(\s+([0-9\.]+))?\s*$/ or die;
-    push @machines,
-        { hostName => $1
-        , systemTypes => [split(/,/, $2)]
-        , sshKeys => $3
-        , maxJobs => $4
-        , speedFactor => 1.0 * ($6 || 1)
-        , enabled => 1
-        };
-}
-
-close CONF;
-
-
-# Acquire the exclusive lock on $currentLoad/main-lock.
-my $mainLock = "$currentLoad/main-lock";
-open MAINLOCK, ">>$mainLock" or die;
-flock(MAINLOCK, LOCK_EX) or die;
 
 
 sub openSlotLock {
@@ -91,150 +57,213 @@ sub openSlotLock {
     open $slotLock, ">>$slotLockFn" or die;
     return $slotLock;
 }
+
+
+# Read the list of machines.
+my @machines;
+if (defined $conf && -e $conf) {
+    open CONF, "< $conf" or die;
+    while (<CONF>) {
+        chomp;
+        s/\#.*$//g;
+        next if /^\s*$/;
+        my @tokens = split /\s/, $_;
+        push @machines,
+            { hostName => $tokens[0]
+            , systemTypes => [ split(/,/, $tokens[1]) ]
+            , sshKeys => $tokens[2]
+            , maxJobs => int($tokens[3])
+            , speedFactor => 1.0 * (defined $tokens[4] ? int($tokens[4]) : 1)
+            , features => [ split(/,/, $tokens[5] || "") ]
+            , enabled => 1
+            };
+    }
+    close CONF;
+}
+
+
+
+# Wait for the calling process to ask us whether we can build some derivation.
+my ($drvPath, $hostName, $slotLock);
+
+REQ: while (1) {
+    $_ = <STDIN> || exit 0;
+    my ($amWilling, $neededSystem);
+    ($amWilling, $neededSystem, $drvPath, $requiredFeatures) = split;
+    my @requiredFeatures = split /,/, $requiredFeatures;
+
+    my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
+
+    if (!defined $currentLoad) {
+        sendReply "decline";
+        next;
+    }
     
-
-my $hostName;
-my $slotLock;
-
-while (1) {
+    # Acquire the exclusive lock on $currentLoad/main-lock.
+    mkdir $currentLoad, 0777 or die unless -d $currentLoad;
+    my $mainLock = "$currentLoad/main-lock";
+    open MAINLOCK, ">>$mainLock" or die;
+    flock(MAINLOCK, LOCK_EX) or die;
     
-    # Find all machine that can execute this build, i.e., that support
-    # builds for the given platform and are not at their job limit.
-    my $rightType = 0;
-    my @available = ();
-    LOOP: foreach my $cur (@machines) {
-        if ($cur->{enabled} && grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) {
-            $rightType = 1;
+    
+    while (1) {
+        # Find all machine that can execute this build, i.e., that
+        # support builds for the given platform and features, and are
+        # not at their job limit.
+        my $rightType = 0;
+        my @available = ();
+        LOOP: foreach my $cur (@machines) {
+            if ($cur->{enabled}
+                && (grep { $neededSystem eq $_ } @{$cur->{systemTypes}})
+                && all(map { my $f = $_; 0 != grep { $f eq $_ } @{$cur->{features}} } @requiredFeatures))
+            {
+                $rightType = 1;
 
-            # We have a machine of the right type.  Determine the load on
-            # the machine.
-            my $slot = 0;
-            my $load = 0;
-            my $free;
-            while ($slot < $cur->{maxJobs}) {
-                my $slotLock = openSlotLock($cur, $slot);
-                if (flock($slotLock, LOCK_EX | LOCK_NB)) {
-                    $free = $slot unless defined $free;
-                    flock($slotLock, LOCK_UN) or die;
-                } else {
-                    $load++;
+                # We have a machine of the right type.  Determine the load on
+                # the machine.
+                my $slot = 0;
+                my $load = 0;
+                my $free;
+                while ($slot < $cur->{maxJobs}) {
+                    my $slotLock = openSlotLock($cur, $slot);
+                    if (flock($slotLock, LOCK_EX | LOCK_NB)) {
+                        $free = $slot unless defined $free;
+                        flock($slotLock, LOCK_UN) or die;
+                    } else {
+                        $load++;
+                    }
+                    close $slotLock;
+                    $slot++;
                 }
-                close $slotLock;
-                $slot++;
+                
+                push @available, { machine => $cur, load => $load, free => $free }
+                if $load < $cur->{maxJobs};
             }
-
-            push @available, { machine => $cur, load => $load, free => $free }
-            if $load < $cur->{maxJobs};
         }
-    }
 
-    if (defined $ENV{NIX_DEBUG_HOOK}) {
-        print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n"
-            foreach @available;
-    }
-
-
-    # Didn't find any available machine?  Then decline or postpone.
-    if (scalar @available == 0) {
-        # Postpone if we have a machine of the right type, except if the
-        # local system can and wants to do the build.
-        if ($rightType && !$canBuildLocally) {
-            sendReply "postpone";
-            exit 0;
-        } else {
-            decline;
+        if (defined $ENV{NIX_DEBUG_HOOK}) {
+            print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n"
+                foreach @available;
         }
-    }
 
 
-    # Prioritise the available machines as follows:
-    # - First by load divided by speed factor, rounded to the nearest
-    #   integer.  This causes fast machines to be preferred over slow
-    #   machines with similar loads.
-    # - Then by speed factor.
-    # - Finally by load.
-    sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); }
-    @available = sort
-        { lf($a) <=> lf($b)
-              || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor}
-              || $a->{load} <=> $b->{load}
-        } @available;
+        # Didn't find any available machine?  Then decline or postpone.
+        if (scalar @available == 0) {
+            # Postpone if we have a machine of the right type, except
+            # if the local system can and wants to do the build.
+            if ($rightType && !$canBuildLocally) {
+                sendReply "postpone";
+            } else {
+                sendReply "decline";                
+            }
+            close MAINLOCK;
+            next REQ;
+        }
 
 
-    # Select the best available machine and lock a free slot.
-    my $selected = $available[0]; 
-    my $machine = $selected->{machine};
-
-    $slotLock = openSlotLock($machine, $selected->{free});
-    flock($slotLock, LOCK_EX | LOCK_NB) or die;
-    utime undef, undef, $slotLock;
-
-    close MAINLOCK;
+        # Prioritise the available machines as follows:
+        # - First by load divided by speed factor, rounded to the nearest
+        #   integer.  This causes fast machines to be preferred over slow
+        #   machines with similar loads.
+        # - Then by speed factor.
+        # - Finally by load.
+        sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); }
+        @available = sort
+            { lf($a) <=> lf($b)
+                  || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor}
+                  || $a->{load} <=> $b->{load}
+            } @available;
 
 
-    # Connect to the selected machine.
-    @sshOpts = ("-i", $machine->{sshKeys}, "-x");
-    $hostName = $machine->{hostName};
-    last if openSSHConnection $hostName;
+        # Select the best available machine and lock a free slot.
+        my $selected = $available[0]; 
+        my $machine = $selected->{machine};
+        
+        $slotLock = openSlotLock($machine, $selected->{free});
+        flock($slotLock, LOCK_EX | LOCK_NB) or die;
+        utime undef, undef, $slotLock;
+
+        close MAINLOCK;
+
+
+        # Connect to the selected machine.
+        @sshOpts = ("-i", $machine->{sshKeys}, "-x");
+        $hostName = $machine->{hostName};
+        last REQ if openSSHConnection $hostName;
     
-    warn "unable to open SSH connection to $hostName, trying other available machines...\n";
-    $machine->{enabled} = 0;
+        warn "unable to open SSH connection to $hostName, trying other available machines...\n";
+        $machine->{enabled} = 0;
+    }
 }
 
 
 # Tell Nix we've accepted the build.
 sendReply "accept";
-my $x = <STDIN>;
-chomp $x;
-
-if ($x ne "okay") {
-    exit 0;
-}
+my @inputs = split /\s/, readline(STDIN);
+my @outputs = split /\s/, readline(STDIN);
 
 
-# Do the actual build.
 print STDERR "building `$drvPath' on `$hostName'\n";
+print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace;
 
-my $inputs = `cat inputs`; die if ($? != 0);
-$inputs =~ s/\n/ /g;
-
-my $outputs = `cat outputs`; die if ($? != 0);
-$outputs =~ s/\n/ /g;
-
-print "copying inputs...\n";
 
 my $maybeSign = "";
 $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
 
-system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath $inputs") == 0
+
+# Register the derivation as a temporary GC root.  Note that $PPID is
+# the PID of the remote SSH process, which, due to the use of a
+# persistant SSH connection, should be the same across all remote
+# command invocations for this session.
+my $rootsDir = "@localstatedir@/nix/gcroots/tmp";
+system("ssh $hostName @sshOpts 'mkdir -m 1777 -p $rootsDir; ln -sfn $drvPath $rootsDir/\$PPID.drv'");
+
+sub removeRoots {
+    system("ssh $hostName @sshOpts 'rm -f $rootsDir/\$PPID.drv $rootsDir/\$PPID.out'");
+}
+
+
+# Copy the derivation and its dependencies to the build machine.
+system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0
     or die "cannot copy inputs to $hostName: $?";
 
-print "building...\n";
 
-my $buildFlags = "--max-silent-time $maxSilentTime";
+# Perform the build.
+my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsDir/\$PPID.out --option verbosity 0";
 
-# `-tt' forces allocation of a pseudo-terminal.  This is required to
-# make the remote nix-store process receive a signal when the
-# connection dies.  Without it, the remote process might continue to
-# run indefinitely (that is, until it next tries to write to
-# stdout/stderr).
-if (system("ssh $hostName @sshOpts -tt 'nix-store --realise $buildFlags $drvPath > /dev/null'") != 0) {
-    # If we couldn't run ssh or there was an ssh problem (indicated by
-    # exit code 255), then we return exit code 1; otherwise we assume
-    # that the builder failed, which we indicate to Nix using exit
-    # code 100.  It's important to distinguish between the two because
-    # the first is a transient failure and the latter is permanent.
-    my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100;
-    print STDERR "build of `$drvPath' on `$hostName' failed with exit code $?\n";
+# We let the remote side kill its process group when the connection is
+# closed unexpectedly.  This is necessary to ensure that no processes
+# are left running on the remote system if the local Nix process is
+# killed.  (SSH itself doesn't kill child processes if the connection
+# is interrupted unless the `-tt' flag is used to force a pseudo-tty,
+# in which case every child receives SIGHUP; however, `-tt' doesn't
+# work on some platforms when connection sharing is used.)
+pipe STDIN, DUMMY; # make sure we have a readable STDIN
+if (system("ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) {
+    # Note that if we get exit code 100 from `nix-store -r', it
+    # denotes a permanent build failure (as opposed to an SSH problem
+    # or a temporary Nix problem).  We propagate this to the caller to
+    # allow it to distinguish between transient and permanent
+    # failures.
+    my $res = $? >> 8;
+    print STDERR "build of `$drvPath' on `$hostName' failed with exit code $res\n";
+    removeRoots;
     exit $res;
 }
 
-print "build of `$drvPath' on `$hostName' succeeded\n";
+#print "build of `$drvPath' on `$hostName' succeeded\n";
 
-foreach my $output (split '\n', $outputs) {
+
+# Copy the output from the build machine.
+foreach my $output (@outputs) {
     my $maybeSignRemote = "";
     $maybeSignRemote = "--sign" if $UID != 0;
     
-    system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output' | @bindir@/nix-store --import > /dev/null") == 0
+    system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" .
+           "| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0
 	or die "cannot copy $output from $hostName: $?";
 }
+
+
+# Get rid of the temporary GC roots.
+removeRoots;

scripts/copy-from-other-stores.pl.in

@@ -17,25 +17,19 @@ foreach my $dir (@remoteStoresAll) {
 }
 
 
+$ENV{"NIX_REMOTE"} = "";
+
+
 sub findStorePath {
     my $storePath = shift;
-    
-    my $storePathName = basename $storePath;
-    
     foreach my $store (@remoteStores) {
-        # Determine whether $storePath exists by looking for the
-        # existence of the info file, and if so, get store path info
-        # from that file.  This rather breaks abstraction: we should
-        # be using `nix-store' for that.  But right now there is no
-        # good way to tell nix-store to access a store mounted under a
-        # different location (there's $NIX_STORE, but that only works
-        # if the remote store is mounted under its "real" location).
-        my $infoFile = "$store/var/nix/db/info/$storePathName";
-        my $storePath2 = "$store/store/$storePathName";
-        if (-f $infoFile && -e $storePath2) {
-            return ($infoFile, $storePath2);
-        }
+        my $sourcePath = "$store/store/" . basename $storePath;
+        next unless -e $sourcePath || -l $sourcePath;
+        $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
+        return ($store, $sourcePath) if
+            system("@bindir@/nix-store --check-validity $storePath") == 0;
     }
+    return undef;
 }
 
 
@@ -46,37 +40,38 @@ if ($ARGV[0] eq "--query") {
 
         if ($cmd eq "have") {
             my $storePath = <STDIN>; chomp $storePath;
-            (my $infoFile) = findStorePath $storePath;
-            print STDOUT ($infoFile ? "1\n" : "0\n");
+            print STDOUT (defined findStorePath($storePath) ? "1\n" : "0\n");
         }
 
         elsif ($cmd eq "info") {
             my $storePath = <STDIN>; chomp $storePath;
-            (my $infoFile) = findStorePath $storePath;
-            if (!$infoFile) {
+            my ($store, $sourcePath) = findStorePath($storePath);
+            if (!defined $store) {
                 print "0\n";
                 next; # not an error
             }
             print "1\n";
 
-            my $deriver = "";
-            my @references = ();
+            $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
+            
+            my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
+            die "cannot query deriver of `$storePath'" if $? != 0;
+            chomp $deriver;
+            $deriver = "" if $deriver eq "unknown-deriver";
 
-            open INFO, "<$infoFile" or die "cannot read info file $infoFile\n";
-            while (<INFO>) {
-                chomp;
-                /^([\w-]+): (.*)$/ or die "bad info file";
-                my $key = $1;
-                my $value = $2;
-                if ($key eq "Deriver") { $deriver = $value; }
-                elsif ($key eq "References") { @references = split ' ', $value; }
-            }
-            close INFO;
+            my @references = split "\n",
+                `@bindir@/nix-store --query --references $storePath`;
+            die "cannot query references of `$storePath'" if $? != 0;
+
+            my $narSize = `@bindir@/nix-store --query --size $storePath`;
+            die "cannot query size of `$storePath'" if $? != 0;
+            chomp $narSize;
 
             print "$deriver\n";
             print scalar @references, "\n";
             print "$_\n" foreach @references;
-            print "0\n"; # !!! showing size not supported (yet)
+            print "$narSize\n";
+            print "$narSize\n";
         }
 
         else { die "unknown command `$cmd'"; }
@@ -87,8 +82,8 @@ if ($ARGV[0] eq "--query") {
 elsif ($ARGV[0] eq "--substitute") {
     die unless scalar @ARGV == 2;
     my $storePath = $ARGV[1];
-    (my $infoFile, my $sourcePath) = findStorePath $storePath;
-    die unless $infoFile;
+    my ($store, $sourcePath) = findStorePath $storePath;
+    die unless $store;
     print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
     system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
         or die "cannot copy `$sourcePath' to `$storePath'";

scripts/download-using-manifests.pl.in

@@ -1,7 +1,7 @@
 #! @perl@ -w -I@libexecdir@/nix
 
 use strict;
-use readmanifest;
+use NixManifest;
 use POSIX qw(strftime);
 use File::Temp qw(tempdir);
 
@@ -12,6 +12,10 @@ STDOUT->autoflush(1);
 my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests");
 my $logFile = "@localstatedir@/log/nix/downloads";
 
+# For queries, skip expensive calls to nix-hash etc.  We're just
+# estimating the expected download size.
+my $fast = 1;
+
 
 # Load all manifests.
 my %narFiles;
@@ -31,6 +35,151 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") {
 }
 
 
+sub isValidPath {
+    my $p = shift;
+    if ($fast) {
+        return -e $p;
+    } else {
+        return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
+    }
+}
+
+
+sub parseHash {
+    my $hash = shift;
+    if ($hash =~ /^(.+):(.+)$/) {
+        return ($1, $2);
+    } else {
+        return ("md5", $hash);
+    }
+}
+
+
+# Compute the most efficient sequence of downloads to produce the
+# given path.
+sub computeSmallestDownload {
+    my $targetPath = shift;
+    
+    # Build a graph of all store paths that might contribute to the
+    # construction of $targetPath, and the special node "start".  The
+    # edges are either patch operations, or downloads of full NAR
+    # files.  The latter edges only occur between "start" and a store
+    # path.
+    my %graph;
+
+    $graph{"start"} = {d => 0, pred => undef, edges => []};
+
+    my @queue = ();
+    my $queueFront = 0;
+    my %done;
+
+    sub addNode {
+        my $graph = shift;
+        my $u = shift;
+        $$graph{$u} = {d => 999999999999, pred => undef, edges => []}
+            unless defined $$graph{$u};
+    }
+
+    sub addEdge {
+        my $graph = shift;
+        my $u = shift;
+        my $v = shift;
+        my $w = shift;
+        my $type = shift;
+        my $info = shift;
+        addNode $graph, $u;
+        push @{$$graph{$u}->{edges}},
+            {weight => $w, start => $u, end => $v, type => $type, info => $info};
+        my $n = scalar @{$$graph{$u}->{edges}};
+    }
+
+    push @queue, $targetPath;
+
+    while ($queueFront < scalar @queue) {
+        my $u = $queue[$queueFront++];
+        next if defined $done{$u};
+        $done{$u} = 1;
+
+        addNode \%graph, $u;
+
+        # If the path already exists, it has distance 0 from the
+        # "start" node.
+        if (isValidPath($u)) {
+            addEdge \%graph, "start", $u, 0, "present", undef;
+        }
+
+        else {
+
+            # Add patch edges.
+            my $patchList = $patches{$u};
+            foreach my $patch (@{$patchList}) {
+                if (isValidPath($patch->{basePath})) {
+                    # !!! this should be cached
+                    my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
+                    my $format = "--base32";
+                    $format = "" if $baseHashAlgo eq "md5";
+                    my $hash = $fast && $baseHashAlgo eq "sha256"
+                        ? `$binDir/nix-store -q --hash "$patch->{basePath}"`
+                        : `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
+                    chomp $hash;
+                    $hash =~ s/.*://;
+                    next if $hash ne $baseHash;
+                }
+                push @queue, $patch->{basePath};
+                addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
+            }
+
+            # Add NAR file edges to the start node.
+            my $narFileList = $narFiles{$u};
+            foreach my $narFile (@{$narFileList}) {
+                # !!! how to handle files whose size is not known in advance?
+                # For now, assume some arbitrary size (1 MB).
+                addEdge \%graph, "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
+            }
+        }
+    }
+
+
+    # Run Dijkstra's shortest path algorithm to determine the shortest
+    # sequence of download and/or patch actions that will produce
+    # $targetPath.
+
+    my @todo = keys %graph;
+
+    while (scalar @todo > 0) {
+
+        # Remove the closest element from the todo list.
+        # !!! inefficient, use a priority queue
+        @todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo;
+        my $u = pop @todo;
+
+        my $u_ = $graph{$u};
+
+        foreach my $edge (@{$u_->{edges}}) {
+            my $v_ = $graph{$edge->{end}};
+            if ($v_->{d} > $u_->{d} + $edge->{weight}) {
+                $v_->{d} = $u_->{d} + $edge->{weight};
+                # Store the edge; to edge->start is actually the
+                # predecessor.
+                $v_->{pred} = $edge; 
+            }
+        }
+    }
+
+
+    # Retrieve the shortest path from "start" to $targetPath.
+    my @path = ();
+    my $cur = $targetPath;
+    return () unless defined $graph{$targetPath}->{pred};
+    while ($cur ne "start") {
+        push @path, $graph{$cur}->{pred};
+        $cur = $graph{$cur}->{pred}->{start};
+    }
+
+    return @path;
+}
+
+
 # Parse the arguments.
 
 if ($ARGV[0] eq "--query") {
@@ -46,6 +195,7 @@ if ($ARGV[0] eq "--query") {
 
         elsif ($cmd eq "info") {
             my $storePath = <STDIN>; chomp $storePath;
+
             my $info;
             if (defined $narFiles{$storePath}) {
                 $info = @{$narFiles{$storePath}}[0];
@@ -57,13 +207,32 @@ if ($ARGV[0] eq "--query") {
                 print "0\n";
                 next; # not an error
             }
+
             print "1\n";
             print "$info->{deriver}\n";
             my @references = split " ", $info->{references};
             print scalar @references, "\n";
             print "$_\n" foreach @references;
-            my $size = $info->{size} || 0;
-            print "$size\n";
+
+            my @path = computeSmallestDownload $storePath;
+            
+            my $downloadSize = 0;
+            while (scalar @path > 0) {
+                my $edge = pop @path;
+                my $u = $edge->{start};
+                my $v = $edge->{end};
+                if ($edge->{type} eq "patch") {
+                    $downloadSize += $edge->{info}->{size} || 0;
+                }
+                elsif ($edge->{type} eq "narfile") {
+                    $downloadSize += $edge->{info}->{size} || 0;
+                }
+            }
+                
+            print "$downloadSize\n";
+            
+            my $narSize = $info->{narSize} || 0;
+            print "$narSize\n";
         }
         
         else { die "unknown command `$cmd'"; }
@@ -79,6 +248,7 @@ elsif ($ARGV[0] ne "--substitute") {
 
 die unless scalar @ARGV == 2;
 my $targetPath = $ARGV[1];
+$fast = 0;
 
 
 # Create a temporary directory.
@@ -110,148 +280,9 @@ foreach my $localPath (@{$localPathList}) {
 }
 
 
-# Build a graph of all store paths that might contribute to the
-# construction of $targetPath, and the special node "start".  The
-# edges are either patch operations, or downloads of full NAR files.
-# The latter edges only occur between "start" and a store path.
-
-my %graph;
-
-$graph{"start"} = {d => 0, pred => undef, edges => []};
-
-my @queue = ();
-my $queueFront = 0;
-my %done;
-
-sub addToQueue {
-    my $v = shift;
-    return if defined $done{$v};
-    $done{$v} = 1;
-    push @queue, $v;
-}
-
-sub addNode {
-    my $u = shift;
-    $graph{$u} = {d => 999999999999, pred => undef, edges => []}
-        unless defined $graph{$u};
-}
-
-sub addEdge {
-    my $u = shift;
-    my $v = shift;
-    my $w = shift;
-    my $type = shift;
-    my $info = shift;
-    addNode $u;
-    push @{$graph{$u}->{edges}},
-        {weight => $w, start => $u, end => $v, type => $type, info => $info};
-    my $n = scalar @{$graph{$u}->{edges}};
-}
-
-addToQueue $targetPath;
-
-sub isValidPath {
-    my $p = shift;
-    return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
-}
-
-sub parseHash {
-    my $hash = shift;
-    if ($hash =~ /^(.+):(.+)$/) {
-        return ($1, $2);
-    } else {
-        return ("md5", $hash);
-    }
-}
-
-while ($queueFront < scalar @queue) {
-    my $u = $queue[$queueFront++];
-#    print "$u\n";
-
-    addNode $u;
-
-    # If the path already exists, it has distance 0 from the "start"
-    # node.
-    if (isValidPath($u)) {
-        addEdge "start", $u, 0, "present", undef;
-    }
-
-    else {
-
-        # Add patch edges.
-        my $patchList = $patches{$u};
-        foreach my $patch (@{$patchList}) {
-            if (isValidPath($patch->{basePath})) {
-                # !!! this should be cached
-                my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
-                my $format = "--base32";
-                $format = "" if $baseHashAlgo eq "md5";
-                my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
-                chomp $hash;
-                if ($hash ne $baseHash) {
-                    print LOGFILE "$$ rejecting $patch->{basePath}\n";
-                    next;
-                }
-            }
-            addToQueue $patch->{basePath};
-            addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
-        }
-
-        # Add NAR file edges to the start node.
-        my $narFileList = $narFiles{$u};
-        foreach my $narFile (@{$narFileList}) {
-            # !!! how to handle files whose size is not known in advance?
-            # For now, assume some arbitrary size (1 MB).
-            addEdge "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
-            if ($u eq $targetPath) {
-                my $size = $narFile->{size} || -1;
-                print LOGFILE "$$ full-download-would-be $size\n";
-            }
-        }
-
-    }
-}
-
-
-# Run Dijkstra's shortest path algorithm to determine the shortest
-# sequence of download and/or patch actions that will produce
-# $targetPath.
-
-sub byDistance { # sort by distance, reversed
-    return -($graph{$a}->{d} <=> $graph{$b}->{d});
-}
-
-my @todo = keys %graph;
-
-while (scalar @todo > 0) {
-
-    # Remove the closest element from the todo list.
-    @todo = sort byDistance @todo;
-    my $u = pop @todo;
-
-    my $u_ = $graph{$u};
-
-    foreach my $edge (@{$u_->{edges}}) {
-        my $v_ = $graph{$edge->{end}};
-        if ($v_->{d} > $u_->{d} + $edge->{weight}) {
-            $v_->{d} = $u_->{d} + $edge->{weight};
-            # Store the edge; to edge->start is actually the
-            # predecessor.
-            $v_->{pred} = $edge; 
-        }
-    }
-}
-
-
-# Retrieve the shortest path from "start" to $targetPath.
-my @path = ();
-my $cur = $targetPath;
-die "don't know how to produce $targetPath\n"
-    unless defined $graph{$targetPath}->{pred};
-while ($cur ne "start") {
-    push @path, $graph{$cur}->{pred};
-    $cur = $graph{$cur}->{pred}->{start};
-}
+# Compute the shortest path.
+my @path = computeSmallestDownload $targetPath;
+die "don't know how to produce $targetPath\n" if scalar @path == 0;
 
 
 # Traverse the shortest path, perform the actions described by the

scripts/generate-patches.pl.in

@@ -1,408 +0,0 @@
-#! @perl@ -w -I@libexecdir@/nix
-
-use strict;
-use File::Temp qw(tempdir);
-use readmanifest;
-
-
-# Some patch generations options.
-
-# Max size of NAR archives to generate patches for.
-my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
-$maxNarSize = 100 * 1024 * 1024 if !defined $maxNarSize;
-
-# If patch is bigger than this fraction of full archive, reject.
-my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
-$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
-
-
-die unless scalar @ARGV == 5;
-
-my $hashAlgo = "sha256";
-
-my $cacheDir = $ARGV[0];
-my $patchesDir = $ARGV[1];
-my $patchesURL = $ARGV[2];
-my $srcDir = $ARGV[3];
-my $dstDir = $ARGV[4];
-
-my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-print "TEMP = $tmpDir\n";
-
-#END { rmdir $tmpDir; }
-
-my %srcNarFiles;
-my %srcLocalPaths;
-my %srcPatches;
-
-my %dstNarFiles;
-my %dstLocalPaths;
-my %dstPatches;
-
-readManifest "$srcDir/MANIFEST",
-    \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
-
-readManifest "$dstDir/MANIFEST",
-    \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
-
-
-sub findOutputPaths {
-    my $narFiles = shift;
-
-    my %outPaths;
-    
-    foreach my $p (keys %{$narFiles}) {
-
-        # Ignore store expressions.
-        next if ($p =~ /\.store$/);
-        next if ($p =~ /\.drv$/);
-        
-        # Ignore builders (too much ambiguity -- they're all called
-        # `builder.sh').
-        next if ($p =~ /\.sh$/);
-        next if ($p =~ /\.patch$/);
-        
-        # Don't bother including tar files etc.
-        next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
-
-        $outPaths{$p} = 1;
-    }
-
-    return %outPaths;
-}
-
-print "finding src output paths...\n";
-my %srcOutPaths = findOutputPaths \%srcNarFiles;
-
-print "finding dst output paths...\n";
-my %dstOutPaths = findOutputPaths \%dstNarFiles;
-
-
-sub getNameVersion {
-    my $p = shift;
-    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
-    my $name = $1;
-    my $version = $2;
-    $name =~ s/^-//;
-    $version =~ s/^-//;
-    return ($name, $version);
-}
-
-
-# A quick hack to get a measure of the `distance' between two
-# versions: it's just the position of the first character that differs
-# (or 999 if they are the same).
-sub versionDiff {
-    my $s = shift;
-    my $t = shift;
-    my $i;
-    return 999 if $s eq $t;
-    for ($i = 0; $i < length $s; $i++) {
-        return $i if $i >= length $t or
-            substr($s, $i, 1) ne substr($t, $i, 1);
-    }
-    return $i;
-}
-
-
-sub getNarBz2 {
-    my $narFiles = shift;
-    my $storePath = shift;
-    
-    my $narFileList = $$narFiles{$storePath};
-    die "missing store expression $storePath" unless defined $narFileList;
-
-    my $narFile = @{$narFileList}[0];
-    die unless defined $narFile;
-
-    $narFile->{url} =~ /\/([^\/]+)$/;
-    die unless defined $1;
-    return "$cacheDir/$1";
-}
-
-
-sub containsPatch {
-    my $patches = shift;
-    my $storePath = shift;
-    my $basePath = shift;
-    my $patchList = $$patches{$storePath};
-    return 0 if !defined $patchList;
-    my $found = 0;
-    foreach my $patch (@{$patchList}) {
-        # !!! baseHash might differ
-        return 1 if $patch->{basePath} eq $basePath;
-    }
-    return 0;
-}
-
-
-# Compute the "weighted" number of uses of a path in the build graph.
-sub computeUses {
-    my $narFiles = shift;
-    my $path = shift;
-
-    # Find the deriver of $path.
-    return 1 unless defined $$narFiles{$path};
-    my $deriver = @{$$narFiles{$path}}[0]->{deriver};
-    return 1 unless defined $deriver && $deriver ne "";
-
-#    print "  DERIVER $deriver\n";
-
-    # Optimisation: build the referrers graph from the references
-    # graph.
-    my %referrers;
-    foreach my $q (keys %{$narFiles}) {
-        my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
-        foreach my $r (@refs) {
-            $referrers{$r} = [] unless defined $referrers{$r};
-            push @{$referrers{$r}}, $q;
-        }
-    }
-
-    # Determine the shortest path from $deriver to all other reachable
-    # paths in the `referrers' graph.
-
-    my %dist;
-    $dist{$deriver} = 0;
-
-    my @queue = ($deriver);
-    my $pos = 0;
-    
-    while ($pos < scalar @queue) {
-        my $p = $queue[$pos];
-        $pos++;
-
-        foreach my $q (@{$referrers{$p}}) {
-            if (!defined $dist{$q}) {
-                $dist{$q} = $dist{$p} + 1;
-#                print "    $q $dist{$q}\n";
-                push @queue, $q;
-            }
-        }
-    }
-
-    my $wuse = 1.0;
-    foreach my $user (keys %dist) {
-        next if $user eq $deriver;
-#        print "    $user $dist{$user}\n";
-        $wuse += 1.0 / 2.0**$dist{$user};
-    }
-
-#    print "    XXX $path $wuse\n";
-    
-    return $wuse;
-}
-
-
-# For each output path in the destination, see if we need to / can
-# create a patch.
-
-print "creating patches...\n";
-
-foreach my $p (keys %dstOutPaths) {
-
-    # If exactly the same path already exists in the source, skip it.
-    next if defined $srcOutPaths{$p};
-    
-    print "  $p\n";
-
-    # If not, then we should find the paths in the source that are
-    # `most' likely to be present on a system that wants to install
-    # this path.
-
-    (my $name, my $version) = getNameVersion $p;
-
-    my @closest = ();
-    my $closestVersion;
-    my $minDist = -1; # actually, larger means closer
-
-    # Find all source paths with the same name.
-
-    foreach my $q (keys %srcOutPaths) {
-        (my $name2, my $version2) = getNameVersion $q;
-        if ($name eq $name2) {
-
-            # If the sizes differ too much, then skip.  This
-            # disambiguates between, e.g., a real component and a
-            # wrapper component (cf. Firefox in Nixpkgs).
-            my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
-            my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
-            my $ratio = $srcSize / $dstSize;
-            $ratio = 1 / $ratio if $ratio < 1;
-#            print "  SIZE $srcSize $dstSize $ratio $q\n";
-
-            if ($ratio >= 3) {
-                print "  SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
-                next;
-            }
-
-            # If the numbers of weighted uses differ too much, then
-            # skip.  This disambiguates between, e.g., the bootstrap
-            # GCC and the final GCC in Nixpkgs.
-            my $srcUses = computeUses \%srcNarFiles, $q;
-            my $dstUses = computeUses \%dstNarFiles, $p;
-            $ratio = $srcUses / $dstUses;
-            $ratio = 1 / $ratio if $ratio < 1;
-            print "  USE $srcUses $dstUses $ratio $q\n";
-            
-#            if ($ratio >= 2) {
-#                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
-#                next;
-#            }
-
-            # If there are multiple matching names, include the ones
-            # with the closest version numbers.
-            my $dist = versionDiff $version, $version2;
-            if ($dist > $minDist) {
-                $minDist = $dist;
-                @closest = ($q);
-                $closestVersion = $version2;
-            } elsif ($dist == $minDist) {
-                push @closest, $q;
-            }
-        }
-    }
-
-    if (scalar(@closest) == 0) {
-        print "  NO BASE: $p\n";
-        next;
-    }
-
-    foreach my $closest (@closest) {
-
-        # Generate a patch between $closest and $p.
-        print "  $p <- $closest\n";
-
-        # If the patch already exists, skip it.
-        if (containsPatch(\%srcPatches, $p, $closest) ||
-            containsPatch(\%dstPatches, $p, $closest))
-        {
-            print "    skipping, already exists\n";
-            next;
-        }
-
-#        next;
-        
-        my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
-        my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
-
-        system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
-            or die "cannot unpack $srcNarBz2";
-
-        if ((stat "$tmpDir/A")[7] >= $maxNarSize) {
-            print "    skipping, source is too large\n";
-            next;
-        }
-        
-        system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0
-            or die "cannot unpack $dstNarBz2";
-
-        if ((stat "$tmpDir/B")[7] >= $maxNarSize) {
-            print "    skipping, destination is too large\n";
-            next;
-        }
-        
-        system("@libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF") == 0
-            or die "cannot compute binary diff";
-
-        my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die;
-        chomp $baseHash;
-
-        my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die;
-        chomp $narHash;
-
-        my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die;
-        chomp $narDiffHash;
-
-        my $narDiffSize = (stat "$tmpDir/DIFF")[7];
-        my $dstNarBz2Size = (stat $dstNarBz2)[7];
-
-        print "    size $narDiffSize; full size $dstNarBz2Size\n";
-        
-        if ($narDiffSize >= $dstNarBz2Size) {
-            print "    rejecting; patch bigger than full archive\n";
-            next;
-        }
-    
-        if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
-            print "    rejecting; patch too large relative to full archive\n";
-            next;
-        }
-    
-        my $finalName =
-            "$narDiffHash.nar-bsdiff";
-
-        if (-e "$patchesDir/$finalName") {
-            print "    not copying, already exists\n";
-        }
-
-        else {
-
-            system("cp '$tmpDir/DIFF' '$patchesDir/$finalName.tmp'") == 0
-                or die "cannot copy diff";
-            
-            rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
-                or die "cannot rename $patchesDir/$finalName.tmp";
-            
-        }
-        
-        # Add the patch to the manifest.
-        addPatch \%dstPatches, $p,
-            { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
-            , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
-            , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
-            }, 0;
-    }
-}
-
-
-# Add in any potentially useful patches in the source (namely, those
-# patches that produce either paths in the destination or paths that
-# can be used as the base for other useful patches).
-
-print "propagating patches...\n";
-
-my $changed;
-do {
-    # !!! we repeat this to reach the transitive closure; inefficient
-    $changed = 0;
-
-    print "loop\n";
-
-    my %dstBasePaths;
-    foreach my $q (keys %dstPatches) {
-	foreach my $patch (@{$dstPatches{$q}}) {
-	    $dstBasePaths{$patch->{basePath}} = 1;
-	}
-    }
-
-    foreach my $p (keys %srcPatches) {
-        my $patchList = $srcPatches{$p};
-
-        my $include = 0;
-
-        # Is path $p included in the destination?  If so, include
-        # patches that produce it.
-        $include = 1 if defined $dstNarFiles{$p};
-
-        # Is path $p a path that serves as a base for paths in the
-        # destination?  If so, include patches that produce it.
-	# !!! check baseHash
-        $include = 1 if defined $dstBasePaths{$p};
-
-        if ($include) {
-            foreach my $patch (@{$patchList}) {
-                $changed = 1 if addPatch \%dstPatches, $p, $patch;
-            }
-        }
-        
-    }
-    
-} while $changed;
-
-
-# Rewrite the manifest of the destination (with the new patches).
-writeManifest "$dstDir/MANIFEST",
-    \%dstNarFiles, \%dstPatches;

scripts/nix-build.in

@@ -102,7 +102,7 @@ EOF
         $n += 2;
     }
 
-    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type") {
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @buildArgs, ($arg, $ARGV[$n]);
@@ -123,6 +123,11 @@ EOF
         $verbose = 1;
     }
     
+    elsif ($arg eq "--quiet") {
+        push @buildArgs, $arg;
+        push @instArgs, $arg;
+    }
+    
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -165,7 +170,7 @@ foreach my $expr (@exprs) {
 
     # Build.
     my @outPaths;
-    $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-rv",
+    $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-r",
         @buildArgs, @drvPaths;
     while (<OUTPATHS>) {chomp; push @outPaths, $_;}
     if (!close OUTPATHS) {

scripts/nix-copy-closure.in

@@ -1,6 +1,6 @@
 #! @perl@ -w -I@libexecdir@/nix
 
-use ssh;
+use SSH;
 
 my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
@@ -61,7 +61,7 @@ if ($toMode) { # Copy TO the remote machine.
     my @allStorePaths;
 
     # Get the closure of this path.
-    my $pid = open(READ, "$binDir/nix-store --query --requisites @storePaths|") or die;
+    my $pid = open(READ, "set -f; $binDir/nix-store --query --requisites @storePaths|") or die;
     
     while (<READ>) {
         chomp;
@@ -73,7 +73,7 @@ if ($toMode) { # Copy TO the remote machine.
 
 
     # Ask the remote host which paths are invalid.
-    open(READ, "ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
+    open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
     my @missing = ();
     while (<READ>) {
         chomp;
@@ -88,7 +88,7 @@ if ($toMode) { # Copy TO the remote machine.
         print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
+        system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
             or die "copying store paths to remote machine `$sshHost' failed: $?";
     }
 
@@ -101,7 +101,7 @@ else { # Copy FROM the remote machine.
     # machine.  Paths are assumed to be store paths; there is no
     # resolution (following of symlinks).
     my $pid = open(READ,
-        "ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
+        "set -f; ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
     
     my @allStorePaths;
 
@@ -115,7 +115,7 @@ else { # Copy FROM the remote machine.
 
 
     # What paths are already valid locally?
-    open(READ, "@bindir@/nix-store --check-validity --print-invalid @allStorePaths|");
+    open(READ, "set -f; @bindir@/nix-store --check-validity --print-invalid @allStorePaths|");
     my @missing = ();
     while (<READ>) {
         chomp;
@@ -130,7 +130,7 @@ else { # Copy FROM the remote machine.
         print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0
+        system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0
             or die "copying store paths from remote machine `$sshHost' failed: $?";
     }
 

scripts/nix-generate-patches.in

@@ -0,0 +1,42 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use File::Temp qw(tempdir);
+use NixManifest;
+use GeneratePatches;
+
+if (scalar @ARGV != 5) {
+    print STDERR <<EOF;
+Usage: nix-generate-patches NAR-DIR PATCH-DIR PATCH-URI OLD-MANIFEST NEW-MANIFEST
+
+This command generates binary patches between NAR files listed in
+OLD-MANIFEST and NEW-MANIFEST.  The patches are written to the
+directory PATCH-DIR, and the prefix PATCH-URI is used to generate URIs
+for the patches.  The patches are added to NEW-MANIFEST.  All NARs are
+required to exist in NAR-DIR.  Patches are generated between
+succeeding versions of packages with the same name.
+EOF
+    exit 1;
+}
+
+my $narPath = $ARGV[0];
+my $patchesPath = $ARGV[1];
+my $patchesURL = $ARGV[2];
+my $srcManifest = $ARGV[3];
+my $dstManifest = $ARGV[4];
+
+my (%srcNarFiles, %srcLocalPaths, %srcPatches);
+readManifest $srcManifest, \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
+
+my (%dstNarFiles, %dstLocalPaths, %dstPatches);
+readManifest $dstManifest, \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
+
+my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+    or die "cannot create a temporary directory";
+
+generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches,
+    $narPath, $patchesPath, $patchesURL, $tmpDir;
+
+propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches;
+
+writeManifest $dstManifest, \%dstNarFiles, \%dstPatches;

scripts/nix-pull.in

@@ -2,7 +2,7 @@
 
 use strict;
 use File::Temp qw(tempdir);
-use readmanifest;
+use NixManifest;
 
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";

scripts/nix-push.in

@@ -2,7 +2,7 @@
 
 use strict;
 use File::Temp qw(tempdir);
-use readmanifest;
+use NixManifest;
 
 my $hashAlgo = "sha256";
 
@@ -172,12 +172,6 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
     $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
     close HASH;
 
-    open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
-    my $narHash = <HASH>;
-    chomp $narHash;
-    $narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
-    close HASH;
-    
     my $narName = "$narbz2Hash.nar.bz2";
 
     my $narFile = "$narDir/$narName";
@@ -195,6 +189,14 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
     chomp $deriver;
     $deriver = "" if $deriver eq "unknown-deriver";
 
+    my $narHash = `$binDir/nix-store --query --hash '$storePath'`;
+    die "cannot query hash for `$storePath'" if $? != 0;
+    chomp $narHash;
+
+    my $narSize = `$binDir/nix-store --query --size '$storePath'`;
+    die "cannot query size for `$storePath'" if $? != 0;
+    chomp $narSize;
+
     my $url;
     if ($localCopy) {
         $url = "$targetArchivesUrl/$narName";
@@ -205,7 +207,8 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
         { url => $url
         , hash => "$hashAlgo:$narbz2Hash"
         , size => $narbz2Size
-        , narHash => "$hashAlgo:$narHash"
+        , narHash => "$narHash"
+        , narSize => $narSize
         , references => $references
         , deriver => $deriver
         }

src/bin2c/bin2c.c

@@ -14,10 +14,10 @@ int main(int argc, char * * argv)
 {
     int c;
     if (argc != 2) abort();
-    print("static unsigned char %s[] = {", argv[1]);
+    print("static unsigned char %s[] = { ", argv[1]);
     while ((c = getchar()) != EOF) {
         print("0x%02x, ", (unsigned char) c);
     }
-    print("};\n");
+    print("0 };\n");
     return 0;
 }

src/bsdiff-4.3/Makefile.am

@@ -1,3 +1,5 @@
+EXTRA_DIST = compat-include
+
 libexec_PROGRAMS = bsdiff bspatch
 
 bsdiff_SOURCES = bsdiff.c
@@ -8,4 +10,4 @@ bspatch_SOURCES = bspatch.c
 
 bspatch_LDADD = ${bzip2_lib}
 
-AM_CFLAGS = -O3 ${bzip2_include}
+AM_CFLAGS = -O3 ${bzip2_include} ${bsddiff_compat_include}

src/bsdiff-4.3/bsdiff.c

@@ -277,6 +277,7 @@ int main(int argc,char *argv[])
 		for(scsc=scan+=len;scan<newsize;scan++) {
 			len=search(I,old,oldsize,new+scan,newsize-scan,
 					0,oldsize,&pos);
+			if (len > 64 * 1024) break;
 
 			for(;scsc<scan+len;scsc++)
 			if((scsc+lastoffset<oldsize) &&

src/bsdiff-4.3/compat-include/err.h

@@ -0,0 +1,12 @@
+/* Simulate BSD's <err.h> functionality. */
+
+#ifndef COMPAT_ERR_H_INCLUDED
+#define COMPAT_ERR_H_INCLUDED 1
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define err(rc,...)  do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
+#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
+
+#endif

src/libexpr/Makefile.am

@@ -11,7 +11,7 @@ pkginclude_HEADERS = \
  names.hh symbol-table.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
- ../boost/format/libformat.la
+ ../boost/format/libformat.la @boehmgc_lib@
 
 BUILT_SOURCES = \
  parser-tab.hh lexer-tab.hh parser-tab.cc lexer-tab.cc

src/libexpr/attr-path.cc

@@ -5,8 +5,9 @@
 namespace nix {
 
 
+// !!! Shouldn't we return a pointer to a Value?
 void findAlongAttrPath(EvalState & state, const string & attrPath,
-    const Bindings & autoArgs, Expr * e, Value & v)
+    Bindings & autoArgs, Expr * e, Value & v)
 {
     Strings tokens = tokenizeString(attrPath, ".");
 
@@ -48,7 +49,7 @@ void findAlongAttrPath(EvalState & state, const string & attrPath,
             Bindings::iterator a = v.attrs->find(state.symbols.create(attr));
             if (a == v.attrs->end())
                 throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % curPath);
-            v = a->second.value;
+            v = *a->value;
         }
 
         else if (apType == apIndex) {

src/libexpr/attr-path.hh

@@ -1,17 +1,17 @@
 #ifndef __ATTR_PATH_H
 #define __ATTR_PATH_H
 
+#include "eval.hh"
+
 #include <string>
 #include <map>
 
-#include "eval.hh"
-
 
 namespace nix {
 
     
 void findAlongAttrPath(EvalState & state, const string & attrPath,
-    const Bindings & autoArgs, Expr * e, Value & v);
+    Bindings & autoArgs, Expr * e, Value & v);
 
     
 }

src/libexpr/common-opts.cc

@@ -20,13 +20,17 @@ bool parseOptionArg(const string & arg, Strings::iterator & i,
     if (i == argsEnd) throw error;
     string value = *i++;
 
-    Value & v(autoArgs[state.symbols.create(name)].value);
+    /* !!! check for duplicates! */
+    Value * v = state.allocValue();
+    autoArgs.push_back(Attr(state.symbols.create(name), v));
 
     if (arg == "--arg")
-        state.mkThunk_( v, parseExprFromString(state, value, absPath(".")));
+        state.mkThunk_(*v, parseExprFromString(state, value, absPath(".")));
     else
-        mkString(v, value);
-    
+        mkString(*v, value);
+
+    autoArgs.sort(); // !!! inefficient
+
     return true;
 }
 

src/libexpr/eval.cc

@@ -8,15 +8,46 @@
 
 #include <cstring>
 
+#if HAVE_BOEHMGC
+
+#include <gc/gc.h>
+#include <gc/gc_cpp.h>
+
+#define NEW new (UseGC)
+
+#else
+
+#define GC_STRDUP strdup
+#define GC_MALLOC malloc
+
+#define NEW new
+
+#endif
+
 
 #define LocalNoInline(f) static f __attribute__((noinline)); f
 #define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f
 
 
 namespace nix {
+
+
+Bindings::iterator Bindings::find(const Symbol & name)
+{
+    Attr key(name, 0);
+    iterator i = lower_bound(begin(), end(), key);
+    if (i != end() && i->name == name) return i;
+    return end();
+}
+
+
+void Bindings::sort()
+{
+    std::sort(begin(), end());
+}
     
 
-std::ostream & operator << (std::ostream & str, Value & v)
+std::ostream & operator << (std::ostream & str, const Value & v)
 {
     switch (v.type) {
     case tInt:
@@ -39,14 +70,14 @@ std::ostream & operator << (std::ostream & str, Value & v)
         str << v.path; // !!! escaping?
         break;
     case tNull:
-        str << "true";
+        str << "null";
         break;
     case tAttrs: {
         str << "{ ";
         typedef std::map<string, Value *> Sorted;
         Sorted sorted;
         foreach (Bindings::iterator, i, *v.attrs)
-            sorted[i->first] = &i->second.value;
+            sorted[i->name] = i->value;
         foreach (Sorted::iterator, i, sorted)
             str << i->first << " = " << *i->second << "; ";
         str << "}";
@@ -59,7 +90,7 @@ std::ostream & operator << (std::ostream & str, Value & v)
         str << "]";
         break;
     case tThunk:
-    case tCopy:
+    case tApp:
         str << "<CODE>";
         break;
     case tLambda:
@@ -91,7 +122,6 @@ string showType(const Value & v)
         case tThunk: return "a thunk";
         case tApp: return "a function application";
         case tLambda: return "a function";
-        case tCopy: return "a copy";
         case tBlackhole: return "a black hole";
         case tPrimOp: return "a built-in function";
         case tPrimOpApp: return "a partially applied built-in function";
@@ -108,12 +138,14 @@ EvalState::EvalState()
     , sMeta(symbols.create("meta"))
     , sName(symbols.create("name"))
     , sSystem(symbols.create("system"))
+    , sOverrides(symbols.create("__overrides"))
     , baseEnv(allocEnv(128))
     , baseEnvDispl(0)
     , staticBaseEnv(false, 0)
 {
     nrEnvs = nrValuesInEnvs = nrValues = nrListElems = 0;
     nrEvaluated = recursionDepth = maxRecursionDepth = 0;
+    nrAttrsets = nrOpUpdates = nrOpUpdateValuesCopied = 0;
     deepestStack = (char *) -1;
 
     createBaseEnv();
@@ -130,25 +162,26 @@ EvalState::~EvalState()
 
 void EvalState::addConstant(const string & name, Value & v)
 {
+    Value * v2 = allocValue();
+    *v2 = v;
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
-    baseEnv.values[baseEnvDispl++] = v;
+    baseEnv.values[baseEnvDispl++] = v2;
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    (*baseEnv.values[0].attrs)[symbols.create(name2)].value = v;
+    baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2));
 }
 
 
 void EvalState::addPrimOp(const string & name,
-    unsigned int arity, PrimOp primOp)
+    unsigned int arity, PrimOpFun primOp)
 {
-    Value v;
+    Value * v = allocValue();
     string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    v.type = tPrimOp;
-    v.primOp.arity = arity;
-    v.primOp.fun = primOp;
-    v.primOp.name = strdup(name2.c_str());
+    Symbol sym = symbols.create(name2);
+    v->type = tPrimOp;
+    v->primOp = NEW PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
     baseEnv.values[baseEnvDispl++] = v;
-    (*baseEnv.values[0].attrs)[symbols.create(name2)].value = v;
+    baseEnv.values[0]->attrs->push_back(Attr(sym, v));
 }
 
 
@@ -216,7 +249,7 @@ LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2,
 void mkString(Value & v, const char * s)
 {
     v.type = tString;
-    v.string.s = strdup(s);
+    v.string.s = GC_STRDUP(s);
     v.string.context = 0;
 }
 
@@ -226,18 +259,28 @@ void mkString(Value & v, const string & s, const PathSet & context)
     mkString(v, s.c_str());
     if (!context.empty()) {
         unsigned int n = 0;
-        v.string.context = new const char *[context.size() + 1];
-        foreach (PathSet::const_iterator, i, context) 
-            v.string.context[n++] = strdup(i->c_str());
+        v.string.context = (const char * *)
+            GC_MALLOC((context.size() + 1) * sizeof(char *));
+        foreach (PathSet::const_iterator, i, context)  
+            v.string.context[n++] = GC_STRDUP(i->c_str());
         v.string.context[n] = 0;
     }
 }
 
 
+void mkString(Value & v, const Symbol & s)
+{
+    v.type = tString;
+    v.string.s = ((string) s).c_str();
+    v.string.context = 0;
+}
+
+
 void mkPath(Value & v, const char * s)
 {
+    clearValue(v);
     v.type = tPath;
-    v.path = strdup(s);
+    v.path = GC_STRDUP(s);
 }
 
 
@@ -247,22 +290,22 @@ Value * EvalState::lookupVar(Env * env, const VarRef & var)
     
     if (var.fromWith) {
         while (1) {
-            Bindings::iterator j = env->values[0].attrs->find(var.name);
-            if (j != env->values[0].attrs->end())
-                return &j->second.value;
+            Bindings::iterator j = env->values[0]->attrs->find(var.name);
+            if (j != env->values[0]->attrs->end())
+                return j->value;
             if (env->prevWith == 0)
                 throwEvalError("undefined variable `%1%'", var.name);
             for (unsigned int l = env->prevWith; l; --l, env = env->up) ;
         }
     } else
-        return &env->values[var.displ];
+        return env->values[var.displ];
 }
 
 
-Value * EvalState::allocValues(unsigned int count)
+Value * EvalState::allocValue()
 {
-    nrValues += count;
-    return new Value[count]; // !!! check destructor
+    nrValues++;
+    return (Value *) GC_MALLOC(sizeof(Value));
 }
 
 
@@ -270,24 +313,51 @@ Env & EvalState::allocEnv(unsigned int size)
 {
     nrEnvs++;
     nrValuesInEnvs += size;
-    Env * env = (Env *) malloc(sizeof(Env) + size * sizeof(Value));
+    Env * env = (Env *) GC_MALLOC(sizeof(Env) + size * sizeof(Value *));
+
+    /* Clear the values because maybeThunk() expects this. */
+    for (unsigned i = 0; i < size; ++i)
+        env->values[i] = 0;
+    
     return *env;
 }
 
 
+Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
+{
+    Value * v = allocValue();
+    vAttrs.attrs->push_back(Attr(name, v));
+    return v;
+}
+
+    
 void EvalState::mkList(Value & v, unsigned int length)
 {
     v.type = tList;
     v.list.length = length;
-    v.list.elems = new Value *[length];
+    v.list.elems = (Value * *) GC_MALLOC(length * sizeof(Value *));
     nrListElems += length;
 }
 
 
-void EvalState::mkAttrs(Value & v)
+void EvalState::mkAttrs(Value & v, unsigned int expected)
 {
+    clearValue(v);
     v.type = tAttrs;
-    v.attrs = new Bindings;
+    v.attrs = NEW Bindings;
+    v.attrs->reserve(expected);
+    nrAttrsets++;
+}
+
+
+unsigned long nrThunks = 0;
+
+static inline void mkThunk(Value & v, Env & env, Expr * expr)
+{
+    v.type = tThunk;
+    v.thunk.env = &env;
+    v.thunk.expr = expr;
+    nrThunks++;
 }
 
 
@@ -297,14 +367,28 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 }
 
 
-void EvalState::cloneAttrs(Value & src, Value & dst)
+unsigned long nrAvoided = 0;
+
+/* Create a thunk for the delayed computation of the given expression
+   in the given environment.  But if the expression is a variable,
+   then look it up right away.  This significantly reduces the number
+   of thunks allocated. */
+Value * EvalState::maybeThunk(Env & env, Expr * expr)
 {
-    mkAttrs(dst);
-    foreach (Bindings::iterator, i, *src.attrs) {
-        Attr & a = (*dst.attrs)[i->first];
-        mkCopy(a.value, i->second.value);
-        a.pos = i->second.pos;
+    ExprVar * var;
+    /* Ignore variables from `withs' because they can throw an
+       exception. */
+    if ((var = dynamic_cast<ExprVar *>(expr))) {
+        Value * v = lookupVar(&env, var->info);
+        /* The value might not be initialised in the environment yet.
+           In that case, ignore it. */
+        if (v) { nrAvoided++; return v; }
     }
+
+    Value * v = allocValue();
+    mkThunk(*v, env, expr);
+
+    return v;
 }
 
 
@@ -402,7 +486,7 @@ void ExprInt::eval(EvalState & state, Env & env, Value & v)
 
 void ExprString::eval(EvalState & state, Env & env, Value & v)
 {
-    mkString(v, s.c_str());
+    mkString(v, s);
 }
 
 
@@ -414,49 +498,67 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
 
 void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 {
-    state.mkAttrs(v);
+    state.mkAttrs(v, attrs.size());
 
     if (recursive) {
         /* Create a new environment that contains the attributes in
            this `rec'. */
-        Env & env2(state.allocEnv(attrs.size() + inherited.size()));
+        Env & env2(state.allocEnv(attrs.size()));
         env2.up = &env;
 
-        unsigned int displ = 0;
-        
+        AttrDefs::iterator overrides = attrs.find(state.sOverrides);
+        bool hasOverrides = overrides != attrs.end();
+
         /* The recursive attributes are evaluated in the new
-           environment. */
-        foreach (Attrs::iterator, i, attrs) {
-            nix::Attr & a = (*v.attrs)[i->first];
-            mkCopy(a.value, env2.values[displ]);
-            mkThunk(env2.values[displ++], env2, i->second.first);
-            a.pos = &i->second.second;
+           environment, while the inherited attributes are evaluated
+           in the original environment. */
+        unsigned int displ = 0;
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited) {
+                /* !!! handle overrides? */
+                Value * vAttr = state.lookupVar(&env, i->second.var);
+                env2.values[displ++] = vAttr;
+                v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos));
+            } else {
+                Value * vAttr;
+                if (hasOverrides) {
+                    vAttr = state.allocValue();
+                    mkThunk(*vAttr, env2, i->second.e);
+                } else
+                    vAttr = state.maybeThunk(env2, i->second.e);
+                env2.values[displ++] = vAttr;
+                v.attrs->push_back(Attr(i->first, vAttr, &i->second.pos));
+            }
+        
+        /* If the rec contains an attribute called `__overrides', then
+           evaluate it, and add the attributes in that set to the rec.
+           This allows overriding of recursive attributes, which is
+           otherwise not possible.  (You can use the // operator to
+           replace an attribute, but other attributes in the rec will
+           still reference the original value, because that value has
+           been substituted into the bodies of the other attributes.
+           Hence we need __overrides.) */
+        if (hasOverrides) {
+            Value * vOverrides = (*v.attrs)[overrides->second.displ].value;
+            state.forceAttrs(*vOverrides);
+            foreach (Bindings::iterator, i, *vOverrides->attrs) {
+                AttrDefs::iterator j = attrs.find(i->name);
+                if (j != attrs.end()) {
+                    (*v.attrs)[j->second.displ] = *i;
+                    env2.values[j->second.displ] = i->value;
+                } else
+                    v.attrs->push_back(*i);
+            }
+            v.attrs->sort();
         }
-
-        /* The inherited attributes, on the other hand, are
-           evaluated in the original environment. */
-        foreach (list<Inherited>::iterator, i, inherited) {
-            nix::Attr & a = (*v.attrs)[i->first.name];
-            Value * v2 = state.lookupVar(&env, i->first);
-            mkCopy(a.value, *v2);
-            mkCopy(env2.values[displ++], *v2);
-            a.pos = &i->second;
-        }
-
     }
 
     else {
-        foreach (Attrs::iterator, i, attrs) {
-            nix::Attr & a = (*v.attrs)[i->first];
-            mkThunk(a.value, env, i->second.first);
-            a.pos = &i->second.second;
-        }
-
-        foreach (list<Inherited>::iterator, i, inherited) {
-            nix::Attr & a = (*v.attrs)[i->first.name];
-            mkCopy(a.value, *state.lookupVar(&env, i->first));
-            a.pos = &i->second;
-        }
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited)
+                v.attrs->push_back(Attr(i->first, state.lookupVar(&env, i->second.var), &i->second.pos));
+            else
+                v.attrs->push_back(Attr(i->first, state.maybeThunk(env, i->second.e), &i->second.pos));
     }
 }
 
@@ -465,20 +567,18 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 {
     /* Create a new environment that contains the attributes in this
        `let'. */
-    Env & env2(state.allocEnv(attrs->attrs.size() + attrs->inherited.size()));
+    Env & env2(state.allocEnv(attrs->attrs.size()));
     env2.up = &env;
 
-    unsigned int displ = 0;
-
-    /* The recursive attributes are evaluated in the new
+    /* The recursive attributes are evaluated in the new environment,
+       while the inherited attributes are evaluated in the original
        environment. */
-    foreach (ExprAttrs::Attrs::iterator, i, attrs->attrs)
-        mkThunk(env2.values[displ++], env2, i->second.first);
-
-    /* The inherited attributes, on the other hand, are evaluated in
-       the original environment. */
-    foreach (list<ExprAttrs::Inherited>::iterator, i, attrs->inherited)
-        mkCopy(env2.values[displ++], *state.lookupVar(&env, i->first));
+    unsigned int displ = 0;
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        if (i->second.inherited)
+            env2.values[displ++] = state.lookupVar(&env, i->second.var);
+        else
+            env2.values[displ++] = state.maybeThunk(env2, i->second.e);
 
     state.eval(env2, body, v);
 }
@@ -487,11 +587,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
     state.mkList(v, elems.size());
-    Value * vs = state.allocValues(v.list.length);
-    for (unsigned int n = 0; n < v.list.length; ++n) {
-        v.list.elems[n] = &vs[n];
-        mkThunk(vs[n], env, elems[n]);
-    }
+    for (unsigned int n = 0; n < v.list.length; ++n)
+        v.list.elems[n] = state.maybeThunk(env, elems[n]);
 }
 
 
@@ -503,21 +600,26 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+unsigned long nrLookups = 0;
+unsigned long nrLookupSize = 0;
+
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
+    nrLookups++;
     Value v2;
     state.evalAttrs(env, e, v2);
+    nrLookupSize += v2.attrs->size();
     Bindings::iterator i = v2.attrs->find(name);
     if (i == v2.attrs->end())
         throwEvalError("attribute `%1%' missing", name);
     try {            
-        state.forceValue(i->second.value);
+        state.forceValue(*i->value);
     } catch (Error & e) {
         addErrorPrefix(e, "while evaluating the attribute `%1%' at %2%:\n",
-            name, *i->second.pos);
+            name, *i->pos);
         throw;
     }
-    v = i->second.value;
+    v = *i->value;
 }
 
 
@@ -541,24 +643,27 @@ void ExprApp::eval(EvalState & state, Env & env, Value & v)
 {
     Value vFun;
     state.eval(env, e1, vFun);
-    Value vArg;
-    mkThunk(vArg, env, e2); // !!! should this be on the heap?
-    state.callFunction(vFun, vArg, v);
+    state.callFunction(vFun, *state.maybeThunk(env, e2), v);
 }
 
 
 void EvalState::callFunction(Value & fun, Value & arg, Value & v)
 {
     if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
-        unsigned int argsLeft =
-            fun.type == tPrimOp ? fun.primOp.arity : fun.primOpApp.argsLeft;
+
+        /* Figure out the number of arguments still needed. */
+        unsigned int argsDone = 0;
+        Value * primOp = &fun;
+        while (primOp->type == tPrimOpApp) {
+            argsDone++;
+            primOp = primOp->primOpApp.left;
+        }
+        assert(primOp->type == tPrimOp);
+        unsigned int arity = primOp->primOp->arity;
+        unsigned int argsLeft = arity - argsDone;
+        
         if (argsLeft == 1) {
-            /* We have all the arguments, so call the primop.  First
-               find the primop. */
-            Value * primOp = &fun;
-            while (primOp->type == tPrimOpApp) primOp = primOp->primOpApp.left;
-            assert(primOp->type == tPrimOp);
-            unsigned int arity = primOp->primOp.arity;
+            /* We have all the arguments, so call the primop. */
                 
             /* Put all the arguments in an array. */
             Value * vArgs[arity];
@@ -569,19 +674,16 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
 
             /* And call the primop. */
             try {
-                primOp->primOp.fun(*this, vArgs, v);
+                primOp->primOp->fun(*this, vArgs, v);
             } catch (Error & e) {
-                addErrorPrefix(e, "while evaluating the builtin function `%1%':\n", primOp->primOp.name);
+                addErrorPrefix(e, "while evaluating the builtin function `%1%':\n", primOp->primOp->name);
                 throw;
             }
         } else {
-            Value * v2 = allocValues(2);
-            v2[0] = fun;
-            v2[1] = arg;
             v.type = tPrimOpApp;
-            v.primOpApp.left = &v2[0];
-            v.primOpApp.right = &v2[1];
-            v.primOpApp.argsLeft = argsLeft - 1;
+            v.primOpApp.left = allocValue();
+            *v.primOpApp.left = fun;
+            v.primOpApp.right = &arg;
         }
         return;
     }
@@ -599,13 +701,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
     unsigned int displ = 0;
 
     if (!fun.lambda.fun->matchAttrs)
-        env2.values[displ++] = arg;
+        env2.values[displ++] = &arg;
 
     else {
         forceAttrs(arg);
         
         if (!fun.lambda.fun->arg.empty())
-            env2.values[displ++] = arg;
+            env2.values[displ++] = &arg;
 
         /* For each formal argument, get the actual argument.  If
            there is no matching actual argument but the formal
@@ -615,11 +717,11 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
             Bindings::iterator j = arg.attrs->find(i->name);
             if (j == arg.attrs->end()) {
                 if (!i->def) throwTypeError("function at %1% called without required argument `%2%'",
-                    fun.lambda.fun->pos, i->name);   
-                mkThunk(env2.values[displ++], env2, i->def);
+                    fun.lambda.fun->pos, i->name);
+                env2.values[displ++] = maybeThunk(env2, i->def);
             } else {
                 attrsUsed++;
-                mkCopy(env2.values[displ++], j->second.value);
+                env2.values[displ++] = j->value;
             }
         }
 
@@ -627,7 +729,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
            argument (unless the attribute match specifies a `...').
            TODO: show the names of the expected/unexpected
            arguments. */
-        if (!fun.lambda.fun->formals->ellipsis && attrsUsed != arg.attrs->size())
+        if (!fun.lambda.fun->formals->ellipsis && attrsUsed != arg.attrs->size()) 
             throwTypeError("function at %1% called with unexpected argument", fun.lambda.fun->pos);
     }
 
@@ -640,7 +742,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v)
 }
 
 
-void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res)
+void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 {
     forceValue(fun);
 
@@ -650,16 +752,18 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res
     }
 
     Value actualArgs;
-    mkAttrs(actualArgs);
+    mkAttrs(actualArgs, fun.lambda.fun->formals->formals.size());
 
     foreach (Formals::Formals_::iterator, i, fun.lambda.fun->formals->formals) {
-        Bindings::const_iterator j = args.find(i->name);
+        Bindings::iterator j = args.find(i->name);
         if (j != args.end())
-            (*actualArgs.attrs)[i->name] = j->second;
+            actualArgs.attrs->push_back(*j);
         else if (!i->def)
             throwTypeError("cannot auto-call a function that has an argument without a default value (`%1%')", i->name);
     }
 
+    actualArgs.attrs->sort();
+
     callFunction(fun, actualArgs, res);
 }
 
@@ -670,7 +774,8 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     env2.up = &env;
     env2.prevWith = prevWith;
 
-    state.evalAttrs(env, attrs, env2.values[0]);
+    env2.values[0] = state.allocValue();
+    state.evalAttrs(env, attrs, *env2.values[0]);
 
     state.eval(env2, body, v);
 }
@@ -732,18 +837,38 @@ void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 {
-    Value v2;
-    state.evalAttrs(env, e1, v2);
-        
-    state.cloneAttrs(v2, v);
-        
+    Value v1, v2;
+    state.evalAttrs(env, e1, v1);
     state.evalAttrs(env, e2, v2);
-    
-    foreach (Bindings::iterator, i, *v2.attrs) {
-        Attr & a = (*v.attrs)[i->first];
-        mkCopy(a.value, i->second.value);
-        a.pos = i->second.pos;
+
+    state.nrOpUpdates++;
+
+    if (v1.attrs->size() == 0) { v = v2; return; }
+    if (v2.attrs->size() == 0) { v = v1; return; }
+
+    state.mkAttrs(v, v1.attrs->size() + v2.attrs->size());
+
+    /* Merge the attribute sets, preferring values from the second
+       set.  Make sure to keep the resulting vector in sorted
+       order. */
+    Bindings::iterator i = v1.attrs->begin();
+    Bindings::iterator j = v2.attrs->begin();
+
+    while (i != v1.attrs->end() && j != v2.attrs->end()) {
+        if (i->name == j->name) {
+            v.attrs->push_back(*j);
+            ++i; ++j;
+        }
+        else if (i->name < j->name)
+            v.attrs->push_back(*i++);
+        else
+            v.attrs->push_back(*j++);
     }
+
+    while (i != v1.attrs->end()) v.attrs->push_back(*i++);
+    while (j != v2.attrs->end()) v.attrs->push_back(*j++);
+    
+    state.nrOpUpdateValuesCopied += v.attrs->size();
 }
 
 
@@ -806,10 +931,6 @@ void EvalState::forceValue(Value & v)
             throw;
         }
     }
-    else if (v.type == tCopy) {
-        forceValue(*v.val);
-        v = *v.val;
-    }
     else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v);
     else if (v.type == tBlackhole)
@@ -823,7 +944,7 @@ void EvalState::strictForceValue(Value & v)
     
     if (v.type == tAttrs) {
         foreach (Bindings::iterator, i, *v.attrs)
-            strictForceValue(i->second.value);
+            strictForceValue(*i->value);
     }
     
     else if (v.type == tList) {
@@ -884,12 +1005,18 @@ string EvalState::forceString(Value & v)
 }
 
 
-string EvalState::forceString(Value & v, PathSet & context)
+void copyContext(const Value & v, PathSet & context)
 {
-    string s = forceString(v);
     if (v.string.context)
         for (const char * * p = v.string.context; *p; ++p) 
             context.insert(*p);
+}
+
+
+string EvalState::forceString(Value & v, PathSet & context)
+{
+    string s = forceString(v);
+    copyContext(v, context);
     return s;
 }
 
@@ -908,7 +1035,7 @@ bool EvalState::isDerivation(Value & v)
 {
     if (v.type != tAttrs) return false;
     Bindings::iterator i = v.attrs->find(sType);
-    return i != v.attrs->end() && forceStringNoCtx(i->second.value) == "derivation";
+    return i != v.attrs->end() && forceStringNoCtx(*i->value) == "derivation";
 }
 
 
@@ -920,9 +1047,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
     string s;
 
     if (v.type == tString) {
-        if (v.string.context) 
-            for (const char * * p = v.string.context; *p; ++p) 
-                context.insert(*p);
+        copyContext(v, context);
         return v.string.s;
     }
 
@@ -954,7 +1079,7 @@ string EvalState::coerceToString(Value & v, PathSet & context,
         Bindings::iterator i = v.attrs->find(sOutPath);
         if (i == v.attrs->end())
             throwTypeError("cannot coerce an attribute set (except a derivation) to a string");
-        return coerceToString(i->second.value, context, coerceMore, copyToStore);
+        return coerceToString(*i->value, context, coerceMore, copyToStore);
     }
 
     if (coerceMore) {
@@ -1040,9 +1165,9 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 
         case tAttrs: {
             if (v1.attrs->size() != v2.attrs->size()) return false;
-            Bindings::iterator i, j;
-            for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j)
-                if (i->first != j->first || !eqValues(i->second.value, j->second.value))
+            Bindings::iterator i = v1.attrs->begin(), j = v2.attrs->begin();
+            for ( ; i != v1.attrs->end(); ++i, ++j)
+                if (i->name != j->name || !eqValues(*i->value, *j->value))
                     return false;
             return true;
         }
@@ -1065,20 +1190,26 @@ void EvalState::printStats()
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
     Verbosity v = showStats ? lvlInfo : lvlDebug;
     printMsg(v, "evaluation statistics:");
+    printMsg(v, format("  size of a value: %1%") % sizeof(Value));
     printMsg(v, format("  expressions evaluated: %1%") % nrEvaluated);
     printMsg(v, format("  stack space used: %1% bytes") % (&x - deepestStack));
     printMsg(v, format("  max eval() nesting depth: %1%") % maxRecursionDepth);
     printMsg(v, format("  stack space per eval() level: %1% bytes")
         % ((&x - deepestStack) / (float) maxRecursionDepth));
     printMsg(v, format("  environments allocated: %1% (%2% bytes)")
-        % nrEnvs % (nrEnvs * sizeof(Env)));
-    printMsg(v, format("  values allocated in environments: %1% (%2% bytes)")
-        % nrValuesInEnvs % (nrValuesInEnvs * sizeof(Value)));
+        % nrEnvs % (nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *)));
     printMsg(v, format("  list elements: %1% (%2% bytes)")
         % nrListElems % (nrListElems * sizeof(Value *)));
-    printMsg(v, format("  misc. values allocated: %1% (%2% bytes)")
+    printMsg(v, format("  values allocated: %1% (%2% bytes)")
         % nrValues % (nrValues * sizeof(Value)));
+    printMsg(v, format("  attribute sets allocated: %1%") % nrAttrsets);
+    printMsg(v, format("  right-biased unions: %1%") % nrOpUpdates);
+    printMsg(v, format("  values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
     printMsg(v, format("  symbols in symbol table: %1%") % symbols.size());
+    printMsg(v, format("  number of thunks: %1%") % nrThunks);
+    printMsg(v, format("  number of thunks avoided: %1%") % nrAvoided);
+    printMsg(v, format("  number of attr lookups: %1%") % nrLookups);
+    printMsg(v, format("  attr lookup size: %1%") % nrLookupSize);
 }
 
 

src/libexpr/eval.hh

@@ -1,22 +1,41 @@
 #ifndef __EVAL_H
 #define __EVAL_H
 
-#include <map>
-
 #include "nixexpr.hh"
 #include "symbol-table.hh"
+#include "hash.hh"
+
+#include <map>
+
+#if HAVE_BOEHMGC
+#include <gc/gc_allocator.h>
+#endif
 
 
 namespace nix {
 
 
-class Hash;
 class EvalState;
 struct Env;
 struct Value;
 struct Attr;
 
-typedef std::map<Symbol, Attr> Bindings;
+
+/* Attribute sets are represented as a vector of attributes, sorted by
+   symbol (i.e. pointer to the attribute name in the symbol table). */
+#if HAVE_BOEHMGC
+typedef std::vector<Attr, gc_allocator<Attr> > BindingsBase;
+#else
+typedef std::vector<Attr> BindingsBase;
+#endif
+
+
+class Bindings : public BindingsBase
+{
+public:
+    iterator find(const Symbol & name);
+    void sort();
+};
 
 
 typedef enum {
@@ -30,14 +49,23 @@ typedef enum {
     tThunk,
     tApp,
     tLambda,
-    tCopy,
     tBlackhole,
     tPrimOp,
     tPrimOpApp,
 } ValueType;
 
 
-typedef void (* PrimOp) (EvalState & state, Value * * args, Value & v);
+typedef void (* PrimOpFun) (EvalState & state, Value * * args, Value & v);
+
+
+struct PrimOp
+{
+    PrimOpFun fun;
+    unsigned int arity;
+    Symbol name;
+    PrimOp(PrimOpFun fun, unsigned int arity, Symbol name)
+        : fun(fun), arity(arity), name(name) { }
+};
 
 
 struct Value
@@ -90,15 +118,9 @@ struct Value
             Env * env;
             ExprLambda * fun;
         } lambda;
-        Value * val;
-        struct {
-            PrimOp fun;
-            char * name;
-            unsigned int arity;
-        } primOp;
+        PrimOp * primOp;
         struct {
             Value * left, * right;
-            unsigned int argsLeft;
         } primOpApp;
     };
 };
@@ -108,20 +130,36 @@ struct Env
 {
     Env * up;
     unsigned int prevWith; // nr of levels up to next `with' environment
-    Value values[0];
+    Value * values[0];
 };
 
 
 struct Attr
 {
-    Value value;
+    Symbol name;
+    Value * value;
     Pos * pos;
+    Attr(Symbol name, Value * value, Pos * pos = &noPos)
+        : name(name), value(value), pos(pos) { };
     Attr() : pos(&noPos) { };
+    bool operator < (const Attr & a) const
+    {
+        return name < a.name;
+    }
 };
 
 
+/* After overwriting an app node, be sure to clear pointers in the
+   Value to ensure that the target isn't kept alive unnecessarily. */
+static inline void clearValue(Value & v)
+{
+    v.app.right = 0;
+}
+
+
 static inline void mkInt(Value & v, int n)
 {
+    clearValue(v);
     v.type = tInt;
     v.integer = n;
 }
@@ -129,26 +167,12 @@ static inline void mkInt(Value & v, int n)
 
 static inline void mkBool(Value & v, bool b)
 {
+    clearValue(v);
     v.type = tBool;
     v.boolean = b;
 }
 
 
-static inline void mkThunk(Value & v, Env & env, Expr * expr)
-{
-    v.type = tThunk;
-    v.thunk.env = &env;
-    v.thunk.expr = expr;
-}
-
-
-static inline void mkCopy(Value & v, Value & src)
-{
-    v.type = tCopy;
-    v.val = &src;
-}
-
-
 static inline void mkApp(Value & v, Value & left, Value & right)
 {
     v.type = tApp;
@@ -161,6 +185,8 @@ void mkString(Value & v, const char * s);
 void mkString(Value & v, const string & s, const PathSet & context = PathSet());
 void mkPath(Value & v, const char * s);
 
+void copyContext(const Value & v, PathSet & context);
+
 
 typedef std::map<Path, Hash> DrvHashes;
 
@@ -171,7 +197,7 @@ typedef std::map<Path, Path> SrcToStore;
 struct EvalState;
 
 
-std::ostream & operator << (std::ostream & str, Value & v);
+std::ostream & operator << (std::ostream & str, const Value & v);
 
 
 class EvalState 
@@ -181,7 +207,8 @@ public:
 
     SymbolTable symbols;
 
-    const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sSystem;
+    const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName,
+        sSystem, sOverrides;
 
 private:
     SrcToStore srcToStore; 
@@ -265,7 +292,7 @@ private:
     void addConstant(const string & name, Value & v);
 
     void addPrimOp(const string & name,
-        unsigned int arity, PrimOp primOp);
+        unsigned int arity, PrimOpFun primOp);
 
     Value * lookupVar(Env * env, const VarRef & var);
     
@@ -283,18 +310,20 @@ public:
 
     /* Automatically call a function for which each argument has a
        default value or has a binding in the `args' map. */
-    void autoCallFunction(const Bindings & args, Value & fun, Value & res);
+    void autoCallFunction(Bindings & args, Value & fun, Value & res);
     
     /* Allocation primitives. */
-    Value * allocValues(unsigned int count);
+    Value * allocValue();
     Env & allocEnv(unsigned int size);
 
-    void mkList(Value & v, unsigned int length);
-    void mkAttrs(Value & v);
-    void mkThunk_(Value & v, Expr * expr);
-    
-    void cloneAttrs(Value & src, Value & dst);
+    Value * allocAttr(Value & vAttrs, const Symbol & name);
 
+    void mkList(Value & v, unsigned int length);
+    void mkAttrs(Value & v, unsigned int expected);
+    void mkThunk_(Value & v, Expr * expr);
+
+    Value * maybeThunk(Env & env, Expr * expr);
+    
     /* Print statistics. */
     void printStats();
 
@@ -305,11 +334,15 @@ private:
     unsigned long nrValues;
     unsigned long nrListElems;
     unsigned long nrEvaluated;
+    unsigned long nrAttrsets;
+    unsigned long nrOpUpdates;
+    unsigned long nrOpUpdateValuesCopied;
     unsigned int recursionDepth;
     unsigned int maxRecursionDepth;
     char * deepestStack; /* for measuring stack usage */
     
     friend class RecursionCounter;
+    friend class ExprOpUpdate;
 };
 
 

src/libexpr/get-drvs.cc

@@ -10,7 +10,7 @@ string DrvInfo::queryDrvPath(EvalState & state) const
     if (drvPath == "" && attrs) {
         Bindings::iterator i = attrs->find(state.sDrvPath);
         PathSet context;
-        (string &) drvPath = i != attrs->end() ? state.coerceToPath(i->second.value, context) : "";
+        (string &) drvPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : "";
     }
     return drvPath;
 }
@@ -21,7 +21,7 @@ string DrvInfo::queryOutPath(EvalState & state) const
     if (outPath == "" && attrs) {
         Bindings::iterator i = attrs->find(state.sOutPath);
         PathSet context;
-        (string &) outPath = i != attrs->end() ? state.coerceToPath(i->second.value, context) : "";
+        (string &) outPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : "";
     }
     return outPath;
 }
@@ -36,23 +36,23 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
     Bindings::iterator a = attrs->find(state.sMeta);
     if (a == attrs->end()) return meta; /* fine, empty meta information */
 
-    state.forceAttrs(a->second.value);
+    state.forceAttrs(*a->value);
 
-    foreach (Bindings::iterator, i, *a->second.value.attrs) {
+    foreach (Bindings::iterator, i, *a->value->attrs) {
         MetaValue value;
-        state.forceValue(i->second.value);
-        if (i->second.value.type == tString) {
+        state.forceValue(*i->value);
+        if (i->value->type == tString) {
             value.type = MetaValue::tpString;
-            value.stringValue = i->second.value.string.s;
-        } else if (i->second.value.type == tInt) {
+            value.stringValue = i->value->string.s;
+        } else if (i->value->type == tInt) {
             value.type = MetaValue::tpInt;
-            value.intValue = i->second.value.integer;
-        } else if (i->second.value.type == tList) {
+            value.intValue = i->value->integer;
+        } else if (i->value->type == tList) {
             value.type = MetaValue::tpStrings;
-            for (unsigned int j = 0; j < i->second.value.list.length; ++j)
-                value.stringValues.push_back(state.forceStringNoCtx(*i->second.value.list.elems[j]));
+            for (unsigned int j = 0; j < i->value->list.length; ++j)
+                value.stringValues.push_back(state.forceStringNoCtx(*i->value->list.elems[j]));
         } else continue;
-        ((MetaInfo &) meta)[i->first] = value;
+        ((MetaInfo &) meta)[i->name] = value;
     }
 
     return meta;
@@ -99,13 +99,13 @@ static bool getDerivation(EvalState & state, Value & v,
         Bindings::iterator i = v.attrs->find(state.sName);
         /* !!! We really would like to have a decent back trace here. */
         if (i == v.attrs->end()) throw TypeError("derivation name missing");
-        drv.name = state.forceStringNoCtx(i->second.value);
+        drv.name = state.forceStringNoCtx(*i->value);
 
-        i = v.attrs->find(state.sSystem);
-        if (i == v.attrs->end())
+        Bindings::iterator i2 = v.attrs->find(state.sSystem);
+        if (i2 == v.attrs->end())
             drv.system = "unknown";
         else
-            drv.system = state.forceStringNoCtx(i->second.value);
+            drv.system = state.forceStringNoCtx(*i2->value);
 
         drv.attrs = v.attrs;
 
@@ -138,7 +138,7 @@ static string addToPath(const string & s1, const string & s2)
 
 
 static void getDerivations(EvalState & state, Value & vIn,
-    const string & pathPrefix, const Bindings & autoArgs,
+    const string & pathPrefix, Bindings & autoArgs,
     DrvInfos & drvs, Done & done)
 {
     Value v;
@@ -163,12 +163,12 @@ static void getDerivations(EvalState & state, Value & vIn,
         typedef std::map<string, Symbol> SortedSymbols;
         SortedSymbols attrs;
         foreach (Bindings::iterator, i, *v.attrs)
-            attrs.insert(std::pair<string, Symbol>(i->first, i->first));
+            attrs.insert(std::pair<string, Symbol>(i->name, i->name));
 
         foreach (SortedSymbols::iterator, i, attrs) {
             startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first);
             string pathPrefix2 = addToPath(pathPrefix, i->first);
-            Value & v2((*v.attrs)[i->second].value);
+            Value & v2(*v.attrs->find(i->second)->value);
             if (combineChannels)
                 getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done);
             else if (getDerivation(state, v2, pathPrefix2, drvs, done)) {
@@ -178,7 +178,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                    attribute. */
                 if (v2.type == tAttrs) {
                     Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations"));
-                    if (j != v2.attrs->end() && state.forceBool(j->second.value))
+                    if (j != v2.attrs->end() && state.forceBool(*j->value))
                         getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done);
                 }
             }
@@ -200,7 +200,7 @@ static void getDerivations(EvalState & state, Value & vIn,
 
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
-    const Bindings & autoArgs, DrvInfos & drvs)
+    Bindings & autoArgs, DrvInfos & drvs)
 {
     Done done;
     getDerivations(state, v, pathPrefix, autoArgs, drvs, done);

src/libexpr/get-drvs.hh

@@ -1,13 +1,13 @@
 #ifndef __GET_DRVS_H
 #define __GET_DRVS_H
 
+#include "eval.hh"
+
 #include <string>
 #include <map>
 
 #include <boost/shared_ptr.hpp>
 
-#include "eval.hh"
-
 
 namespace nix {
 
@@ -62,7 +62,11 @@ public:
 };
 
 
+#if HAVE_BOEHMGC
+typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
+#else
 typedef list<DrvInfo> DrvInfos;
+#endif
 
 
 /* If value `v' denotes a derivation, store information about the
@@ -70,7 +74,7 @@ typedef list<DrvInfo> DrvInfos;
 bool getDerivation(EvalState & state, Value & v, DrvInfo & drv);
 
 void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
-    const Bindings & autoArgs, DrvInfos & drvs);
+    Bindings & autoArgs, DrvInfos & drvs);
 
  
 }

src/libexpr/lexer.l

@@ -46,7 +46,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
-static Expr * unescapeStr(const char * s)
+static Expr * unescapeStr(SymbolTable & symbols, const char * s)
 {
     string t;
     char c;
@@ -66,7 +66,7 @@ static Expr * unescapeStr(const char * s)
         }
         else t += c;
     }
-    return new ExprString(t);
+    return new ExprString(symbols.create(t));
 }
 
  
@@ -119,7 +119,7 @@ inherit     { return INHERIT; }
                  "$\"" will be consumed as part of a string, rather
                  than a "$" followed by the string terminator.
                  Disallow "$\"" for now. */
-              yylval->e = unescapeStr(yytext);
+              yylval->e = unescapeStr(data->symbols, yytext);
               return STR;
             }
 <STRING>\$\{  { BEGIN(INITIAL); return DOLLAR_CURLY; }
@@ -140,7 +140,7 @@ inherit     { return INHERIT; }
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\. {
-                   yylval->e = unescapeStr(yytext + 2);
+                   yylval->e = unescapeStr(data->symbols, yytext + 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; }

src/libexpr/nixexpr.cc

@@ -55,10 +55,11 @@ void ExprAttrs::show(std::ostream & str)
 {
     if (recursive) str << "rec ";
     str << "{ ";
-    foreach (list<Inherited>::iterator, i, inherited)
-        str << "inherit " << i->first.name << "; ";
-    foreach (Attrs::iterator, i, attrs)
-        str << i->first << " = " << *i->second.first << "; ";
+    foreach (AttrDefs::iterator, i, attrs)
+        if (i->second.inherited)
+            str << "inherit " << i->first << " " << "; ";
+        else
+            str << i->first << " = " << *i->second.e << "; ";
     str << "}";
 }
 
@@ -91,10 +92,11 @@ void ExprLambda::show(std::ostream & str)
 void ExprLet::show(std::ostream & str)
 {
     str << "let ";
-    foreach (list<ExprAttrs::Inherited>::iterator, i, attrs->inherited)
-        str << "inherit " << i->first.name << "; ";
-    foreach (ExprAttrs::Attrs::iterator, i, attrs->attrs)
-        str << i->first << " = " << *i->second.first << "; ";
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        if (i->second.inherited)
+            str << "inherit " << i->first << "; ";
+        else
+            str << i->first << " = " << *i->second.e << "; ";
     str << "in " << *body;
 }
 
@@ -211,26 +213,18 @@ void ExprAttrs::bindVars(const StaticEnv & env)
         StaticEnv newEnv(false, &env);
     
         unsigned int displ = 0;
-
-        foreach (ExprAttrs::Attrs::iterator, i, attrs)
-            newEnv.vars[i->first] = displ++;
-
-        foreach (list<Inherited>::iterator, i, inherited) {
-            newEnv.vars[i->first.name] = displ++;
-            i->first.bind(env);
-        }
-
-        foreach (ExprAttrs::Attrs::iterator, i, attrs)
-            i->second.first->bindVars(newEnv);
+        foreach (AttrDefs::iterator, i, attrs)
+            newEnv.vars[i->first] = i->second.displ = displ++;
+        
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited) i->second.var.bind(env);
+            else i->second.e->bindVars(newEnv);
     }
 
-    else {
-        foreach (ExprAttrs::Attrs::iterator, i, attrs)
-            i->second.first->bindVars(env);
-
-        foreach (list<Inherited>::iterator, i, inherited)
-            i->first.bind(env);
-    }
+    else
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited) i->second.var.bind(env);
+            else i->second.e->bindVars(env);
 }
 
 void ExprList::bindVars(const StaticEnv & env)
@@ -263,17 +257,12 @@ void ExprLet::bindVars(const StaticEnv & env)
     StaticEnv newEnv(false, &env);
     
     unsigned int displ = 0;
-
-    foreach (ExprAttrs::Attrs::iterator, i, attrs->attrs)
-        newEnv.vars[i->first] = displ++;
-
-    foreach (list<ExprAttrs::Inherited>::iterator, i, attrs->inherited) {
-        newEnv.vars[i->first.name] = displ++;
-        i->first.bind(env);
-    }
-
-    foreach (ExprAttrs::Attrs::iterator, i, attrs->attrs)
-        i->second.first->bindVars(newEnv);
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        newEnv.vars[i->first] = i->second.displ = displ++;
+    
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        if (i->second.inherited) i->second.var.bind(env);
+        else i->second.e->bindVars(newEnv);
     
     body->bindVars(newEnv);
 }

src/libexpr/nixexpr.hh

@@ -1,10 +1,10 @@
 #ifndef __NIXEXPR_H
 #define __NIXEXPR_H
 
-#include <map>
-
 #include "symbol-table.hh"
 
+#include <map>
+
 
 namespace nix {
 
@@ -15,6 +15,7 @@ MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
 MakeError(TypeError, EvalError)
+MakeError(ImportError, EvalError) // error building an imported derivation
 
 
 /* Position objects. */
@@ -64,8 +65,8 @@ struct ExprInt : Expr
 
 struct ExprString : Expr
 {
-    string s;
-    ExprString(const string & s) : s(s) { };
+    Symbol s;
+    ExprString(const Symbol & s) : s(s) { };
     COMMON_METHODS
 };
 
@@ -100,6 +101,7 @@ struct VarRef
     unsigned int level;
     unsigned int displ;
 
+    VarRef() { };
     VarRef(const Symbol & name) : name(name) { };
     void bind(const StaticEnv & env);
 };
@@ -130,12 +132,18 @@ struct ExprOpHasAttr : Expr
 struct ExprAttrs : Expr
 {
     bool recursive;
-    typedef std::pair<Expr *, Pos> Attr;
-    typedef std::pair<VarRef, Pos> Inherited;
-    typedef std::map<Symbol, Attr> Attrs;
-    Attrs attrs;
-    list<Inherited> inherited;
-    std::map<Symbol, Pos> attrNames; // used during parsing
+    struct AttrDef {
+        bool inherited;
+        Expr * e; // if not inherited
+        VarRef var; // if inherited
+        Pos pos;
+        unsigned int displ; // displacement
+        AttrDef(Expr * e, const Pos & pos) : inherited(false), e(e), pos(pos) { };
+        AttrDef(const Symbol & name, const Pos & pos) : inherited(true), var(name), pos(pos) { };
+        AttrDef() { };
+    };
+    typedef std::map<Symbol, AttrDef> AttrDefs;
+    AttrDefs attrs;
     ExprAttrs() : recursive(false) { };
     COMMON_METHODS
 };

src/libexpr/parser.y

@@ -7,48 +7,59 @@
 %parse-param { yyscan_t scanner }
 %parse-param { ParseData * data }
 %lex-param { yyscan_t scanner }
+%lex-param { ParseData * data }
 
-
-%{
-/* Newer versions of Bison copy the declarations below to
-   parser-tab.hh, which sucks bigtime since lexer.l doesn't want that
-   stuff.  So allow it to be excluded. */
-#ifndef BISON_HEADER_HACK
-#define BISON_HEADER_HACK
+%code requires {
+    
+#ifndef BISON_HEADER
+#define BISON_HEADER
     
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
 #include "util.hh"
     
 #include "nixexpr.hh"
 
+namespace nix {
+
+    struct ParseData 
+    {
+        SymbolTable & symbols;
+        Expr * result;
+        Path basePath;
+        Path path;
+        string error;
+        Symbol sLetBody;
+        ParseData(SymbolTable & symbols)
+            : symbols(symbols)
+            , sLetBody(symbols.create("<let-body>"))
+            { };
+    };
+    
+}
+
+#define YY_DECL int yylex \
+    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data)
+
+#endif
+
+}
+
+%{
+
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
 #define YYSTYPE YYSTYPE // workaround a bug in Bison 2.4
 
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+YY_DECL;
 
 using namespace nix;
 
 
 namespace nix {
-
     
-struct ParseData 
-{
-    SymbolTable & symbols;
-    Expr * result;
-    Path basePath;
-    Path path;
-    string error;
-    Symbol sLetBody;
-    ParseData(SymbolTable & symbols)
-        : symbols(symbols)
-        , sLetBody(symbols.create("<let-body>"))
-    { };
-};
-
 
 static string showAttrPath(const vector<Symbol> & attrPath)
 {
@@ -82,20 +93,20 @@ static void addAttr(ExprAttrs * attrs, const vector<Symbol> & attrPath,
     unsigned int n = 0;
     foreach (vector<Symbol>::const_iterator, i, attrPath) {
         n++;
-        ExprAttrs::Attrs::iterator j = attrs->attrs.find(*i);
+        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(*i);
         if (j != attrs->attrs.end()) {
-            ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.first);
-            if (!attrs2 || n == attrPath.size()) dupAttr(attrPath, pos, j->second.second);
-            attrs = attrs2;
+            if (!j->second.inherited) {
+                ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
+                if (!attrs2 || n == attrPath.size()) dupAttr(attrPath, pos, j->second.pos);
+                attrs = attrs2;
+            } else
+                dupAttr(attrPath, pos, j->second.pos);
         } else {
-            if (attrs->attrNames.find(*i) != attrs->attrNames.end())
-                dupAttr(attrPath, pos, attrs->attrNames[*i]);
-            attrs->attrNames[*i] = pos;
             if (n == attrPath.size())
-                attrs->attrs[*i] = ExprAttrs::Attr(e, pos);
+                attrs->attrs[*i] = ExprAttrs::AttrDef(e, pos);
             else {
                 ExprAttrs * nested = new ExprAttrs;
-                attrs->attrs[*i] = ExprAttrs::Attr(nested, pos);
+                attrs->attrs[*i] = ExprAttrs::AttrDef(nested, pos);
                 attrs = nested;
             }
         }
@@ -113,9 +124,9 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 }
 
 
-static Expr * stripIndentation(vector<Expr *> & es)
+static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
 {
-    if (es.empty()) return new ExprString("");
+    if (es.empty()) return new ExprString(symbols.create(""));
     
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
@@ -195,7 +206,7 @@ static Expr * stripIndentation(vector<Expr *> & es)
                 s2 = string(s2, 0, p + 1);
         }
 
-        es2->push_back(new ExprString(s2));
+        es2->push_back(new ExprString(symbols.create(s2)));
     }
 
     return new ExprConcatStrings(es2);
@@ -224,9 +235,6 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 }
 
 
-#endif
-
-
 %}
 
 %union {
@@ -337,15 +345,15 @@ expr_simple
   | INT { $$ = new ExprInt($1); }
   | '"' string_parts '"' {
       /* For efficiency, and to simplify parse trees a bit. */
-      if ($2->empty()) $$ = new ExprString("");
+      if ($2->empty()) $$ = new ExprString(data->symbols.create(""));
       else if ($2->size() == 1) $$ = $2->front();
       else $$ = new ExprConcatStrings($2);
   }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = stripIndentation(*$2);
+      $$ = stripIndentation(data->symbols, *$2);
   }
   | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
-  | URI { $$ = new ExprString($1); }
+  | URI { $$ = new ExprString(data->symbols.create($1)); }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
@@ -375,21 +383,19 @@ binds
   | binds INHERIT ids ';'
     { $$ = $1;
       foreach (vector<Symbol>::iterator, i, *$3) {
-          if ($$->attrNames.find(*i) != $$->attrNames.end())
-              dupAttr(*i, makeCurPos(@3, data), $$->attrNames[*i]);
+          if ($$->attrs.find(*i) != $$->attrs.end())
+              dupAttr(*i, makeCurPos(@3, data), $$->attrs[*i].pos);
           Pos pos = makeCurPos(@3, data);
-          $$->inherited.push_back(ExprAttrs::Inherited(*i, pos));
-          $$->attrNames[*i] = pos;
+          $$->attrs[*i] = ExprAttrs::AttrDef(*i, pos);
       }
     }
   | binds INHERIT '(' expr ')' ids ';'
     { $$ = $1;
       /* !!! Should ensure sharing of the expression in $4. */
       foreach (vector<Symbol>::iterator, i, *$6) {
-          if ($$->attrNames.find(*i) != $$->attrNames.end())
-              dupAttr(*i, makeCurPos(@6, data), $$->attrNames[*i]);
-          $$->attrs[*i] = ExprAttrs::Attr(new ExprSelect($4, *i), makeCurPos(@6, data));
-          $$->attrNames[*i] = makeCurPos(@6, data);
+          if ($$->attrs.find(*i) != $$->attrs.end())
+              dupAttr(*i, makeCurPos(@6, data), $$->attrs[*i].pos);
+          $$->attrs[*i] = ExprAttrs::AttrDef(new ExprSelect($4, *i), makeCurPos(@6, data));
       }}
 
   | { $$ = new ExprAttrs; }
@@ -480,8 +486,6 @@ Expr * parseExprFromFile(EvalState & state, Path path)
     }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (stat(path.c_str(), &st))
-        throw SysError(format("getting status of `%1%'") % path);
     if (S_ISDIR(st.st_mode))
         path = canonPath(path + "/default.nix");
 

src/libexpr/primops.cc

@@ -1,5 +1,5 @@
-#include "misc.hh"
 #include "eval.hh"
+#include "misc.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
@@ -37,7 +37,11 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
             throw EvalError(format("cannot import `%1%', since path `%2%' is not valid")
                 % path % *i);
         if (isDerivation(*i))
-            store->buildDerivations(singleton<PathSet>(*i));
+            try {
+                store->buildDerivations(singleton<PathSet>(*i));
+            } catch (Error & e) {
+                throw ImportError(e.msg());
+            }
     }
 
     state.evalFile(path, v);
@@ -115,24 +119,24 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
         args[0]->attrs->find(state.symbols.create("startSet"));
     if (startSet == args[0]->attrs->end())
         throw EvalError("attribute `startSet' required");
-    state.forceList(startSet->second.value);
+    state.forceList(*startSet->value);
 
     list<Value *> workSet;
-    for (unsigned int n = 0; n < startSet->second.value.list.length; ++n)
-        workSet.push_back(startSet->second.value.list.elems[n]);
+    for (unsigned int n = 0; n < startSet->value->list.length; ++n)
+        workSet.push_back(startSet->value->list.elems[n]);
 
     /* Get the operator. */
     Bindings::iterator op =
         args[0]->attrs->find(state.symbols.create("operator"));
     if (op == args[0]->attrs->end())
         throw EvalError("attribute `operator' required");
-    state.forceValue(op->second.value);
+    state.forceValue(*op->value);
 
     /* Construct the closure by applying the operator to element of
        `workSet', adding the result to `workSet', continuing until
        no new elements are found. */
     list<Value> res;
-    set<Value, CompareValues> doneKeys;
+    set<Value, CompareValues> doneKeys; // !!! use Value *?
     while (!workSet.empty()) {
 	Value * e = *(workSet.begin());
 	workSet.pop_front();
@@ -143,15 +147,15 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
             e->attrs->find(state.symbols.create("key"));
         if (key == e->attrs->end())
             throw EvalError("attribute `key' required");
-        state.forceValue(key->second.value);
+        state.forceValue(*key->value);
 
-        if (doneKeys.find(key->second.value) != doneKeys.end()) continue;
-        doneKeys.insert(key->second.value);
+        if (doneKeys.find(*key->value) != doneKeys.end()) continue;
+        doneKeys.insert(*key->value);
         res.push_back(*e);
         
         /* Call the `operator' function with `e' as argument. */
         Value call;
-        mkApp(call, op->second.value, *e);
+        mkApp(call, *op->value, *e);
         state.forceList(call);
 
         /* Add the values returned by the operator to the work set. */
@@ -163,13 +167,9 @@ static void prim_genericClosure(EvalState & state, Value * * args, Value & v)
 
     /* Create the result list. */
     state.mkList(v, res.size());
-    Value * vs = state.allocValues(res.size());
-
     unsigned int n = 0;
-    foreach (list<Value>::iterator, i, res) {
-        v.list.elems[n] = &vs[n];
-        vs[n++] = *i;
-    }
+    foreach (list<Value>::iterator, i, res)
+        *(v.list.elems[n++] = state.allocValue()) = *i;
 }
 
 
@@ -206,15 +206,16 @@ static void prim_addErrorContext(EvalState & state, Value * * args, Value & v)
  * else => {success=false; value=false;} */
 static void prim_tryEval(EvalState & state, Value * * args, Value & v)
 {
-    state.mkAttrs(v);
+    state.mkAttrs(v, 2);
     try {
         state.forceValue(*args[0]);
-        (*v.attrs)[state.symbols.create("value")].value = *args[0];
-        mkBool((*v.attrs)[state.symbols.create("success")].value, true);
+        v.attrs->push_back(Attr(state.symbols.create("value"), args[0]));
+        mkBool(*state.allocAttr(v, state.symbols.create("success")), true);
     } catch (AssertionError & e) {
-        mkBool((*v.attrs)[state.symbols.create("value")].value, false);
-        mkBool((*v.attrs)[state.symbols.create("success")].value, false);
+        mkBool(*state.allocAttr(v, state.symbols.create("value")), false);
+        mkBool(*state.allocAttr(v, state.symbols.create("success")), false);
     }
+    v.attrs->sort();
 }
 
 
@@ -320,9 +321,9 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
     if (attr == args[0]->attrs->end())
         throw EvalError("required attribute `name' missing");
     string drvName;
-    Pos & posDrvName(*attr->second.pos);
+    Pos & posDrvName(*attr->pos);
     try {        
-        drvName = state.forceStringNoCtx(attr->second.value);
+        drvName = state.forceStringNoCtx(*attr->value);
     } catch (Error & e) {
         e.addPrefix(format("while evaluating the derivation attribute `name' at %1%:\n") % posDrvName);
         throw;
@@ -337,7 +338,7 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
     bool outputHashRecursive = false;
 
     foreach (Bindings::iterator, i, *args[0]->attrs) {
-        string key = i->first;
+        string key = i->name;
         startNest(nest, lvlVomit, format("processing attribute `%1%'") % key);
 
         try {
@@ -345,9 +346,9 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
             if (key == "args") {
-                state.forceList(i->second.value);
-                for (unsigned int n = 0; n < i->second.value.list.length; ++n) {
-                    string s = state.coerceToString(*i->second.value.list.elems[n], context, true);
+                state.forceList(*i->value);
+                for (unsigned int n = 0; n < i->value->list.length; ++n) {
+                    string s = state.coerceToString(*i->value->list.elems[n], context, true);
                     drv.args.push_back(s);
                 }
             }
@@ -355,11 +356,11 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
             /* All other attributes are passed to the builder through
                the environment. */
             else {
-                string s = state.coerceToString(i->second.value, context, true);
+                string s = state.coerceToString(*i->value, context, true);
                 drv.env[key] = s;
                 if (key == "builder") drv.builder = s;
-                else if (i->first == state.sSystem) drv.platform = s;
-                else if (i->first == state.sName) drvName = s;
+                else if (i->name == state.sSystem) drv.platform = s;
+                else if (i->name == state.sName) drvName = s;
                 else if (key == "outputHash") outputHash = s;
                 else if (key == "outputHashAlgo") outputHashAlgo = s;
                 else if (key == "outputHashMode") {
@@ -371,7 +372,7 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
 
         } catch (Error & e) {
             e.addPrefix(format("while evaluating the derivation attribute `%1%' at %2%:\n")
-                % key % *i->second.pos);
+                % key % *i->pos);
             e.addPrefix(format("while instantiating the derivation named `%1%' at %2%:\n")
                 % drvName % posDrvName);
             throw;
@@ -483,9 +484,10 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
     state.drvHashes[drvPath] = hashDerivationModulo(state, drv);
 
     /* !!! assumes a single output */
-    state.mkAttrs(v);
-    mkString((*v.attrs)[state.sOutPath].value, outPath, singleton<PathSet>(drvPath));
-    mkString((*v.attrs)[state.sDrvPath].value, drvPath, singleton<PathSet>("=" + drvPath));
+    state.mkAttrs(v, 2);
+    mkString(*state.allocAttr(v, state.sOutPath), outPath, singleton<PathSet>(drvPath));
+    mkString(*state.allocAttr(v, state.sDrvPath), drvPath, singleton<PathSet>("=" + drvPath));
+    v.attrs->sort();
 }
 
 
@@ -685,17 +687,14 @@ static void prim_attrNames(EvalState & state, Value * * args, Value & v)
     state.forceAttrs(*args[0]);
 
     state.mkList(v, args[0]->attrs->size());
-    Value * vs = state.allocValues(v.list.length);
 
     StringSet names;
     foreach (Bindings::iterator, i, *args[0]->attrs)
-        names.insert(i->first);
+        names.insert(i->name);
 
     unsigned int n = 0;
-    foreach (StringSet::iterator, i, names) {
-        v.list.elems[n] = &vs[n];
-        mkString(vs[n++], *i);
-    }
+    foreach (StringSet::iterator, i, names)
+        mkString(*(v.list.elems[n++] = state.allocValue()), *i);
 }
 
 
@@ -709,8 +708,8 @@ static void prim_getAttr(EvalState & state, Value * * args, Value & v)
     if (i == args[1]->attrs->end())
         throw EvalError(format("attribute `%1%' missing") % attr);
     // !!! add to stack trace?
-    state.forceValue(i->second.value);
-    v = i->second.value;
+    state.forceValue(*i->value);
+    v = *i->value;
 }
 
 
@@ -736,11 +735,20 @@ static void prim_removeAttrs(EvalState & state, Value * * args, Value & v)
     state.forceAttrs(*args[0]);
     state.forceList(*args[1]);
 
-    state.cloneAttrs(*args[0], v);
-
+    /* Get the attribute names to be removed. */
+    std::set<Symbol> names;
     for (unsigned int i = 0; i < args[1]->list.length; ++i) {
         state.forceStringNoCtx(*args[1]->list.elems[i]);
-        v.attrs->erase(state.symbols.create(args[1]->list.elems[i]->string.s));
+        names.insert(state.symbols.create(args[1]->list.elems[i]->string.s));
+    }
+
+    /* Copy all attributes not in that set.  Note that we don't need
+       to sort v.attrs because it's a subset of an already sorted
+       vector. */
+    state.mkAttrs(v, args[0]->attrs->size());
+    foreach (Bindings::iterator, i, *args[0]->attrs) {
+        if (names.find(i->name) == names.end())
+            v.attrs->push_back(*i);
     }
 }
 
@@ -753,7 +761,9 @@ static void prim_listToAttrs(EvalState & state, Value * * args, Value & v)
 {
     state.forceList(*args[0]);
 
-    state.mkAttrs(v);
+    state.mkAttrs(v, args[0]->list.length);
+
+    std::set<Symbol> seen;
 
     for (unsigned int i = 0; i < args[0]->list.length; ++i) {
         Value & v2(*args[0]->list.elems[i]);
@@ -762,16 +772,21 @@ static void prim_listToAttrs(EvalState & state, Value * * args, Value & v)
         Bindings::iterator j = v2.attrs->find(state.sName);
         if (j == v2.attrs->end())
             throw TypeError("`name' attribute missing in a call to `listToAttrs'");
-        string name = state.forceStringNoCtx(j->second.value);
+        string name = state.forceStringNoCtx(*j->value);
         
-        j = v2.attrs->find(state.symbols.create("value"));
-        if (j == v2.attrs->end())
+        Bindings::iterator j2 = v2.attrs->find(state.symbols.create("value"));
+        if (j2 == v2.attrs->end())
             throw TypeError("`value' attribute missing in a call to `listToAttrs'");
 
-        Attr & a = (*v.attrs)[state.symbols.create(name)];
-        mkCopy(a.value, j->second.value);
-        a.pos = j->second.pos;
+        Symbol sym = state.symbols.create(name);
+        if (seen.find(sym) == seen.end()) {
+            v.attrs->push_back(Attr(sym, j2->value, j2->pos));
+            seen.insert(sym);
+        }
+        /* !!! Throw an error if `name' already exists? */
     }
+
+    v.attrs->sort();
 }
 
 
@@ -783,15 +798,12 @@ static void prim_intersectAttrs(EvalState & state, Value * * args, Value & v)
     state.forceAttrs(*args[0]);
     state.forceAttrs(*args[1]);
         
-    state.mkAttrs(v);
-    
-    foreach (Bindings::iterator, i, *args[1]->attrs) {
-        Bindings::iterator j = args[0]->attrs->find(i->first);
-        if (j != args[0]->attrs->end()) {
-            Attr & a = (*v.attrs)[i->first];
-            mkCopy(a.value, i->second.value);
-            a.pos = i->second.pos;
-        }
+    state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size()));
+
+    foreach (Bindings::iterator, i, *args[0]->attrs) {
+        Bindings::iterator j = args[1]->attrs->find(i->name);
+        if (j != args[1]->attrs->end())
+            v.attrs->push_back(*j);
     }
 }
 
@@ -815,12 +827,16 @@ static void prim_functionArgs(EvalState & state, Value * * args, Value & v)
     if (args[0]->type != tLambda)
         throw TypeError("`functionArgs' requires a function");
 
-    state.mkAttrs(v);
-
-    if (!args[0]->lambda.fun->matchAttrs) return;
+    if (!args[0]->lambda.fun->matchAttrs) {
+        state.mkAttrs(v, 0);
+        return;
+    }
 
+    state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size());
     foreach (Formals::Formals_::iterator, i, args[0]->lambda.fun->formals->formals)
-        mkBool((*v.attrs)[i->name].value, i->def);
+        // !!! should optimise booleans (allocate only once)
+        mkBool(*state.allocAttr(v, i->name), i->def);
+    v.attrs->sort();
 }
 
 
@@ -868,12 +884,10 @@ static void prim_map(EvalState & state, Value * * args, Value & v)
     state.forceList(*args[1]);
 
     state.mkList(v, args[1]->list.length);
-    Value * vs = state.allocValues(v.list.length);
 
-    for (unsigned int n = 0; n < v.list.length; ++n) {
-        v.list.elems[n] = &vs[n];
-        mkApp(vs[n], *args[0], *args[1]->list.elems[n]);
-    }
+    for (unsigned int n = 0; n < v.list.length; ++n)
+        mkApp(*(v.list.elems[n] = state.allocValue()), 
+            *args[0], *args[1]->list.elems[n]);
 }
 
 
@@ -951,7 +965,7 @@ static void prim_substring(EvalState & state, Value * * args, Value & v)
 
     if (start < 0) throw EvalError("negative start position in `substring'");
 
-    mkString(v, string(s, start, len), context);
+    mkString(v, start >= s.size() ? "" : string(s, start, len), context);
 }
 
 
@@ -1002,9 +1016,10 @@ static void prim_parseDrvName(EvalState & state, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0]);
     DrvName parsed(name);
-    state.mkAttrs(v);
-    mkString((*v.attrs)[state.sName].value, parsed.name);
-    mkString((*v.attrs)[state.symbols.create("version")].value, parsed.version);
+    state.mkAttrs(v, 2);
+    mkString(*state.allocAttr(v, state.sName), parsed.name);
+    mkString(*state.allocAttr(v, state.symbols.create("version")), parsed.version);
+    v.attrs->sort();
 }
 
 
@@ -1029,7 +1044,7 @@ void EvalState::createBaseEnv()
     Value v;
 
     /* `builtins' must be first! */
-    mkAttrs(v);
+    mkAttrs(v, 128);
     addConstant("builtins", v);
 
     mkBool(v, true);
@@ -1068,7 +1083,7 @@ void EvalState::createBaseEnv()
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
     string s = "attrs: let res = derivationStrict attrs; in attrs // { drvPath = res.drvPath; outPath = res.outPath; type = \"derivation\"; }";
-    mkThunk(v, baseEnv, parseExprFromString(*this, s, "/"));
+    mkThunk_(v, parseExprFromString(*this, s, "/"));
     addConstant("derivation", v);
 
     // Paths
@@ -1117,7 +1132,11 @@ void EvalState::createBaseEnv()
 
     // Versions
     addPrimOp("__parseDrvName", 1, prim_parseDrvName);
-    addPrimOp("__compareVersions", 2, prim_compareVersions);    
+    addPrimOp("__compareVersions", 2, prim_compareVersions);
+
+    /* Now that we've added all primops, sort the `builtins' attribute
+       set, because attribute lookups expect it to be sorted. */
+    baseEnv.values[0]->attrs->sort();
 }
 
 

src/libexpr/symbol-table.hh

@@ -1,8 +1,13 @@
 #ifndef __SYMBOL_TABLE_H
 #define __SYMBOL_TABLE_H
 
+#include "config.h"
+
 #include <map>
+
+#if HAVE_TR1_UNORDERED_SET
 #include <tr1/unordered_set>
+#endif
 
 #include "types.hh"
 
@@ -23,6 +28,8 @@ private:
     friend class SymbolTable;
 
 public:
+    Symbol() : s(0) { };
+    
     bool operator == (const Symbol & s2) const
     {
         return s == s2.s;
@@ -60,7 +67,11 @@ inline std::ostream & operator << (std::ostream & str, const Symbol & sym)
 class SymbolTable
 {
 private:
+#if HAVE_TR1_UNORDERED_SET 
     typedef std::tr1::unordered_set<string> Symbols;
+#else
+    typedef std::set<string> Symbols;
+#endif
     Symbols symbols;
 
 public:

src/libexpr/value-to-xml.cc

@@ -34,10 +34,10 @@ static void showAttrs(EvalState & state, bool strict, bool location,
     StringSet names;
     
     foreach (Bindings::iterator, i, attrs)
-        names.insert(i->first);
+        names.insert(i->name);
     
     foreach (StringSet::iterator, i, names) {
-        Attr & a(attrs[state.symbols.create(*i)]);
+        Attr & a(*attrs.find(state.symbols.create(*i)));
         
         XMLAttrs xmlAttrs;
         xmlAttrs["name"] = *i;
@@ -45,7 +45,7 @@ static void showAttrs(EvalState & state, bool strict, bool location,
         
         XMLOpenElement _(doc, "attr", xmlAttrs);
         printValueAsXML(state, strict, location,
-            a.value, doc, context, drvsSeen);
+            *a.value, doc, context, drvsSeen);
     }
 }
 
@@ -69,6 +69,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
         case tString:
             /* !!! show the context? */
+            copyContext(v, context);
             doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
             break;
 
@@ -89,16 +90,16 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 Path drvPath;
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
-                    if (strict) state.forceValue(a->second.value);
-                    if (a->second.value.type == tString)
-                        xmlAttrs["drvPath"] = drvPath = a->second.value.string.s;
+                    if (strict) state.forceValue(*a->value);
+                    if (a->value->type == tString)
+                        xmlAttrs["drvPath"] = drvPath = a->value->string.s;
                 }
         
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
-                    if (strict) state.forceValue(a->second.value);
-                    if (a->second.value.type == tString)
-                        xmlAttrs["outPath"] = a->second.value.string.s;
+                    if (strict) state.forceValue(*a->value);
+                    if (a->value->type == tString)
+                        xmlAttrs["outPath"] = a->value->string.s;
                 }
 
                 XMLOpenElement _(doc, "derivation", xmlAttrs);

src/libexpr/value-to-xml.hh

@@ -1,12 +1,12 @@
 #ifndef __VALUE_TO_XML_H
 #define __VALUE_TO_XML_H
 
-#include <string>
-#include <map>
-
 #include "nixexpr.hh"
 #include "eval.hh"
 
+#include <string>
+#include <map>
+
 namespace nix {
 
 void printValueAsXML(EvalState & state, bool strict, bool location,

src/libmain/Makefile.am

@@ -2,7 +2,7 @@ pkglib_LTLIBRARIES = libmain.la
 
 libmain_la_SOURCES = shared.cc
 
-libmain_la_LIBADD = ../libstore/libstore.la
+libmain_la_LIBADD = ../libstore/libstore.la @boehmgc_lib@
 
 pkginclude_HEADERS = shared.hh
 

src/libmain/shared.cc

@@ -13,6 +13,10 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#if HAVE_BOEHMGC
+#include <gc/gc.h>
+#endif
+
 
 namespace nix {
 
@@ -50,25 +54,26 @@ void printGCWarning()
 
 void printMissing(const PathSet & paths)
 {
-    unsigned long long downloadSize;
+    unsigned long long downloadSize, narSize;
     PathSet willBuild, willSubstitute, unknown;
-    queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize);
+    queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     if (!willBuild.empty()) {
-        printMsg(lvlInfo, format("the following derivations will be built:"));
+        printMsg(lvlInfo, format("these derivations will be built:"));
         foreach (PathSet::iterator, i, willBuild)
             printMsg(lvlInfo, format("  %1%") % *i);
     }
 
     if (!willSubstitute.empty()) {
-        printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") %
-            (downloadSize / (1024.0 * 1024.0)));
+        printMsg(lvlInfo, format("these paths will be downloaded/copied (%.2f MiB download, %.2f MiB unpacked):")
+            % (downloadSize / (1024.0 * 1024.0))
+            % (narSize / (1024.0 * 1024.0)));
         foreach (PathSet::iterator, i, willSubstitute)
             printMsg(lvlInfo, format("  %1%") % *i);
     }
 
     if (!unknown.empty()) {
-        printMsg(lvlInfo, format("don't know how to build the following paths%1%:")
+        printMsg(lvlInfo, format("don't know how to build these paths%1%:")
             % (readOnlyMode ? " (may be caused by read-only store access)" : ""));
         foreach (PathSet::iterator, i, unknown)
             printMsg(lvlInfo, format("  %1%") % *i);
@@ -135,6 +140,7 @@ static void initAndRun(int argc, char * * argv)
     /* Get some settings from the configuration file. */
     thisSystem = querySetting("system", SYSTEM);
     maxBuildJobs = queryIntSetting("build-max-jobs", 1);
+    buildCores = queryIntSetting("build-cores", 1);
     maxSilentTime = queryIntSetting("build-max-silent-time", 0);
 
     /* Catch SIGINT. */
@@ -195,17 +201,16 @@ static void initAndRun(int argc, char * * argv)
     remaining.clear();
 
     /* Process default options. */
+    int verbosityDelta = 0;
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
-        if (arg == "--verbose" || arg == "-v")
-            verbosity = (Verbosity) ((int) verbosity + 1);
+        if (arg == "--verbose" || arg == "-v") verbosityDelta++;
+        else if (arg == "--quiet") verbosityDelta--;
         else if (arg == "--log-type") {
             ++i;
             if (i == args.end()) throw UsageError("`--log-type' requires an argument");
             setLogType(*i);
         }
-        else if (arg == "--build-output" || arg == "-B")
-            ; /* !!! obsolete - remove eventually */
         else if (arg == "--no-build-output" || arg == "-Q")
             buildVerbosity = lvlVomit;
         else if (arg == "--print-build-trace")
@@ -226,6 +231,8 @@ static void initAndRun(int argc, char * * argv)
             tryFallback = true;
         else if (arg == "--max-jobs" || arg == "-j")
             maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end());
+        else if (arg == "--cores")
+            buildCores = getIntArg<unsigned int>(arg, i, args.end());
         else if (arg == "--readonly-mode")
             readOnlyMode = true;
         else if (arg == "--max-silent-time")
@@ -244,6 +251,9 @@ static void initAndRun(int argc, char * * argv)
         else remaining.push_back(arg);
     }
 
+    verbosityDelta += queryIntSetting("verbosity", lvlInfo);
+    verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
+    
     /* Automatically clean up the temporary roots file when we
        exit. */
     RemoveTempRoots removeTempRoots __attribute__((unused));
@@ -311,6 +321,14 @@ static void setuidInit()
 }
 
 
+/* Called when the Boehm GC runs out of memory. */
+static void * oomHandler(size_t requested)
+{
+    /* Convert this to a proper C++ exception. */
+    throw std::bad_alloc();
+}
+
+
 }
 
 
@@ -332,6 +350,26 @@ int main(int argc, char * * argv)
 
     std::ios::sync_with_stdio(false);
 
+#if HAVE_BOEHMGC
+    /* Initialise the Boehm garbage collector.  This isn't necessary
+       on most platforms, but for portability we do it anyway. */
+    GC_INIT();
+
+    GC_oom_fn = oomHandler;
+
+    /* Set the initial heap size to something fairly big (384 MiB) so
+       that in most cases we don't need to garbage collect at all.
+       (Collection has a fairly significant overhead, some.)  The heap
+       size can be overriden through libgc's GC_INITIAL_HEAP_SIZE
+       environment variable.  We should probably also provide a
+       nix.conf setting for this.  Note that GC_expand_hp() causes a
+       lot of virtual, but not physical (resident) memory to be
+       allocated.  This might be a problem on systems that don't
+       overcommit. */
+    if (!getenv("GC_INITIAL_HEAP_SIZE"))
+        GC_expand_hp(384 * 1024 * 1024);
+#endif
+
     try {
         try {
             initAndRun(argc, argv);
@@ -355,7 +393,7 @@ int main(int argc, char * * argv)
         printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg());
         if (e.prefix() != "" && !showTrace)
             printMsg(lvlError, "(use `--show-trace' to show detailed location information)");
-        return 1;
+        return e.status;
     } catch (std::exception & e) {
         printMsg(lvlError, format("error: %1%") % e.what());
         return 1;

src/libmain/shared.hh

@@ -1,7 +1,7 @@
 #ifndef __SHARED_H
 #define __SHARED_H
 
-#include "types.hh"
+#include "util.hh"
 
 #include <signal.h>
 

src/libstore/Makefile.am

@@ -10,7 +10,14 @@ pkginclude_HEADERS = \
   globals.hh references.hh pathlocks.hh \
   worker-protocol.hh
 
-libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
+libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib}
+
+EXTRA_DIST = schema.sql
 
 AM_CXXFLAGS = -Wall \
- -I$(srcdir)/.. -I$(srcdir)/../libutil
+ ${sqlite_include} -I$(srcdir)/.. -I$(srcdir)/../libutil
+
+local-store.lo: schema.sql.hh
+
+%.sql.hh: %.sql
+	../bin2c/bin2c schema < $< > $@ || (rm $@ && exit 1)

src/libstore/build.cc

@@ -25,6 +25,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <stdio.h>
+#include <cstring>
 
 #include <pwd.h>
 #include <grp.h>
@@ -63,6 +64,7 @@ static const uid_t rootUserId = 0;
 
 /* Forward definition. */
 class Worker;
+class HookInstance;
 
 
 /* A pointer to a goal. */
@@ -212,8 +214,14 @@ public:
 
     bool cacheFailure;
 
+    /* Set if at least one derivation had a BuildError (i.e. permanent
+       failure). */
+    bool permanentFailure;
+
     LocalStore & store;
 
+    boost::shared_ptr<HookInstance> hook;
+    
     Worker(LocalStore & store);
     ~Worker();
 
@@ -262,12 +270,13 @@ public:
 
     /* Wait for input to become available. */
     void waitForInput();
-    
+
+    unsigned int exitStatus();
 };
 
 
 MakeError(SubstError, Error)
-MakeError(BuildError, Error) /* denoted a permanent build failure */
+MakeError(BuildError, Error) /* denotes a permanent build failure */
 
 
 //////////////////////////////////////////////////////////////////////
@@ -614,6 +623,107 @@ void deletePathWrapped(const Path & path)
 //////////////////////////////////////////////////////////////////////
 
 
+struct HookInstance
+{
+    /* Pipes for talking to the build hook. */
+    Pipe toHook;
+
+    /* Pipe for the hook's standard output/error. */
+    Pipe fromHook;
+
+    /* Pipe for the builder's standard output/error. */
+    Pipe builderOut;
+    
+    /* The process ID of the hook. */
+    Pid pid;
+
+    HookInstance();
+
+    ~HookInstance();
+};
+
+
+HookInstance::HookInstance()
+{
+    debug("starting build hook");
+    
+    Path buildHook = absPath(getEnv("NIX_BUILD_HOOK"));
+    
+    /* Create a pipe to get the output of the child. */
+    fromHook.create();
+    
+    /* Create the communication pipes. */
+    toHook.create();
+
+    /* Create a pipe to get the output of the builder. */
+    builderOut.create();
+
+    /* Fork the hook. */
+    pid = fork();
+    switch (pid) {
+        
+    case -1:
+        throw SysError("unable to fork");
+
+    case 0:
+        try { /* child */
+
+            commonChildInit(fromHook);
+
+            if (chdir("/") == -1) throw SysError("changing into `/");
+            
+            /* Dup the communication pipes. */
+            toHook.writeSide.close();
+            if (dup2(toHook.readSide, STDIN_FILENO) == -1)
+                throw SysError("dupping to-hook read side");
+
+            /* Use fd 4 for the builder's stdout/stderr. */
+            builderOut.readSide.close();
+            if (dup2(builderOut.writeSide, 4) == -1)
+                throw SysError("dupping builder's stdout/stderr");
+            
+            execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(),
+                (format("%1%") % maxSilentTime).str().c_str(),
+                (format("%1%") % printBuildTrace).str().c_str(),
+                NULL);
+            
+            throw SysError(format("executing `%1%'") % buildHook);
+            
+        } catch (std::exception & e) {
+            std::cerr << format("build hook error: %1%") % e.what() << std::endl;
+        }
+        quickExit(1);
+    }
+    
+    /* parent */
+    pid.setSeparatePG(true);
+    pid.setKillSignal(SIGTERM);
+    fromHook.writeSide.close();
+    toHook.readSide.close();
+}
+
+
+HookInstance::~HookInstance()
+{
+    try {
+        /* Cleanly shut down the hook by closing its stdin if it's not
+           already building.  Otherwise pid's destructor will kill
+           it. */
+        if (pid != -1 && toHook.writeSide != -1) {
+            toHook.writeSide.close();
+            pid.wait(true);
+        }
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
+typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
+
 class DerivationGoal : public Goal
 {
 private:
@@ -648,14 +758,11 @@ private:
     AutoCloseFD fdLogFile;
 
     /* Pipe for the builder's standard output/error. */
-    Pipe logPipe;
-
-    /* Whether we're building using a build hook. */
-    bool usingBuildHook;
-
-    /* Pipes for talking to the build hook (if any). */
-    Pipe toHook;
+    Pipe builderOut;
 
+    /* The build hook. */
+    boost::shared_ptr<HookInstance> hook;
+    
     /* Whether we're currently doing a chroot build. */
     bool useChroot;
     
@@ -693,12 +800,8 @@ private:
     void buildDone();
 
     /* Is the build hook willing to perform the build? */
-    typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
     HookReply tryBuildHook();
 
-    /* Synchronously wait for a build hook to finish. */
-    void terminateBuildHook(bool kill = false);
-
     /* Start building a derivation. */
     void startBuilder();
 
@@ -710,10 +813,6 @@ private:
     /* Open a log file and a pipe to it. */
     Path openLogFile();
 
-    /* Common initialisation to be performed in child processes (i.e.,
-       both in builders and in build hooks). */
-    void initChild();
-    
     /* Delete the temporary directory, if we have one. */
     void deleteTmpDir(bool force);
 
@@ -741,6 +840,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker)
     trace("created");
 }
 
+
 DerivationGoal::~DerivationGoal()
 {
     /* Careful: we should never ever throw an exception from a
@@ -753,6 +853,7 @@ DerivationGoal::~DerivationGoal()
     }
 }
 
+
 void DerivationGoal::killChild()
 {
     if (pid != -1) {
@@ -777,6 +878,8 @@ void DerivationGoal::killChild()
         
         assert(pid == -1);
     }
+
+    hook.reset();
 }
 
 
@@ -886,7 +989,10 @@ void DerivationGoal::outputsSubstituted()
     foreach (PathSet::iterator, i, drv.inputSrcs)
         addWaitee(worker.makeSubstitutionGoal(*i));
 
-    state = &DerivationGoal::inputsRealised;
+    if (waitees.empty()) /* to prevent hang (no wake-up event) */
+        inputsRealised();
+    else
+        state = &DerivationGoal::inputsRealised;
 }
 
 
@@ -960,6 +1066,16 @@ PathSet outputPaths(const DerivationOutputs & outputs)
 }
 
 
+static bool canBuildLocally(const string & platform)
+{
+    return platform == thisSystem 
+#ifdef CAN_DO_LINUX32_BUILDS
+        || (platform == "i686-linux" && thisSystem == "x86_64-linux")
+#endif
+        ;
+}
+
+
 void DerivationGoal::tryToBuild()
 {
     trace("trying to build");
@@ -1027,28 +1143,36 @@ void DerivationGoal::tryToBuild()
     foreach (DerivationOutputs::iterator, i, drv.outputs)
         if (pathFailed(i->second.path)) return;
 
+    /* Don't do a remote build if the derivation has the attribute
+       `preferLocalBuild' set. */
+    bool preferLocalBuild =
+        drv.env["preferLocalBuild"] == "1" && canBuildLocally(drv.platform);
+
     /* Is the build hook willing to accept this job? */
-    usingBuildHook = true;
-    switch (tryBuildHook()) {
-        case rpAccept:
-            /* Yes, it has started doing so.  Wait until we get EOF
-               from the hook. */
-            state = &DerivationGoal::buildDone;
-            return;
-        case rpPostpone:
-            /* Not now; wait until at least one child finishes. */
-            worker.waitForAWhile(shared_from_this());
-            outputLocks.unlock();
-            return;
-        case rpDecline:
-            /* We should do it ourselves. */
-            break;
+    if (!preferLocalBuild) {
+        switch (tryBuildHook()) {
+            case rpAccept:
+                /* Yes, it has started doing so.  Wait until we get
+                   EOF from the hook. */
+                state = &DerivationGoal::buildDone;
+                return;
+            case rpPostpone:
+                /* Not now; wait until at least one child finishes or
+                   the wake-up timeout expires. */
+                worker.waitForAWhile(shared_from_this());
+                outputLocks.unlock();
+                return;
+            case rpDecline:
+                /* We should do it ourselves. */
+                break;
+        }
     }
-
-    usingBuildHook = false;
-
-    /* Make sure that we are allowed to start a build. */
-    if (worker.getNrLocalBuilds() >= maxBuildJobs) {
+    
+    /* Make sure that we are allowed to start a build.  If this
+       derivation prefers to be done locally, do it even if
+       maxBuildJobs is 0. */
+    unsigned int curBuilds = worker.getNrLocalBuilds();
+    if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) {
         worker.waitForBuildSlot(shared_from_this());
         outputLocks.unlock();
         return;
@@ -1066,6 +1190,7 @@ void DerivationGoal::tryToBuild()
         if (printBuildTrace)
             printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%")
                 % drvPath % drv.outputs["out"].path % 0 % e.msg());
+        worker.permanentFailure = true;
         amDone(ecFailed);
         return;
     }
@@ -1084,18 +1209,29 @@ void DerivationGoal::buildDone()
        to have terminated.  In fact, the builder could also have
        simply have closed its end of the pipe --- just don't do that
        :-) */
-    /* !!! this could block! security problem! solution: kill the
-       child */
-    pid_t savedPid = pid;
-    int status = pid.wait(true);
+    int status;
+    pid_t savedPid;
+    if (hook) {
+        savedPid = hook->pid;
+        status = hook->pid.wait(true);
+    } else {
+        /* !!! this could block! security problem! solution: kill the
+           child */
+        savedPid = pid;
+        status = pid.wait(true);
+    }
 
     debug(format("builder process for `%1%' finished") % drvPath);
 
     /* So the child is gone now. */
     worker.childTerminated(savedPid);
-
+    
     /* Close the read side of the logger pipe. */
-    logPipe.readSide.close();
+    if (hook) {
+        hook->builderOut.readSide.close();
+        hook->fromHook.readSide.close();
+    }
+    else builderOut.readSide.close();
 
     /* Close the log file. */
     fdLogFile.close();
@@ -1168,11 +1304,11 @@ void DerivationGoal::buildDone()
         /* When using a build hook, the hook will return a remote
            build failure using exit code 100.  Anything else is a hook
            problem. */
-        bool hookError = usingBuildHook &&
+        bool hookError = hook &&
             (!WIFEXITED(status) || WEXITSTATUS(status) != 100);
         
         if (printBuildTrace) {
-            if (usingBuildHook && hookError)
+            if (hook && hookError)
                 printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%")
                     % drvPath % drv.outputs["out"].path % status % e.msg());
             else
@@ -1191,6 +1327,7 @@ void DerivationGoal::buildDone()
             foreach (DerivationOutputs::iterator, i, drv.outputs)
                 worker.store.registerFailedPath(i->second.path);
         
+        worker.permanentFailure = !hookError && !fixedOutput;
         amDone(ecFailed);
         return;
     }
@@ -1207,162 +1344,85 @@ void DerivationGoal::buildDone()
 }
 
 
-DerivationGoal::HookReply DerivationGoal::tryBuildHook()
+HookReply DerivationGoal::tryBuildHook()
 {
-    if (!useBuildHook) return rpDecline;
-    Path buildHook = getEnv("NIX_BUILD_HOOK");
-    if (buildHook == "") return rpDecline;
-    buildHook = absPath(buildHook);
+    if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
 
-    /* Create a directory where we will store files used for
-       communication between us and the build hook. */
-    tmpDir = createTempDir();
-    
-    /* Create the log file and pipe. */
-    Path logFile = openLogFile();
+    if (!worker.hook)
+        worker.hook = boost::shared_ptr<HookInstance>(new HookInstance);
 
-    /* Create the communication pipes. */
-    toHook.create();
+    /* Tell the hook about system features (beyond the system type)
+       required from the build machine.  (The hook could parse the
+       drv file itself, but this is easier.) */
+    Strings features = tokenizeString(drv.env["requiredSystemFeatures"]);
+    foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */
 
-    /* Fork the hook. */
-    pid = fork();
-    switch (pid) {
-        
-    case -1:
-        throw SysError("unable to fork");
-
-    case 0:
-        try { /* child */
-
-            initChild();
-
-            string s;
-            foreach (DerivationOutputs::const_iterator, i, drv.outputs)
-                s += i->second.path + " ";
-            if (setenv("NIX_HELD_LOCKS", s.c_str(), 1))
-                throw SysError("setting an environment variable");
-
-            execl(buildHook.c_str(), buildHook.c_str(),
-                (worker.getNrLocalBuilds() < maxBuildJobs ? (string) "1" : "0").c_str(),
-                thisSystem.c_str(),
-                drv.platform.c_str(),
-                drvPath.c_str(),
-                (format("%1%") % maxSilentTime).str().c_str(),
-                NULL);
-            
-            throw SysError(format("executing `%1%'") % buildHook);
-            
-        } catch (std::exception & e) {
-            std::cerr << format("build hook error: %1%") % e.what() << std::endl;
-        }
-        quickExit(1);
-    }
-    
-    /* parent */
-    pid.setSeparatePG(true);
-    pid.setKillSignal(SIGTERM);
-    logPipe.writeSide.close();
-    worker.childStarted(shared_from_this(),
-        pid, singleton<set<int> >(logPipe.readSide), false, false);
-
-    toHook.readSide.close();
+    /* Send the request to the hook. */
+    writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
+        % (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0")
+        % drv.platform % drvPath % concatStringsSep(",", features)).str());
 
     /* Read the first line of input, which should be a word indicating
        whether the hook wishes to perform the build. */
     string reply;
-    try {
-        while (true) {
-            string s = readLine(logPipe.readSide);
-            if (string(s, 0, 2) == "# ") {
-                reply = string(s, 2);
-                break;
-            }
-            handleChildOutput(logPipe.readSide, s + "\n");
+    while (true) {
+        string s = readLine(worker.hook->fromHook.readSide);
+        if (string(s, 0, 2) == "# ") {
+            reply = string(s, 2);
+            break;
         }
-    } catch (Error & e) {
-        terminateBuildHook(true);
-        throw;
+        s += "\n";
+        writeToStderr((unsigned char *) s.c_str(), s.size());
     }
 
     debug(format("hook reply is `%1%'") % reply);
 
-    if (reply == "decline" || reply == "postpone") {
-        /* Clean up the child.  !!! hacky / should verify */
-        terminateBuildHook();
+    if (reply == "decline" || reply == "postpone")
         return reply == "decline" ? rpDecline : rpPostpone;
-    }
+    else if (reply != "accept")
+        throw Error(format("bad hook reply `%1%'") % reply);
 
-    else if (reply == "accept") {
+    printMsg(lvlTalkative, format("using hook to build path(s) %1%")
+        % showPaths(outputPaths(drv.outputs)));
 
-        printMsg(lvlInfo, format("using hook to build path(s) %1%")
-            % showPaths(outputPaths(drv.outputs)));
+    hook = worker.hook;
+    worker.hook.reset();
         
-        /* Write the information that the hook needs to perform the
-           build, i.e., the set of input paths, the set of output
-           paths, and the references (pointer graph) in the input
-           paths. */
+    /* Tell the hook all the inputs that have to be copied to the
+       remote system.  This unfortunately has to contain the entire
+       derivation closure to ensure that the validity invariant holds
+       on the remote system.  (I.e., it's unfortunate that we have to
+       list it since the remote system *probably* already has it.) */
+    PathSet allInputs;
+    allInputs.insert(inputPaths.begin(), inputPaths.end());
+    computeFSClosure(drvPath, allInputs);
         
-        Path inputListFN = tmpDir + "/inputs";
-        Path outputListFN = tmpDir + "/outputs";
-        Path referencesFN = tmpDir + "/references";
-
-        /* The `inputs' file lists all inputs that have to be copied
-           to the remote system.  This unfortunately has to contain
-           the entire derivation closure to ensure that the validity
-           invariant holds on the remote system.  (I.e., it's
-           unfortunate that we have to list it since the remote system
-           *probably* already has it.) */
-        PathSet allInputs;
-        allInputs.insert(inputPaths.begin(), inputPaths.end());
-        computeFSClosure(drvPath, allInputs);
+    string s;
+    foreach (PathSet::iterator, i, allInputs) s += *i + " ";
+    writeLine(hook->toHook.writeSide, s);
         
-        string s;
-        foreach (PathSet::iterator, i, allInputs) s += *i + "\n";
+    /* Tell the hooks the outputs that have to be copied back from the
+       remote system. */
+    s = "";
+    foreach (DerivationOutputs::iterator, i, drv.outputs)
+        s += i->second.path + " ";
+    writeLine(hook->toHook.writeSide, s);
+    
+    hook->toHook.writeSide.close();
+
+    /* Create the log file and pipe. */
+    Path logFile = openLogFile();
+
+    set<int> fds;
+    fds.insert(hook->fromHook.readSide);
+    fds.insert(hook->builderOut.readSide);
+    worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
+    
+    if (printBuildTrace)
+        printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
+            % drvPath % drv.outputs["out"].path % drv.platform % logFile);
         
-        writeFile(inputListFN, s);
-
-        /* The `outputs' file lists all outputs that have to be copied
-           from the remote system. */
-        s = "";
-        foreach (DerivationOutputs::iterator, i, drv.outputs)
-            s += i->second.path + "\n";
-        writeFile(outputListFN, s);
-
-        /* The `references' file has exactly the format accepted by
-           `nix-store --register-validity'. */
-        writeFile(referencesFN,
-            makeValidityRegistration(allInputs, true, false));
-
-        /* Tell the hook to proceed. */
-        writeLine(toHook.writeSide, "okay");
-        toHook.writeSide.close();
-
-        if (printBuildTrace)
-            printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
-                % drvPath % drv.outputs["out"].path % drv.platform % logFile);
-        
-        return rpAccept;
-    }
-
-    else throw Error(format("bad hook reply `%1%'") % reply);
-}
-
-
-void DerivationGoal::terminateBuildHook(bool kill)
-{
-    debug("terminating build hook");
-    pid_t savedPid = pid;
-    if (kill)
-        pid.kill();
-    else
-        pid.wait(true);
-    /* `false' means don't wake up waiting goals, since we want to
-       keep this build slot ourselves. */
-    worker.childTerminated(savedPid, false);
-    toHook.writeSide.close();
-    fdLogFile.close();
-    logPipe.readSide.close();
-    deleteTmpDir(true); /* get rid of the hook's temporary directory */
+    return rpAccept;    
 }
 
 
@@ -1379,11 +1439,7 @@ void DerivationGoal::startBuilder()
         format("building path(s) %1%") % showPaths(outputPaths(drv.outputs)))
     
     /* Right platform? */
-    if (drv.platform != thisSystem 
-#ifdef CAN_DO_LINUX32_BUILDS
-        && !(drv.platform == "i686-linux" && thisSystem == "x86_64-linux")
-#endif
-        )
+    if (!canBuildLocally(drv.platform))
         throw Error(
             format("a `%1%' is required to build `%3%', but I am a `%2%'")
             % drv.platform % thisSystem % drvPath);
@@ -1411,6 +1467,9 @@ void DerivationGoal::startBuilder()
        in the store or in the build directory). */
     env["NIX_STORE"] = nixStore;
 
+    /* The maximum number of cores to utilize for parallel building. */
+    env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str();
+
     /* Add all bindings specified in the derivation. */
     foreach (StringPairs::iterator, i, drv.env)
         env[i->first] = i->second;
@@ -1495,7 +1554,7 @@ void DerivationGoal::startBuilder()
 
         /* Write closure info to `fileName'. */
         writeFile(tmpDir + "/" + fileName,
-            makeValidityRegistration(paths, false, false));
+            worker.store.makeValidityRegistration(paths, false, false));
     }
 
     
@@ -1545,6 +1604,9 @@ void DerivationGoal::startBuilder()
 
     if (fixedOutput) useChroot = false;
 
+    /* Hack to allow derivations to disable chroot builds. */
+    if (drv.env["__noChroot"] == "1") useChroot = false;
+
     if (useChroot) {
 #if CHROOT_ENABLED
         /* Create a temporary directory in which we set up the chroot
@@ -1568,7 +1630,7 @@ void DerivationGoal::startBuilder()
 
         /* Create a /etc/passwd with entries for the build user and the
            nobody account.  The latter is kind of a hack to support
-           Samba-in-QEMU.  */
+           Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
 
         writeFile(chrootRootDir + "/etc/passwd",
@@ -1576,13 +1638,13 @@ void DerivationGoal::startBuilder()
                 "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n"
                 "nobody:x:65534:65534:Nobody:/:/noshell\n")
                 % (buildUser.enabled() ? buildUser.getUID() : getuid())
-	        % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
+                % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
 
 	/* Declare the build user's group so that programs get a consistent
-	   view of the system (e.g., "id -gn").  */
-	writeFile(chrootRootDir + "/etc/group",
-		  (format("nixbld:!:%1%:\n")
-		   % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
+	   view of the system (e.g., "id -gn"). */
+        writeFile(chrootRootDir + "/etc/group",
+            (format("nixbld:!:%1%:\n")
+                % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
 
         /* Bind-mount a user-configurable set of directories from the
            host file system.  The `/dev/pts' directory must be mounted
@@ -1641,9 +1703,12 @@ void DerivationGoal::startBuilder()
     printMsg(lvlChatty, format("executing builder `%1%'") %
         drv.builder);
 
-    /* Create the log file and pipe. */
+    /* Create the log file. */
     Path logFile = openLogFile();
     
+    /* Create a pipe to get the output of the builder. */
+    builderOut.create();
+
     /* Fork a child to build the package.  Note that while we
        currently use forks to run and wait for the children, it
        shouldn't be hard to use threads for this on systems where
@@ -1657,7 +1722,7 @@ void DerivationGoal::startBuilder()
     case 0:
 
         /* Warning: in the child we should absolutely not make any
-           Berkeley DB calls! */
+           SQLite calls! */
 
         try { /* child */
 
@@ -1684,18 +1749,23 @@ void DerivationGoal::startBuilder()
                         throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target);
                 }
                     
-                /* Do the chroot().  initChild() will do a chdir() to
-                   the temporary build directory to make sure the
-                   current directory is in the chroot.  (Actually the
-                   order doesn't matter, since due to the bind mount
-                   tmpDir and tmpRootDit/tmpDir are the same
-                   directories.) */
+                /* Do the chroot().  Below we do a chdir() to the
+                   temporary build directory to make sure the current
+                   directory is in the chroot.  (Actually the order
+                   doesn't matter, since due to the bind mount tmpDir
+                   and tmpRootDit/tmpDir are the same directories.) */
                 if (chroot(chrootRootDir.c_str()) == -1)
                     throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir);
             }
 #endif
             
-            initChild();
+            commonChildInit(builderOut);
+    
+            if (chdir(tmpDir.c_str()) == -1)
+                throw SysError(format("changing into `%1%'") % tmpDir);
+
+            /* Close all other file descriptors. */
+            closeMostFDs(set<int>());
 
 #ifdef CAN_DO_LINUX32_BUILDS
             if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") {
@@ -1716,10 +1786,10 @@ void DerivationGoal::startBuilder()
             
             /* If we are running in `build-users' mode, then switch to
                the user we allocated above.  Make sure that we drop
-               all root privileges.  Note that initChild() above has
-               closed all file descriptors except std*, so that's
-               safe.  Also note that setuid() when run as root sets
-               the real, effective and saved UIDs. */
+               all root privileges.  Note that above we have closed
+               all file descriptors except std*, so that's safe.  Also
+               note that setuid() when run as root sets the real,
+               effective and saved UIDs. */
             if (buildUser.enabled()) {
                 printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser());
 
@@ -1773,9 +1843,9 @@ void DerivationGoal::startBuilder()
     
     /* parent */
     pid.setSeparatePG(true);
-    logPipe.writeSide.close();
+    builderOut.writeSide.close();
     worker.childStarted(shared_from_this(), pid,
-        singleton<set<int> >(logPipe.readSide), true, true);
+        singleton<set<int> >(builderOut.readSide), true, true);
 
     if (printBuildTrace) {
         printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
@@ -1807,12 +1877,12 @@ PathSet parseReferenceSpecifiers(const Derivation & drv, string attr)
 void DerivationGoal::computeClosure()
 {
     map<Path, PathSet> allReferences;
-    map<Path, Hash> contentHashes;
+    map<Path, HashResult> contentHashes;
 
     /* When using a build hook, the build hook can register the output
        as valid (by doing `nix-store --import').  If so we don't have
        to do anything here. */
-    if (usingBuildHook) {
+    if (hook) {
         bool allValid = true;
         foreach (DerivationOutputs::iterator, i, drv.outputs)
             if (!worker.store.isValidPath(i->second.path)) allValid = false;
@@ -1864,7 +1934,7 @@ void DerivationGoal::computeClosure()
             if (ht == htUnknown)
                 throw BuildError(format("unknown hash algorithm `%1%'") % algo);
             Hash h = parseHash(ht, i->second.hash);
-            Hash h2 = recursive ? hashPath(ht, path) : hashFile(ht, path);
+            Hash h2 = recursive ? hashPath(ht, path).first : hashFile(ht, path);
             if (h != h2)
                 throw BuildError(
                     format("output path `%1%' should have %2% hash `%3%', instead has `%4%'")
@@ -1878,7 +1948,7 @@ void DerivationGoal::computeClosure()
 	   contained in it.  Compute the SHA-256 NAR hash at the same
 	   time.  The hash is stored in the database so that we can
 	   verify later on whether nobody has messed with the store. */
-        Hash hash;
+        HashResult hash;
         PathSet references = scanForReferences(path, allPaths, hash);
         contentHashes[path] = hash;
 
@@ -1907,14 +1977,18 @@ void DerivationGoal::computeClosure()
     }
 
     /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  !!! this should be
-       atomic so that either all paths are registered as valid, or
-       none are. */
-    foreach (DerivationOutputs::iterator, i, drv.outputs)
-        worker.store.registerValidPath(i->second.path,
-            contentHashes[i->second.path],
-            allReferences[i->second.path],
-            drvPath);
+       paths referenced by each of them. */
+    ValidPathInfos infos;
+    foreach (DerivationOutputs::iterator, i, drv.outputs) {
+        ValidPathInfo info;
+        info.path = i->second.path;
+        info.hash = contentHashes[i->second.path].first;
+        info.narSize = contentHashes[i->second.path].second;
+        info.references = allReferences[i->second.path];
+        info.deriver = drvPath;
+        infos.push_back(info);
+    }
+    worker.store.registerValidPaths(infos);
 
     /* It is now safe to delete the lock files, since all future
        lockers will see that the output paths are valid; they will not
@@ -1940,32 +2014,10 @@ Path DerivationGoal::openLogFile()
     if (fdLogFile == -1)
         throw SysError(format("creating log file `%1%'") % logFileName);
 
-    /* Create a pipe to get the output of the child. */
-    logPipe.create();
-
     return logFileName;
 }
 
 
-void DerivationGoal::initChild()
-{
-    commonChildInit(logPipe);
-    
-    if (chdir(tmpDir.c_str()) == -1)
-        throw SysError(format("changing into `%1%'") % tmpDir);
-
-    /* When running a hook, dup the communication pipes. */
-    if (usingBuildHook) {
-        toHook.writeSide.close();
-        if (dup2(toHook.readSide, STDIN_FILENO) == -1)
-            throw SysError("dupping to-hook read side");
-    }
-
-    /* Close all other file descriptors. */
-    closeMostFDs(set<int>());
-}
-
-
 void DerivationGoal::deleteTmpDir(bool force)
 {
     if (tmpDir != "") {
@@ -1985,19 +2037,22 @@ void DerivationGoal::deleteTmpDir(bool force)
 
 void DerivationGoal::handleChildOutput(int fd, const string & data)
 {
-    if (fd == logPipe.readSide) {
+    if ((hook && fd == hook->builderOut.readSide) ||
+        (!hook && fd == builderOut.readSide))
+    {
         if (verbosity >= buildVerbosity)
             writeToStderr((unsigned char *) data.c_str(), data.size());
         writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size());
     }
 
-    else abort();
+    if (hook && fd == hook->fromHook.readSide)
+        writeToStderr((unsigned char *) data.c_str(), data.size());
 }
 
 
 void DerivationGoal::handleEOF(int fd)
 {
-    if (fd == logPipe.readSide) worker.wakeUp(shared_from_this());
+    worker.wakeUp(shared_from_this());
 }
 
 
@@ -2341,10 +2396,15 @@ void SubstitutionGoal::finished()
 
     canonicalisePathMetaData(storePath);
 
-    Hash contentHash = hashPath(htSHA256, storePath);
-
-    worker.store.registerValidPath(storePath, contentHash,
-        info.references, info.deriver);
+    HashResult hash = hashPath(htSHA256, storePath);
+    
+    ValidPathInfo info2;
+    info2.path = storePath;
+    info2.hash = hash.first;
+    info2.narSize = hash.second;
+    info2.references = info.references;
+    info2.deriver = info.deriver;
+    worker.store.registerValidPath(info2);
 
     outputLock->setDeletion(true);
     
@@ -2391,6 +2451,7 @@ Worker::Worker(LocalStore & store)
     nrLocalBuilds = 0;
     lastWokenUp = 0;
     cacheFailure = queryBoolSetting("build-cache-failure", false);
+    permanentFailure = false;
 }
 
 
@@ -2635,6 +2696,7 @@ void Worker::waitForInput()
         timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before);
     } else lastWokenUp = 0;
 
+    using namespace std;
     /* Use select() to wait for the input side of any logger pipe to
        become `available'.  Note that `available' (i.e., non-blocking)
        includes EOF. */
@@ -2716,6 +2778,11 @@ void Worker::waitForInput()
 }
 
 
+unsigned int Worker::exitStatus()
+{
+    return permanentFailure ? 100 : 1;
+}
+
 
 //////////////////////////////////////////////////////////////////////
 
@@ -2742,7 +2809,7 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
         }
             
     if (!failed.empty())
-        throw Error(format("build of %1% failed") % showPaths(failed));
+        throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus());
 }
 
 
@@ -2758,7 +2825,7 @@ void LocalStore::ensurePath(const Path & path)
     worker.run(goals);
 
     if (goal->getExitCode() != Goal::ecSuccess)
-        throw Error(format("path `%1%' does not exist and cannot be created") % path);
+        throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus());
 }
 
  

src/libstore/derivations.hh

@@ -1,10 +1,10 @@
 #ifndef __DERIVATIONS_H
 #define __DERIVATIONS_H
 
-#include "hash.hh"
-
 #include <map>
 
+#include "types.hh"
+
 
 namespace nix {
 

src/libstore/gc.cc

@@ -1,6 +1,5 @@
 #include "globals.hh"
 #include "misc.hh"
-#include "pathlocks.hh"
 #include "local-store.hh"
 
 #include <boost/shared_ptr.hpp>
@@ -31,7 +30,7 @@ static const int defaultGcLevel = 1000;
    read.  To be precise: when they try to create a new temporary root
    file, they will block until the garbage collector has finished /
    yielded the GC lock. */
-static int openGCLock(LockType lockType)
+int LocalStore::openGCLock(LockType lockType)
 {
     Path fnGCLock = (format("%1%/%2%")
         % nixStateDir % gcLockName).str();
@@ -127,7 +126,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
        Instead of reading all the roots, it would be more efficient to
        check if the root is in a directory in or linked from the
        gcroots directory. */
-    if (queryBoolSetting("gc-check-reachability", true)) {
+    if (queryBoolSetting("gc-check-reachability", false)) {
         Roots roots = store->findRoots();
         if (roots.find(gcRoot) == roots.end())
             printMsg(lvlError, 
@@ -136,7 +135,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
                     "therefore, `%2%' might be removed by the garbage collector")
                 % gcRoot % storePath);
     }
-        
+
     /* Grab the global GC root, causing us to block while a GC is in
        progress.  This prevents the set of permanent roots from
        increasing while a GC is in progress. */
@@ -416,18 +415,13 @@ struct LocalStore::GCState
     PathSet busy;
     bool gcKeepOutputs;
     bool gcKeepDerivations;
-
-    bool drvsIndexed;
-    typedef std::multimap<string, Path> DrvsByName;
-    DrvsByName drvsByName; // derivation paths hashed by name attribute
-
-    GCState(GCResults & results_) : results(results_), drvsIndexed(false)
+    GCState(GCResults & results_) : results(results_)
     {
     }
 };
 
 
-static bool doDelete(GCOptions::GCAction action)
+static bool shouldDelete(GCOptions::GCAction action)
 {
     return action == GCOptions::gcDeleteDead
         || action == GCOptions::gcDeleteSpecific;
@@ -441,45 +435,11 @@ bool LocalStore::isActiveTempFile(const GCState & state,
         && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
 }
 
-
-/* Return all the derivations in the Nix store that have `path' as an
-   output.  This function assumes that derivations have the same name
-   as their outputs. */
-PathSet LocalStore::findDerivers(GCState & state, const Path & path)
-{
-    PathSet derivers;
-
-    Path deriver = queryDeriver(path);
-    if (deriver != "") derivers.insert(deriver);
-
-    if (!state.drvsIndexed) {
-        Paths entries = readDirectory(nixStore);
-        foreach (Paths::iterator, i, entries)
-            if (isDerivation(*i))
-                state.drvsByName.insert(std::pair<string, Path>(
-                        getNameOfStorePath(*i), nixStore + "/" + *i));
-        state.drvsIndexed = true;
-    }
-    
-    string name = getNameOfStorePath(path);
-
-    // Urgh, I should have used Haskell...
-    std::pair<GCState::DrvsByName::iterator, GCState::DrvsByName::iterator> range =
-        state.drvsByName.equal_range(name);
-
-    for (GCState::DrvsByName::iterator i = range.first; i != range.second; ++i)
-        if (isValidPath(i->second)) {
-            Derivation drv = derivationFromPath(i->second);
-            foreach (DerivationOutputs::iterator, j, drv.outputs)
-                if (j->second.path == path) derivers.insert(i->second);
-        }
-
-    return derivers;
-}
-
     
 bool LocalStore::tryToDelete(GCState & state, const Path & path)
 {
+    checkInterrupt();
+    
     if (!pathExists(path)) return true;
     if (state.deleted.find(path) != state.deleted.end()) return true;
     if (state.live.find(path) != state.live.end()) return false;
@@ -508,10 +468,10 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
            then don't delete the derivation if any of the outputs are
            live. */
         if (state.gcKeepDerivations && isDerivation(path)) {
-            Derivation drv = derivationFromPath(path);
-            foreach (DerivationOutputs::iterator, i, drv.outputs)
-                if (!tryToDelete(state, i->second.path)) {
-                    printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output is alive") % path);
+            PathSet outputs = queryDerivationOutputs(path);
+            foreach (PathSet::iterator, i, outputs)
+                if (!tryToDelete(state, *i)) {
+                    printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output `%2%' is alive") % path % *i);
                     goto isLive;
                 }
         }
@@ -522,18 +482,9 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
         if (!pathExists(path)) return true;
 
         /* If gc-keep-outputs is set, then don't delete this path if
-           its deriver is not garbage.  !!! Nix does not reliably
-           store derivers, so we have to look at all derivations to
-           determine which of them derive `path'.  Since this makes
-           the garbage collector very slow to start on large Nix
-           stores, here we just look for all derivations that have the
-           same name as `path' (where the name is the part of the
-           filename after the hash, i.e. the `name' attribute of the
-           derivation).  This is somewhat hacky: currently, the
-           deriver of a path always has the same name as the output,
-           but this might change in the future. */
+           there are derivers of this path that are not garbage. */
         if (state.gcKeepOutputs) {
-            PathSet derivers = findDerivers(state, path);
+            PathSet derivers = queryValidDerivers(path);
             foreach (PathSet::iterator, deriver, derivers) {
                 /* Break an infinite recursion if gc-keep-derivations
                    and gc-keep-outputs are both set by tentatively
@@ -567,7 +518,7 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path)
     }
 
     /* The path is garbage, so delete it. */
-    if (doDelete(state.options.action)) {
+    if (shouldDelete(state.options.action)) {
         printMsg(lvlInfo, format("deleting `%1%'") % path);
 
         unsigned long long bytesFreed, blocksFreed;
@@ -613,6 +564,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     
     state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false);
     state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true);
+
+    /* Using `--ignore-liveness' with `--delete' can have unintended
+       consequences if `gc-keep-outputs' or `gc-keep-derivations' are
+       true (the garbage collector will recurse into deleting the
+       outputs or derivers, respectively).  So disable them. */
+    if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
+        state.gcKeepOutputs = false;
+        state.gcKeepDerivations = false;
+    }
     
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
@@ -667,7 +627,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         vector<Path> entries_(entries.begin(), entries.end());
         random_shuffle(entries_.begin(), entries_.end());
 
-        if (doDelete(state.options.action))
+        if (shouldDelete(state.options.action))
             printMsg(lvlError, format("deleting garbage..."));
         else
             printMsg(lvlError, format("determining live/dead paths..."));

src/libstore/globals.cc

@@ -20,8 +20,9 @@ string nixBinDir = "/UNINIT";
 bool keepFailed = false;
 bool keepGoing = false;
 bool tryFallback = false;
-Verbosity buildVerbosity = lvlInfo;
+Verbosity buildVerbosity = lvlError;
 unsigned int maxBuildJobs = 1;
+unsigned int buildCores = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
 time_t maxSilentTime = 0;

src/libstore/globals.hh

@@ -55,6 +55,11 @@ extern Verbosity buildVerbosity;
 /* Maximum number of parallel build jobs.  0 means unlimited. */
 extern unsigned int maxBuildJobs;
 
+/* Number of CPU cores to utilize in parallel within a build, i.e. by passing
+   this number to Make via '-j'. 0 means that the number of actual CPU cores on
+   the local host ought to be auto-detected. */
+extern unsigned int buildCores;
+
 /* Read-only mode.  Don't copy stuff to the store, don't change the
    database. */
 extern bool readOnlyMode;

src/libstore/local-store.hh

@@ -5,6 +5,11 @@
 
 #include "store-api.hh"
 #include "util.hh"
+#include "pathlocks.hh"
+
+
+class sqlite3;
+class sqlite3_stmt;
 
 
 namespace nix {
@@ -12,8 +17,9 @@ namespace nix {
 
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
    0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
-   Version 4 is Nix 0.11.  Version 5 is Nix 0.12*/
-const int nixSchemaVersion = 5;
+   Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
+   Nix 1.0. */
+const int nixSchemaVersion = 6;
 
 
 extern string drvsLogDir;
@@ -41,6 +47,34 @@ struct RunningSubstituter
 };
 
 
+/* Wrapper object to close the SQLite database automatically. */
+struct SQLite
+{
+    sqlite3 * db;
+    SQLite() { db = 0; }
+    ~SQLite();
+    operator sqlite3 * () { return db; }
+};
+
+
+/* Wrapper object to create and destroy SQLite prepared statements. */
+struct SQLiteStmt
+{
+    sqlite3 * db;
+    sqlite3_stmt * stmt;
+    unsigned int curArg;
+    SQLiteStmt() { stmt = 0; }
+    void create(sqlite3 * db, const string & s);
+    void reset();
+    ~SQLiteStmt();
+    operator sqlite3_stmt * () { return stmt; }
+    void bind(const string & value);
+    void bind(int value);
+    void bind64(long long value);
+    void bind();
+};
+    
+
 class LocalStore : public StoreAPI
 {
 private:
@@ -64,6 +98,8 @@ public:
 
     PathSet queryValidPaths();
     
+    ValidPathInfo queryPathInfo(const Path & path);
+
     Hash queryPathHash(const Path & path);
 
     void queryReferences(const Path & path, PathSet & references);
@@ -71,6 +107,14 @@ public:
     void queryReferrers(const Path & path, PathSet & referrers);
 
     Path queryDeriver(const Path & path);
+
+    /* Return all currently valid derivations that have `path' as an
+       output.  (Note that the result of `queryDeriver()' is the
+       derivation that was actually used to produce `path', which may
+       not exist anymore.) */
+    PathSet queryValidDerivers(const Path & path);
+
+    PathSet queryDerivationOutputs(const Path & path);
     
     PathSet querySubstitutablePaths();
     
@@ -132,8 +176,7 @@ public:
        execution of the derivation (or something equivalent).  Also
        register the hash of the file system contents of the path.  The
        hash must be a SHA-256 hash. */
-    void registerValidPath(const Path & path,
-        const Hash & hash, const PathSet & references, const Path & deriver);
+    void registerValidPath(const ValidPathInfo & info);
 
     void registerValidPaths(const ValidPathInfos & infos);
 
@@ -144,6 +187,10 @@ public:
     /* Query whether `path' previously failed to build. */
     bool hasPathFailed(const Path & path);
 
+    PathSet queryFailedPaths();
+
+    void clearFailedPaths(const PathSet & paths);
+
 private:
 
     Path schemaPath;
@@ -151,45 +198,63 @@ private:
     /* Lock file used for upgrading. */
     AutoCloseFD globalLock;
 
-    /* !!! The cache can grow very big.  Maybe it should be pruned
-       every once in a while. */
-    std::map<Path, ValidPathInfo> pathInfoCache;
+    /* The SQLite database object. */
+    SQLite db;
 
-    /* Store paths for which the referrers file must be purged. */
-    PathSet delayedUpdates;
-
-    /* Whether to do an fsync() after writing Nix metadata. */
-    bool doFsync;
+    /* Some precompiled SQLite statements. */
+    SQLiteStmt stmtRegisterValidPath;
+    SQLiteStmt stmtUpdatePathInfo;
+    SQLiteStmt stmtAddReference;
+    SQLiteStmt stmtQueryPathInfo;
+    SQLiteStmt stmtQueryReferences;
+    SQLiteStmt stmtQueryReferrers;
+    SQLiteStmt stmtInvalidatePath;
+    SQLiteStmt stmtRegisterFailedPath;
+    SQLiteStmt stmtHasPathFailed;
+    SQLiteStmt stmtQueryFailedPaths;
+    SQLiteStmt stmtClearFailedPath;
+    SQLiteStmt stmtAddDerivationOutput;
+    SQLiteStmt stmtQueryValidDerivers;
+    SQLiteStmt stmtQueryDerivationOutputs;
 
     int getSchema();
 
-    void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false);
+    void openDB(bool create);
 
-    ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false);
+    unsigned long long queryValidPathId(const Path & path);
 
+    unsigned long long addValidPath(const ValidPathInfo & info);
+        
+    void addReference(unsigned long long referrer, unsigned long long reference);
+    
     void appendReferrer(const Path & from, const Path & to, bool lock);
     
     void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
 
-    void flushDelayedUpdates();
-    
-    bool queryReferrersInternal(const Path & path, PathSet & referrers);
-    
     void invalidatePath(const Path & path);
-    
-    void upgradeStore12();
+
+    void verifyPath(const Path & path, const PathSet & store,
+        PathSet & done, PathSet & validPaths);
+
+    void updatePathInfo(const ValidPathInfo & info);
+
+    void upgradeStore6();
+    PathSet queryValidPathsOld();
+    ValidPathInfo queryPathInfoOld(const Path & path);
 
     struct GCState;
 
     bool tryToDelete(GCState & state, const Path & path);
     
-    PathSet findDerivers(GCState & state, const Path & path);
-    
     bool isActiveTempFile(const GCState & state,
         const Path & path, const string & suffix);
         
+    int openGCLock(LockType lockType);
+    
     void startSubstituter(const Path & substituter,
         RunningSubstituter & runningSubstituter);
+
+    Path createTempDirInStore();
 };
 
 

src/libstore/misc.cc

@@ -27,10 +27,10 @@ void computeFSClosure(const Path & storePath,
         store->queryReferences(storePath, references);
 
     if (includeOutputs && isDerivation(storePath)) {
-        Derivation drv = derivationFromPath(storePath);
-        foreach (DerivationOutputs::iterator, i, drv.outputs)
-            if (store->isValidPath(i->second.path))
-                computeFSClosure(i->second.path, paths, flipDirection, true);
+        PathSet outputs = store->queryDerivationOutputs(storePath);
+        foreach (PathSet::iterator, i, outputs)
+            if (store->isValidPath(*i))
+                computeFSClosure(*i, paths, flipDirection, true);
     }
 
     foreach (PathSet::iterator, i, references)
@@ -48,9 +48,9 @@ Path findOutput(const Derivation & drv, string id)
 
 void queryMissing(const PathSet & targets,
     PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
-    unsigned long long & downloadSize)
+    unsigned long long & downloadSize, unsigned long long & narSize)
 {
-    downloadSize = 0;
+    downloadSize = narSize = 0;
     
     PathSet todo(targets.begin(), targets.end()), done;
 
@@ -88,6 +88,7 @@ void queryMissing(const PathSet & targets,
             if (store->querySubstitutablePathInfo(p, info)) {
                 willSubstitute.insert(p);
                 downloadSize += info.downloadSize;
+                narSize += info.narSize;
                 todo.insert(info.references.begin(), info.references.end());
             } else
                 unknown.insert(p);

src/libstore/misc.hh

@@ -31,7 +31,7 @@ Path findOutput(const Derivation & drv, string id);
    will be substituted. */
 void queryMissing(const PathSet & targets,
     PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
-    unsigned long long & downloadSize);
+    unsigned long long & downloadSize, unsigned long long & narSize);
 
 
 }

src/libstore/optimise-store.cc

@@ -68,7 +68,7 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath,
            the contents of the symlink (i.e. the result of
            readlink()), not the contents of the target (which may not
            even exist). */
-        Hash hash = hashPath(htSHA256, path);
+        Hash hash = hashPath(htSHA256, path).first;
         stats.totalFiles++;
         printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
 
@@ -119,9 +119,23 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath,
             }
 
             /* Atomically replace the old file with the new hard link. */
-            if (rename(tempLink.c_str(), path.c_str()) == -1)
+            if (rename(tempLink.c_str(), path.c_str()) == -1) {
+                if (errno == EMLINK) {
+                    /* Some filesystems generate too many links on the
+                       rename, rather than on the original link.
+                       (Probably it temporarily increases the st_nlink
+                       field before decreasing it again.) */
+                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
+                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
+
+                    /* Unlink the temp link. */
+                    if (unlink(tempLink.c_str()) == -1)
+                        printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
+                    return;
+                }
                 throw SysError(format("cannot rename `%1%' to `%2%'")
                     % tempLink % path);
+            }
         } else
             printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
         

src/libstore/references.cc

@@ -81,7 +81,7 @@ void RefScanSink::operator () (const unsigned char * data, unsigned int len)
 
 
 PathSet scanForReferences(const string & path,
-    const PathSet & refs, Hash & hash)
+    const PathSet & refs, HashResult & hash)
 {
     RefScanSink sink;
     std::map<string, Path> backMap;

src/libstore/references.hh

@@ -7,7 +7,7 @@
 namespace nix {
 
 PathSet scanForReferences(const Path & path, const PathSet & refs,
-    Hash & hash);
+    HashResult & hash);
     
 }
 

src/libstore/remote-store.cc

@@ -13,6 +13,7 @@
 
 #include <iostream>
 #include <unistd.h>
+#include <cstring>
 
 
 namespace nix {
@@ -96,10 +97,6 @@ void RemoteStore::forkSlave()
     if (worker == "")
         worker = nixBinDir + "/nix-worker";
 
-    string verbosityArg = "-";
-    for (int i = 1; i < verbosity; ++i)
-        verbosityArg += "v";
-
     child = fork();
     
     switch (child) {
@@ -119,10 +116,7 @@ void RemoteStore::forkSlave()
             close(fdSocket);
             close(fdChild);
 
-            execlp(worker.c_str(), worker.c_str(), "--slave",
-                /* hacky - must be at the end */
-                verbosityArg == "-" ? NULL : verbosityArg.c_str(),
-                NULL);
+            execlp(worker.c_str(), worker.c_str(), "--slave", NULL);
 
             throw SysError(format("executing `%1%'") % worker);
             
@@ -158,6 +152,7 @@ void RemoteStore::connectToDaemon()
     addr.sun_family = AF_UNIX;
     if (socketPathRel.size() >= sizeof(addr.sun_path))
         throw Error(format("socket path `%1%' is too long") % socketPathRel);
+    using namespace std;
     strcpy(addr.sun_path, socketPathRel.c_str());
     
     if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1)
@@ -196,6 +191,8 @@ void RemoteStore::setOptions()
         writeInt(logType, to);
         writeInt(printBuildTrace, to);
     }
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 6)
+        writeInt(buildCores, to);
     processStderr();
 }
 
@@ -214,7 +211,9 @@ bool RemoteStore::isValidPath(const Path & path)
 PathSet RemoteStore::queryValidPaths()
 {
     openConnection();
-    throw Error("not implemented");
+    writeInt(wopQueryValidPaths, to);
+    processStderr();
+    return readStorePaths(from);
 }
 
 
@@ -243,10 +242,29 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path,
     if (info.deriver != "") assertStorePath(info.deriver);
     info.references = readStorePaths(from);
     info.downloadSize = readLongLong(from);
+    info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
     return true;
 }
 
 
+ValidPathInfo RemoteStore::queryPathInfo(const Path & path)
+{
+    openConnection();
+    writeInt(wopQueryPathInfo, to);
+    writeString(path, to);
+    processStderr();
+    ValidPathInfo info;
+    info.path = path;
+    info.deriver = readString(from);
+    if (info.deriver != "") assertStorePath(info.deriver);
+    info.hash = parseHash(htSHA256, readString(from));
+    info.references = readStorePaths(from);
+    info.registrationTime = readInt(from);
+    info.narSize = readLongLong(from);
+    return info;
+}
+
+
 Hash RemoteStore::queryPathHash(const Path & path)
 {
     openConnection();
@@ -294,6 +312,16 @@ Path RemoteStore::queryDeriver(const Path & path)
 }
 
 
+PathSet RemoteStore::queryDerivationOutputs(const Path & path)
+{
+    openConnection();
+    writeInt(wopQueryDerivationOutputs, to);
+    writeString(path, to);
+    processStderr();
+    return readStorePaths(from);
+}
+
+
 Path RemoteStore::addToStore(const Path & _srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter)
 {
@@ -439,6 +467,25 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 }
 
 
+PathSet RemoteStore::queryFailedPaths()
+{
+    openConnection();
+    writeInt(wopQueryFailedPaths, to);
+    processStderr();
+    return readStorePaths(from);
+}
+
+
+void RemoteStore::clearFailedPaths(const PathSet & paths)
+{
+    openConnection();
+    writeInt(wopClearFailedPaths, to);
+    writeStringSet(paths, to);
+    processStderr();
+    readInt(from);
+}
+
+
 void RemoteStore::processStderr(Sink * sink, Source * source)
 {
     unsigned int msg;
@@ -462,8 +509,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source)
             writeToStderr((const unsigned char *) s.c_str(), s.size());
         }
     }
-    if (msg == STDERR_ERROR)
-        throw Error(readString(from));
+    if (msg == STDERR_ERROR) {
+        string error = readString(from);
+        unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1;
+        throw Error(error, status);
+    }
     else if (msg != STDERR_LAST)
         throw Error("protocol error processing standard error");
 }

src/libstore/remote-store.hh

@@ -29,6 +29,8 @@ public:
 
     PathSet queryValidPaths();
     
+    ValidPathInfo queryPathInfo(const Path & path);
+
     Hash queryPathHash(const Path & path);
 
     void queryReferences(const Path & path, PathSet & references);
@@ -37,6 +39,8 @@ public:
 
     Path queryDeriver(const Path & path);
     
+    PathSet queryDerivationOutputs(const Path & path);
+    
     bool hasSubstitutes(const Path & path);
     
     bool querySubstitutablePathInfo(const Path & path,
@@ -68,6 +72,10 @@ public:
 
     void collectGarbage(const GCOptions & options, GCResults & results);
     
+    PathSet queryFailedPaths();
+
+    void clearFailedPaths(const PathSet & paths);
+    
 private:
     AutoCloseFD fdSocket;
     FdSink to;

src/libstore/schema.sql

@@ -0,0 +1,44 @@
+create table if not exists ValidPaths (
+    id               integer primary key autoincrement not null,
+    path             text unique not null,
+    hash             text not null,
+    registrationTime integer not null,
+    deriver          text,
+    narSize          integer
+);
+
+create table if not exists Refs (
+    referrer  integer not null,
+    reference integer not null,
+    primary key (referrer, reference),
+    foreign key (referrer) references ValidPaths(id) on delete cascade,
+    foreign key (reference) references ValidPaths(id) on delete restrict
+);
+
+create index if not exists IndexReferrer on Refs(referrer);
+create index if not exists IndexReference on Refs(reference);
+
+-- Paths can refer to themselves, causing a tuple (N, N) in the Refs
+-- table.  This causes a deletion of the corresponding row in
+-- ValidPaths to cause a foreign key constraint violation (due to `on
+-- delete restrict' on the `reference' column).  Therefore, explicitly
+-- get rid of self-references.
+create trigger if not exists DeleteSelfRefs before delete on ValidPaths
+  begin
+    delete from Refs where referrer = old.id and reference = old.id;
+  end;
+
+create table if not exists DerivationOutputs (
+    drv  integer not null,
+    id   text not null, -- symbolic output id, usually "out"
+    path text not null,
+    primary key (drv, id),
+    foreign key (drv) references ValidPaths(id) on delete cascade
+);
+
+create index if not exists IndexDerivationOutputs on DerivationOutputs(path);
+
+create table if not exists FailedPaths (
+    path text primary key not null,
+    time integer not null
+);

src/libstore/store-api.cc

@@ -1,7 +1,6 @@
 #include "store-api.hh"
 #include "globals.hh"
 #include "util.hh"
-#include "derivations.hh"
 
 #include <limits.h>
 
@@ -53,18 +52,6 @@ Path toStorePath(const Path & path)
 }
 
 
-string getNameOfStorePath(const Path & path)
-{
-    Path::size_type slash = path.rfind('/');
-    string p = slash == Path::npos ? path : string(path, slash + 1);
-    Path::size_type dash = p.find('-');
-    assert(dash != Path::npos);
-    string p2 = string(p, dash + 1);
-    if (isDerivation(p2)) p2 = string(p2, 0, p2.size() - 4);
-    return p2;
-}
-
-
 Path followLinksToStore(const Path & _path)
 {
     Path path = absPath(_path);
@@ -203,7 +190,7 @@ std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter)
 {
     HashType ht(hashAlgo);
-    Hash h = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath);
+    Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath);
     string name = baseNameOf(srcPath);
     Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
     return std::pair<Path, Hash>(dstPath, h);
@@ -229,7 +216,7 @@ Path computeStorePathForText(const string & name, const string & s,
 /* Return a string accepted by decodeValidPathInfo() that
    registers the specified paths as valid.  Note: it's the
    responsibility of the caller to provide a closure. */
-string makeValidityRegistration(const PathSet & paths,
+string StoreAPI::makeValidityRegistration(const PathSet & paths,
     bool showDerivers, bool showHash)
 {
     string s = "";
@@ -237,18 +224,19 @@ string makeValidityRegistration(const PathSet & paths,
     foreach (PathSet::iterator, i, paths) {
         s += *i + "\n";
 
-        if (showHash)
-            s += printHash(store->queryPathHash(*i)) + "\n";
+        ValidPathInfo info = queryPathInfo(*i);
 
-        Path deriver = showDerivers ? store->queryDeriver(*i) : "";
+        if (showHash) {
+            s += printHash(info.hash) + "\n";
+            s += (format("%1%\n") % info.narSize).str();
+        }
+
+        Path deriver = showDerivers ? info.deriver : "";
         s += deriver + "\n";
 
-        PathSet references;
-        store->queryReferences(*i, references);
+        s += (format("%1%\n") % info.references.size()).str();
 
-        s += (format("%1%\n") % references.size()).str();
-            
-        foreach (PathSet::iterator, j, references)
+        foreach (PathSet::iterator, j, info.references)
             s += *j + "\n";
     }
 
@@ -265,6 +253,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
         string s;
         getline(str, s);
         info.hash = parseHash(htSHA256, s);
+        getline(str, s);
+        if (!string2Int(s, info.narSize)) throw Error("number expected");
     }
     getline(str, info.deriver);
     string s; int n;

src/libstore/store-api.hh

@@ -1,14 +1,14 @@
 #ifndef __STOREAPI_H
 #define __STOREAPI_H
 
+#include "hash.hh"
+#include "serialise.hh"
+
 #include <string>
 #include <map>
 
 #include <boost/shared_ptr.hpp>
 
-#include "hash.hh"
-#include "serialise.hh"
-
 
 namespace nix {
 
@@ -16,8 +16,6 @@ namespace nix {
 typedef std::map<Path, Path> Roots;
 
 
-
-
 struct GCOptions
 {
     /* Garbage collector operation:
@@ -89,9 +87,25 @@ struct SubstitutablePathInfo
     Path deriver;
     PathSet references;
     unsigned long long downloadSize; /* 0 = unknown or inapplicable */
+    unsigned long long narSize; /* 0 = unknown */
 };
 
 
+struct ValidPathInfo 
+{
+    Path path;
+    Path deriver;
+    Hash hash;
+    PathSet references;
+    time_t registrationTime;
+    unsigned long long narSize; // 0 = unknown
+    unsigned long long id; // internal use only
+    ValidPathInfo() : registrationTime(0), narSize(0) { }
+};
+
+typedef list<ValidPathInfo> ValidPathInfos;
+
+
 class StoreAPI 
 {
 public:
@@ -104,6 +118,9 @@ public:
     /* Query the set of valid paths. */
     virtual PathSet queryValidPaths() = 0;
 
+    /* Query information about a valid path. */
+    virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
+
     /* Queries the hash of a valid path. */ 
     virtual Hash queryPathHash(const Path & path) = 0;
 
@@ -112,33 +129,18 @@ public:
     virtual void queryReferences(const Path & path,
         PathSet & references) = 0;
 
-    /* Like queryReferences, but with self-references filtered out. */
-    PathSet queryReferencesNoSelf(const Path & path)
-    {
-        PathSet res;
-        queryReferences(path, res);
-        res.erase(path);
-        return res;
-    }
-
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
     virtual void queryReferrers(const Path & path,
         PathSet & referrers) = 0;
 
-    /* Like queryReferrers, but with self-references filtered out. */
-    PathSet queryReferrersNoSelf(const Path & path)
-    {
-        PathSet res;
-        queryReferrers(path, res);
-        res.erase(path);
-        return res;
-    }
-
     /* Query the deriver of a store path.  Return the empty string if
        no deriver has been set. */
     virtual Path queryDeriver(const Path & path) = 0;
 
+    /* Query the outputs of the derivation denoted by `path'. */
+    virtual PathSet queryDerivationOutputs(const Path & path) = 0;
+    
     /* Query whether a path has substitutes. */
     virtual bool hasSubstitutes(const Path & path) = 0;
 
@@ -224,6 +226,19 @@ public:
 
     /* Perform a garbage collection. */
     virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
+
+    /* Return the set of paths that have failed to build.*/
+    virtual PathSet queryFailedPaths() = 0;
+
+    /* Clear the "failed" status of the given paths.  The special
+       value `*' causes all failed paths to be cleared. */
+    virtual void clearFailedPaths(const PathSet & paths) = 0;
+
+    /* Return a string representing information about the path that
+       can be loaded into the database using `nix-store --load-db' or
+       `nix-store --register-validity'. */
+    string makeValidityRegistration(const PathSet & paths,
+        bool showDerivers, bool showHash);
 };
 
 
@@ -243,12 +258,6 @@ void checkStoreName(const string & name);
 Path toStorePath(const Path & path);
 
 
-/* Get the "name" part of a store path, that is, the part after the
-   hash and the dash, and with any ".drv" suffix removed
-   (e.g. /nix/store/<hash>-foo-1.2.3.drv => foo-1.2.3). */
-string getNameOfStorePath(const Path & path);
-
-
 /* Follow symlinks until we end up with a path in the Nix store. */
 Path followLinksToStore(const Path & path);
 
@@ -323,21 +332,6 @@ boost::shared_ptr<StoreAPI> openStore();
 string showPaths(const PathSet & paths);
 
 
-string makeValidityRegistration(const PathSet & paths,
-    bool showDerivers, bool showHash);
-    
-struct ValidPathInfo 
-{
-    Path path;
-    Path deriver;
-    Hash hash;
-    PathSet references;
-    time_t registrationTime;
-    ValidPathInfo() : registrationTime(0) { }
-};
-
-typedef list<ValidPathInfo> ValidPathInfos;
-
 ValidPathInfo decodeValidPathInfo(std::istream & str,
     bool hashGiven = false);
 

src/libstore/worker-protocol.hh

@@ -8,7 +8,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x105
+#define PROTOCOL_VERSION 0x108
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -34,6 +34,11 @@ typedef enum {
     wopSetOptions = 19,
     wopCollectGarbage = 20,
     wopQuerySubstitutablePathInfo = 21,
+    wopQueryDerivationOutputs = 22,
+    wopQueryValidPaths = 23,
+    wopQueryFailedPaths = 24,
+    wopClearFailedPaths = 25,
+    wopQueryPathInfo = 26,
 } WorkerOp;
 
 

src/libutil/Makefile.am

@@ -3,7 +3,7 @@ pkglib_LTLIBRARIES = libutil.la
 libutil_la_SOURCES = util.cc hash.cc serialise.cc \
   archive.cc xml-writer.cc
 
-libutil_la_LIBADD = ../boost/format/libformat.la
+libutil_la_LIBADD = ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib}
 
 pkginclude_HEADERS = util.hh hash.hh serialise.hh \
   archive.hh xml-writer.hh types.hh

src/libutil/archive.cc

@@ -181,8 +181,6 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
         left -= n;
     }
 
-    sink.finalizeContents(size);
-
     readPadding(size, source);
 }
 
@@ -317,12 +315,6 @@ struct RestoreSink : ParseSink
         writeFull(fd, data, len);
     }
 
-    void finalizeContents(unsigned long long size)
-    {
-        errno = ftruncate(fd, size);
-        if (errno) throw SysError(format("truncating file to its allocated length of %1% bytes") % size);
-    }
-
     void createSymlink(const Path & path, const string & target)
     {
         Path p = dstPath + path;

src/libutil/archive.hh

@@ -64,7 +64,6 @@ struct ParseSink
     virtual void isExecutable() { };
     virtual void preallocateContents(unsigned long long size) { };
     virtual void receiveContents(unsigned char * data, unsigned int len) { };
-    virtual void finalizeContents(unsigned long long size) { };
 
     virtual void createSymlink(const Path & path, const string & target) { };
 };

src/libutil/hash.cc

@@ -286,9 +286,18 @@ Hash hashFile(HashType ht, const Path & path)
 HashSink::HashSink(HashType ht) : ht(ht)
 {
     ctx = new Ctx;
+    bytes = 0;
     start(ht, *ctx);
 }
     
+HashSink::HashSink(const HashSink & h)
+{
+    ht = h.ht;
+    bytes = h.bytes;
+    ctx = new Ctx;
+    *ctx = *h.ctx;
+}
+    
 HashSink::~HashSink()
 {
     delete ctx;
@@ -297,18 +306,20 @@ HashSink::~HashSink()
 void HashSink::operator ()
     (const unsigned char * data, unsigned int len)
 {
+    bytes += len;
     update(ht, *ctx, data, len);
 }
 
-Hash HashSink::finish()
+HashResult HashSink::finish()
 {
     Hash hash(ht);
     nix::finish(ht, *ctx, hash.hash);
-    return hash;
+    return HashResult(hash, bytes);
 }
 
 
-Hash hashPath(HashType ht, const Path & path, PathFilter & filter)
+HashResult hashPath(
+    HashType ht, const Path & path, PathFilter & filter)
 {
     HashSink sink(ht);
     dumpPath(path, sink, filter);

src/libutil/hash.hh

@@ -40,7 +40,6 @@ struct Hash
 
     /* For sorting. */
     bool operator < (const Hash & h) const;
-
 };
 
 
@@ -72,7 +71,8 @@ Hash hashFile(HashType ht, const Path & path);
    (essentially) hashString(ht, dumpPath(path)). */
 struct PathFilter;
 extern PathFilter defaultPathFilter;
-Hash hashPath(HashType ht, const Path & path,
+typedef std::pair<Hash, unsigned long long> HashResult;
+HashResult hashPath(HashType ht, const Path & path,
     PathFilter & filter = defaultPathFilter);
 
 /* Compress a hash to the specified number of bytes by cyclically
@@ -93,16 +93,18 @@ class HashSink : public Sink
 private:
     HashType ht;
     Ctx * ctx;
+    unsigned long long bytes;
 
 public:
     HashSink(HashType ht);
+    HashSink(const HashSink & h);
     ~HashSink();
     virtual void operator () (const unsigned char * data, unsigned int len);
-    Hash finish();
+    HashResult finish();
 };
 
 
 }
 
-    
+
 #endif /* !__HASH_H */

src/libutil/types.hh

@@ -27,7 +27,8 @@ protected:
     string prefix_; // used for location traces etc.
     string err;
 public:
-    BaseError(const format & f);
+    unsigned int status; // exit status
+    BaseError(const format & f, unsigned int status = 1);
     ~BaseError() throw () { };
     const char * what() const throw () { return err.c_str(); }
     const string & msg() const throw () { return err; }
@@ -39,7 +40,7 @@ public:
     class newClass : public superClass                  \
     {                                                   \
     public:                                             \
-        newClass(const format & f) : superClass(f) { }; \
+        newClass(const format & f, unsigned int status = 1) : superClass(f, status) { }; \
     };
 
 MakeError(Error, BaseError)
@@ -63,7 +64,7 @@ typedef set<Path> PathSet;
 
  
 typedef enum { 
-    lvlError,
+    lvlError = 0,
     lvlInfo,
     lvlTalkative,
     lvlChatty,

src/libutil/util.cc

@@ -7,11 +7,9 @@
 #include <sstream>
 #include <cstring>
 
-#include <sys/stat.h>
 #include <sys/wait.h>
-#include <sys/types.h>
 #include <fcntl.h>
-#include <unistd.h>
+#include <limits.h>
 
 #include "util.hh"
 
@@ -22,7 +20,8 @@ extern char * * environ;
 namespace nix {
 
 
-BaseError::BaseError(const format & f)
+BaseError::BaseError(const format & f, unsigned int status)
+    : status(status)
 {
     err = f.str();
 }
@@ -148,6 +147,15 @@ string baseNameOf(const Path & path)
 }
 
 
+struct stat lstat(const Path & path)
+{
+    struct stat st;
+    if (lstat(path.c_str(), &st))
+        throw SysError(format("getting status of `%1%'") % path);
+    return st;
+}
+
+
 bool pathExists(const Path & path)
 {
     int res;
@@ -163,9 +171,7 @@ bool pathExists(const Path & path)
 Path readLink(const Path & path)
 {
     checkInterrupt();
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError(format("getting status of `%1%'") % path);
+    struct stat st = lstat(path);
     if (!S_ISLNK(st.st_mode))
         throw Error(format("`%1%' is not a symlink") % path);
     char buf[st.st_size];
@@ -177,9 +183,7 @@ Path readLink(const Path & path)
 
 bool isLink(const Path & path)
 {
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError(format("getting status of `%1%'") % path);
+    struct stat st = lstat(path);
     return S_ISLNK(st.st_mode);
 }
 
@@ -227,13 +231,12 @@ string readFile(const Path & path)
 }
 
 
-void writeFile(const Path & path, const string & s, bool doFsync)
+void writeFile(const Path & path, const string & s)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
     if (fd == -1)
         throw SysError(format("opening file `%1%'") % path);
     writeFull(fd, (unsigned char *) s.c_str(), s.size());
-    if (doFsync) fsync(fd);
 }
 
 
@@ -269,9 +272,7 @@ static void _computePathSize(const Path & path,
 {
     checkInterrupt();
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-	throw SysError(format("getting attributes of path `%1%'") % path);
+    struct stat st = lstat(path);
 
     bytes += st.st_size;
     blocks += st.st_blocks;
@@ -301,9 +302,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed,
 
     printMsg(lvlVomit, format("%1%") % path);
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-	throw SysError(format("getting attributes of path `%1%'") % path);
+    struct stat st = lstat(path);
 
     if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) {
         bytesFreed += st.st_size;
@@ -350,9 +349,7 @@ void makePathReadOnly(const Path & path)
 {
     checkInterrupt();
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-	throw SysError(format("getting attributes of path `%1%'") % path);
+    struct stat st = lstat(path);
 
     if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) {
 	if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1)
@@ -411,12 +408,18 @@ Paths createDirs(const Path & path)
 {
     Paths created;
     if (path == "/") return created;
-    if (!pathExists(path)) {
+
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1) {
         created = createDirs(dirOf(path));
-        if (mkdir(path.c_str(), 0777) == -1)
+        if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST)
             throw SysError(format("creating directory `%1%'") % path);
+        st = lstat(path);
         created.push_back(path);
     }
+
+    if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path);
+    
     return created;
 }
 
@@ -481,7 +484,16 @@ void printMsg_(Verbosity level, const format & f)
     else if (logType == ltEscapes && level != lvlInfo)
         prefix = "\033[" + escVerbosity(level) + "s";
     string s = (format("%1%%2%\n") % prefix % f.str()).str();
-    writeToStderr((const unsigned char *) s.c_str(), s.size());
+    try {
+        writeToStderr((const unsigned char *) s.c_str(), s.size());
+    } catch (SysError & e) {
+        /* Ignore failing writes to stderr if we're in an exception
+           handler, otherwise throw an exception.  We need to ignore
+           write errors in exception handlers to ensure that cleanup
+           code runs to completion if the other side of stderr has
+           been closed unexpectedly. */
+        if (!std::uncaught_exception()) throw;
+    }
 }
 
 
@@ -966,6 +978,17 @@ Strings tokenizeString(const string & s, const string & separators)
 }
 
 
+string concatStringsSep(const string & sep, const Strings & ss)
+{
+    string s;
+    foreach (Strings::const_iterator, i, ss) {
+        if (s.size() != 0) s += sep;
+        s += *i;
+    }
+    return s;
+}
+
+
 string statusToString(int status)
 {
     if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

src/libutil/util.hh

@@ -4,6 +4,7 @@
 #include "types.hh"
 
 #include <sys/types.h>
+#include <sys/stat.h>
 #include <dirent.h>
 #include <unistd.h>
 #include <signal.h>
@@ -42,6 +43,9 @@ Path dirOf(const Path & path);
    following the final `/'. */
 string baseNameOf(const Path & path);
 
+/* Get status of `path'. */
+struct stat lstat(const Path & path);
+
 /* Return true iff the given path exists. */
 bool pathExists(const Path & path);
 
@@ -60,7 +64,7 @@ string readFile(int fd);
 string readFile(const Path & path);
 
 /* Write a string to a file. */
-void writeFile(const Path & path, const string & s, bool doFsync = false);
+void writeFile(const Path & path, const string & s);
 
 /* Read a line from a file descriptor. */
 string readLine(int fd);
@@ -280,6 +284,11 @@ MakeError(Interrupted, BaseError)
 Strings tokenizeString(const string & s, const string & separators = " \t\n\r");
 
 
+/* Concatenate the given strings with a separator between the
+   elements. */
+string concatStringsSep(const string & sep, const Strings & ss);
+
+
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);

src/nix-env/Makefile.am

@@ -4,7 +4,7 @@ nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh user-env.cc user-env.hh hel
 
 nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \
  ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
+ ../boost/format/libformat.la
 
 nix-env.o: help.txt.hh
 

src/nix-env/nix-env.cc

@@ -69,7 +69,7 @@ typedef void (* Operation) (Globals & globals,
 
 void printHelp()
 {
-    cout << string((char *) helpText, sizeof helpText);
+    cout << string((char *) helpText);
 }
 
 
@@ -132,7 +132,7 @@ static void getAllExprs(EvalState & state,
             if (hasSuffix(attrName, ".nix"))
                 attrName = string(attrName, 0, attrName.size() - 4);
             attrs.attrs[state.symbols.create(attrName)] =
-                ExprAttrs::Attr(parseExprFromFile(state, absPath(path2)), noPos);
+                ExprAttrs::AttrDef(parseExprFromFile(state, absPath(path2)), noPos);
         }
         else
             /* `path2' is a directory (with no default.nix in it);
@@ -154,14 +154,14 @@ static Expr * loadSourceExpr(EvalState & state, const Path & path)
        some system-wide directory). */
     ExprAttrs * attrs = new ExprAttrs;
     attrs->attrs[state.symbols.create("_combineChannels")] =
-        ExprAttrs::Attr(new ExprList(), noPos);
+        ExprAttrs::AttrDef(new ExprList(), noPos);
     getAllExprs(state, path, *attrs);
     return attrs;
 }
 
 
 static void loadDerivations(EvalState & state, Path nixExprPath,
-    string systemFilter, const Bindings & autoArgs,
+    string systemFilter, Bindings & autoArgs,
     const string & pathPrefix, DrvInfos & elems)
 {
     Value v;
@@ -247,11 +247,12 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
         }
 
         /* If `newestOnly', if a selector matches multiple derivations
-           with the same name, pick the one with the highest priority.
-           If there are multiple derivations with the same priority,
-           pick the one with the highest version.  If there are
-           multiple derivations with the same priority and name and
-           version, then pick the first one. */
+           with the same name, pick the one matching the current
+           system.  If there are still multiple derivations, pick the
+           one with the highest priority.  If there are still multiple
+           derivations, pick the one with the highest version.
+           Finally, if there are still multiple derivations,
+           arbitrarily pick the first one. */
         if (newestOnly) {
 
             /* Map from package names to derivations. */
@@ -266,7 +267,11 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
                 Newest::iterator k = newest.find(drvName.name);
                 
                 if (k != newest.end()) {
-                    d = comparePriorities(state, j->first, k->second.first);
+                    d = j->first.system == k->second.first.system ? 0 :
+                        j->first.system == thisSystem ? 1 :
+                        k->second.first.system == thisSystem ? -1 : 0;
+                    if (d == 0)
+                        d = comparePriorities(state, j->first, k->second.first);
                     if (d == 0)
                         d = compareVersions(drvName.version, DrvName(k->second.first.name).version);
                 }
@@ -316,7 +321,7 @@ static bool isPath(const string & s)
 
 
 static void queryInstSources(EvalState & state,
-    const InstallSourceInfo & instSource, const Strings & args,
+    InstallSourceInfo & instSource, const Strings & args,
     DrvInfos & elems, bool newestOnly)
 {
     InstallSourceType type = instSource.type;
@@ -1230,7 +1235,7 @@ void run(Strings args)
     
     globals.instSource.type = srcUnknown;
     globals.instSource.nixExprPath = getDefNixExprPath();
-    globals.instSource.systemFilter = thisSystem;
+    globals.instSource.systemFilter = "*";
     
     globals.dryRun = false;
     globals.preserveInstalled = false;

src/nix-env/user-env.cc

@@ -25,7 +25,8 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv)
     if (pathExists(manifestFile)) {
         Value v;
         state.eval(parseExprFromFile(state, manifestFile), v);
-        getDerivations(state, v, "", Bindings(), elems);
+        Bindings bindings;
+        getDerivations(state, v, "", bindings, elems);
     } else if (pathExists(oldManifestFile))
         readLegacyManifest(oldManifestFile, elems);
 
@@ -58,23 +59,24 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
            the meta attributes. */
         Path drvPath = keepDerivations ? i->queryDrvPath(state) : "";
 
-        Value & v(*state.allocValues(1));
+        Value & v(*state.allocValue());
         manifest.list.elems[n++] = &v;
-        state.mkAttrs(v);
+        state.mkAttrs(v, 8);
 
-        mkString((*v.attrs)[state.sType].value, "derivation");
-        mkString((*v.attrs)[state.sName].value, i->name);
-        mkString((*v.attrs)[state.sSystem].value, i->system);
-        mkString((*v.attrs)[state.sOutPath].value, i->queryOutPath(state));
+        mkString(*state.allocAttr(v, state.sType), "derivation");
+        mkString(*state.allocAttr(v, state.sName), i->name);
+        mkString(*state.allocAttr(v, state.sSystem), i->system);
+        mkString(*state.allocAttr(v, state.sOutPath), i->queryOutPath(state));
         if (drvPath != "")
-            mkString((*v.attrs)[state.sDrvPath].value, i->queryDrvPath(state));
-        
-        state.mkAttrs((*v.attrs)[state.sMeta].value);
-        
+            mkString(*state.allocAttr(v, state.sDrvPath), i->queryDrvPath(state));
+
+        Value & vMeta = *state.allocAttr(v, state.sMeta);
+        state.mkAttrs(vMeta, 16);
+
         MetaInfo meta = i->queryMetaInfo(state);
 
         foreach (MetaInfo::const_iterator, j, meta) {
-            Value & v2((*(*v.attrs)[state.sMeta].value.attrs)[state.symbols.create(j->first)].value);
+            Value & v2(*state.allocAttr(vMeta, state.symbols.create(j->first)));
             switch (j->second.type) {
                 case MetaValue::tpInt: mkInt(v2, j->second.intValue); break;
                 case MetaValue::tpString: mkString(v2, j->second.stringValue); break;
@@ -82,7 +84,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
                     state.mkList(v2, j->second.stringValues.size());
                     unsigned int m = 0;
                     foreach (Strings::const_iterator, k, j->second.stringValues) {
-                        v2.list.elems[m] = state.allocValues(1);
+                        v2.list.elems[m] = state.allocValue();
                         mkString(*v2.list.elems[m++], *k);
                     }
                     break;
@@ -91,6 +93,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
             }
         }
     
+        vMeta.attrs->sort();
+        v.attrs->sort();
+        
         /* This is only necessary when installing store paths, e.g.,
            `nix-env -i /nix/store/abcd...-foo'. */
         store->addTempRoot(i->queryOutPath(state));
@@ -106,8 +111,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     Path manifestFile = store->addTextToStore("env-manifest.nix",
         (format("%1%") % manifest).str(), references);
 
-    printMsg(lvlError, manifestFile);
-
     /* Get the environment builder expression. */
     Value envBuilder;
     state.eval(parseExprFromFile(state, nixDataDir + "/nix/corepkgs/buildenv"), envBuilder);
@@ -115,11 +118,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
     /* Construct a Nix expression that calls the user environment
        builder with the manifest as argument. */
     Value args, topLevel;
-    state.mkAttrs(args);
-    mkString((*args.attrs)[state.sSystem].value, thisSystem);
-    mkString((*args.attrs)[state.symbols.create("manifest")].value,
+    state.mkAttrs(args, 3);
+    mkString(*state.allocAttr(args, state.sSystem), thisSystem);
+    mkString(*state.allocAttr(args, state.symbols.create("manifest")),
         manifestFile, singleton<PathSet>(manifestFile));
-    (*args.attrs)[state.symbols.create("derivations")].value = manifest;
+    args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest));
+    args.attrs->sort();
     mkApp(topLevel, envBuilder, args);
         
     /* Evaluate it. */

src/nix-hash/Makefile.am

@@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash
 
 nix_hash_SOURCES = nix-hash.cc help.txt
 nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
+ ../boost/format/libformat.la
 
 nix-hash.o: help.txt.hh
 

src/nix-hash/nix-hash.cc

@@ -1,16 +1,16 @@
-#include <iostream>
-
 #include "hash.hh"
 #include "shared.hh"
 #include "help.txt.hh"
 
+#include <iostream>
+
 
 using namespace nix;
 
 
 void printHelp()
 {
-    std::cout << string((char *) helpText, sizeof helpText);
+    std::cout << string((char *) helpText);
 }
 
 
@@ -44,7 +44,7 @@ void run(Strings args)
 
     if (op == opHash) {
         for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
-            Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i);
+            Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i).first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             std::cout << format("%1%\n") %
                 (base32 ? printHash32(h) : printHash(h));

src/nix-instantiate/Makefile.am

@@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate
 nix_instantiate_SOURCES = nix-instantiate.cc help.txt
 nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \
  ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
+ ../boost/format/libformat.la
 
 nix-instantiate.o: help.txt.hh
 

src/nix-instantiate/nix-instantiate.cc

@@ -1,6 +1,3 @@
-#include <map>
-#include <iostream>
-
 #include "globals.hh"
 #include "shared.hh"
 #include "eval.hh"
@@ -13,13 +10,16 @@
 #include "common-opts.hh"
 #include "help.txt.hh"
 
+#include <map>
+#include <iostream>
+
 
 using namespace nix;
 
 
 void printHelp()
 {
-    std::cout << string((char *) helpText, sizeof helpText);
+    std::cout << string((char *) helpText);
 }
 
 
@@ -38,7 +38,7 @@ static bool indirectRoot = false;
 
 
 void processExpr(EvalState & state, const Strings & attrPaths,
-    bool parseOnly, bool strict, const Bindings & autoArgs,
+    bool parseOnly, bool strict, Bindings & autoArgs,
     bool evalOnly, bool xmlOutput, bool location, Expr * e)
 {
     if (parseOnly)

src/nix-log2xml/log2xml.cc

@@ -18,6 +18,8 @@ struct Decoder
     int priority;
     bool ignoreLF;
     int lineNo, charNo;
+    bool warning;
+    bool error;
 
     Decoder()
     {
@@ -29,6 +31,8 @@ struct Decoder
         ignoreLF = false;
         lineNo = 1;
         charNo = 0;
+        warning = false;
+        error = false;
     }
 
     void pushChar(char c);
@@ -95,6 +99,12 @@ void Decoder::pushChar(char c)
                     case 'b':
                         ignoreLF = false;
                         break;
+                    case 'e':
+                        error = true;
+                        break;
+                    case 'w':
+                        warning = true;
+                        break;
                 }
             } else if (c >= '0' && c <= '9') {
                 int n = 0;
@@ -118,6 +128,8 @@ void Decoder::finishLine()
     string tag = inHeader ? "head" : "line";
     cout << "<" << tag;
     if (priority != 1) cout << " priority='" << priority << "'";
+    if (warning) cout << " warning='1'";
+    if (error) cout << " error='1'";
     cout << ">";
 
     for (unsigned int i = 0; i < line.size(); i++) {
@@ -158,6 +170,8 @@ void Decoder::finishLine()
     line = "";
     inHeader = false;
     priority = 1;
+    warning = false;
+    error = false;
 }
 
 

src/nix-setuid-helper/Makefile.am

@@ -4,5 +4,4 @@ nix_setuid_helper_SOURCES = nix-setuid-helper.cc
 nix_setuid_helper_LDADD = ../libutil/libutil.la \
  ../boost/format/libformat.la
 
-AM_CXXFLAGS = \
- -I$(srcdir)/.. -I$(srcdir)/../libutil
+AM_CXXFLAGS = -I$(srcdir)/.. -I$(srcdir)/../libutil

src/nix-store/Makefile.am

@@ -1,8 +1,11 @@
 bin_PROGRAMS = nix-store
 
-nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt
+nix_store_SOURCES =				\
+  nix-store.cc dotgraph.cc dotgraph.hh help.txt	\
+  xmlgraph.cc xmlgraph.hh
+
 nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
+ ../boost/format/libformat.la
 
 nix-store.o: help.txt.hh
 

src/nix-store/dotgraph.cc

@@ -52,13 +52,13 @@ static string symbolicName(const string & path)
 }
 
 
+#if 0
 string pathLabel(const Path & nePath, const string & elemPath)
 {
     return (string) nePath + "-" + elemPath;
 }
 
 
-#if 0
 void printClosure(const Path & nePath, const StoreExpr & fs)
 {
     PathSet workList(fs.closure.roots);

src/nix-store/help.txt

@@ -16,7 +16,7 @@ Operations:
 
   --gc: run the garbage collector
 
-  --dump: dump a path as a Nix archive, forgetting dependencies
+  --dump: dump a path as a Nix archive (NAR), forgetting dependencies
   --restore: restore a path from a Nix archive, without
       registering validity
 
@@ -27,6 +27,9 @@ Operations:
   --verify: verify Nix structures
   --optimise: optimise the Nix store by hard-linking identical files
 
+  --query-failed-paths: list paths that failed to build (if enabled)
+  --clear-failed-paths: clear the failed status of the given paths
+
   --version: output version information
   --help: display help
 
@@ -39,7 +42,9 @@ Query flags:
   --referrers-closure: print all paths (in)directly refering to the path
   --tree: print a tree showing the dependency graph of the path
   --graph: print a dot graph rooted at given path
+  --xml: emit an XML representation of the graph rooted at the given path
   --hash: print the SHA-256 hash of the contents of the path
+  --size: print the size of the NAR dump of the path
   --roots: print the garbage collector roots that point to the path
 
 Query switches (not applicable to all queries):

src/nix-store/nix-store.cc

@@ -1,15 +1,16 @@
-#include <iostream>
-#include <algorithm>
-
 #include "globals.hh"
 #include "misc.hh"
 #include "archive.hh"
 #include "shared.hh"
 #include "dotgraph.hh"
+#include "xmlgraph.hh"
 #include "local-store.hh"
 #include "util.hh"
 #include "help.txt.hh"
 
+#include <iostream>
+#include <algorithm>
+
 
 using namespace nix;
 using std::cin;
@@ -21,7 +22,7 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs);
 
 void printHelp()
 {
-    cout << string((char *) helpText, sizeof helpText);
+    cout << string((char *) helpText);
 }
 
 
@@ -33,7 +34,7 @@ static bool indirectRoot = false;
 LocalStore & ensureLocalStore()
 {
     LocalStore * store2(dynamic_cast<LocalStore *>(store.get()));
-    if (!store2) throw Error("you don't have sufficient rights to use --verify");
+    if (!store2) throw Error("you don't have sufficient rights to use this command");
     return *store2;
 }
 
@@ -225,8 +226,8 @@ static void printTree(const Path & path,
 static void opQuery(Strings opFlags, Strings opArgs)
 {
     enum { qOutputs, qRequisites, qReferences, qReferrers
-         , qReferrersClosure, qDeriver, qBinding, qHash
-         , qTree, qGraph, qResolve, qRoots } query = qOutputs;
+         , qReferrersClosure, qDeriver, qBinding, qHash, qSize
+         , qTree, qGraph, qXml, qResolve, qRoots } query = qOutputs;
     bool useOutput = false;
     bool includeOutputs = false;
     bool forceRealise = false;
@@ -247,8 +248,10 @@ static void opQuery(Strings opFlags, Strings opArgs)
             query = qBinding;
         }
         else if (*i == "--hash") query = qHash;
+        else if (*i == "--size") query = qSize;
         else if (*i == "--tree") query = qTree;
         else if (*i == "--graph") query = qGraph;
+        else if (*i == "--xml") query = qXml;
         else if (*i == "--resolve") query = qResolve;
         else if (*i == "--roots") query = qRoots;
         else if (*i == "--use-output" || *i == "-u") useOutput = true;
@@ -308,11 +311,15 @@ static void opQuery(Strings opFlags, Strings opArgs)
             break;
 
         case qHash:
+        case qSize:
             foreach (Strings::iterator, i, opArgs) {
                 Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise);
-                Hash hash = store->queryPathHash(path);
-                assert(hash.type == htSHA256);
-                cout << format("sha256:%1%\n") % printHash32(hash);
+                ValidPathInfo info = store->queryPathInfo(path);
+                if (query == qHash) {
+                    assert(info.hash.type == htSHA256);
+                    cout << format("sha256:%1%\n") % printHash32(info.hash);
+                } else if (query == qSize) 
+                    cout << format("%1%\n") % info.narSize;
             }
             break;
 
@@ -327,7 +334,15 @@ static void opQuery(Strings opFlags, Strings opArgs)
             PathSet roots;
             foreach (Strings::iterator, i, opArgs)
                 roots.insert(maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise));
-	    printDotGraph(roots);
+            printDotGraph(roots);
+            break;
+        }
+
+        case qXml: {
+            PathSet roots;
+            foreach (Strings::iterator, i, opArgs)
+                roots.insert(maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise));
+            printXmlGraph(roots);
             break;
         }
 
@@ -383,9 +398,8 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
     if (!opArgs.empty())
         throw UsageError("no arguments expected");
     PathSet validPaths = store->queryValidPaths();
-    foreach (PathSet::iterator, i, validPaths) {
-        cout << makeValidityRegistration(singleton<PathSet>(*i), true, true);
-    }
+    foreach (PathSet::iterator, i, validPaths)
+        cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
 }
 
 
@@ -400,8 +414,11 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
             /* !!! races */
             if (canonicalise)
                 canonicalisePathMetaData(info.path);
-            if (!hashGiven)
-                info.hash = hashPath(htSHA256, info.path);
+            if (!hashGiven) {
+                HashResult hash = hashPath(htSHA256, info.path);
+                info.hash = hash.first;
+                info.narSize = hash.second;
+            }
             infos.push_back(info);
         }
     }
@@ -651,8 +668,7 @@ static void opOptimise(Strings opFlags, Strings opArgs)
 
     bool dryRun = false;
 
-    for (Strings::iterator i = opFlags.begin();
-         i != opFlags.end(); ++i)
+    foreach (Strings::iterator, i, opFlags)
         if (*i == "--dry-run") dryRun = true;
         else throw UsageError(format("unknown flag `%1%'") % *i);
 
@@ -667,6 +683,24 @@ static void opOptimise(Strings opFlags, Strings opArgs)
 }
 
 
+static void opQueryFailedPaths(Strings opFlags, Strings opArgs)
+{
+    if (!opArgs.empty() || !opFlags.empty())
+        throw UsageError("no arguments expected");
+    PathSet failed = store->queryFailedPaths();
+    foreach (PathSet::iterator, i, failed)
+        cout << format("%1%\n") % *i;
+}
+
+
+static void opClearFailedPaths(Strings opFlags, Strings opArgs)
+{
+    if (!opFlags.empty())
+        throw UsageError("no flags expected");
+    store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end()));
+}
+
+
 /* Scan the arguments; find the operation, set global flags, put all
    other flags in a list, and put all other arguments in another
    list. */
@@ -718,6 +752,10 @@ void run(Strings args)
             op = opVerify;
         else if (arg == "--optimise")
             op = opOptimise;
+        else if (arg == "--query-failed-paths")
+            op = opQueryFailedPaths;
+        else if (arg == "--clear-failed-paths")
+            op = opClearFailedPaths;
         else if (arg == "--add-root") {
             if (i == args.end())
                 throw UsageError("`--add-root requires an argument");

src/nix-store/xmlgraph.cc

@@ -0,0 +1,71 @@
+#include "xmlgraph.hh"
+#include "util.hh"
+#include "store-api.hh"
+
+#include <iostream>
+
+
+using std::cout;
+
+namespace nix {
+
+
+static inline const string & xmlQuote(const string & s)
+{
+    // Luckily, store paths shouldn't contain any character that needs to be
+    // quoted.
+    return s;
+}
+
+
+static string makeEdge(const string & src, const string & dst)
+{
+    format f = format("  <edge src=\"%1%\" dst=\"%2%\"/>\n")
+      % xmlQuote(src) % xmlQuote(dst);
+    return f.str();
+}
+
+
+static string makeNode(const string & id)
+{
+    format f = format("  <node name=\"%1%\"/>\n") % xmlQuote(id);
+    return f.str();
+}
+
+
+void printXmlGraph(const PathSet & roots)
+{
+    PathSet workList(roots);
+    PathSet doneSet;
+
+    cout << "<?xml version='1.0' encoding='utf-8'?>\n"
+	 << "<nix>\n";
+
+    while (!workList.empty()) {
+	Path path = *(workList.begin());
+	workList.erase(path);
+
+	if (doneSet.find(path) != doneSet.end()) continue;
+	doneSet.insert(path);
+
+	cout << makeNode(path);
+
+	PathSet references;
+	store->queryReferences(path, references);
+
+	for (PathSet::iterator i = references.begin();
+	     i != references.end(); ++i)
+	{
+	    if (*i != path) {
+		workList.insert(*i);
+		cout << makeEdge(*i, path);
+	    }
+	}
+
+    }
+
+    cout << "</nix>\n";
+}
+
+
+}

src/nix-store/xmlgraph.hh

@@ -0,0 +1,12 @@
+#ifndef __XMLGRAPH_H
+#define __XMLGRAPH_H
+
+#include "types.hh"
+
+namespace nix {
+
+void printXmlGraph(const PathSet & roots);
+
+}
+
+#endif /* !__XMLGRAPH_H */

src/nix-worker/Makefile.am

@@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker
 
 nix_worker_SOURCES = nix-worker.cc help.txt
 nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \
- ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
+ ../boost/format/libformat.la
 
 nix-worker.o: help.txt.hh