Fix typo.

useful for fields like meta.maintainers, meta.priority (which can be
  a proper integer now) and even meta.license (if there are multiple
  licenses).

Makefile.am

@@ -1,23 +1,19 @@
 SUBDIRS = externals src scripts corepkgs doc misc tests
 EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
-  svn-revision nix.conf.example NEWS
+  nix.conf.example NEWS version
 
 include ./substitute.mk
 
 nix.spec: nix.spec.in
 
-rpm: nix.spec dist
-	rpm $(EXTRA_RPM_FLAGS) -ta $(distdir).tar.gz
-
-relname:
-	echo -n $(distdir) > relname
-
 install-data-local: init-state
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 	$(INSTALL_DATA) $(srcdir)/nix.conf.example $(DESTDIR)$(sysconfdir)/nix
 	if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
 		$(INSTALL_DATA) $(srcdir)/nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
 	fi
+	$(INSTALL) -d $(DESTDIR)$(docdir)
+	$(INSTALL_DATA) README $(DESTDIR)$(docdir)/
 
 if INIT_STATE
 
@@ -37,19 +33,16 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
-	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(prefix)/store
+	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
-#	$(bindir)/nix-store --init
 
 else
+
 init-state:
+
 endif
 
-svn-revision:
-	svnversion . > svn-revision
-
-all-local: NEWS
-
-NEWS: doc/manual/NEWS.txt
+NEWS:
+	$(MAKE) -C doc/manual NEWS.txt
 	cp $(srcdir)/doc/manual/NEWS.txt NEWS

README

@@ -1,9 +1,10 @@
-For installation and usage instructions, please read the manual, which
-can be found in `docs/manual/manual.html', and additionally at the Nix
-website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
+Nix is a purely functional package manager.  For installation and
+usage instructions, please read the manual, which can be found in
+`docs/manual/manual.html', and additionally at the Nix website at
+<http://nixos.org/>.
 
 
 Acknowledgments
 
 This product includes software developed by the OpenSSL Project for
-use in the OpenSSL Toolkit (http://www.OpenSSL.org/)
+use in the OpenSSL Toolkit (http://www.OpenSSL.org/).

aterm-gc.supp

@@ -1,184 +1,117 @@
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:mark_memory_young
-   fun:mark_phase_young
+   fun:*
+   fun:AT_collect_minor
 }
 
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm_young
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_markTerm_young
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:mark_phase_young
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Value4
-   fun:mark_phase_young
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
-   Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_phase_young
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
+   ATerm library conservatively scans for GC roots
    Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_phase_young
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_phase_young
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr4
+   fun:*
+   fun:AT_collect_minor
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr8
+   fun:*
+   fun:AT_collect_minor
+}
+
+{
+   ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_phase_young
-   fun:AT_collect_minor
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr4
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr8
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Cond
+   fun:*
+   fun:*
+   fun:AT_collect
 }

configure.ac

@@ -1,25 +1,10 @@
-AC_INIT(nix, 0.12)
+AC_INIT(nix, m4_esyscmd([echo -n $(cat ./version)$VERSION_SUFFIX]))
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
 AM_INIT_AUTOMAKE([dist-bzip2 foreign])
-        
-# Change to `1' to produce a `stable' release (i.e., the `preREVISION'
-# suffix is not added).
-STABLE=0
-
-# Put the revision number in the version.
-if test "$STABLE" != "1"; then
-    if REVISION=`test -d $srcdir/.svn && svnversion -n $srcdir 2> /dev/null`; then
-        VERSION=${VERSION}pre${REVISION}
-    elif REVISION=`cat $srcdir/svn-revision 2> /dev/null`; then
-        VERSION=${VERSION}pre${REVISION}
-    fi
-fi
 
 AC_DEFINE_UNQUOTED(NIX_VERSION, ["$VERSION"], [Nix version.])
 
-AC_PREFIX_DEFAULT(/nix)
-
 AC_CANONICAL_HOST
 
 
@@ -61,14 +46,34 @@ AC_SUBST(system)
 AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
 
 
+# State should be stored in /nix/var, unless the user overrides it explicitly.
+test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
+
+
+# Whether to produce a statically linked binary.  On Cygwin, this is
+# the default: dynamically linking against the ATerm DLL does work,
+# except that it requires the ATerm "lib" directory to be in $PATH, as
+# Windows doesn't have anything like an RPATH embedded in executable.
+# Since this is kind of annoying, we use static libraries for now.
+    
+AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
+  [produce statically linked binaries]),
+  static_nix=$enableval, static_nix=no)
+
+if test "$sys_name" = cygwin; then
+   static_nix=yes
+fi
+  
+if test "$static_nix" = yes; then
+    AC_DISABLE_SHARED
+    AC_ENABLE_STATIC
+fi
+
+  
 # Windows-specific stuff.
 if test "$sys_name" = "cygwin"; then
     # We cannot delete open files.
     AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.])
-
-    # Shared libraries don't work, currently.
-    AC_DISABLE_SHARED
-    AC_ENABLE_STATIC
 fi
 
 
@@ -101,6 +106,8 @@ AC_LANG_POP(C++)
 
 # Check for chroot support (requires chroot() and bind mounts).
 AC_CHECK_FUNCS([chroot])
+AC_CHECK_FUNCS([unshare])
+AC_CHECK_HEADERS([sched.h], [], [], [])
 AC_CHECK_HEADERS([sys/param.h], [], [], [])
 AC_CHECK_HEADERS([sys/mount.h], [], [],
 [#ifdef HAVE_SYS_PARAM_H
@@ -109,12 +116,17 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
 ])
 
 
-# Check for <locale>
+# Check for <locale>.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([locale], [], [], [])
 AC_LANG_POP(C++)
 
 
+# Check whether we have the personality() syscall, which allows us to
+# do i686-linux builds on x86_64-linux machines.
+AC_CHECK_HEADERS([sys/personality.h])
+
+
 AC_DEFUN([NEED_PROG],
 [
 AC_PATH_PROG($1, $2)
@@ -124,7 +136,7 @@ fi
 ])
 
 NEED_PROG(curl, curl)
-NEED_PROG(shell, bash)
+NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
@@ -136,6 +148,7 @@ NEED_PROG(perl, perl)
 NEED_PROG(tar, tar)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(dblatex, dblatex)
+AC_PATH_PROG(gzip, gzip)
 
 AC_PATH_PROG(openssl_prog, openssl, openssl) # if not found, call openssl in $PATH
 AC_SUBST(openssl_prog)
@@ -173,7 +186,7 @@ AC_SUBST(xmlflags)
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
   [path of the Nix store]),
-  storedir=$withval, storedir='${prefix}/store')
+  storedir=$withval, storedir='/nix/store')
 AC_SUBST(storedir)
 
 AC_ARG_ENABLE(old-db-compat, AC_HELP_STRING([--disable-old-db-compat],
@@ -239,7 +252,7 @@ if test -z "$bzip2"; then
   bzip2_lib='-L${top_builddir}/externals/inst-bzip2/lib -lbz2'
   bzip2_include='-I${top_builddir}/externals/inst-bzip2/include'
   # The binary will be copied to $libexecdir.
-  bzip2_bin='${libexecdir}'
+  bzip2_bin='${libexecdir}/nix'
   # But for testing, we have to use the temporary copy :-(
   bzip2_bin_test='${top_builddir}/externals/inst-bzip2/bin'
 else
@@ -269,6 +282,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
 
 # Nice to have, but not essential.
 AC_CHECK_FUNCS([strsignal])
+AC_CHECK_FUNCS([posix_fallocate])
 
 
 # This is needed if ATerm, Berkeley DB or bzip2 are static libraries,
@@ -278,6 +292,14 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
+if test "$static_nix" = yes; then
+    # `-all-static' has to be added at the end of configure, because
+    # the C compiler doesn't know about -all-static (it's filtered out
+    # by libtool, but configure doesn't use libtool).
+    LDFLAGS="-all-static $LDFLAGS"
+fi
+
+
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    externals/Makefile

corepkgs/channels/unpack.sh.in

@@ -8,9 +8,15 @@ inputs=($inputs)
 for ((n = 0; n < ${#inputs[*]}; n += 2)); do
     channelName=${inputs[n]}
     channelTarball=${inputs[n+1]}
+    
     echo "unpacking channel $channelName"
+    
     @bunzip2@ < $channelTarball | @tar@ xf -
 
+    if test -e */channel-name; then
+        channelName="$(@coreutils@/cat */channel-name)"
+    fi
+
     nr=1
     attrName=$(echo $channelName | @tr@ -- '- ' '__')
     dirName=$attrName

doc/manual/Makefile.am

@@ -14,15 +14,16 @@ XSLTPROC = $(xsltproc) $(xmlflags) \
 man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
  nix-prefetch-url.1 nix-channel.1 \
- nix-pack-closure.1 nix-unpack-closure.1 \
  nix-install-package.1 nix-hash.1 nix-copy-closure.1
 
+man8_MANS = nix-worker.8
+
 FIGURES = figures/user-environments.png
 
 MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  package-management.xml writing-nix-expressions.xml builtins.xml \
  build-farm.xml \
- $(man1_MANS:.1=.xml) \
+ $(man1_MANS:.1=.xml) $(man8_MANS:.8=.xml) \
  troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
  env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
  conf-file.xml release-notes.xml \
@@ -76,12 +77,16 @@ NEWS.txt: release-notes.xml
 all-local: manual.html NEWS.html NEWS.txt
 
 install-data-local: manual.html
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/manual
-	$(INSTALL_DATA) manual.html $(DESTDIR)$(datadir)/nix/manual
-	$(INSTALL_DATA) style.css $(DESTDIR)$(datadir)/nix/manual
-	cp -r images $(DESTDIR)$(datadir)/nix/manual/images
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/manual/figures
-	$(INSTALL_DATA) $(FIGURES) $(DESTDIR)$(datadir)/nix/manual/figures
+	$(INSTALL) -d $(DESTDIR)$(docdir)/manual
+	$(INSTALL_DATA) manual.html $(DESTDIR)$(docdir)/manual
+	ln -sf manual.html $(DESTDIR)$(docdir)/manual/index.html
+	$(INSTALL_DATA) style.css $(DESTDIR)$(docdir)/manual
+	cp -r images $(DESTDIR)$(docdir)/manual/images
+	$(INSTALL) -d $(DESTDIR)$(docdir)/manual/figures
+	$(INSTALL_DATA) $(FIGURES) $(DESTDIR)$(docdir)/manual/figures
+	$(INSTALL) -d $(DESTDIR)$(docdir)/release-notes
+	$(INSTALL_DATA) NEWS.html $(DESTDIR)$(docdir)/release-notes/index.html
+	$(INSTALL_DATA) style.css $(DESTDIR)$(docdir)/release-notes/
 
 images:
 	mkdir images

doc/manual/build-farm.xml

@@ -56,7 +56,7 @@ build farm, since:
 <para>TODO</para>
 
 <para>The sources of the Nix build farm are at <link
-xlink:href='https://svn.cs.uu.nl:12443/repos/trace/release/trunk'/>.</para>
+xlink:href='https://svn.nixos.org/repos/nix/release/trunk'/>.</para>
 
 </section>
 
@@ -85,7 +85,7 @@ nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto
 
 <para>An example build hook can be found in the Nix build farm
 sources: <link
-xlink:href='https://svn.cs.uu.nl:12443/repos/trace/release/trunk/common/distributed/build-remote.pl'
+xlink:href='https://svn.nixos.org/repos/nix/release/trunk/common/distributed/build-remote.pl'
 />.  It should be suitable for most purposes, with maybe some minor
 adjustments.  It uses <command>ssh</command> and
 <command>rsync</command> to copy the build inputs and outputs and

doc/manual/builtins.xml

@@ -77,18 +77,25 @@ attrValues = attrs: map (name: builtins.getAttr name attrs) (builtins.attrNames
 if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
     This allows a Nix expression to fall back gracefully on older Nix
-    installations that don’t have the desired built-in function.
-    However, in that case you should not write
+    installations that don’t have the desired built-in
+    function.</para></listitem>
 
-<programlisting>
-if builtins ? getEnv then __getEnv "PATH" else ""</programlisting>
+  </varlistentry>
 
-    This Nix expression will trigger an “undefined variable” error on
-    older Nix versions since <function>__getEnv</function> doesn’t
-    exist.  <literal>builtins.getEnv</literal>, on the other hand, is
-    safe since <literal>builtins</literal> always exists and attribute
-    selection is lazy, so it’s only performed if the test
-    succeeds.</para></listitem>
+  
+  <varlistentry><term><function>builtins.compareVersions</function>
+  <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
+
+    <listitem><para>Compare two strings representing versions and
+    return <literal>-1</literal> if version
+    <replaceable>s1</replaceable> is older than version
+    <replaceable>s2</replaceable>, <literal>0</literal> if they are
+    the same, and <literal>1</literal> if
+    <replaceable>s1</replaceable> is newer than
+    <replaceable>s2</replaceable>.  The version comparison algorithm
+    is the same as the one used by <link
+    linkend="ssec-version-comparisons"><command>nix-env
+    -u</command></link>.</para></listitem>
 
   </varlistentry>
 
@@ -146,6 +153,16 @@ if builtins ? getEnv then __getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.div</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the quotient of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.filterSource</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
@@ -337,6 +354,36 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.isString</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a string, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
+
+  
+  <varlistentry><term><function>builtins.isInt</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a int, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
+
+  
+  <varlistentry><term><function>builtins.isBool</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a bool, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>isNull</function>
   <replaceable>e</replaceable></term>
 
@@ -352,6 +399,15 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.length</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return the length of the list
+    <replaceable>e</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.lessThan</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
@@ -408,6 +464,31 @@ map (x: "foo" + x) ["bar" "bla" "abc"]</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.mul</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the product of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
+  <varlistentry><term><function>builtins.parseDrvName</function>
+  <replaceable>s</replaceable></term>
+
+    <listitem><para>Split the string <replaceable>s</replaceable> into
+    a package name and version.  The package name is everything up to
+    but not including the first dash followed by a digit, and the
+    version is everything following that dash.  The result is returned
+    in an attribute set <literal>{name, version}</literal>.  Thus,
+    <literal>builtins.parseDrvName "nix-0.12pre12876"</literal>
+    returns <literal>{name = "nix"; version =
+    "0.12pre12876";}</literal>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.pathExists</function>
   <replaceable>path</replaceable></term>
 

doc/manual/conf-file.xml

@@ -60,27 +60,6 @@ env-keep-derivations = false
   </varlistentry>
 
   
-  <varlistentry xml:id="conf-gc-reserved-space"><term><literal>gc-reserved-space</literal></term>
-
-    <listitem><para>This option specifies how much space should be
-    reserved in normal use so that the garbage collector can run
-    succesfully.  Since the garbage collector must perform Berkeley DB
-    transactions, it needs some disk space for itself.  However, when
-    the disk is full, this space is not available, so the collector
-    would not be able to run precisely when it is most needed.</para>
-
-    <para>For this reason, when Nix is run, it allocates a file
-    <filename>/nix/var/nix/db/reserved</filename> of the size
-    specified by this option.  When the garbage collector is run, this
-    file is deleted before the Berkeley DB environment is opened.
-    This should give it enough room to proceed.</para>
-
-    <para>The default is <literal>1048576</literal> (1
-    MiB).</para></listitem>
-
-  </varlistentry>
-
-  
   <varlistentry><term><literal>env-keep-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations

doc/manual/env-common.xml

@@ -151,12 +151,12 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
   <para>On the basis of this information, and whatever persistent
   state the build hook keeps about other machines and their current
   load, it has to decide what to do with the build.  It should print
-  out on file descriptor 3 one of the following responses (terminated
-  by a newline, <literal>"\n"</literal>):
+  out on standard error one of the following responses (terminated by
+  a newline, <literal>"\n"</literal>):
 
   <variablelist>
 
-    <varlistentry><term><literal>decline</literal></term>
+    <varlistentry><term><literal># decline</literal></term>
 
       <listitem><para>The build hook is not willing or able to perform
       the build; the calling Nix process should do the build itself,
@@ -164,7 +164,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
     </varlistentry>
 
-    <varlistentry><term><literal>postpone</literal></term>
+    <varlistentry><term><literal># postpone</literal></term>
 
       <listitem><para>The build hook cannot perform the build now, but
       can do so in the future (e.g., because all available build slots
@@ -174,7 +174,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
     </varlistentry>
 
-    <varlistentry><term><literal>accept</literal></term>
+    <varlistentry><term><literal># accept</literal></term>
 
       <listitem><para>The build hook has accepted the
       build.</para></listitem>
@@ -185,37 +185,12 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
   </para>
 
-  <para>If the build hook accepts the build, it is possible that it is
-  no longer necessary to do the build because some other process has
-  performed the build in the meantime.  To prevent races, the hook
-  must read from file descriptor 4 a single line that tells it whether
-  to continue:
-
-  <variablelist>
-
-    <varlistentry><term><literal>cancel</literal></term>
-
-      <listitem><para>The build has already been done, so the hook
-      should exit.</para></listitem>
-
-    </varlistentry>
-  
-    <varlistentry><term><literal>okay</literal></term>
-
-      <listitem><para>The hook should proceed with the build.  At this
-      point, the calling Nix process has acquired locks on the output
-      path, so no other Nix process will perform the
-      build.</para></listitem>
-
-    </varlistentry>
-
-  </variablelist>
-
-  </para>
-
-  <para>If the hook has been told to proceed, Nix will store in the
-  hook’s current directory a number of text files that contain
-  information about the derivation:
+  <para>After sending <literal># accept</literal>, the hook should
+  read one line from standard input, which will be the string
+  <literal>okay</literal>.  It can then proceed with the build.
+  Before sending <literal>okay</literal>, Nix will store in the hook’s
+  current directory a number of text files that contain information
+  about the derivation:
 
   <variablelist>
 
@@ -255,7 +230,9 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
   <para>The hook should copy the inputs to the remote machine,
   register the validity of the inputs, perform the remote build, and
   copy the outputs back to the local machine.  An exit code other than
-  <literal>0</literal> indicates that the hook has failed.</para>
+  <literal>0</literal> indicates that the hook has failed.  An exit
+  code equal to 100 means that the remote build failed (as opposed to,
+  e.g., a network error).</para>
 
   </listitem>
 
@@ -274,6 +251,27 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
     
+<varlistentry xml:id="envar-other-stores"><term><envar>NIX_OTHER_STORES</envar></term>
+
+  <listitem><para>This variable contains the paths of remote Nix
+  installations from whichs paths can be copied, separated by colons.
+  See <xref linkend="sec-sharing-packages" /> for details.  Each path
+  should be the <filename>/nix</filename> directory of a remote Nix
+  installation (i.e., not the <filename>/nix/store</filename>
+  directory).  The paths are subject to globbing, so you can set it so
+  something like <literal>/var/run/nix/remote-stores/*/nix</literal>
+  and mount multiple remote filesystems in
+  <literal>/var/run/nix/remote-stores</literal>.</para>
+
+  <para>Note that if you’re building through the <link
+  linkend="sec-nix-worker">Nix daemon</link>, the only setting for
+  this variable that matters is the one that the
+  <command>nix-worker</command> process uses.  So if you want to
+  change it, you have to restart the daemon.</para></listitem>
+
+</varlistentry>
+
+    
 </variablelist>
 
 

doc/manual/installation.xml

@@ -42,25 +42,22 @@ platforms as well.</para>
 <section><title>Obtaining Nix</title>
 
 <para>The easiest way to obtain Nix is to download a <link
-xlink:href="http://nix.cs.uu.nl/">source distribution</link>.  RPMs
+xlink:href="http://nixos.org/">source distribution</link>.  RPMs
 for Red Hat, SuSE, and Fedora Core are also available.</para>
 
 <para>Alternatively, the most recent sources of Nix can be obtained
 from its <link
-xlink:href="https://svn.cs.uu.nl:12443/repos/trace/nix/trunk">Subversion
+xlink:href="https://svn.nixos.org/repos/nix/nix/trunk">Subversion
 repository</link>.  For example, the following command will check out
 the latest revision into a directory called
 <filename>nix</filename>:</para>
 
 <screen>
-$ svn checkout https://svn.cs.uu.nl:12443/repos/trace/nix/trunk nix</screen>
+$ svn checkout https://svn.nixos.org/repos/nix/nix/trunk nix</screen>
 
 <para>Likewise, specific releases can be obtained from the <link
-xlink:href="https://svn.cs.uu.nl:12443/repos/trace/nix/tags">tags
-directory</link> of the repository.  If you don't have Subversion, you
-can also download an automatically generated <link
-xlink:href="https://svn.cs.uu.nl:12443/dist/trace/">compressed
-tar-file</link> of the head revision of the trunk.</para>
+xlink:href="https://svn.nixos.org/repos/nix/nix/tags">tags
+directory</link> of the repository.</para>
 
 </section>
 
@@ -99,18 +96,23 @@ ubiquitous 2.5.4a won't.  Note that these are only required if you
 modify the parser or when you are building from the Subversion
 repository.</para>
 
-<para>Nix uses Sleepycat's Berkeley DB, CWI's ATerm library and the
-bzip2 compressor (including the bzip2 library).  These are included in
-the Nix source distribution.  If you build from the Subversion
-repository, you must download them yourself and place them in the
+<para>Nix uses CWI's ATerm library and the bzip2 compressor (including
+the bzip2 library).  These are included in the Nix source
+distribution.  If you build from the Subversion repository, you must
+download them yourself and place them in the
 <filename>externals/</filename> directory.  See
 <filename>externals/Makefile.am</filename> for the precise URLs of
 these packages.  Alternatively, if you already have them installed,
 you can use <command>configure</command>'s
-<option>--with-bdb</option>, <option>--with-aterm</option> and
-<option>--with-bzip2</option> options to point to their respective
-locations.  Note that Berkeley DB <emphasis>must</emphasis> be version
-4.5; other versions may not have compatible database formats.</para>
+<option>--with-aterm</option> and <option>--with-bzip2</option>
+options to point to their respective locations.</para>
+
+<para>If you want to be able to upgrade Nix stores from before version
+0.12pre12020, you need Sleepycat's Berkeley DB version version 4.5.
+(Other versions may not have compatible database formats.).  Berkeley
+DB 4.5 is included in the Nix source distribution.  If you do not need
+this ability, you can build Nix with the
+<option>--disable-old-db-compat</option> configure option.</para>
 
 </section>
 
@@ -161,7 +163,7 @@ options.</para>
 <section><title>Installing from RPMs</title>
 
 <para>RPM packages of Nix can be downloaded from <link
-xlink:href="http://nix.cs.uu.nl/" />.  These RPMs should work for most
+xlink:href="http://nixos.org/" />.  These RPMs should work for most
 fairly recent releases of SuSE and Red Hat Linux.  They have been
 known to work work on SuSE Linux 8.1 and 9.0, and Red Hat 9.0.  In
 fact, it should work on any RPM-based Linux distribution based on
@@ -193,12 +195,12 @@ $ rm -rf /nix/var</screen>
 
 <para>You can install the latest stable version of Nix through Nix
 itself by subscribing to the channel <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-stable" />,
+xlink:href="http://nixos.org/releases/nix/channels/nix-stable" />,
 or the latest unstable version by subscribing to the channel <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-unstable" />.
+xlink:href="http://nixos.org/releases/nix/channels/nix-unstable" />.
 You can also do a <link linkend="sec-one-click">one-click
 installation</link> by clicking on the package links at <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/" />.</para>
+xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para>
 
 </section>
 
@@ -334,8 +336,8 @@ $ chown -R root /nix/store /nix/var/nix</screen>
 
 </para>
 
-<para>The Nix daemon should be started as follows (as
-<literal>root</literal>):
+<para>The <link linkend="sec-nix-worker">Nix daemon</link> should be
+started as follows (as <literal>root</literal>):
 
 <screen>
 $ nix-worker --daemon</screen>

doc/manual/introduction.xml

@@ -240,7 +240,7 @@ configuration (e.g., to build configuration files in
 possible to easily roll back the entire configuration of the system to
 an earlier state.  Also, users can install software without root
 privileges.  For more information and downloads, see the <link
-xlink:href="http://nix.cs.uu.nl/nixos/">NixOS homepage</link>.</para>
+xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 
 </simplesect>
 
@@ -257,14 +257,15 @@ xlink:href="http://nix.cs.uu.nl/nixos/">NixOS homepage</link>.</para>
 
 <section><title>About us</title>
 
-<para>Nix was developed at the <link
+<para>Nix was originally developed at the <link
 xlink:href="http://www.cs.uu.nl/">Department of Information and
 Computing Sciences</link>, Utrecht University by the <link
 xlink:href="http://www.cs.uu.nl/wiki/Trace/WebHome">TraCE
-project</link>.  The project is funded by the Software Engineering
-Research Program <link
+project</link> (2003-2008).  The project was funded by the Software
+Engineering Research Program <link
 xlink:href="http://www.jacquard.nl/">Jacquard</link> to improve the
-support for variability in software systems.</para>
+support for variability in software systems.  Further funding is now
+provided by the NIRICT LaQuSo Build Farm project.</para>
 
 </section>
 
@@ -299,24 +300,24 @@ Lesser General Public License for more details.</para>
 
 <para>Some background information on Nix can be found in a number of
 papers.  The ICSE 2004 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'>Imposing
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/immdsd-icse2004-final.pdf'>Imposing
 a Memory Management Discipline on Software Deployment</citetitle>
 discusses the hashing mechanism used to ensure reliable dependency
 identification and non-interference between different versions and
 variants of packages.  The LISA 2004 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'>Nix:
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/nspfssd-lisa2004-final.pdf'>Nix:
 A Safe and Policy-Free System for Software Deployment</citetitle>
 gives a more general discussion of Nix from a system-administration
 perspective.  The CBSE 2005 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'>Efficient
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/eupfcdm-cbse2005-final.pdf'>Efficient
 Upgrading in a Purely Functional Component Deployment Model
 </citetitle> is about transparent patch deployment in Nix.  The SCM-12
 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/servicecm-scm12-final.pdf'>
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/servicecm-scm12-final.pdf'>
 Service Configuration Management</citetitle> shows how services (e.g.,
 web servers) can be deployed and managed through Nix.  A short
 overview of NixOS is given in the HotOS XI paper <citetitle
-xlink:href="http://www.cs.uu.nl/~eelco/pubs/hotos-final.pdf">Purely
+xlink:href="http://www.st.ewi.tudelft.nl/~dolstra/pubs/hotos-final.pdf">Purely
 Functional System Configuration Management</citetitle>.  The Nix
 homepage has <link
 xlink:href="http://nix.cs.uu.nl/docs/papers.html">an up-to-date list
@@ -328,7 +329,7 @@ Purely Functional Software Deployment Model</citetitle>, which
 contains most of the papers listed above.</para>
 
 <para>Nix has a homepage at <link
-xlink:href="http://nix.cs.uu.nl/"/>.</para>
+xlink:href="http://nixos.org/"/>.</para>
 
 </section>
 

doc/manual/manual.xml

@@ -14,8 +14,8 @@
         <surname>Dolstra</surname>
       </personname>
       <affiliation>
-        <orgname>Utrecht University</orgname>
-        <orgdiv>Faculty of Science, Department of Information and Computing Sciences</orgdiv>
+        <orgname>Delft University of Technology</orgname>
+        <orgdiv>Department of Software Technology</orgdiv>
       </affiliation>
     </author>
 
@@ -24,10 +24,11 @@
       <year>2005</year>
       <year>2006</year>
       <year>2007</year>
+      <year>2008</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>September 2007</date>
+    <date>November 2008</date>
     
   </info>
 
@@ -88,10 +89,6 @@
         <title>nix-install-package</title>
         <xi:include href="nix-install-package.xml" />
       </section>
-      <section xml:id="sec-nix-pack-closure">
-        <title>nix-pack-closure</title>
-        <xi:include href="nix-pack-closure.xml" />
-      </section>
       <section xml:id="sec-nix-prefetch-url">
         <title>nix-prefetch-url</title>
         <xi:include href="nix-prefetch-url.xml" />
@@ -104,16 +101,16 @@
         <title>nix-push</title>
         <xi:include href="nix-push.xml" />
       </section>
-      <section xml:id="sec-nix-unpack-closure">
-        <title>nix-unpack-closure</title>
-        <xi:include href="nix-unpack-closure.xml" />
+      <section xml:id="sec-nix-worker">
+        <title>nix-worker</title>
+        <xi:include href="nix-worker.xml" />
       </section>
     </section>
     
   </appendix>
 
   <xi:include href="troubleshooting.xml" />
-  <xi:include href="bugs.xml" />
+  <!-- <xi:include href="bugs.xml" /> -->
   <xi:include href="glossary.xml" />
 
   <appendix>

doc/manual/nix-copy-closure.xml

@@ -128,7 +128,7 @@ those paths.  If this bothers you, use
 <para>Copy Firefox with all its dependencies to a remote machine:
 
 <screen>
-$ nix-copy-closure alice@itchy.labs $(type -tP firefox)</screen>
+$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)</screen>
 
 </para>
 

doc/manual/nix-env.xml

@@ -526,7 +526,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4'</screen>
 
 </refsection>
     
-<refsection><title>Versions</title>
+<refsection xml:id="ssec-version-comparisons"><title>Versions</title>
 
 <para>The upgrade operation determines whether a derivation
 <varname>y</varname> is an upgrade of a derivation

doc/manual/nix-pack-closure.xml

@@ -1,82 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<refmeta>
-  <refentrytitle>nix-pack-closure</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-pack-closure</refname>
-  <refpurpose>pack the closure of a store path into a single file that
-  can be unpacked with
-  <command>nix-unpack-closure</command></refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-pack-closure</command>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-pack-closure</command> packs the
-contents of the store paths <replaceable>paths</replaceable> and
-<emphasis>all their dependencies</emphasis> into a single file, which
-is written to standard output.  (That is, it
-<emphasis>serialises</emphasis> <replaceable>paths</replaceable>.)
-The output can then be unpacked into the Nix store of another machine
-using <command>nix-unpack-closure</command>.</para>
-
-<para>Together, <command>nix-pack-closure</command> and
-<command>nix-unpack-closure</command> provide a quick and easy way to
-deploy a package to a different machine.  However, as the output of
-<command>nix-pack-closure</command> tends to be rather large (since it
-contains all dependencies), it’s not very efficient.
-<command>nix-push</command> and <command>nix-pull</command> are more
-efficient, but are also a bit more cumbersome to use.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>To copy some instance of Subversion with all its dependencies to
-another machine:
-
-<screen>
-$ nix-pack-closure /nix/store/hj232g1r...-subversion-1.3.0 > svn.closure
-
-<lineannotation>Copy <!-- !!! <filename> -->svn.closure to the remote machine, then on the remote machine do:</lineannotation>
-$ nix-unpack-closure &lt; svn.closure</screen>
-
-</para>
-
-<para>Copy the program <command>azureus</command> with all its
-dependencies to the machine <literal>scratchy</literal>:
-
-<screen>
-$ nix-pack-closure $(which azureus) | ssh scratchy nix-unpack-closure</screen>
-    
-</para>
-
-<para>As a variation on the previous example, copy
-<command>azureus</command>, and also install it in the user’s profile
-on the target machine:
-
-<screen>
-$ nix-pack-closure $(which azureus) | ssh scratchy 'nix-env -i $(nix-unpack-closure)'</screen>
-
-</para>
- 
-
-</refsection>
-
-
-</refentry>

doc/manual/nix-store.xml

@@ -128,6 +128,7 @@ lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134
     <arg choice='plain'><option>-r</option></arg>
   </group>
   <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg><option>--dry-run</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -166,6 +167,11 @@ the specified store paths.  Realisation is a somewhat overloaded term:
 output.  (For non-derivations argument, the argument itself is
 printed.)</para>
 
+<para>If the <option>--dry-run</option> option is used, then
+<command>nix-store</command> will print on standard error a
+description of what packages would be built or downloaded, and then
+quit.</para>
+
 </refsection>
             
 
@@ -204,6 +210,10 @@ linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
     <arg choice='plain'><option>--print-dead</option></arg>
     <arg choice='plain'><option>--delete</option></arg>
   </group>
+  <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
+  <arg><option>--max-links</option> <replaceable>nrlinks</replaceable></arg>
+  <arg><option>--max-atime</option> <replaceable>atime</replaceable></arg>
+  <arg><option>--use-atime</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -258,7 +268,70 @@ the Nix store not reachable via file system references from a set of
 
 </variablelist>
 
-<para>The behaviour of the collector is influenced by the <link
+<para>By default, all unreachable paths are deleted.  The following
+options control what gets deleted and in what order:
+
+<variablelist>
+
+  <varlistentry><term><option>--max-freed</option> <replaceable>bytes</replaceable></term>
+  
+    <listitem><para>Keep deleting paths until at least
+    <replaceable>bytes</replaceable> bytes have been
+    deleted, then stop.</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--max-links</option> <replaceable>nrlinks</replaceable></term>
+  
+    <listitem><para>Keep deleting paths until the hard link count on
+    <filename>/nix/store</filename> is less than
+    <replaceable>nrlinks</replaceable>, then stop.  This is useful for
+    very large Nix stores on filesystems with a 32000 subdirectories
+    limit (like <literal>ext3</literal>).</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--max-atime</option> <replaceable>atime</replaceable></term>
+  
+    <listitem><para>Only delete a store path if its last-accessed time
+    is less than <replaceable>atime</replaceable>.  This allows you to
+    garbage-collect only packages that haven’t been used recently.
+    The time is expressed as the number of seconds in the Unix epoch,
+    i.e., since 1970-01-01 00:00:00 UTC.  An easy way to convert to
+    this format is <literal>date +%s -d "<replaceable>date
+    specification</replaceable>"</literal>.</para>
+
+    <para>For directories, the last-accessed time is the highest
+    last-accessed time of any regular file in the directory (or in any
+    of its subdirectories).  That is, the <literal>atime</literal>
+    field maintained by the filesystem is ignored for directories.
+    This is because operations such as rebuilding the
+    <command>locate</command> database tend to update the
+    <literal>atime</literal> values of all directories, so they’re not
+    a useful indicator of whether a package was recently used.</para>
+
+    <para>Note that <command>nix-store --optimise</command> reads all
+    regular files in the Nix store, and so causes all last-accessed
+    times to be set to the present time.  This makes
+    <option>--max-atime</option> ineffective (for a while at
+    least).</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--use-atime</option></term>
+
+    <listitem><para>Delete store paths in order of ascending
+    last-accessed time.  This is useful in conjunction with the other
+    options to delete only the least recently used
+    packages.</para></listitem>
+
+  </varlistentry>
+  
+</variablelist>
+
+</para>
+
+<para>The behaviour of the collector is also influenced by the <link
 linkend="conf-gc-keep-outputs"><literal>gc-keep-outputs</literal></link>
 and <link
 linkend="conf-gc-keep-derivations"><literal>gc-keep-derivations</literal></link>
@@ -284,6 +357,20 @@ deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
 
 </para>
 
+<para>To delete unreachable paths not accessed in the last two months:
+
+<screen>
+$ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
+
+</para>
+
+<para>To delete at least 100 MiBs of unreachable paths:
+
+<screen>
+$ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
+
+</para>
+
 </refsection>
 
 
@@ -950,7 +1037,51 @@ ktorrent-2.2.1/NEWS
 </refsection>
 
 
-<!-- TODO: export, import operations -->
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--dump-db</option></title>
+
+<refsection>
+  <title>Synopsis</title>
+  <cmdsynopsis>
+    <command>nix-store</command>
+    <arg choice='plain'><option>--dump-db</option></arg>
+  </cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+            
+<para>The operation <option>--dump-db</option> writes a dump of the
+Nix database to standard output.  It can be loaded into an empty Nix
+store using <option>--load-db</option>.  This is useful for making
+backups and when migrating to different database schemas.</para>
+
+</refsection>
+            
+</refsection>
+
+
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--dump-db</option></title>
+
+<refsection>
+  <title>Synopsis</title>
+  <cmdsynopsis>
+    <command>nix-store</command>
+    <arg choice='plain'><option>--load-db</option></arg>
+  </cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+            
+<para>The operation <option>--load-db</option> reads a dump of the Nix
+database created by <option>--dump-db</option> from standard input and
+loads it into the Nix database.</para>
+
+</refsection>
+            
+</refsection>
 
 
 </refentry>

doc/manual/nix-unpack-closure.xml

@@ -1,42 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<refmeta>
-  <refentrytitle>nix-unpack-closure</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-unpack-closure</refname>
-  <refpurpose>unpack the closure of a store path created by <command>nix-pack-closure</command> into the Nix store</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-unpack-closure</command>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-unpack-closure</command> unpacks the
-closure of a set of store paths created by
-<command>nix-pack-closure</command> into the local Nix store.  The
-closure is a single file read from standard input.  See the
-description of <command>nix-pack-closure</command> for details and
-examples.</para>
-
-<para>The top-level paths in the closure (i.e., the paths passed to
-the original <command>nix-pack-closure</command> call that created the
-closure) are printed on standard output.  These paths can be passed,
-for instance, to <literal>nix-env -i</literal> to install them into a
-user environment on the target machine.</para>
-
-</refsection>
-
-
-</refentry>

doc/manual/nix-worker.xml

@@ -0,0 +1,34 @@
+<refentry xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-worker</refentrytitle>
+  <manvolnum>8</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
+<refnamediv>
+  <refname>nix-worker</refname>
+  <refpurpose>Nix multi-user support daemon</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-worker</command>
+    <arg choice="plain"><option>--daemon</option></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+
+<refsection><title>Description</title>
+
+<para>The Nix daemon is necessary in multi-user Nix installations.  It
+performs build actions and other operations on the Nix store on behalf
+of unprivileged users.</para>
+
+
+</refsection>
+
+</refentry>

doc/manual/opt-common-syn.xml

@@ -24,6 +24,7 @@
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
 <arg><option>--log-type</option> <replaceable>type</replaceable></arg>
+<arg><option>--show-trace</option></arg>
 <sbr />
 
 </nop>

doc/manual/opt-common.xml

@@ -251,14 +251,14 @@
 
 <programlisting>
 { # The system (e.g., `i686-linux') for which to build the packages.
-  system ? __currentSystem
+  system ? builtins.currentSystem
   <replaceable>...</replaceable>
 }: <replaceable>...</replaceable></programlisting>
 
   So if you call this Nix expression (e.g., when you do
   <literal>nix-env -i <replaceable>pkgname</replaceable></literal>),
   the function will be called automatically using the value <link
-  linkend='builtin-currentSystem'><literal>__currentSystem</literal></link>
+  linkend='builtin-currentSystem'><literal>builtins.currentSystem</literal></link>
   for the <literal>system</literal> argument.  You can override this
   using <option>--arg</option>, e.g., <literal>nix-env -i
   <replaceable>pkgname</replaceable> --arg system
@@ -305,6 +305,14 @@
 </varlistentry>
 
 
+<varlistentry><term><option>--show-trace</option></term>
+  
+  <listitem><para>Causes Nix to print out a stack trace in case of Nix
+  expression evaluation errors.</para></listitem>
+  
+</varlistentry>
+
+
 </variablelist>
 
 

doc/manual/package-management.xml

@@ -38,7 +38,7 @@ to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
 expressions based on it, or completely new ones.)  You can download
 the latest version from <link
-xlink:href='http://nix.cs.uu.nl/dist/nix' />.</para>
+xlink:href='http://nixos.org/releases/full-index-nixpkgs.html' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available packages in the release:
@@ -118,15 +118,15 @@ available somewhere.  This is done using the
 containing a <emphasis>manifest</emphasis> describing what binaries
 are available.  This URL should correspond to the Nix Packages release
 that you’re using.  For instance, if you obtained a release from <link
-xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />, then
-you should do:
+xlink:href='http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x'
+/>, then you should do:
 
 <screen>
-$ nix-pull http://nix.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/MANIFEST</screen>
+$ nix-pull http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x/MANIFEST</screen>
 
 If you then issue the installation command, it should start
 downloading binaries from <systemitem
-class='fqdomainname'>nix.cs.uu.nl</systemitem>, instead of building
+class='fqdomainname'>nixos.org</systemitem>, instead of building
 them from source.  This might still take a while since all
 dependencies must be downloaded, but on a reasonably fast connection
 such as an DSL line it’s on the order of a few minutes.</para>
@@ -458,7 +458,7 @@ URL.</para>
 <command>nix-channel --add</command>, e.g.,
 
 <screen>
-$ nix-channel --add http://nix.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable</screen>
+$ nix-channel --add http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable</screen>
 
 subscribes you to a channel that always contains that latest version
 of the Nix Packages collection.  (Instead of
@@ -496,20 +496,19 @@ available in the subscribed channels.</para>
 
 <para>Often, when you want to install a specific package (e.g., from
 the <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/">Nix
-Packages collection</link> or from our <link
-xlink:href='http://nix.cs.uu.nl/dist/'>release server</link>),
-subscribing to a channel is a bit cumbersome.  And channels don’t help
-you at all if you want to install an older version of a package than
-the one provided by the current contents of the channel, or a package
-that has been removed from the channel.  That’s when
-<emphasis>one-click installs</emphasis> come in handy: you can just go
-to the web page that contains the package, click on it, and it will be
-installed with all the necessary dependencies.</para>
+xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/">Nix
+Packages collection</link>), subscribing to a channel is a bit
+cumbersome.  And channels don’t help you at all if you want to install
+an older version of a package than the one provided by the current
+contents of the channel, or a package that has been removed from the
+channel.  That’s when <emphasis>one-click installs</emphasis> come in
+handy: you can just go to the web page that contains the package,
+click on it, and it will be installed with all the necessary
+dependencies.</para>
 
 <para>For instance, you can go to <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/" />
-— or to any older release of Nix Packages — and click on any link for
+xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/" /> —
+or to any older release of Nix Packages — and click on any link for
 the individual packages for your platform (say, <link
 xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.10pre6622/pkgs/subversion-1.4.0-i686-linux.nixpkg'><literal>subversion-1.4.0</literal>
 for <literal>i686-linux</literal></link>).  The first time you do
@@ -529,4 +528,82 @@ linkend='sec-nix-install-package' /> for details.</para>
 </section>
 
 
+<section xml:id="sec-sharing-packages"><title>Sharing packages between machines</title>
+
+<para>Sometimes you want to copy a package from one machine to
+another.  Or, you want to install some packages and you know that
+another machine already has some or all of those packages or their
+dependencies.  In that case there are mechanisms to quickly copy
+packages between machines.</para>
+
+<para>The command <command
+linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix
+store path along with all its dependencies to or from another machine
+via the SSH protocol.  It doesn’t copy store paths that are already
+present on the target machine.  For example, the following command
+copies Firefox with all its dependencies:
+
+<screen>
+$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)</screen>
+
+See <xref linkend='sec-nix-copy-closure' /> for details.</para>
+
+<para>With <command linkend='refsec-nix-store-export'>nix-store
+--export</command> and <command
+linkend='refsec-nix-store-import'>nix-store --import</command> you can
+write the closure of a store path (that is, the path and all its
+dependencies) to a file, and then unpack that file into another Nix
+store.  For example,
+
+<screen>
+$ nix-store --export $(type -p firefox) > firefox.closure</screen>
+
+writes the closure of Firefox to a file.  You can then copy this file
+to another machine and install the closure:
+
+<screen>
+$ nix-store --import &lt; firefox.closure</screen>
+
+Any store paths in the closure that are already present in the target
+store are ignored.  It is also possible to pipe the export into
+another command, e.g. to copy and install a closure directly to/on
+another machine:
+
+<screen>
+$ nix-store --export $(type -p firefox) | bzip2 | \
+    ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen>
+
+But note that <command>nix-copy-closure</command> is generally more
+efficient in this example because it only copies paths that are not
+already present in the target Nix store.</para>
+
+<para>Finally, if you can mount the Nix store of a remote machine in
+your local filesystem, Nix can copy paths from the remote Nix store to
+the local Nix store <emphasis>on demand</emphasis>.  For instance,
+suppose that you mount a remote machine containing a Nix store via
+<command
+xlink:href="http://fuse.sourceforge.net/sshfs.html">sshfs</command>:
+
+<screen>
+$ sshfs alice@itchy.example.org:/ /mnt</screen>
+
+You should then set the <envar>NIX_OTHER_STORES</envar> environment
+variable to tell Nix about this remote Nix store:
+
+<screen>
+$ export NIX_OTHER_STORES=/mnt/nix</screen>
+
+Then if you do any Nix operation, e.g.
+
+<screen>
+$ nix-env -i firefox</screen>
+
+and Nix has to build a path that it sees is already present in
+<filename>/mnt/nix</filename>, then it will just copy from there
+instead of building it from source.</para>
+ 
+
+</section>
+
+
 </chapter>

doc/manual/quick-start.xml

@@ -11,7 +11,7 @@ to the following chapters.</para>
 <orderedlist>
 
 <listitem><para>Download a source tarball or RPM from <link
-xlink:href='http://nix.cs.uu.nl/'/>.  Build source
+xlink:href='http://nixos.org/'/>.  Build source
 distributions using the regular sequence:
         
 <screen>
@@ -33,7 +33,7 @@ file).</para></listitem>
 
 <screen>
 $ nix-channel --add \
-    http://nix.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable</screen>
+    http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable</screen>
 
 </para></listitem>
 

doc/manual/release-notes.xml

@@ -8,11 +8,259 @@
 
 <!--==================================================================-->
 
-<section xml:id="ssec-relnotes-0.12"><title>Release 0.12 (TBA)</title>
+<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (April NN,
+2009)</title>
+
+<para>This is primarily a bug fix release.  It has some new
+features:</para>
 
 <itemizedlist>
 
-  <listitem><para><command>nix-store --dump-db / --load-db</command>.</para></listitem>
+  <listitem>
+    <para>Syntactic sugar for writing nested attribute sets.  Instead of
+
+<programlisting>
+{
+  foo = {
+    bar = 123;
+    xyzzy = true;
+  };
+  a = { b = { c = "d"; }; };
+}
+</programlisting>
+
+    you can write
+
+<programlisting>
+{
+  foo.bar = 123;
+  foo.xyzzy = true;
+  a.b.c = "d";
+}
+</programlisting>
+
+    This is useful, for instance, in NixOS configuration files.</para>
+    
+  </listitem>
+
+  <listitem>
+    <para>Support for Nix channels generated by Hydra, the Nix-based
+    continuous build system.  (Hydra generates NAR archives on the
+    fly, so the size and hash of these archives isn’t known in
+    advance.)</para>
+  </listitem>
+
+  <listitem>
+    <para>Support <literal>i686-linux</literal> builds directly on
+    <literal>x86_64-linux</literal> Nix installations.  This is
+    implemented using the <function>personality()</function> syscall,
+    which causes <command>uname</command> to return
+    <literal>i686</literal> in child processes.</para>
+  </listitem>
+  
+  <listitem>
+    <para>Various improvements to the <literal>chroot</literal>
+    support.  Building in a <literal>chroot</literal> works quite well
+    now.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix no longer blocks if it tries to build a path and another
+    process is already building the same path.  Instead it tries to
+    build another buildable path first.  This improves
+    parallelism.</para>
+  </listitem>
+
+  <listitem>
+    <para>Support for large (> 4 GiB) files in NAR archives.</para>
+  </listitem>
+    
+  <listitem>
+    <para>Various (performance) improvements to the remote build
+    mechanism.</para>
+  </listitem>
+    
+  <listitem>
+    <para>New primops: <varname>builtins.addErrorContext</varname> (to
+    add a string to stack traces — useful for debugging),
+    <varname>builtins.isBool</varname>,
+    <varname>builtins.isString</varname>,
+    <varname>builtins.isInt</varname>.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.12"><title>Release 0.12 (November 20,
+2008)</title>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix no longer uses Berkeley DB to store Nix store metadata.
+    The principal advantages of the new storage scheme are: it works
+    properly over decent implementations of NFS (allowing Nix stores
+    to be shared between multiple machines); no recovery is needed
+    when a Nix process crashes; no write access is needed for
+    read-only operations; no more running out of Berkeley DB locks on
+    certain operations.</para>
+
+    <para>You still need to compile Nix with Berkeley DB support if
+    you want Nix to automatically convert your old Nix store to the
+    new schema.  If you don’t need this, you can build Nix with the
+    <filename>configure</filename> option
+    <option>--disable-old-db-compat</option>.</para>
+
+    <para>After the automatic conversion to the new schema, you can
+    delete the old Berkeley DB files:
+
+    <screen>
+$ cd /nix/var/nix/db
+$ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG</screen>
+
+    The new metadata is stored in the directories
+    <filename>/nix/var/nix/db/info</filename> and
+    <filename>/nix/var/nix/db/referrer</filename>.  Though the
+    metadata is stored in human-readable plain-text files, they are
+    not intended to be human-editable, as Nix is rather strict about
+    the format.</para>
+
+    <para>The new storage schema may or may not require less disk
+    space than the Berkeley DB environment, mostly depending on the
+    cluster size of your file system.  With 1 KiB clusters (which
+    seems to be the <literal>ext3</literal> default nowadays) it
+    usually takes up much less space.</para>
+  </listitem>
+
+  <listitem><para>There is a new substituter that copies paths
+  directly from other (remote) Nix stores mounted somewhere in the
+  filesystem.  For instance, you can speed up an installation by
+  mounting some remote Nix store that already has the packages in
+  question via NFS or <literal>sshfs</literal>.  The environment
+  variable <envar>NIX_OTHER_STORES</envar> specifies the locations of
+  the remote Nix directories,
+  e.g. <literal>/mnt/remote-fs/nix</literal>.</para></listitem>
+
+  <listitem><para>New <command>nix-store</command> operations
+  <option>--dump-db</option> and <option>--load-db</option> to dump
+  and reload the Nix database.</para></listitem>
+
+  <listitem><para>The garbage collector has a number of new options to
+  allow only some of the garbage to be deleted.  The option
+  <option>--max-freed <replaceable>N</replaceable></option> tells the
+  collector to stop after at least <replaceable>N</replaceable> bytes
+  have been deleted.  The option <option>--max-links
+  <replaceable>N</replaceable></option> tells it to stop after the
+  link count on <filename>/nix/store</filename> has dropped below
+  <replaceable>N</replaceable>.  This is useful for very large Nix
+  stores on filesystems with a 32000 subdirectories limit (like
+  <literal>ext3</literal>).  The option <option>--use-atime</option>
+  causes store paths to be deleted in order of ascending last access
+  time.  This allows non-recently used stuff to be deleted.  The
+  option <option>--max-atime <replaceable>time</replaceable></option>
+  specifies an upper limit to the last accessed time of paths that may
+  be deleted.  For instance,
+
+    <screen>
+    $ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
+
+  deletes everything that hasn’t been accessed in two months.</para></listitem>
+
+  <listitem><para><command>nix-env</command> now uses optimistic
+  profile locking when performing an operation like installing or
+  upgrading, instead of setting an exclusive lock on the profile.
+  This allows multiple <command>nix-env -i / -u / -e</command>
+  operations on the same profile in parallel.  If a
+  <command>nix-env</command> operation sees at the end that the profile
+  was changed in the meantime by another process, it will just
+  restart.  This is generally cheap because the build results are
+  still in the Nix store.</para></listitem>
+
+  <listitem><para>The option <option>--dry-run</option> is now
+  supported by <command>nix-store -r</command> and
+  <command>nix-build</command>.</para></listitem>
+
+  <listitem><para>The information previously shown by
+  <option>--dry-run</option> (i.e., which derivations will be built
+  and which paths will be substituted) is now always shown by
+  <command>nix-env</command>, <command>nix-store -r</command> and
+  <command>nix-build</command>.  The total download size of
+  substitutable paths is now also shown.  For instance, a build will
+  show something like
+
+    <screen>
+the following derivations will be built:
+  /nix/store/129sbxnk5n466zg6r1qmq1xjv9zymyy7-activate-configuration.sh.drv
+  /nix/store/7mzy971rdm8l566ch8hgxaf89x7lr7ik-upstart-jobs.drv
+  ...
+the following paths will be downloaded/copied (30.02 MiB):
+  /nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4
+  /nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6
+  ...</screen>      
+  
+  </para></listitem>
+
+  <listitem><para>Language features:
+
+    <itemizedlist>
+
+      <listitem><para>@-patterns as in Haskell.  For instance, in a
+      function definition
+
+      <programlisting>f = args @ {x, y, z}: <replaceable>...</replaceable>;</programlisting>
+
+      <varname>args</varname> refers to the argument as a whole, which
+      is further pattern-matched against the attribute set pattern
+      <literal>{x, y, z}</literal>.</para></listitem>
+
+      <listitem><para>“<literal>...</literal>” (ellipsis) patterns.
+      An attribute set pattern can now say <literal>...</literal>  at
+      the end of the attribute name list to specify that the function
+      takes <emphasis>at least</emphasis> the listed attributes, while
+      ignoring additional attributes.  For instance,
+
+      <programlisting>{stdenv, fetchurl, fuse, ...}: <replaceable>...</replaceable></programlisting>
+
+      defines a function that accepts any attribute set that includes
+      at least the three listed attributes.</para></listitem>
+
+      <listitem><para>New primops:
+      <varname>builtins.parseDrvName</varname> (split a package name
+      string like <literal>"nix-0.12pre12876"</literal> into its name
+      and version components, e.g. <literal>"nix"</literal> and
+      <literal>"0.12pre12876"</literal>),
+      <varname>builtins.compareVersions</varname> (compare two version
+      strings using the same algorithm that <command>nix-env</command>
+      uses), <varname>builtins.length</varname> (efficiently compute
+      the length of a list), <varname>builtins.mul</varname> (integer
+      multiplication), <varname>builtins.div</varname> (integer
+      division).
+      <!-- <varname>builtins.genericClosure</varname> -->
+      </para></listitem>
+      
+    </itemizedlist>
+
+  </para></listitem>
+
+  <listitem><para><command>nix-prefetch-url</command> now supports
+  <literal>mirror://</literal> URLs, provided that the environment
+  variable <envar>NIXPKGS_ALL</envar> points at a Nixpkgs
+  tree.</para></listitem>
+
+  <listitem><para>Removed the commands
+  <command>nix-pack-closure</command> and
+  <command>nix-unpack-closure</command>.   You can do almost the same
+  thing but much more efficiently by doing <literal>nix-store --export
+  $(nix-store -qR <replaceable>paths</replaceable>) > closure</literal> and
+  <literal>nix-store --import &lt;
+  closure</literal>.</para></listitem>
+
+  <listitem><para>Lots of bug fixes, including a big performance bug in
+  the handling of <literal>with</literal>-expressions.</para></listitem>
 
 </itemizedlist>
 

doc/manual/troubleshooting.xml

@@ -5,121 +5,10 @@
 
 
 <para>This section provides solutions for some common problems.  See
-the <link xlink:href="https://bugs.cs.uu.nl/browse/NIX">Nix
+the <link xlink:href="http://bugs.strategoxt.org/browse/NIX">Nix
 bug tracker</link> for a list of currently known issues.</para>
 
 
-<section><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
-
-<para>Symptom: Nix operations (in particular the
-<command>nix-store</command> operations <option>--gc</option>,
-<option>--verify</option>, and <option>--clear-substitutes</option> —
-the latter being called by <command>nix-channel --update</command>)
-failing:
-
-<screen>
-$ nix-store --verify
-error: Db::del: Cannot allocate memory</screen>
-
-</para>
-
-<para>Possible solution: make sure that no Nix processes are running,
-then do:
-
-<screen>
-$ cd /nix/var/nix/db
-$ rm __db.00*</screen>
-
-</para>
-
-</section>
-
-
-<section><title>Berkeley DB gives weird error messages</title>
-
-<para>Symptom: you get error messages such as
-
-<screen>
-Berkeley DB message: Finding last valid log LSN: file: 1 offset 28
-Berkeley DB error: file validpaths (meta pgno = 0) has LSN [483][34721].
-Berkeley DB error: end of log is [1][28]
-Berkeley DB error: /nix/var/nix/db/validpaths: unexpected file type or format</screen>
-
-or other weird Berkeley DB errors, and they don’t go away (i.e.,
-automatic recovery doesn’t work).  This may be the case after a system
-crash.</para>
-
-<para>Solution: first try to run <command>db_recover</command> and
-then <link linkend='refsec-nix-store-verify'><command>nix-store
---verify</command></link>:
-
-<screen>
-$ db_recover -h /nix/var/nix/db
-$ nix-store --verify</screen>
-
-(Make sure that you have the right version of
-<command>db_recover</command>, namely, Berkeley DB 4.4 for Nix 0.10,
-and 4.5 for Nix 0.11.)</para>
-
-<para>If that doesn’t work, it’s time to bring out the big guns:
-
-<screen>
-$ cd /nix/var/nix
-$ cp -pr db db-backup  <lineannotation>(making a backup just in case)</lineannotation>
-$ cd db
-$ rm __db.* log*  <lineannotation>(removing the Berkeley DB environment)</lineannotation>
-$ mkdir tmp
-$ for i in *; do db_dump $i | (cd tmp &amp;&amp; db_load $i); done 
-<lineannotation>(ignore error messages about non-database files like “reserved”)</lineannotation>
-$ mv tmp/* .
-$ nix-store --verify</screen>
-
-</para>
-
-</section>
-
-
-<section><title>Berkeley DB out of locks</title>
-
-<para>It is possible, especially in <command>nix-store
---verify</command> or when running the garbage collector, to run out
-of Berkeley DB locks, like this:
-
-<screen>
-$ nix-store --verify
-checking path existence
-checking path realisability
-checking the derivers table
-checking the references table
-Berkeley DB error: Lock table is out of available object entries
-error: Db::get: Cannot allocate memory</screen>
-
-</para>
-
-<para>A workaround is to increase the number of locks that Berkeley DB
-allocates.  (The real solution would be for Nix to not use so many
-locks.)  This can be done by putting the following in the file
-<filename>/nix/var/nix/db/<link
-xlink:href="http://www.oracle.com/technology/documentation/berkeley-db/db/ref/env/db_config.html">DB_CONFIG</link></filename>:
-
-<programlisting>
-set_lk_max_locks   100000
-set_lk_max_lockers 100000
-set_lk_max_objects 100000
-</programlisting>
-
-(Increase these numbers if necessary.)  Then make sure that there are
-no running Nix processes and delete the Berkeley DB environment:
-
-<screen>
-$ rm /nix/var/nix/db/__db.*</screen>
-
-The Berkeley DB environment is automatically recreated with the new
-limits when you run any Nix command.</para>
-
-</section>
-
-
 <section><title>Collisions in <command>nix-env</command></title>
 
 <para>Symptom: when installing or upgrading, you get an error message such as
@@ -187,7 +76,8 @@ Furthermore, the <literal>st_nlink</literal> field of the
 <para>This only happens on very large Nix installations (such as build
 machines).</para>
 
-<para>Quick solution: run the garbage collector.</para>
+<para>Quick solution: run the garbage collector.  You may want to use
+the <option>--max-links</option> option.</para>
 
 <para>Real solution: put the Nix store on a file system that supports
 more than 32,000 subdirectories per directory, such as ReiserFS.

doc/manual/writing-nix-expressions.xml

@@ -731,9 +731,9 @@ stdenv.mkDerivation {
     xlink:href='http://www.ietf.org/rfc/rfc2396.txt'>RFC 2396</link>
     can be written <emphasis>as is</emphasis>, without quotes.  For
     instance, the string
-    <literal>"https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2"</literal>
+    <literal>"http://example.org/foo.tar.bz2"</literal>
     can also be written as
-    <literal>https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2</literal>.</para>
+    <literal>http://example.org/foo.tar.bz2</literal>.</para>
 
   </listitem>
 
@@ -937,17 +937,84 @@ set.</para>
 <para>Functions have the following form:
 
 <programlisting>
-{<replaceable>params</replaceable>}: <replaceable>body</replaceable></programlisting>
+<replaceable>pattern</replaceable>: <replaceable>body</replaceable></programlisting>
 
-This defines a function that must be called with an attribute set
-containing the attributes listed in <replaceable>params</replaceable>,
-which is a comma-separated list of attribute names.  Optionally, for
-each parameter a <emphasis>default value</emphasis> may be specified
-by writing <literal><replaceable>param</replaceable> ?
-<replaceable>e</replaceable></literal>, where
-<replaceable>e</replaceable> is an arbitrary expression.  If a
-parameter has a default, the corresponding attribute may be omitted in
-function calls.</para>
+The pattern specifies what the argument of the function must look
+like, and binds variables in the body to (parts of) the
+argument.  There are three kinds of patterns:</para>
+
+<itemizedlist>
+
+  
+  <listitem><para>If a pattern is a single identifier, then the
+  function matches any argument.  Example:
+
+  <programlisting>
+let negate = x: !x;
+    concat = x: y: x + y;
+in if negate true then concat "foo" "bar" else ""</programlisting>
+
+  Note that <function>concat</function> is a function that takes one
+  argument and returns a function that takes another argument.  This
+  allows partial parameterisation (i.e., only filling some of the
+  arguments of a function); e.g.,
+
+  <programlisting>
+map (concat "foo") ["bar" "bla" "abc"]</programlisting>
+
+  evaluates to <literal>["foobar" "foobla"
+  "fooabc"]</literal>.</para></listitem>
+
+  
+  <listitem><para>An <emphasis>attribute set pattern</emphasis> of the
+  form <literal>{name1, name2, …, nameN}</literal>
+  matches an attribute set containing the listed attributes, and binds
+  the values of those attributes to variables in the function body.
+  For example, the function
+
+<programlisting>
+{x, y, z}: z + y + x</programlisting>
+
+  can only be called with a set containing exactly the attributes
+  <varname>x</varname>, <varname>y</varname> and
+  <varname>z</varname>.  No other attributes are allowed.  If you want
+  to allow additional arguments, you can use an ellipsis
+  (<literal>...</literal>): 
+
+<programlisting>
+{x, y, z, ....}: z + y + x</programlisting>
+
+  This works on any set that contains at least the three named
+  attributes.</para>
+
+  <para>It is possible to provide <emphasis>default values</emphasis>
+  for attributes, in which case they are allowed to be missing.  A
+  default value is specified by writing
+  <literal><replaceable>name</replaceable> ?
+  <replaceable>e</replaceable></literal>, where
+  <replaceable>e</replaceable> is an arbitrary expression.  For example,
+
+<programlisting>
+{x, y ? "foo", z ? "bar"}: z + y + x</programlisting>
+
+  specifies a function that only requires an attribute named
+  <varname>x</varname>, but optionally accepts <varname>y</varname>
+  and <varname>z</varname>.</para></listitem>
+
+
+  <listitem><para>An <literal>@</literal>-pattern requires that the
+  argument matches with the patterns on the left- and right-hand side
+  of the <literal>@</literal>-sign.  For example:
+
+<programlisting>
+args@{x, y, z, ...}: z + y + x + args.a</programlisting>
+
+  Here <varname>args</varname> is bound to the entire argument, which
+  is further matches against the pattern <literal>{x, y, z,
+  ...}</literal>.</para></listitem>
+
+
+</itemizedlist>
 
 <para>Note that functions do not have names.  If you want to give them
 a name, you can bind them to an attribute, e.g.,
@@ -958,31 +1025,6 @@ in concat {x = "foo"; y = "bar";}</programlisting>
 
 </para>
 
-<para>It is also possible to define a function that takes a single
-argument and that does not need to be called with an attribute set as
-argument.  The syntax is
-
-<programlisting>
-<replaceable>var</replaceable>: <replaceable>body</replaceable></programlisting>
-
-where <replaceable>var</replaceable> is the name of the argument.  It
-is not possible to define a default.  Example:
-
-<programlisting>
-let negate = x: !x;
-    concat = x: y: x + y;
-in if negate true then concat "foo" "bar" else ""</programlisting>
-
-Note that <function>concat</function> is a function that takes one
-arguments and returns a function that takes another argument.  This
-allows partial parameterisation (i.e., only filling some of the
-arguments of a function); e.g.,
-
-<programlisting>
-map (concat "foo") ["bar" "bla" "abc"]</programlisting>
-
-evaluates to <literal>["foobar" "foobla" "fooabc"]</literal>.</para>
-
 </simplesect>
 
 
@@ -1390,7 +1432,7 @@ command-line argument.  See <xref linkend='sec-standard-environment'
   inputs.</para></listitem>
 
   <listitem><para>After the build, Nix sets the last-modified
-  timestamp on all files in the build result to 0 (00:00:00 1/1/1970
+  timestamp on all files in the build result to 1 (00:00:01 1/1/1970
   UTC), sets the group to the default group, and sets the mode of the
   file to 0444 or 0555 (i.e., read-only, with execute permission
   enabled if the file was originally executable).  Note that possible
@@ -1644,59 +1686,6 @@ impureEnvVars = ["http_proxy" "https_proxy" <replaceable>...</replaceable>];
 
 <section xml:id='sec-standard-environment'><title>The standard environment</title>
 
-<para>The standard build environment in the Nix Packages collection
-provides a basic environment for building Unix packages.  It consists
-of the following packages:
-
-<itemizedlist>
-
-  <listitem><para>The GNU C Compiler, configured with C and C++
-  support.  On Linux, the compiler has been patched to provide greater
-  <quote>purity</quote> assurance.  For instance, the compiler doesn't
-  search in locations such as <filename>/usr/include</filename>.  In
-  fact, attempts to add such directories through the
-  <option>-I</option> flag are filtered out.  Likewise, the linker
-  (from GNU binutils) doesn't search in standard locations such as
-  <filename>/usr/lib</filename>.  Programs built on Linux are linked
-  against a GNU C Library that likewise doesn't search in the default
-  system locations.</para></listitem>
-
-  <listitem><para>GNU coreutils (contains a few dozen standard Unix
-  commands).</para></listitem>
-
-  <listitem><para>GNU findutils (contains
-  <command>find</command>).</para></listitem>
-
-  <listitem><para>GNU diffutils (contains <command>diff</command>,
-  <command>cmp</command>).</para></listitem>
-
-  <listitem><para>GNU <command>sed</command>.</para></listitem>
-
-  <listitem><para>GNU <command>grep</command>.</para></listitem>
-
-  <listitem><para>GNU <command>awk</command>.</para></listitem>
-
-  <listitem><para>GNU <command>tar</command>.</para></listitem>
-
-  <listitem><para><command>gzip</command> and
-  <command>bzip2</command>.</para></listitem>
-
-  <listitem><para>GNU Make.  It has been patched to provide
-  <quote>nested</quote> output that can be fed into the
-  <command>nix-log2xml</command> command and
-  <command>log2html</command> stylesheet to create a structured,
-  readable output of the build steps performed by
-  Make.</para></listitem>
-
-  <listitem><para>Bash.  This is the shell used for all builders in
-  the Nix Packages collection.  Not using <command>/bin/sh</command>
-  removes a large source of portability problems.</para></listitem>
-
-  <listitem><para>Patch.</para></listitem>
-
-</itemizedlist>
-
-</para>
 
 <para>The standard environment is used by passing it as an input
 called <envar>stdenv</envar> to the derivation, and then doing
@@ -1765,115 +1754,6 @@ myPostInstall() {
 
 </para>
 
-<para>The generic builder has a number of <emphasis>phases</emphasis>,
-each of which can be override in its entirety by setting the indicated
-variable.  The phases are:
-
-<itemizedlist>
-
-  <listitem>
-
-    <para><function>unpackPhase</function> unpacks the source files
-    listed in the <envar>src</envar> environment variable to the
-    current directory.  It supports <filename>tar</filename> files,
-    optionally compressed with <command>gzip</command> or
-    <command>bzip2</command>; Zip files (but note that the
-    <command>unzip</command> command is not a part of the standard
-    environment; you should add it as a build input yourself); and
-    unpacked source trees (i.e., directories; they are copied
-    verbatim).  You can add support for other file types by setting
-    the <varname>findUnpacker</varname> hook.  This hook should set
-    the variable <varname>unpackCmd</varname> to contain the command
-    to be executed to unpack the file.</para>
-
-    <para>After unpacking all source files,
-    <function>unpackPhase</function> changes the current directory to
-    the directory created by unpacking the sources.  If there are
-    multiple source directories, you should set
-    <varname>sourceRoot</varname> to the name of the intended
-    directory.</para>
-
-    <para>It also calls the hook <varname>postUnpack</varname> after
-    unpacking.</para>
-
-  </listitem>
-
-  <listitem><para><function>patchPhase</function> calls the
-  <command>patch</command> command with the <option>-p1</option>
-  option for each patch file listed in the <envar>patches</envar>
-  variable.</para></listitem>
-
-  <listitem>
-
-    <para><function>configurePhase</function> runs the script called
-    <filename>configure</filename> in the current directory with a
-    <option>--prefix</option> set to the output path.  You can add
-    additional flags through the <varname>configureFlags</varname>
-    variable.  If <filename>configure</filename> does not exist,
-    nothing happens.</para>
-
-    <para>Before and after running <filename>configure</filename>, the
-    hooks <varname>preConfigure</varname> and
-    <varname>postConfigure</varname> are called, respectively.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para><function>buildPhase</function> calls
-    <command>make</command>.  You can set flags for
-    <command>make</command> through the <varname>makeFlags</varname>
-    variable.</para>
-
-    <para>Before and after running <command>make</command>, the hooks
-    <varname>preBuild</varname> and <varname>postBuild</varname> are
-    called, respectively.</para>
-    
-  </listitem>
-
-  <listitem><para><function>checkPhase</function> calls <command>make
-  check</command>, but only if the <varname>doCheck</varname> variable
-  is set to <literal>1</literal>.  Additional flags can be set through
-  the <varname>checkFlags</varname> variable.</para></listitem>
-
-  <listitem>
-
-    <para><function>installPhase</function> calls <command>make
-    install</command>.  Additional flags can be set through the
-    <varname>installFlags</varname> variable.  It also strips any
-    static libraries in the output path of debug information unless
-    <varname>dontStrip</varname> is set to
-    <literal>1</literal>.</para>
-
-    <para>Before and after running <command>make install</command>,
-    the hooks <varname>preInstall</varname> and
-    <varname>postInstall</varname> are called, respectively.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para><function>distPhase</function> calls <command>make
-    dist</command>, but only if the <varname>doDist</varname> variable
-    is set to <literal>1</literal>.  Additional flags can be set
-    through the <varname>distFlags</varname> variable.  The resulting
-    tarball is copied to the <filename>/tarballs</filename>
-    subdirectory of the output path.</para>
-
-    <para>Before and after running <command>make dist</command>, the
-    hooks <varname>preDist</varname> and <varname>postDist</varname>
-    are called, respectively.</para>
-    
-  </listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>You can change the order in which phases are executed, or add
-new phases, by setting the <varname>phases</varname> variable.  The
-default is <literal>patchPhase configurePhase buildPhase checkPhase
-installPhase distPhase</literal>.</para>
 
 </section>
 

externals/Makefile.am

@@ -48,7 +48,7 @@ ATERM = aterm-2.4.2-fixes-r2
 $(ATERM).tar.bz2:
 	@echo "Nix requires the CWI ATerm library to build."
 	@echo "Please download version 2.4.2-fixes-r2 from"
-	@echo "  http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes-r2.tar.bz2"
+	@echo "  http://nixos.org/tarballs/aterm-2.4.2-fixes-r2.tar.bz2"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -75,12 +75,12 @@ endif
 
 # bzip2
 
-BZIP2 = bzip2-1.0.4
+BZIP2 = bzip2-1.0.5
 
 $(BZIP2).tar.gz:
 	@echo "Nix requires bzip2 to build."
-	@echo "Please download version 1.0.4 from"
-	@echo "  http://www.bzip.org/1.0.4/bzip2-1.0.4.tar.gz"
+	@echo "Please download version 1.0.5 from"
+	@echo "  http://www.bzip.org/1.0.5/bzip2-1.0.5.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -113,5 +113,5 @@ EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.bz2 $(BZIP2).tar.gz \
   bdb-cygwin.patch
 
 ext-clean:
-	$(RM) -f have-db build-db have-aterm build-aterm
+	$(RM) -f have-db build-db have-aterm build-aterm have-bzip2 build-bzip2
 	$(RM) -rf $(DB) $(ATERM) $(BZIP2)

misc/emacs/nix-mode.el

@@ -76,10 +76,10 @@ The hook `nix-mode-hook' is run when Nix mode is started.
     ("\\<baseNameOf\\>" . font-lock-builtin-face)
     ("\\<toString\\>" . font-lock-builtin-face)
     ("\\<isNull\\>" . font-lock-builtin-face)
-    ("\\<\\([a-zA-Z_][a-zA-Z0-9_']*\\)[ \t]*="
-     (1 font-lock-variable-name-face nil nil))
     ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+"
      . font-lock-constant-face)
+    ("\\<\\([a-zA-Z_][a-zA-Z0-9_'\.]*\\)[ \t]*="
+     (1 font-lock-variable-name-face nil nil))
     ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+"
      . font-lock-constant-face)
     ))

misc/vim/syntax/nix.vim

@@ -21,12 +21,14 @@ syn match nixFuncArg "\zs\w\+\ze\s*:"
 syn region nixStringParam start=+\${+ end=+}+
 syn region nixMultiLineComment start=+/\*+ skip=+\\"+ end=+\*/+
 syn match  nixEndOfLineComment "#.*$"
-syn region nixString      start=+"+ skip=+\\"+ end=+"+ contains=nixStringParam
+syn region nixStringIndented start=+''+ skip=+'''\|''${\|"+ end=+''+ contains=nixStringParam
+syn region nixString         start=+"+ skip=+\\"+ end=+"+ contains=nixStringParam
 
 hi def link nixKeyword       Keyword
 hi def link nixConditional   Conditional
 hi def link nixBrace         Special
 hi def link nixString        String
+hi def link nixStringIndented String
 hi def link nixBuiltin       Special
 hi def link nixStringParam   Macro
 hi def link nixMultiLineComment Comment

nix.conf.example

@@ -29,25 +29,6 @@
 #gc-keep-derivations = true
 
 
-### Option `gc-reserved-space'
-#
-# This option specifies how much space should be reserved in normal
-# use so that the garbage collector can run succesfully.  Since the
-# garbage collector must perform Berkeley DB transactions, it needs
-# some disk space for itself.  However, when the disk is full, this
-# space is not available, so the collector would not be able to run
-# precisely when it is most needed.
-#
-# For this reason, when Nix is run, it allocates a file
-# /nix/var/nix/db/reserved of the size specified by this option.  When
-# the garbage collector is run, this file is deleted before the
-# Berkeley DB environment is opened.  This should give it enough room
-# to proceed.
-#
-# The default is "1048576" (1 MiB).
-#gc-reserved-space = 1048576
-
-
 ### Option `env-keep-derivations'
 #
 # If `false' (default), derivations are not stored in Nix user
@@ -164,30 +145,26 @@
 # (using `mount --bind' on Linux) some directories from the normal
 # file system hierarchy inside the chroot.  These are the Nix store,
 # the temporary build directory (usually /tmp/nix-<pid>-<number>) and
-# the directories listed here.  The default is "/dev /proc".  Files
-# in /dev (such as /dev/null) are needed by many builds, and some
-# files in /proc may also be needed occasionally.
+# the directories listed here.  The default is "/dev /dev/pts /proc".
+# Files in /dev (such as /dev/null) are needed by many builds, and
+# some files in /proc may also be needed occasionally.
 #
 # Example:
 #   build-use-chroot = /dev /proc /bin
-#build-chroot-dirs = /dev /proc
+#build-chroot-dirs = /dev /dev/pts /proc
 
 
-### Option `system'
+### Option `build-cache-failure'
 #
-# This option specifies the canonical Nix system name of the current
-# installation, such as `i686-linux' or `powerpc-darwin'.  Nix can
-# only build derivations whose `system' attribute equals the value
-# specified here.  In general, it never makes sense to modify this
-# value from its default, since you can use it to `lie' about the
-# platform you are building on (e.g., perform a Mac OS build on a
-# Linux machine; the result would obviously be wrong).  It only makes
-# sense if the Nix binaries can run on multiple platforms, e.g.,
-# `universal binaries' that run on `powerpc-darwin' and `i686-darwin'.
-#
-# It defaults to the canonical Nix system name detected by `configure'
-# at build time.
+# If this option is enabled, Nix will do negative caching; that is, it
+# will remember failed builds, and won't attempt to try to build them
+# again if you ask for it.  Negative caching is disabled by default
+# because Nix cannot distinguish between permanent build errors (e.g.,
+# a syntax error in a source file) and transient build errors (e.g., a
+# full disk), as they both cause the builder to return a non-zero exit
+# code.  You can clear the cache by doing `rm -f
+# /nix/var/nix/db/failed/*'.
 #
 # Example:
-#   system = i686-darwin
-#system =
+#   build-cache-failure = true
+#build-cache-failure = false

nix.spec.in

@@ -13,11 +13,10 @@ Version: @version@
 Release: 1
 License: GPL
 Group: Software Deployment
-URL: http://nix.cs.uu.nl/
+URL: http://nixos.org/
 Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
-%define _prefix /nix
-Prefix: %{_prefix}
+Prefix: /usr
 Requires: /usr/bin/perl
 Requires: curl
 
@@ -47,14 +46,14 @@ if test -n "%{enable_setuid}"; then
         extraFlags="$extraFlags --with-nix-group=%{nix_group}"
     fi
 fi
-./configure --prefix=%{_prefix} $extraFlags
+./configure --prefix=%{_prefix} --sysconfdir=/etc $extraFlags
 make
 make check
 
 %install
 rm -rf $RPM_BUILD_ROOT
 make DESTDIR=$RPM_BUILD_ROOT install
-rm $RPM_BUILD_ROOT/%{_prefix}/etc/nix/nix.conf
+rm $RPM_BUILD_ROOT/etc/nix/nix.conf
 strip $RPM_BUILD_ROOT/%{_prefix}/bin/* || true
 
 %clean
@@ -76,10 +75,9 @@ fi
 %{_prefix}/lib
 %{_prefix}/libexec
 %{_prefix}/include
-%{_prefix}/var
 %{_prefix}/share
-%{_prefix}/store
+/etc/profile.d/nix.sh
+/nix/var
+/nix/store
 %config
-%{_prefix}/etc
-#%doc
-#%{_prefix}/share/nix/manual
+/etc/nix

release.nix

@@ -0,0 +1,196 @@
+{ nixpkgs ? ../nixpkgs }:
+
+let
+
+  jobs = rec {
+
+
+    tarball =
+      { nix ? {outPath = ./.; rev = 1234;}
+      , officialRelease ? false
+      }:
+
+      with import nixpkgs {};
+
+      releaseTools.sourceTarball {
+        name = "nix-tarball";
+        version = builtins.readFile ./version;
+        src = nix;
+        inherit officialRelease;
+
+        buildInputs = [curl bison flex2533 perl libxml2 libxslt w3m bzip2 jing_tools tetex dblatex];
+
+        configureFlags = ''
+          --with-docbook-rng=${docbook5}/xml/rng/docbook
+          --with-docbook-xsl=${docbook5_xsl}/xml/xsl/docbook
+          --with-xml-flags=--nonet
+        '';
+
+        # Include the BDB, ATerm and Bzip2 tarballs in the distribution.
+        preConfigure = ''
+          stripHash ${db45.src}
+          # Remove unnecessary stuff from the Berkeley DB tarball.
+          ( mkdir bdb-temp
+            cd bdb-temp
+            tar xfz ${db45.src}
+            cd *
+            rm -rf docs test tcl perl libdb_java java rpc_server build_vxworks \
+              examples_java examples_c examples_cxx dist/tags
+            mkdir test
+            touch test/include.tcl
+            cd ..
+            tar cvfz ../externals/$strippedName *
+          )
+
+          stripHash ${aterm242fixes.src}
+          cp -pv ${aterm242fixes.src} externals/$strippedName
+
+          stripHash ${bzip2.src}
+          cp -pv ${bzip2.src} externals/$strippedName
+        '';
+
+        preDist = ''
+          make -C doc/manual install prefix=$out
+          
+          make -C doc/manual manual.pdf prefix=$out
+          cp doc/manual/manual.pdf $out/manual.pdf
+          
+          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
+          echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products
+          echo "doc release-notes $out/share/doc/nix/release-notes" >> $out/nix-support/hydra-build-products
+        '';
+      };
+
+
+    build =
+      { tarball ? jobs.tarball {}
+      , system ? "i686-linux"
+      }:
+
+      with import nixpkgs {inherit system;};
+
+      releaseTools.nixBuild {
+        name = "nix";
+        src = tarball;
+
+        buildInputs = [curl perl bzip2 openssl];
+
+        configureFlags = ''
+          --disable-init-state
+          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+        '';
+      };
+
+      
+    static =
+      { tarball ? jobs.tarball {}
+      , system ? "i686-linux"
+      }:
+
+      with import nixpkgs {inherit system;};
+
+      releaseTools.binaryTarball {
+        name = "nix-static-tarball";
+        src = tarball;
+
+        buildInputs = [curl perl bzip2];
+
+        configureFlags = ''
+          --disable-init-state
+          --disable-old-db-compat --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --enable-static-nix
+        '';
+      };
+
+      
+    coverage =
+      { tarball ? jobs.tarball {}
+      }:
+
+      with import nixpkgs {};
+
+      releaseTools.coverageAnalysis {
+        name = "nix-build";
+        src = tarball;
+
+        buildInputs = [
+          curl perl bzip2 openssl
+          # These are for "make check" only:
+          graphviz libxml2 libxslt
+        ];
+
+        configureFlags = ''
+          --disable-init-state --disable-shared
+          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+        '';
+
+        lcovFilter = ["*/boost/*" "*-tab.*"];
+
+        # We call `dot', and even though we just use it to
+        # syntax-check generated dot files, it still requires some
+        # fonts.  So provide those.
+        FONTCONFIG_FILE = texFunctions.fontsConf;
+      };
+
+      
+    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 20;
+    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 50;
+    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 50;
+    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 40;
+    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 40;
+    rpm_opensuse103i386 = makeRPM_i686 (diskImages: diskImages.opensuse103i386) 40;
+
+    
+    deb_debian40i386 = makeDeb_i686 (diskImages: diskImages.debian40i386) 40;
+    deb_debian40x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian40x86_64) 40;
+    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 30;
+    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 30;
+    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 50;
+    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 50;
+    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 40;
+    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 40;
+
+
+  };
+
+
+  makeRPM_i686 = makeRPM "i686-linux";
+  makeRPM_x86_64 = makeRPM "x86_64-linux";
+
+  makeRPM = 
+    system: diskImageFun: prio:
+    { tarball ? jobs.tarball {}
+    }:
+
+    with import nixpkgs {inherit system;};
+
+    releaseTools.rpmBuild rec {
+      name = "nix-rpm-${diskImage.name}";
+      src = tarball;
+      diskImage = diskImageFun vmTools.diskImages;
+      memSize = 1024;
+      meta = { schedulingPriority = toString prio; };
+    };
+
+
+  makeDeb_i686 = makeDeb "i686-linux";
+  makeDeb_x86_64 = makeDeb "x86_64-linux";
+  
+  makeDeb =
+    system: diskImageFun: prio:
+    { tarball ? jobs.tarball {}
+    }:
+
+    with import nixpkgs {inherit system;};
+
+    releaseTools.debBuild {
+      name = "nix-deb";
+      src = tarball;
+      diskImage = diskImageFun vmTools.diskImages;
+      memSize = 1024;
+      meta = { schedulingPriority = toString prio; };
+      configureFlags = "--sysconfdir=/etc";
+    };
+
+
+in jobs

scripts/Makefile.am

@@ -1,24 +1,26 @@
 bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build \
-  nix-pack-closure nix-unpack-closure \
   nix-copy-closure 
 
 noinst_SCRIPTS = nix-profile.sh generate-patches.pl \
-  find-runtime-roots.pl build-remote.pl nix-reduce-build
+  find-runtime-roots.pl build-remote.pl nix-reduce-build \
+  copy-from-other-stores.pl nix-http-export.cgi
 
 nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm download-using-manifests.pl find-runtime-roots.pl
+install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
+	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
+	$(INSTALL_PROGRAM) copy-from-other-stores.pl $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 
 include ../substitute.mk
@@ -31,9 +33,10 @@ EXTRA_DIST = nix-collect-garbage.in \
   readconfig.pm.in \
   nix-build.in \
   download-using-manifests.pl.in \
+  copy-from-other-stores.pl.in \
   generate-patches.pl.in \
-  nix-pack-closure.in nix-unpack-closure.in \
   nix-copy-closure.in \
   find-runtime-roots.pl.in \
   build-remote.pl.in \
-  nix-reduce-build.in
+  nix-reduce-build.in \
+  nix-http-export.cgi.in 

scripts/build-remote.pl.in

@@ -3,6 +3,7 @@
 use strict;
 use Fcntl ':flock';
 use English '-no_match_vars';
+use IO::Handle;
 
 # General operation:
 #
@@ -23,16 +24,12 @@ use English '-no_match_vars';
 
 my $loadIncreased = 0;
 
-my $amWilling = shift @ARGV;
-my $localSystem = shift @ARGV;
-my $neededSystem = shift @ARGV;
-my $drvPath = shift @ARGV;
+my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV;
+$maxSilentTime = 0 unless defined $maxSilentTime;
 
 sub sendReply {
     my $reply = shift;
-    open OUT, ">&3" or die;
-    print OUT "$reply\n";
-    close OUT;
+    print STDERR "# $reply\n";
 }
 
 sub decline {
@@ -47,15 +44,11 @@ mkdir $currentLoad, 0777 or die unless -d $currentLoad;
 my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
 decline if !defined $conf || ! -e $conf;
 
-# Decline if the local system can do the build.
-decline if $amWilling && ($localSystem eq $neededSystem);
+my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
 
 
 # Otherwise find a willing remote machine.
-my %machines;
-my %systemTypes;
-my %sshKeys;
-my %maxJobs;
+my @machines;
 my %curJobs;
 
 
@@ -67,10 +60,12 @@ while (<CONF>) {
     s/\#.*$//g;
     next if /^\s*$/;
     /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die;
-    $machines{$1} = "";
-    $systemTypes{$1} = $2;
-    $sshKeys{$1} = $3;
-    $maxJobs{$1} = $4;
+    push @machines,
+        { hostName => $1
+        , systemType => $2
+        , sshKeys => $3
+        , maxJobs => $4
+        };
 }
 
 close CONF;
@@ -85,21 +80,26 @@ flock(MAINLOCK, LOCK_EX) or die;
 # Find a suitable system.
 my $rightType = 0;
 my $machine;
-LOOP: foreach my $cur (keys %machines) {
-    if ($neededSystem eq $systemTypes{$cur}) {
+my $slotLock;
+LOOP: foreach my $cur (@machines) {
+    if ($neededSystem eq $cur->{systemType}
+        || ($neededSystem eq "i686-linux" && $cur->{systemType} eq "x86_64-linux"))
+    {
         $rightType = 1;
 
         # We have a machine of the right type.  Try to get a lock on
         # one of the machine's lock files.
         my $slot = 0;
-        while ($slot < $maxJobs{$cur}) {
-            my $slotLock = "$currentLoad/$cur-$slot";
-            open SLOTLOCK, ">>$slotLock" or die;
-            if (flock(SLOTLOCK, LOCK_EX | LOCK_NB)) {
+        while ($slot < $cur->{maxJobs}) {
+            my $slotLockFn = "$currentLoad/" . $cur->{systemType} . "-" . $cur->{hostName} . "-$slot";
+            $slotLock = new IO::Handle;
+            open $slotLock, ">>$slotLockFn" or die;
+            if (flock($slotLock, LOCK_EX | LOCK_NB)) {
+                utime undef, undef, $slotLock;
                 $machine = $cur;
                 last LOOP;
-            }                
-            close SLOTLOCK;
+            }
+            close $slotLock;
             $slot++;
         }
     }
@@ -108,9 +108,11 @@ LOOP: foreach my $cur (keys %machines) {
 close MAINLOCK;
 
 
-# Didn't find one?
+# Didn't find one?  Then decline or postpone.
 if (!defined $machine) {
-    if ($rightType) {
+    # Postpone if we have a machine of the right type, except if the
+    # local system can and wants to do the build.
+    if ($rightType && !$canBuildLocally) {
         sendReply "postpone";
         exit 0;
     } else {
@@ -120,11 +122,8 @@ if (!defined $machine) {
 
 # Yes we did, accept.
 sendReply "accept";
-open IN, "<&4" or die;
-my $x = <IN>;
+my $x = <STDIN>;
 chomp $x;
-#print "got $x\n";  
-close IN;
 
 if ($x ne "okay") {
     exit 0;
@@ -132,7 +131,8 @@ if ($x ne "okay") {
 
 
 # Do the actual job.
-print "BUILDING REMOTE: $drvPath on $machine\n";
+my $hostName = $machine->{hostName};
+print STDERR "building `$drvPath' on `$hostName'\n";
 
 # Make sure that we don't get any SSH passphrase or host key popups -
 # if there is any problem it should fail, not do something
@@ -141,10 +141,10 @@ $ENV{"DISPLAY"} = "";
 $ENV{"SSH_PASSWORD_FILE="} = "";
 $ENV{"SSH_ASKPASS="} = "";
 
-my $sshOpts = "-i $sshKeys{$machine} -x";
+my $sshOpts = "-i " . $machine->{sshKeys} . " -x";
 
 # Hack to support Cygwin: if we login without a password, we don't
-# have exactly the same right as when we do.  This causes the
+# have exactly the same rights as when we do.  This causes the
 # Microsoft C compiler to fail with certain flags:
 #
 #   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676
@@ -154,8 +154,8 @@ my $sshOpts = "-i $sshKeys{$machine} -x";
 # password.  (It only calls this command when there is no controlling
 # terminal, but Nix ensures that is is the case.  When doing this
 # manually, use setsid(1).)
-if ($sshKeys{$machine} =~ /^password:/) {
-    my $passwordFile = $sshKeys{$machine};
+if ($machine->{sshKeys} =~ /^password:/) {
+    my $passwordFile = $machine->{sshKeys};
     $passwordFile =~ s/^password://;
     $sshOpts = "ssh -x";
     $ENV{"SSH_PASSWORD_FILE"} = $passwordFile;
@@ -173,36 +173,40 @@ $inputs =~ s/\n/ /g;
 my $outputs = `cat outputs`; die if ($? != 0);
 $outputs =~ s/\n/ /g;
 
-print "COPYING INPUTS...\n";
+print "copying inputs...\n";
 
 my $maybeSign = "";
 $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
 
-system("NIX_SSHOPTS=\"$sshOpts\" nix-copy-closure $machine $maybeSign $drvPath $inputs") == 0
-    or die "cannot copy inputs to $machine: $?";
+system("NIX_SSHOPTS=\"$sshOpts\" @bindir@/nix-copy-closure --gzip $hostName $maybeSign $drvPath $inputs") == 0
+    or die "cannot copy inputs to $hostName: $?";
 
-print "BUILDING...\n";
+print "building...\n";
 
-system("ssh $sshOpts $machine 'nix-store -rvvK $drvPath'") == 0
-    or die "remote build on $machine failed: $?";
+my $buildFlags = "--max-silent-time $maxSilentTime";
 
-print "REMOTE BUILD DONE: $drvPath on $machine\n";
+# `-tt' forces allocation of a pseudo-terminal.  This is required to
+# make the remote nix-store process receive a signal when the
+# connection dies.  Without it, the remote process might continue to
+# run indefinitely (that is, until it next tries to write to
+# stdout/stderr).
+if (system("ssh -tt $sshOpts $hostName 'nix-store --realise -K $buildFlags $drvPath > /dev/null'") != 0) {
+    # If we couldn't run ssh or there was an ssh problem (indicated by
+    # exit code 255), then we return exit code 1; otherwise we assume
+    # that the builder failed, which we indicated to Nix using exit
+    # code 100.  It's important to distinguish between the two because
+    # the first is a transient failure and the latter is permanent.
+    my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100;
+    print STDERR "build of `$drvPath' on `$hostName' failed with exit code $?\n";
+    exit $res;
+}
+
+print "build of `$drvPath' on `$hostName' succeeded\n";
 
 foreach my $output (split '\n', $outputs) {
     my $maybeSignRemote = "";
     $maybeSignRemote = "--sign" if $UID != 0;
     
-    system("ssh $sshOpts $machine 'nix-store --export $maybeSignRemote $output' > dump") == 0
-	or die "cannot copy $output from $machine: $?";
-
-    # This doesn't work yet, since the caller has a lock on the output
-    # path.  We should move towards lock-free invocation of build
-    # hooks and substitutes.
-    #system("nix-store --import < dump") == 0
-    #    or die "cannot import $output: $?";
-
-    # Hack: skip the first 8 bytes (the nix-store --export next
-    # archive marker).  The archive follows.
-    system("(dd bs=1 count=8 of=/dev/null && cat) < dump | nix-store --restore $output") == 0
-	or die "cannot restore $output: $?";
+    system("ssh $sshOpts $hostName 'nix-store --export $maybeSignRemote $output | gzip' | gunzip | @bindir@/nix-store --import > /dev/null") == 0
+	or die "cannot copy $output from $hostName: $?";
 }

scripts/copy-from-other-stores.pl.in

@@ -0,0 +1,98 @@
+#! @perl@ -w
+
+use strict;
+use File::Basename;
+use IO::Handle;
+
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
+
+STDOUT->autoflush(1);
+
+my @remoteStoresAll = split ':', ($ENV{"NIX_OTHER_STORES"} or "");
+
+my @remoteStores;
+foreach my $dir (@remoteStoresAll) {
+    push @remoteStores, glob($dir);
+}
+
+
+sub findStorePath {
+    my $storePath = shift;
+    
+    my $storePathName = basename $storePath;
+    
+    foreach my $store (@remoteStores) {
+        # Determine whether $storePath exists by looking for the
+        # existence of the info file, and if so, get store path info
+        # from that file.  This rather breaks abstraction: we should
+        # be using `nix-store' for that.  But right now there is no
+        # good way to tell nix-store to access a store mounted under a
+        # different location (there's $NIX_STORE, but that only works
+        # if the remote store is mounted under its "real" location).
+        my $infoFile = "$store/var/nix/db/info/$storePathName";
+        my $storePath2 = "$store/store/$storePathName";
+        if (-f $infoFile && -e $storePath2) {
+            return ($infoFile, $storePath2);
+        }
+    }
+}
+
+
+if ($ARGV[0] eq "--query") {
+
+    while (<STDIN>) {
+        my $cmd = $_; chomp $cmd;
+
+        if ($cmd eq "have") {
+            my $storePath = <STDIN>; chomp $storePath;
+            (my $infoFile) = findStorePath $storePath;
+            print STDOUT ($infoFile ? "1\n" : "0\n");
+        }
+
+        elsif ($cmd eq "info") {
+            my $storePath = <STDIN>; chomp $storePath;
+            (my $infoFile) = findStorePath $storePath;
+            if (!$infoFile) {
+                print "0\n";
+                next; # not an error
+            }
+            print "1\n";
+
+            my $deriver = "";
+            my @references = ();
+
+            open INFO, "<$infoFile" or die "cannot read info file $infoFile\n";
+            while (<INFO>) {
+                chomp;
+                /^([\w-]+): (.*)$/ or die "bad info file";
+                my $key = $1;
+                my $value = $2;
+                if ($key eq "Deriver") { $deriver = $value; }
+                elsif ($key eq "References") { @references = split ' ', $value; }
+            }
+            close INFO;
+
+            print "$deriver\n";
+            print scalar @references, "\n";
+            print "$_\n" foreach @references;
+            print "0\n"; # !!! showing size not supported (yet)
+        }
+
+        else { die "unknown command `$cmd'"; }
+    }
+}
+
+
+elsif ($ARGV[0] eq "--substitute") {
+    die unless scalar @ARGV == 2;
+    my $storePath = $ARGV[1];
+    (my $infoFile, my $sourcePath) = findStorePath $storePath;
+    die unless $infoFile;
+    print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
+    system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
+        or die "cannot copy `$sourcePath' to `$storePath'";
+}
+
+
+else { die; }

scripts/download-using-manifests.pl.in

@@ -5,20 +5,13 @@ use readmanifest;
 use POSIX qw(strftime);
 use File::Temp qw(tempdir);
 
-my $manifestDir = "@localstatedir@/nix/manifests";
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
+STDOUT->autoflush(1);
+
+my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests");
 my $logFile = "@localstatedir@/log/nix/downloads";
 
-open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
-
-# Create a temporary directory.
-my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-chdir $tmpDir or die "cannot change to `$tmpDir': $!";
-
-my $tmpNar = "$tmpDir/nar";
-my $tmpNar2 = "$tmpDir/nar2";
-
 
 # Load all manifests.
 my %narFiles;
@@ -26,44 +19,56 @@ my %localPaths;
 my %patches;
 
 for my $manifest (glob "$manifestDir/*.nixmanifest") {
-#    print STDERR "reading $manifest\n";
-    if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
+    my $version = readManifest($manifest, \%narFiles, \%localPaths, \%patches);
+    if ($version < 3) {
         print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
         exit 1;
     }
+    if ($version >= 10) {
+        print STDERR "manifest `$manifest' is too new; please delete it or upgrade Nix\n";
+        exit 1;
+    }
 }
 
 
 # Parse the arguments.
 
-if ($ARGV[0] eq "--query-paths") {
-    foreach my $storePath (keys %narFiles) { print "$storePath\n"; }
-    foreach my $storePath (keys %localPaths) { print "$storePath\n"; }
-    exit 0;
-}
+if ($ARGV[0] eq "--query") {
 
-elsif ($ARGV[0] eq "--query-info") {
-    shift @ARGV;
-    foreach my $storePath (@ARGV) {
-        my $info;
-        if (defined $narFiles{$storePath}) {
-            $info = @{$narFiles{$storePath}}[0];
+    while (<STDIN>) {
+        my $cmd = $_; chomp $cmd;
+
+        if ($cmd eq "have") {
+            my $storePath = <STDIN>; chomp $storePath;
+            print STDOUT ((defined $narFiles{$storePath} or defined $localPaths{$storePath})
+                ? "1\n" : "0\n");
         }
-        elsif (defined $localPaths{$storePath}) {
-            $info = @{$localPaths{$storePath}}[0];
-        }
-        else {
-            next; # not an error
-        }
-        print "$storePath\n";
-        print "$info->{deriver}\n";
-        my @references = split " ", $info->{references};
-        my $count = scalar @references;
-        print "$count\n";
-        foreach my $reference (@references) {
-            print "$reference\n";
+
+        elsif ($cmd eq "info") {
+            my $storePath = <STDIN>; chomp $storePath;
+            my $info;
+            if (defined $narFiles{$storePath}) {
+                $info = @{$narFiles{$storePath}}[0];
+            }
+            elsif (defined $localPaths{$storePath}) {
+                $info = @{$localPaths{$storePath}}[0];
+            }
+            else {
+                print "0\n";
+                next; # not an error
+            }
+            print "1\n";
+            print "$info->{deriver}\n";
+            my @references = split " ", $info->{references};
+            print scalar @references, "\n";
+            print "$_\n" foreach @references;
+            my $size = $info->{size} || 0;
+            print "$size\n";
         }
+        
+        else { die "unknown command `$cmd'"; }
     }
+
     exit 0;
 }
 
@@ -76,6 +81,18 @@ die unless scalar @ARGV == 2;
 my $targetPath = $ARGV[1];
 
 
+# Create a temporary directory.
+my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+    or die "cannot create a temporary directory";
+
+chdir $tmpDir or die "cannot change to `$tmpDir': $!";
+
+my $tmpNar = "$tmpDir/nar";
+my $tmpNar2 = "$tmpDir/nar2";
+
+
+open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
+
 my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
 print LOGFILE "$$ get $targetPath $date\n";
 
@@ -88,7 +105,7 @@ foreach my $localPath (@{$localPathList}) {
     my $sourcePath = $localPath->{copyFrom};
     if (-e $sourcePath) {
         print "\n*** Step 1/1: copying from $sourcePath\n";
-        system("@bindir@/nix-store --dump $sourcePath | @bindir@/nix-store --restore $targetPath") == 0
+        system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $targetPath") == 0
             or die "cannot copy `$sourcePath' to `$targetPath'";
         exit 0;
     }
@@ -137,7 +154,7 @@ addToQueue $targetPath;
 
 sub isValidPath {
     my $p = shift;
-    return system("@bindir@/nix-store --check-validity '$p' 2> /dev/null") == 0;
+    return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
 }
 
 sub parseHash {
@@ -171,15 +188,13 @@ while ($queueFront < scalar @queue) {
                 my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
                 my $format = "--base32";
                 $format = "" if $baseHashAlgo eq "md5";
-                my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
+                my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
                 chomp $hash;
-#                print "  MY HASH is $hash\n";
                 if ($hash ne $baseHash) {
                     print LOGFILE "$$ rejecting $patch->{basePath}\n";
                     next;
                 }
             }
-#            print "  PATCH from $patch->{basePath}\n";
             addToQueue $patch->{basePath};
             addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
         }
@@ -187,10 +202,12 @@ while ($queueFront < scalar @queue) {
         # Add NAR file edges to the start node.
         my $narFileList = $narFiles{$u};
         foreach my $narFile (@{$narFileList}) {
-#            print "  NAR from $narFile->{url}\n";
-            addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
+            # !!! how to handle files whose size is not known in advance?
+            # For now, assume some arbitrary size (1 MB).
+            addEdge "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
             if ($u eq $targetPath) {
-                print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
+                my $size = $narFile->{size} || -1;
+                print LOGFILE "$$ full-download-would-be $size\n";
             }
         }
 
@@ -216,8 +233,6 @@ while (scalar @todo > 0) {
 
     my $u_ = $graph{$u};
 
-#    print "IN $u $u_->{d}\n";
-
     foreach my $edge (@{$u_->{edges}}) {
         my $v_ = $graph{$edge->{end}};
         if ($v_->{d} > $u_->{d} + $edge->{weight}) {
@@ -225,7 +240,6 @@ while (scalar @todo > 0) {
             # Store the edge; to edge->start is actually the
             # predecessor.
             $v_->{pred} = $edge; 
-#            print "  RELAX $edge->{end} $v_->{d}\n";
         }
     }
 }
@@ -247,20 +261,18 @@ while ($cur ne "start") {
 my $curStep = 1;
 my $maxStep = scalar @path;
 
-sub downloadFile {
-    my $url = shift;
-    my ($hashAlgo, $hash) = parseHash(shift);
+sub downloadFile { 
+    my $url = shift; 
     $ENV{"PRINT_PATH"} = 1;
     $ENV{"QUIET"} = 1;
-    $ENV{"NIX_HASH_ALGO"} = $hashAlgo;
-    my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
+    my ($hash, $path) = `$binDir/nix-prefetch-url '$url'`;
     die "download of `$url' failed" unless $? == 0;
-    chomp $hash2;
     chomp $path;
-    die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
     return $path;
 }
 
+my $finalNarHash;
+
 while (scalar @path > 0) {
     my $edge = pop @path;
     my $u = $edge->{start};
@@ -277,7 +289,7 @@ while (scalar @path > 0) {
             # as a base to one or more patches.  So turn the base path
             # into a NAR archive, to which we can apply the patch.
             print "  packing base path...\n";
-            system("@bindir@/nix-store --dump $v > $tmpNar") == 0
+            system("$binDir/nix-store --dump $v > $tmpNar") == 0
                 or die "cannot dump `$v'";
         }
     }
@@ -290,7 +302,7 @@ while (scalar @path > 0) {
 
         # Download the patch.
         print "  downloading patch...\n";
-        my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
+        my $patchPath = downloadFile "$patch->{url}";
 
         # Apply the patch to the NAR archive produced in step 1 (for
         # the already present path) or a later step (for patch sequences).
@@ -305,20 +317,23 @@ while (scalar @path > 0) {
             # This was the last patch.  Unpack the final NAR archive
             # into the target path.
             print "  unpacking patched archive...\n";
-            system("@bindir@/nix-store --restore $v < $tmpNar2") == 0
+            system("$binDir/nix-store --restore $v < $tmpNar2") == 0
                 or die "cannot unpack $tmpNar2 into `$v'";
         }
+
+        $finalNarHash = $patch->{narHash};
     }
 
     elsif ($edge->{type} eq "narfile") {
         my $narFile = $edge->{info};
         print "downloading `$narFile->{url}' into `$v'\n";
 
-        print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
+        my $size = $narFile->{size} || -1;
+        print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
         
         # Download the archive.
         print "  downloading archive...\n";
-        my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
+        my $narFilePath = downloadFile "$narFile->{url}";
 
         if ($curStep < $maxStep) {
             # The archive will be used a base to a patch.
@@ -327,14 +342,39 @@ while (scalar @path > 0) {
         } else {
             # Unpack the archive into the target path.
             print "  unpacking archive...\n";
-            system("@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'") == 0
+            system("@bunzip2@ < '$narFilePath' | $binDir/nix-store --restore '$v'") == 0
                 or die "cannot unpack `$narFilePath' into `$v'";
         }
+
+        $finalNarHash = $narFile->{narHash};
     }
 
     $curStep++;
 }
 
 
+# Make sure that the hash declared in the manifest matches what we
+# downloaded and unpacked.
+
+if (defined $finalNarHash) {
+    my ($hashAlgo, $hash) = parseHash $finalNarHash;
+
+    # The hash in the manifest can be either in base-16 or base-32.
+    # Handle both.
+    my $extraFlag =
+        ($hashAlgo eq "sha256" && length($hash) != 64)
+        ? "--base32" : "";
+    
+    my $hash2 = `@bindir@/nix-hash --type $hashAlgo $extraFlag $targetPath`
+        or die "cannot compute hash of path `$targetPath'";
+    chomp $hash2;
+    
+    die "hash mismatch in downloaded path $targetPath; expected $hash, got $hash2"
+        if $hash ne $hash2;
+} else {
+    die "cannot check integrity of the downloaded path since its hash is not known";
+}
+
+
 print LOGFILE "$$ success\n";
 close LOGFILE;

scripts/nix-build.in

@@ -2,6 +2,8 @@
 
 use strict;
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 my $addDrvLink = 0;
 my $addOutLink = 1;
@@ -9,6 +11,8 @@ my $addOutLink = 1;
 my $outLink;
 my $drvLink;
 
+my $dryRun = 0;
+
 my @instArgs = ();
 my @buildArgs = ();
 my @exprs = ();
@@ -73,7 +77,7 @@ EOF
 
     elsif ($arg eq "--attr" or $arg eq "-A") {
         $n++;
-        die "$0: `--attr' requires an argument\n" unless $n < scalar @ARGV;
+        die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @instArgs, ("--attr", $ARGV[$n]);
     }
 
@@ -83,12 +87,31 @@ EOF
         $n += 2;
     }
 
-    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time") {
+    elsif ($arg eq "--log-type") {
+        $n++;
+        die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
+        push @instArgs, ($arg, $ARGV[$n]);
+        push @buildArgs, ($arg, $ARGV[$n]);
+    }
+
+    elsif ($arg eq "--option") {
+        die "$0: `$arg' requires two arguments\n" unless $n + 2 < scalar @ARGV;
+        push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]);
+        push @buildArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]);
+        $n += 2;
+    }
+
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @buildArgs, ($arg, $ARGV[$n]);
     }
 
+    elsif ($arg eq "--dry-run") {
+        push @buildArgs, "--dry-run";
+        $dryRun = 1;
+    }
+    
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -117,9 +140,12 @@ foreach my $expr (@exprs) {
     # Instantiate.
     my @drvPaths;
     # !!! would prefer the perl 5.8.0 pipe open feature here.
-    my $pid = open(DRVPATHS, "-|") || exec "@bindir@/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
+    my $pid = open(DRVPATHS, "-|") || exec "$binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
     while (<DRVPATHS>) {chomp; push @drvPaths, $_;}
-    close DRVPATHS or exit 1;
+    if (!close DRVPATHS) {
+        die "nix-instantiate killed by signal " . ($? & 127) . "\n" if ($? & 127);
+        exit 1;
+    }
 
     foreach my $drvPath (@drvPaths) {
         my $target = readlink $drvPath or die "cannot read symlink `$drvPath'";
@@ -128,10 +154,15 @@ foreach my $expr (@exprs) {
 
     # Build.
     my @outPaths;
-    $pid = open(OUTPATHS, "-|") || exec "@bindir@/nix-store", "--add-root", $outLink, "--indirect", "-rv",
+    $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-rv",
         @buildArgs, @drvPaths;
     while (<OUTPATHS>) {chomp; push @outPaths, $_;}
-    close OUTPATHS or exit 1;
+    if (!close OUTPATHS) {
+        die "nix-store killed by signal " . ($? & 127) . "\n" if ($? & 127);
+        exit 1;
+    }
+
+    next if $dryRun;
 
     foreach my $outPath (@outPaths) {
         my $target = readlink $outPath or die "cannot read symlink `$outPath'";

scripts/nix-channel.in

@@ -125,15 +125,13 @@ sub update {
 
     my $rootFile = "$rootsDir/per-user/$userName/channels";
     
-    # Instantiate the Nix expression.
+    # Build the Nix expression.
     print "unpacking channel Nix expressions...\n";
-    my $storeExpr = `@bindir@/nix-instantiate --add-root '$rootFile'.tmp @datadir@/nix/corepkgs/channels/unpack.nix --argstr system @system@ --arg inputs '$inputs'`
-        or die "cannot instantiate Nix expression";
-    chomp $storeExpr;
-
-    # Build the resulting derivation.
-    my $outPath = `@bindir@/nix-store --add-root '$rootFile' -r '$storeExpr'`
-        or die "cannot realise store expression";
+    my $outPath = `\\
+        @bindir@/nix-build --out-link '$rootFile' --drv-link '$rootFile'.tmp \\
+        @datadir@/nix/corepkgs/channels/unpack.nix \\
+        --argstr system @system@ --arg inputs '$inputs'`
+        or die "cannot unpack the channels";
     chomp $outPath;
 
     unlink "$rootFile.tmp";

scripts/nix-collect-garbage.in

@@ -4,6 +4,8 @@ use strict;
 
 my $profilesDir = "@localstatedir@/nix/profiles";
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 # Process the command line arguments.
 my @args = ();
@@ -34,7 +36,7 @@ sub removeOldGenerations {
         $name = $dir . "/" . $name;
         if (-l $name && (readlink($name) =~ /link/)) {
             print STDERR "removing old generations of profile $name\n";
-            system("@bindir@/nix-env", "-p", $name, "--delete-generations", "old");
+            system("$binDir/nix-env", "-p", $name, "--delete-generations", "old");
         }
         elsif (! -l $name && -d $name) {
             removeOldGenerations $name;
@@ -48,4 +50,4 @@ removeOldGenerations $profilesDir if $removeOld;
 
 
 # Run the actual garbage collector.
-exec "@bindir@/nix-store", "--gc", @args;
+exec "$binDir/nix-store", "--gc", @args;

scripts/nix-copy-closure.in

@@ -1,7 +1,6 @@
 #! @perl@ -w
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 
 if (scalar @ARGV < 1) {
@@ -56,30 +55,18 @@ while (@ARGV) {
 if ($toMode) { # Copy TO the remote machine.
 
     my @allStorePaths;
-    my %storePathsSeen;
 
-    foreach my $storePath (@storePaths) {
-        # $arg might be a symlink to the store, so resolve it.
-        my $storePath2 = (`$binDir/nix-store --query --resolve '$storePath'`
-            or die "cannot resolve `$storePath'");
-        chomp $storePath2;
-
-        # Get the closure of this path.
-        my $pid = open(READ,
-            "$binDir/nix-store --query --requisites '$storePath2'|") or die;
+    # Get the closure of this path.
+    my $pid = open(READ, "$binDir/nix-store --query --requisites @storePaths|") or die;
     
-        while (<READ>) {
-            chomp;
-            die "bad: $_" unless /^\//;
-            if (!defined $storePathsSeen{$_}) {
-                push @allStorePaths, $_;
-                $storePathsSeen{$_} = 1;
-            }
-        }
-
-        close READ or die "nix-store failed: $?";
+    while (<READ>) {
+        chomp;
+        die "bad: $_" unless /^\//;
+        push @allStorePaths, $_;
     }
 
+    close READ or die "nix-store failed: $?";
+
 
     # Ask the remote host which paths are invalid.
     open(READ, "ssh @sshOpts $sshHost nix-store --check-validity --print-invalid @allStorePaths|");
@@ -112,15 +99,11 @@ else { # Copy FROM the remote machine.
         "ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
     
     my @allStorePaths;
-    my %storePathsSeen;
 
     while (<READ>) {
         chomp;
         die "bad: $_" unless /^\//;
-        if (!defined $storePathsSeen{$_}) {
-            push @allStorePaths, $_;
-            $storePathsSeen{$_} = 1;
-        }
+        push @allStorePaths, $_;
     }
 
     close READ or die "nix-store on remote machine `$sshHost' failed: $?";

scripts/nix-http-export.cgi.in

@@ -0,0 +1,51 @@
+#! /bin/sh
+
+export HOME=/tmp
+export NIX_REMOTE=daemon
+
+TMP_DIR="${TMP_DIR:-/tmp/nix-export}"
+
+@coreutils@/mkdir -p "$TMP_DIR" || true
+@coreutils@/chmod a+r "$TMP_DIR"
+
+needed_path="?$QUERY_STRING"
+needed_path="${needed_path#*[?&]needed_path=}"
+needed_path="${needed_path%%&*}"
+#needed_path="$(echo $needed_path  | ./unhttp)"
+needed_path="${needed_path//%2B/+}"
+needed_path="${needed_path//%3D/=}"
+
+echo needed_path: "$needed_path" >&2
+
+NIX_STORE="${NIX_STORE_DIR:-/nix/store}"
+
+echo NIX_STORE: "${NIX_STORE}" >&2
+
+full_path="${NIX_STORE}"/"$needed_path"
+
+if [ "$needed_path" != "${needed_path%.drv}" ]; then
+	echo "Status: 403 You should create the derivation file yourself"
+	echo "Content-Type: text/plain"
+	echo
+	echo "Refusing to disclose derivation contents"
+	exit
+fi
+
+if @bindir@/nix-store --check-validity "$full_path"; then
+	if ! [ -e nix-export/"$needed_path".nar.gz ]; then
+		@bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz
+		@coreutils@/ln -fs  "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz 
+	fi;
+	echo "Status: 301 Moved"
+	echo "Location: nix-export/"$needed_path".nar.gz"
+	echo
+else 
+	echo "Status: 404 No such path found"
+	echo "Content-Type: text/plain"
+	echo
+	echo "Path not found:"
+	echo "$needed_path"
+	echo "checked:"
+	echo "$full_path"
+fi
+

scripts/nix-install-package.in

@@ -3,6 +3,8 @@
 use strict;
 use File::Temp qw(tempdir);
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 sub usageError {
     print STDERR <<EOF;
@@ -59,7 +61,7 @@ if ($interactive && !defined $ENV{"NIX_HAVE_TERMINAL"}) {
     $ENV{"NIX_HAVE_TERMINAL"} = "1";
     $ENV{"LD_LIBRARY_PATH"} = "";
     foreach my $term ("xterm", "konsole", "gnome-terminal", "xterm") {
-        exec($term, "-e", "@bindir@/nix-install-package", @ARGV);
+        exec($term, "-e", "$binDir/nix-install-package", @ARGV);
     }
     die "cannot execute `xterm'";
 }
@@ -121,13 +123,18 @@ if ($interactive) {
 }
 
 
+# Store the manifest in the temporary directory so that we don't
+# pollute /nix/var/nix/manifests.
+$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
+
+
 print "\nPulling manifests...\n";
-system("@bindir@/nix-pull", $manifestURL) == 0
+system("$binDir/nix-pull", $manifestURL) == 0
     or barf "nix-pull failed: $?";
 
 
 print "\nInstalling package...\n";
-system("@bindir@/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0
+system("$binDir/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0
     or barf "nix-env failed: $?";
 
 

scripts/nix-pack-closure.in

@@ -1,75 +0,0 @@
-#! @perl@ -w
-
-# This tool computes the closure of a path (using "nix-store --query
-# --requisites") and puts the contents of each path in the closure in
-# a big NAR archive that can be installed on another Nix installation
-# using "nix-unpack-closure".
-
-# TODO: make this program "streamy", i.e., don't use a temporary
-# directory.
-
-use strict;
-use File::Temp qw(tempdir);
-
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
-
-my $tmpDir = tempdir("nix-pack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-mkdir "$tmpDir/contents", 0755 or die;
-mkdir "$tmpDir/references", 0755 or die;
-mkdir "$tmpDir/derivers", 0755 or die;
-
-open TOPLEVEL, ">$tmpDir/top-level" or die;
-
-
-my %storePaths;
-
-
-while (@ARGV) {
-    my $storePath = shift @ARGV;
-
-    # $storePath might be a symlink to the store, so resolve it.
-    $storePath = (`$binDir/nix-store --query --resolve '$storePath'`
-        or die "cannot resolve `$storePath'");
-    chomp $storePath;
-    print TOPLEVEL $storePath, "\n";
-
-    # Get the closure of this path.
-    my $pid = open(READ,
-        "$binDir/nix-store --query --requisites '$storePath'|") or die;
-    
-    while (<READ>) {
-        chomp;
-        die "bad: $_" unless /^\//;
-        $storePaths{$_} = "";
-    }
-
-    close READ or die "nix-store failed: $?";
-}
-
-
-close TOPLEVEL or die;
-
-
-foreach my $storePath (sort(keys %storePaths)) {
-    print STDERR "packing `$storePath'...\n";
-
-    $storePath =~ /\/([^\/]+)$/;
-    my $name = $1;
-
-    system("$binDir/nix-store --dump '$storePath' > $tmpDir/contents/$name") == 0
-        or die "nix-store --dump failed on `$storePath': $?";
-
-    system("$binDir/nix-store --query --references '$storePath' > $tmpDir/references/$name") == 0
-        or die "nix-store --query --references failed on `$storePath': $?";
-
-    system("$binDir/nix-store --query --deriver '$storePath' > $tmpDir/derivers/$name") == 0
-        or die "nix-store --query --deriver failed on `$storePath': $?";
-}
-
-
-# Write a NAR archive of everything to standard output.
-system("nix-store --dump '$tmpDir'") == 0
-    or die "nix-store --dump failed";

scripts/nix-prefetch-url.in

@@ -37,6 +37,7 @@ fi
 
 
 mkTempDir() {
+    if test -n "$tmpPath"; then return; fi
     local i=0
     while true; do
         if test -z "$TMPDIR"; then TMPDIR=/tmp; fi
@@ -59,11 +60,32 @@ removeTempDir() {
 
 
 doDownload() {
-    @curl@ $cacheFlags --fail -# --location --max-redirs 20 --disable-epsv \
+    @curl@ $cacheFlags --fail --location --max-redirs 20 --disable-epsv \
         --cookie-jar $tmpPath/cookies "$url" -o $tmpFile
 }
 
 
+# Hack to support the mirror:// scheme from Nixpkgs.
+if test "${url:0:9}" = "mirror://"; then
+    if test -z "$NIXPKGS_ALL"; then
+        echo "Resolving mirror:// URLs requires Nixpkgs.  Please point \$NIXPKGS_ALL at a Nixpkgs tree." >&2
+        exit 1
+    fi
+
+    mkTempDir
+    nix-build "$NIXPKGS_ALL" -A resolveMirrorURLs --argstr url "$url" -o $tmpPath/urls > /dev/null
+
+    expanded=($(cat $tmpPath/urls))
+    if test "${#expanded[*]}" = 0; then
+        echo "$0: cannot resolve $url." >&2
+        exit 1
+    fi
+
+    echo "$url expands to ${expanded[*]} (using ${expanded[0]})" >&2
+    url="${expanded[0]}"
+fi
+
+
 # If we don't know the hash or a file with that hash doesn't exist,
 # download the file and add it to the store.
 if test -z "$finalPath"; then
@@ -78,7 +100,7 @@ if test -z "$finalPath"; then
     # garbage-collected independently.
     if test -n "$NIX_DOWNLOAD_CACHE"; then
         echo -n "$url" > $tmpPath/url
-        urlHash=$(nix-hash --type sha256 --base32 --flat $tmpPath/url)
+        urlHash=$(@bindir@/nix-hash --type sha256 --base32 --flat $tmpPath/url)
         echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url"
         cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType"
         cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp"

scripts/nix-profile.sh.in

@@ -7,5 +7,12 @@ if test -n "$HOME"; then
         @coreutils@/ln -s "$_NIX_DEF_LINK" "$NIX_LINK"
     fi
 
-    export PATH=$NIX_LINK/bin:@prefix@/bin:$PATH
+    export PATH=$NIX_LINK/bin:$PATH
+fi
+
+# This is a quick hack to make fontconfig-based packages in Nixpkgs
+# work out of the box on non-NixOS systems.  Of course, we should
+# really fix fontconfig...
+if test -z "$FONTCONFIG_FILE" -a -e /etc/fonts/fonts.conf; then
+    export FONTCONFIG_FILE=/etc/fonts/fonts.conf
 fi

scripts/nix-pull.in

@@ -7,17 +7,11 @@ use readmanifest;
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
-
-my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
-$libexecDir = "@libexecdir@" unless defined $libexecDir;
-
-my $stateDir = $ENV{"NIX_STATE_DIR"};
-$stateDir = "@localstatedir@/nix" unless defined $stateDir;
-
-my $storeDir = $ENV{"NIX_STORE_DIR"};
-$storeDir = "@storedir@" unless defined $storeDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+my $libexecDir = ($ENV{"NIX_LIBEXEC_DIR"} or "@libexecdir@");
+my $storeDir = ($ENV{"NIX_STORE_DIR"} or "@storedir@");
+my $stateDir = ($ENV{"NIX_STATE_DIR"} or "@localstatedir@/nix");
+my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "$stateDir/manifests");
 
 
 # Prevent access problems in shared-stored installations.
@@ -69,10 +63,11 @@ sub processURL {
         print "obtaining list of Nix archives at `$url'...\n";
         $manifest = downloadFile $url;
     }
+
+    my $version = readManifest($manifest, \%narFiles, \%localPaths, \%patches);
     
-    if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
-        die "`$url' is not manifest or it is too old (i.e., for Nix <= 0.7)\n";
-    }
+    die "`$url' is not a manifest or it is too old (i.e., for Nix <= 0.7)\n" if $version < 3;
+    die "manifest `$url' is too new\n" if $version >= 5;
 
     if ($skipWrongStore) {
         foreach my $path (keys %narFiles) {
@@ -92,7 +87,7 @@ sub processURL {
         or die "cannot hash `$manifest'";
     chomp $hash;
     
-    my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
+    my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest";
     
     system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
         or die "cannot link `$finalPath to `$manifest'";

scripts/nix-push.in

@@ -16,8 +16,7 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 my $dataDir = $ENV{"NIX_DATA_DIR"};
 $dataDir = "@datadir@" unless defined $dataDir;
@@ -107,7 +106,7 @@ foreach my $storePath (@storePaths) {
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
         "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
-        "{storePath = builtins.toPath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
+        "{storePath = builtins.storePath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
     
     print NIX $nixexpr;
 }
@@ -129,7 +128,7 @@ while (<READ>) {
 close READ or die "nix-instantiate failed: $?";
 
 
-# Realise the store expressions.
+# Build the derivations.
 print STDERR "creating archives...\n";
 
 my @narPaths;
@@ -141,12 +140,7 @@ while (scalar @tmp > 0) {
     my @tmp2 = @tmp[0..$n - 1];
     @tmp = @tmp[$n..scalar @tmp - 1];
 
-    # Note: we disable build hooks because of the impure path
-    # reference (see above).  Even if that is fixed, using a hook
-    # probably wouldn't make that much sense; pumping lots of data
-    # around just to compress them won't gain that much.
-    $ENV{"NIX_BUILD_HOOK"} = "";
-    my $pid = open(READ, "$binDir/nix-store --no-build-hook --realise @tmp2|")
+    my $pid = open(READ, "$binDir/nix-store --realise @tmp2|")
         or die "cannot run nix-store";
     while (<READ>) {
         chomp;
@@ -224,13 +218,14 @@ writeManifest $manifest, \%narFiles, \%patches;
 sub copyFile {
     my $src = shift;
     my $dst = shift;
-    system("@coreutils@/cp", $src, "$dst.tmp") == 0 or die "cannot copy file";
-    rename("$dst.tmp", "$dst") or die "cannot rename file";
+    my $tmp = "$dst.tmp.$$";
+    system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file";
+    rename($tmp, $dst) or die "cannot rename file: $!";
 }
 
 
-# Upload the archives.
-print STDERR "uploading archives...\n";
+# Upload/copy the archives.
+print STDERR "uploading/copying archives...\n";
 
 sub archiveExists {
     my $name = shift;
@@ -244,9 +239,12 @@ foreach my $narArchive (@narArchives) {
     my $basename = $1;
 
     if ($localCopy) {
+        # Since nix-push creates $dst atomically, if it exists we
+        # don't have to copy again.
+        my $dst = "$localArchivesDir/$basename";
         if (! -f "$localArchivesDir/$basename") {
             print STDERR "  $narArchive\n";
-            copyFile $narArchive, "$localArchivesDir/$basename";
+            copyFile $narArchive, $dst;
         }
     }
     else {

scripts/nix-reduce-build.in

@@ -3,10 +3,30 @@
 WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX);
 cd "$WORKING_DIRECTORY";
 
-if test -z "$1" ; then
-	echo 'nix-reduce-build (paths or Nix expressions) -- (logins at remote computers)' >&2
+if test -z "$1" || test "a--help" = "a$1" ; then
+	echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2
 	echo As in: >&2
-	echo nix-reduce-build /etc/nixos/nixos -- user@somewhere.nowhere.example.org >&2
+	echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2
+	echo nix-reduce-build /etc/nixos/nixos -- \\
+	echo "   " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2
+	echo "  store path name will be added into the end of the URL" >&2
+	echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2
+	echo "  that should be a directory where gzipped 'nix-store --export' ">&2
+	echo "  files are located (they should have .nar.gz extension)"  >&2
+	echo "        Or all together: " >&2
+	echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\
+	"    ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2
+	echo "        Also supports best-effort local builds of failing expression set:" >&2
+	echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2
+	echo "  nix-daemon:// builds using daemon"
+	echo "  nix-self:// builds directly using nix-store from current installation" >&2
+	echo "  nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2;
+	echo "derivations with specified output hash (sha256, sha1 or md5)." >&2
+	echo "  nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2;
+	echo "maximum amount of paths" >&2;
+	echo "  nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2;
+	echo "maximum amount of paths" >&2;
+	echo "        If no package sources are specified, required paths are listed." >&2;
 	exit;
 fi;
 
@@ -19,11 +39,14 @@ echo Will work on $(cat initial | wc -l) targets. >&2
 
 while read ; do
 	case "$REPLY" in 
-		${NIX_STORE_PATH:-/nix/store}/*)
+		${NIX_STORE_DIR:-/nix/store}/*)
 			echo "$REPLY" >> paths; >&2
 			;;
 		*)
-			nix-instantiate "$REPLY" >> paths;
+			(
+				IFS=: ;
+				nix-instantiate $REPLY >> paths;
+			);
 			;;
 	esac;
 done < initial;
@@ -51,14 +74,94 @@ echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2
 cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
 echo We need $(cat needed-paths | wc -l) paths. >&2
 
+egrep '[.]drv$' derivers-closure > critical-derivers;
+
 if test -z "$1" ; then
 	cat needed-paths;	
 fi;
 
+refresh_critical_derivers() {
+    echo "Finding needed derivers..." >&2;
+    cat critical-derivers | while read; do
+        if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then
+            echo "$REPLY";
+        fi;
+    done > new-critical-derivers;
+    mv new-critical-derivers critical-derivers;
+    echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2
+}
+
+build_here() {
+    cat critical-derivers | while read; do 
+        echo "Realising $REPLY using nix-daemon" >&2
+        @bindir@/nix-store -r "${REPLY}"
+    done;
+}
+
+try_to_substitute(){
+    cat needed-paths | while read ; do 
+        echo "Building $REPLY using nix-daemon" >&2
+        @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}"
+    done;
+}
+
 for i in "$@"; do 
-	cat needed-paths | while read; do 
-		nix-copy-closure --from "$i" --gzip "$REPLY" </dev/null || true; 
-	done;
+	sshHost="${i#ssh://}";
+	httpHost="${i#http://}";
+	httpsHost="${i#https://}";
+	filePath="${i#file:/}";
+	if [ "$i" != "$sshHost" ]; then
+		cat needed-paths | while read; do 
+			echo "Getting $REPLY and its closure over ssh" >&2
+			nix-copy-closure --from "$sshHost" --gzip "$REPLY" </dev/null || true; 
+		done;
+	elif [ "$i" != "$httpHost" ] || [ "$i" != "$httpsHost" ]; then
+		cat needed-paths | while read; do
+			echo "Getting $REPLY over http/https" >&2
+			curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import;
+		done;
+	elif [ "$i" != "$filePath" ] ; then
+		cat needed-paths | while read; do 
+			echo "Installing $REPLY from file" >&2
+			gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import;
+		done;
+	elif [ "$i" = "nix-daemon://" ] ; then
+		NIX_REMOTE=daemon try_to_substitute;
+		refresh_critical_derivers;
+		NIX_REMOTE=daemon build_here;
+	elif [ "$i" = "nix-self://" ] ; then
+		NIX_REMOTE= try_to_substitute;
+		refresh_critical_derivers;
+		NIX_REMOTE= build_here;
+	elif [ "$i" = "nix-daemon-fixed://" ] ; then
+		refresh_critical_derivers;
+
+		cat critical-derivers | while read; do 
+			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
+				echo "Realising $REPLY using nix-daemon" >&2
+				NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}"
+			fi;
+		done;
+	elif [ "$i" = "nix-self-fixed://" ] ; then
+		refresh_critical_derivers;
+
+		cat critical-derivers | while read; do 
+			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
+				echo "Realising $REPLY using direct Nix build" >&2
+				NIX_REMOTE= @bindir@/nix-store -r "${REPLY}"
+			fi;
+		done;
+	elif [ "$i" = "nix-daemon-substitute://" ] ; then
+		NIX_REMOTE=daemon try_to_substitute;
+	elif [ "$i" = "nix-self-substitute://" ] ; then
+		NIX_REMOTE= try_to_substitute;
+	elif [ "$i" = "nix-daemon-build://" ] ; then
+		refresh_critical_derivers;
+		NIX_REMOTE=daemon build_here;
+	elif [ "$i" = "nix-self-build://" ] ; then
+		refresh_critical_derivers;
+		NIX_REMOTE= build_here;
+	fi;
 	mv needed-paths wanted-paths;
 	cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
 	echo We still need $(cat needed-paths | wc -l) paths. >&2

scripts/nix-unpack-closure.in

@@ -1,88 +0,0 @@
-#! @perl@ -w
-
-# This tool unpacks the closures created by "nix-pack-closure" and
-# adds them to the Nix store.
-
-# TODO: make this program "streamy", i.e., don't use a temporary
-# directory.
-
-use strict;
-use File::Temp qw(tempdir);
-
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
-
-my $tmpDir = tempdir("nix-unpack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-
-# Unpack the NAR archive on standard input.
-system("nix-store --restore '$tmpDir/unpacked'") == 0
-    or die "nix-store --restore failed";
-
-
-open VALID, ">$tmpDir/validity" or die;
-
-
-# For each path in the closure that is not yet valid, add it to the
-# store.  TODO: use proper locking.  Or even better, let nix-store do
-# this.
-opendir(DIR, "$tmpDir/unpacked/contents") or die "cannot open directory: $!";
-
-foreach my $name (sort(readdir DIR)) {
-    next if $name eq "." or $name eq "..";
-
-    my $storePath = "@storedir@/$name"; # !!!
-
-    # !!! this really isn't a good validity check!
-    system "$binDir/nix-store --check-validity '$storePath' 2> /dev/null";
-    if ($? != 0) {
-        print STDERR "unpacking `$storePath'...\n";
-
-        # !!! race
-        system("@coreutils@/rm -rf '$storePath'") == 0
-            or die "cannot remove `$storePath': $?";
-
-        system("$binDir/nix-store --restore '$storePath' < '$tmpDir/unpacked/contents/$name'") == 0
-            or die "nix-store --dump failed on `$storePath': $?";
-        
-        print VALID "$storePath\n";
-
-        open DRV, "<$tmpDir/unpacked/derivers/$name" or die;
-        my $deriver = <DRV>;
-        chomp $deriver;
-        $deriver = "" if $deriver eq "unknown-deriver";
-        close DRV;
-
-        my @refs;
-        open REFS, "<$tmpDir/unpacked/references/$name" or die;
-        while (<REFS>) {
-            chomp;
-            push @refs, $_;
-        }
-        close REFS;
-
-        print VALID "$deriver\n";
-        
-        print VALID (scalar @refs), "\n";
-        foreach my $ref (@refs) {
-            print VALID "$ref\n";
-        }
-    }
-}
-
-closedir(DIR) or die;
-
-
-# Register the invalid paths as valid.
-system("nix-store --register-validity <'$tmpDir/validity'") == 0
-    or die "nix-store --register-validity failed";
-
-
-# Show the top-level paths so that something useful can be done with
-# them, e.g., passing them to `nix-env -i'.
-if (-e "$tmpDir/unpacked/top-level") {
-    open TOPLEVEL, "<$tmpDir/unpacked/top-level" or die;
-    while (<TOPLEVEL>) { print "$_"; }
-    close TOPLEVEL;
-}

scripts/readmanifest.pm.in

@@ -2,10 +2,7 @@ use strict;
 
 
 sub addPatch {
-    my $patches = shift;
-    my $storePath = shift;
-    my $patch = shift;
-    my $allowConflicts = shift;
+    my ($patches, $storePath, $patch) = @_;
 
     $$patches{$storePath} = []
         unless defined $$patches{$storePath};
@@ -14,15 +11,9 @@ sub addPatch {
 
     my $found = 0;
     foreach my $patch2 (@{$patchList}) {
-        if ($patch2->{url} eq $patch->{url}) {
-            if ($patch2->{hash} eq $patch->{hash}) {
-                $found = 1 if ($patch2->{basePath} eq $patch->{basePath});
-            } else {
-                die "conflicting hashes for URL $patch->{url}, " .
-                    "namely $patch2->{hash} and $patch->{hash}"
-                    unless $allowConflicts;
-            }
-        }
+        $found = 1 if
+            $patch2->{url} eq $patch->{url} &&
+            $patch2->{basePath} eq $patch->{basePath};
     }
     
     push @{$patchList}, $patch if !$found;
@@ -32,12 +23,7 @@ sub addPatch {
 
 
 sub readManifest {
-    my $manifest = shift;
-    my $narFiles = shift;
-    my $localPaths = shift;
-    my $patches = shift;
-    my $allowConflicts = shift;
-    $allowConflicts = 0 unless defined $allowConflicts;
+    my ($manifest, $narFiles, $localPaths, $patches) = @_;
 
     open MANIFEST, "<$manifest"
         or die "cannot open `$manifest': $!";
@@ -98,15 +84,7 @@ sub readManifest {
 
                     my $found = 0;
                     foreach my $narFile (@{$narFileList}) {
-                        if ($narFile->{url} eq $url) {
-                            if ($narFile->{hash} eq $hash) {
-                                $found = 1;
-                            } else {
-                                die "conflicting hashes for URL $url, " .
-                                    "namely $narFile->{hash} and $hash"
-                                    unless $allowConflicts;
-                            }
-                        }
+                        $found = 1 if $narFile->{url} eq $url;
                     }
                     if (!$found) {
                         push @{$narFileList},
@@ -124,7 +102,7 @@ sub readManifest {
                         , basePath => $basePath, baseHash => $baseHash
                         , narHash => $narHash, patchType => $patchType
                         , hashAlgo => $hashAlgo
-                        }, $allowConflicts;
+                        };
                 }
 
                 elsif ($type eq "localPath") {
@@ -171,12 +149,8 @@ sub readManifest {
 }
 
 
-sub writeManifest
-{
-    my $manifest = shift;
-    my $narFiles = shift;
-    my $patches = shift;
-    my $copySources = shift;
+sub writeManifest {
+    my ($manifest, $narFiles, $patches) = @_;
 
     open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
 
@@ -190,9 +164,9 @@ sub writeManifest
             print MANIFEST "{\n";
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $narFile->{url}\n";
-            print MANIFEST "  Hash: $narFile->{hash}\n";
+            print MANIFEST "  Hash: $narFile->{hash}\n" if defined $narFile->{hash};
             print MANIFEST "  NarHash: $narFile->{narHash}\n";
-            print MANIFEST "  Size: $narFile->{size}\n";
+            print MANIFEST "  Size: $narFile->{size}\n" if defined $narFile->{size};
             print MANIFEST "  References: $narFile->{references}\n"
                 if defined $narFile->{references} && $narFile->{references} ne "";
             print MANIFEST "  Deriver: $narFile->{deriver}\n"

src/boost/Makefile.am

@@ -1,6 +1,6 @@
 SUBDIRS = format
 
-pkginclude_HEADERS = assert.hpp checked_delete.hpp format.hpp \
+nobase_pkginclude_HEADERS = assert.hpp checked_delete.hpp format.hpp \
  shared_ptr.hpp weak_ptr.hpp throw_exception.hpp \
  enable_shared_from_this.hpp \
  detail/shared_count.hpp detail/workaround.hpp

src/libexpr/Makefile.am

@@ -2,11 +2,13 @@ pkglib_LTLIBRARIES = libexpr.la
 
 libexpr_la_SOURCES = \
  nixexpr.cc eval.cc primops.cc lexer-tab.cc parser-tab.cc \
- get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc
+ get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc \
+ names.cc normal-forms.cc
 
 pkginclude_HEADERS = \
  nixexpr.hh eval.hh parser.hh lexer-tab.hh parser-tab.hh \
- get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh
+ get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh \
+ names.hh nixexpr-ast.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
  ../boost/format/libformat.la

src/libexpr/eval.cc

@@ -16,13 +16,23 @@ namespace nix {
     
 
 EvalState::EvalState()
-    : normalForms(32768), primOps(128)
+    : sessionNormalForms(32768), normalForms(32768), primOps(128)
 {
-    nrEvaluated = nrCached = 0;
+    nrEvaluated = nrCached = nrDepthAfterReset = 0;
 
     initNixExprHelpers();
 
     addPrimOps();
+
+    allowUnsafeEquality = getEnv("NIX_NO_UNSAFE_EQ", "") == "";
+    safeCache = true;
+
+    loadNormalForms();
+}
+
+EvalState::~EvalState()
+{
+    saveNormalForms();
 }
 
 
@@ -48,6 +58,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
     throw EvalError(format(s) % s2);
 }
 
+LocalNoInlineNoReturn(void throwTypeError(const char * s))
+{
+    throw TypeError(s);
+}
+
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s2))
 {
     throw TypeError(format(s) % s2);
@@ -69,77 +84,91 @@ LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2,
 }
 
 
+/* Pattern-match `pat' against `arg'.  The result is a set of
+   substitutions (`subs') and a set of recursive substitutions
+   (`subsRecursive').  The latter can refer to the variables bound by
+   both `subs' and `subsRecursive'. */
+static void patternMatch(EvalState & state,
+    Pattern pat, Expr arg, ATermMap & subs, ATermMap & subsRecursive)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    
+    if (matchVarPat(pat, name)) 
+        subs.set(name, arg);
+
+    else if (matchAttrsPat(pat, formals, ellipsis)) {
+
+        arg = evalExpr(state, arg);
+
+        /* Get the actual arguments. */
+        ATermMap attrs;
+        queryAllAttrs(arg, attrs);
+        unsigned int nrAttrs = attrs.size();
+
+        /* For each formal argument, get the actual argument.  If
+           there is no matching actual argument but the formal
+           argument has a default, use the default. */
+        unsigned int attrsUsed = 0;
+        for (ATermIterator i(formals); i; ++i) {
+            Expr name, def;
+            DefaultValue def2;
+            if (!matchFormal(*i, name, def2)) abort(); /* can't happen */
+
+            Expr value = attrs[name];
+
+            if (value == 0) {
+                if (!matchDefaultValue(def2, def)) def = 0;
+                if (def == 0) throw TypeError(format("the argument named `%1%' required by the function is missing")
+                    % aterm2String(name));
+                subsRecursive.set(name, def);
+            } else {
+                attrsUsed++;
+                attrs.remove(name);
+                subs.set(name, value);
+            }
+
+        }
+
+        /* Check that each actual argument is listed as a formal
+           argument (unless the attribute match specifies a `...'). */
+        if (ellipsis == eFalse && attrsUsed != nrAttrs)
+            throw TypeError(format("the function does not expect an argument named `%1%'")
+                % aterm2String(attrs.begin()->key));
+    }
+
+    else if (matchAtPat(pat, pat1, pat2)) {
+        patternMatch(state, pat1, arg, subs, subsRecursive);
+        patternMatch(state, pat2, arg, subs, subsRecursive);
+    }
+
+    else abort();
+}
+
+
 /* Substitute an argument set into the body of a function. */
 static Expr substArgs(EvalState & state,
-    Expr body, ATermList formals, Expr arg)
+    Expr body, Pattern pat, Expr arg)
 {
-    unsigned int nrFormals = ATgetLength(formals);
-    ATermMap subs(nrFormals);
-
-    /* Get the actual arguments and put them in the substitution. */
-    ATermMap args;
-    queryAllAttrs(arg, args);
-    for (ATermMap::const_iterator i = args.begin(); i != args.end(); ++i)
-        subs.set(i->key, i->value);
+    ATermMap subs(16), subsRecursive(16);
     
-    /* Get the formal arguments. */
-    ATermVector defsUsed;
-    ATermList recAttrs = ATempty;
-    for (ATermIterator i(formals); i; ++i) {
-        Expr name, def;
-        ValidValues valids2;
-        DefaultValue def2;
-        if (!matchFormal(*i, name, valids2, def2)) abort(); /* can't happen */
+    patternMatch(state, pat, arg, subs, subsRecursive);
 
-        Expr value = subs[name];
-        
-        if (value == 0) {
-            if (!matchDefaultValue(def2, def)) def = 0;
-            if (def == 0) throw TypeError(format("the argument named `%1%' required by the function is missing")
-                % aterm2String(name));
-            value = def;
-            defsUsed.push_back(name);
-            recAttrs = ATinsert(recAttrs, makeBind(name, def, makeNoPos()));
-        }
-
-        ATermList valids;
-        if (matchValidValues(valids2, valids)) {
-            value = evalExpr(state, value);
-            bool found = false;
-            for (ATermIterator j(valids); j; ++j) {
-                Expr v = evalExpr(state, *j);
-                if (value == v) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found) throw TypeError(format("the argument named `%1%' has an illegal value")
-                % aterm2String(name));            
-        }
-    }
-
-    /* Make a recursive attribute set out of the (argument-name,
-       value) tuples.  This is so that we can support default
-       parameters that refer to each other, e.g.  ({x, y ? x + x}: y)
-       {x = "foo";} evaluates to "foofoo". */
-    if (defsUsed.size() != 0) {
-        for (ATermMap::const_iterator i = args.begin(); i != args.end(); ++i)
+    /* If we used any default values, make a recursive attribute set
+       out of the (argument-name, value) tuples.  This is so that we
+       can support default values that refer to each other, e.g.  ({x,
+       y ? x + x}: y) {x = "foo";} evaluates to "foofoo". */
+    if (subsRecursive.size() != 0) {
+        ATermList recAttrs = ATempty;
+        foreach (ATermMap::const_iterator, i, subs)
+            recAttrs = ATinsert(recAttrs, makeBind(i->key, i->value, makeNoPos()));
+        foreach (ATermMap::const_iterator, i, subsRecursive)
             recAttrs = ATinsert(recAttrs, makeBind(i->key, i->value, makeNoPos()));
         Expr rec = makeRec(recAttrs, ATempty);
-        for (ATermVector::iterator i = defsUsed.begin(); i != defsUsed.end(); ++i)
-            subs.set(*i, makeSelect(rec, *i));
-    }
-    
-    if (subs.size() != nrFormals) {
-        /* One or more actual arguments were not declared as formal
-           arguments.  Find out which. */
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            subs.remove(name);
-        }
-        throw TypeError(format("the function does not expect an argument named `%1%'")
-            % aterm2String(subs.begin()->key));
+        foreach (ATermMap::const_iterator, i, subsRecursive)
+            subs.set(i->key, makeSelect(rec, i->key));
     }
 
     return substitute(Substitution(0, &subs), body);
@@ -151,11 +180,12 @@ static Expr substArgs(EvalState & state,
    to attributes substituted with selection expressions on the
    original set.  E.g., e = `rec {x = f x y; y = x;}' becomes `{x = f
    (e.x) (e.y); y = e.x;}'. */
-LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
+LocalNoInline(ATerm expandRec(EvalState & state, ATerm e, ATermList rbnds, ATermList nrbnds))
 {
     ATerm name;
     Expr e2;
     Pos pos;
+    Expr eOverrides = 0;
 
     /* Create the substitution list. */
     ATermMap subs(ATgetLength(rbnds) + ATgetLength(nrbnds));
@@ -165,9 +195,26 @@ LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
     }
     for (ATermIterator i(nrbnds); i; ++i) {
         if (!matchBind(*i, name, e2, pos)) abort(); /* can't happen */
+        if (name == sOverrides) eOverrides = e2;
         subs.set(name, e2);
     }
 
+    /* If the rec contains an attribute called `__overrides', then
+       evaluate it, and add the attributes in that set to the rec.
+       This allows overriding of recursive attributes, which is
+       otherwise not possible.  (You can use the // operator to
+       replace an attribute, but other attributes in the rec will
+       still reference the original value, because that value has been
+       substituted into the bodies of the other attributes.  Hence we
+       need __overrides.) */
+    ATermMap overrides;
+    if (eOverrides) {
+        eOverrides = evalExpr(state, eOverrides);
+        queryAllAttrs(eOverrides, overrides, false);
+        foreach (ATermMap::const_iterator, i, overrides)
+            subs.set(i->key, i->value);
+    }
+
     Substitution subs_(0, &subs);
 
     /* Create the non-recursive set. */
@@ -177,6 +224,10 @@ LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
         as.set(name, makeAttrRHS(substitute(subs_, e2), pos));
     }
 
+    if (eOverrides)
+        foreach (ATermMap::const_iterator, i, overrides)
+            as.set(i->key, makeAttrRHS(i->value, makeNoPos()));
+
     /* Copy the non-recursive bindings.  !!! inefficient */
     for (ATermIterator i(nrbnds); i; ++i) {
         if (!matchBind(*i, name, e2, pos)) abort(); /* can't happen */
@@ -305,9 +356,11 @@ string coerceToString(EvalState & state, Expr e, PathSet & context,
     }
         
     ATermList es;
-    if (matchAttrs(e, es))
-        return coerceToString(state, makeSelect(e, toATerm("outPath")),
-            context, coerceMore, copyToStore);
+    if (matchAttrs(e, es)) {
+        Expr e2 = queryAttr(e, "outPath");
+        if (!e2) throwTypeError("cannot coerce an attribute set (except a derivation) to a string");
+        return coerceToString(state, e2, context, coerceMore, copyToStore);
+    }
 
     if (coerceMore) {
 
@@ -379,15 +432,18 @@ Path coerceToPath(EvalState & state, Expr e, PathSet & context)
 
 Expr autoCallFunction(Expr e, const ATermMap & args)
 {
-    ATermList formals;
+    Pattern pat;
     ATerm body, pos;
+    ATermList formals;
+    ATermBool ellipsis;
     
-    if (matchFunction(e, formals, body, pos)) {
+    /* !!! this should be more general */
+    if (matchFunction(e, pat, body, pos) && matchAttrsPat(pat, formals, ellipsis)) {
         ATermMap actualArgs(ATgetLength(formals));
         
         for (ATermIterator i(formals); i; ++i) {
-            Expr name, def, value; ATerm values, def2;
-            if (!matchFormal(*i, name, values, def2)) abort();
+            Expr name, def, value; ATerm def2;
+            if (!matchFormal(*i, name, def2)) abort();
             if ((value = args.get(name)))
                 actualArgs.set(name, makeAttrRHS(value, makeNoPos()));
             else if (!matchDefaultValue(def2, def))
@@ -420,15 +476,17 @@ LocalNoInline(Expr evalVar(EvalState & state, ATerm name))
     if (arity == 0)
         /* !!! backtrace for primop call */
         return ((PrimOp) ATgetBlobData(fun)) (state, ATermVector());
-    else
+    else {
+        state.safeCache = false;
         return makePrimOp(arity, fun, ATempty);
+    }
 }
 
 
 LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
 {
-    ATermList formals;
-    ATerm pos, name;
+    Pattern pat;
+    ATerm pos;
     Expr body;
         
     /* Evaluate the left-hand side. */
@@ -447,6 +505,7 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
             for (ATermIterator i(args); i; ++i)
                 args2[--arity] = *i;
             /* !!! backtrace for primop call */
+            state.safeCache = true;
             return ((PrimOp) ATgetBlobData(funBlob))
                 (state, args2);
         } else
@@ -454,10 +513,9 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
             return makePrimOp(arity, funBlob, args);
     }
 
-    else if (matchFunction(fun, formals, body, pos)) {
-        arg = evalExpr(state, arg);
+    else if (matchFunction(fun, pat, body, pos)) {
         try {
-            return evalExpr(state, substArgs(state, body, formals, arg));
+            return evalExpr(state, substArgs(state, body, pat, arg));
         } catch (Error & e) {
             addErrorPrefix(e, "while evaluating the function at %1%:\n",
                 showPos(pos));
@@ -465,18 +523,6 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
         }
     }
         
-    else if (matchFunction1(fun, name, body, pos)) {
-        try {
-            ATermMap subs(1);
-            subs.set(name, arg);
-            return evalExpr(state, substitute(Substitution(0, &subs), body));
-        } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating the function at %1%:\n",
-                showPos(pos));
-            throw;
-        }
-    }
-
     else throwTypeError(
         "attempt to call something which is neither a function nor a primop (built-in operation) but %1%",
         showType(fun));
@@ -611,6 +657,48 @@ LocalNoInline(Expr evalOpConcat(EvalState & state, Expr e1, Expr e2))
 }
 
 
+/* Implementation of the `==' and `!=' operators. */
+LocalNoInline(bool areEqual(EvalState & state, Expr e1, Expr e2))
+{
+    e1 = evalExpr(state, e1);
+    e2 = evalExpr(state, e2);
+
+    /* We cannot test functions/primops for equality, and we currently
+       don't support testing equality between attribute sets or lists
+       - that would have to be a deep equality test to be sound. */
+    AFun sym1 = ATgetAFun(e1);
+    AFun sym2 = ATgetAFun(e2);
+
+    if (sym1 != sym2) return false;
+
+    /* Functions are incomparable. */
+    if (sym1 == symFunction || sym1 == symPrimOp) return false;
+
+    if (!state.allowUnsafeEquality && sym1 == symAttrs)
+        throw EvalError("comparison of attribute sets is not implemented");
+
+    /* !!! This allows comparisons of infinite data structures to
+       succeed, such as `let x = [x]; in x == x'.  This is
+       undesirable, since equivalent (?) terms such as `let x = [x]; y
+       = [y]; in x == y' don't terminate. */
+    if (e1 == e2) return true;
+    
+    if (sym1 == symList) {
+        ATermList es1; matchList(e1, es1);
+        ATermList es2; matchList(e2, es2);
+        if (ATgetLength(es1) != ATgetLength(es2)) return false;
+        ATermIterator i(es1), j(es2);
+        while (*i) {
+            if (!areEqual(state, *i, *j)) return false;
+            ++i; ++j;
+        }
+        return true;
+    }
+    
+    return false;
+}
+
+
 static char * deepestStack = (char *) -1; /* for measuring stack usage */
 
 
@@ -633,7 +721,6 @@ Expr evalExpr2(EvalState & state, Expr e)
         sym == symInt ||
         sym == symBool ||
         sym == symFunction ||
-        sym == symFunction1 ||
         sym == symAttrs ||
         sym == symList ||
         sym == symPrimOp)
@@ -657,7 +744,7 @@ Expr evalExpr2(EvalState & state, Expr e)
     /* Mutually recursive sets. */
     ATermList rbnds, nrbnds;
     if (matchRec(e, rbnds, nrbnds))
-        return expandRec(e, rbnds, nrbnds);
+        return expandRec(state, e, rbnds, nrbnds);
 
     /* Conditionals. */
     if (matchIf(e, e1, e2, e3))
@@ -675,12 +762,10 @@ Expr evalExpr2(EvalState & state, Expr e)
        However, we don't want to make (==) strict, because that would
        make operations like `big_derivation == null' very slow (unless
        we were to evaluate them side-by-side). */
-    if (matchOpEq(e, e1, e2))
-        return makeBool(evalExpr(state, e1) == evalExpr(state, e2));
-
-    if (matchOpNEq(e, e1, e2))
-        return makeBool(evalExpr(state, e1) != evalExpr(state, e2));
-
+    if (matchOpEq(e, e1, e2)) return makeBool(areEqual(state, e1, e2));
+        
+    if (matchOpNEq(e, e1, e2)) return makeBool(!areEqual(state, e1, e2));
+        
     /* Negation. */
     if (matchOpNot(e, e1))
         return makeBool(!evalBool(state, e1));
@@ -728,11 +813,19 @@ Expr evalExpr(EvalState & state, Expr e)
         format("evaluating expression: %1%") % e);
 #endif
 
-    state.nrEvaluated++;
+    const unsigned int hugeEvalExpr = 100;
+    unsigned int nrEvaluated = state.nrEvaluated++;
+    state.nrDepthAfterReset++;
 
     /* Consult the memo table to quickly get the normal form of
        previously evaluated expressions. */
     Expr nf = state.normalForms.get(e);
+    if (nf) {
+        state.nrCached++;
+        return nf;
+    }
+
+    nf = state.sessionNormalForms.get(e);
     if (nf) {
         if (nf == makeBlackHole())
             throwEvalError("infinite recursion encountered");
@@ -741,14 +834,25 @@ Expr evalExpr(EvalState & state, Expr e)
     }
 
     /* Otherwise, evaluate and memoize. */
-    state.normalForms.set(e, makeBlackHole());
+    state.sessionNormalForms.set(e, makeBlackHole());
     try {
         nf = evalExpr2(state, e);
     } catch (Error & err) {
-        state.normalForms.remove(e);
+        state.sessionNormalForms.remove(e);
         throw;
     }
-    state.normalForms.set(e, nf);
+
+    // session independent condition
+    if (state.nrDepthAfterReset && state.safeCache &&
+        // heuristic condition
+        state.nrEvaluated - nrEvaluated > hugeEvalExpr) {
+        state.sessionNormalForms.remove(e);
+        state.normalForms.set(e, nf);
+        state.nrDepthAfterReset--;
+    } else {
+        state.sessionNormalForms.set(e, nf);
+    }
+
     return nf;
 }
 
@@ -756,6 +860,7 @@ Expr evalExpr(EvalState & state, Expr e)
 Expr evalFile(EvalState & state, const Path & path)
 {
     startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path);
+    state.nrDepthAfterReset = 0;
     Expr e = parseExprFromFile(state, path);
     try {
         return evalExpr(state, e);
@@ -793,29 +898,6 @@ static Expr strictEvalExpr_(EvalState & state, Expr e, ATermMap & nfs)
         return makeList(ATreverse(es2));
     }
     
-    ATermList formals;
-    ATerm body, pos;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermList formals2 = ATempty;
-        
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ValidValues valids; ATerm dummy;
-            if (!matchFormal(*i, name, valids, dummy)) abort();
-
-            ATermList valids2;
-            if (matchValidValues(valids, valids2)) {
-                ATermList valids3 = ATempty;
-                for (ATermIterator j(valids2); j; ++j)
-                    valids3 = ATinsert(valids3, strictEvalExpr(state, *j, nfs));
-                valids = makeValidValues(ATreverse(valids3));
-            }
-
-            formals2 = ATinsert(formals2, makeFormal(name, valids, dummy));
-        }
-        
-        return makeFunction(ATreverse(formals2), body, pos);
-    }
-    
     return e;
 }
 
@@ -850,11 +932,13 @@ void printEvalStats(EvalState & state)
     char x;
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
     printMsg(showStats ? lvlInfo : lvlDebug,
-        format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space")
+             format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space, %6% cached normal reduction, %7% session dependent reduction.")
         % state.nrEvaluated % state.nrCached
         % ((float) state.nrCached / (float) state.nrEvaluated * 100)
         % AT_calcAllocatedSize()
-        % (&x - deepestStack));
+        % (&x - deepestStack)
+        % state.normalForms.size()
+        % state.sessionNormalForms.size());
     if (showStats)
         printATermMapStats();
 }

src/libexpr/eval.hh

@@ -30,6 +30,7 @@ typedef Expr (* PrimOp) (EvalState &, const ATermVector & args);
 struct EvalState 
 {
     ATermMap normalForms;
+    ATermMap sessionNormalForms;
     ATermMap primOps;
     DrvRoots drvRoots;
     DrvHashes drvHashes; /* normalised derivation hashes */
@@ -37,12 +38,19 @@ struct EvalState
 
     unsigned int nrEvaluated;
     unsigned int nrCached;
+    unsigned int nrDepthAfterReset;
+
+    bool allowUnsafeEquality;
+    bool safeCache;
 
     EvalState();
+    ~EvalState();
 
     void addPrimOps();
     void addPrimOp(const string & name,
         unsigned int arity, PrimOp primOp);
+    void loadNormalForms();
+    void saveNormalForms();
 };
 
 

src/libexpr/expr-to-xml.cc

@@ -4,6 +4,8 @@
 #include "aterm.hh"
 #include "util.hh"
 
+#include <cstdlib>
+
 
 namespace nix {
 
@@ -38,6 +40,31 @@ static void showAttrs(const ATermMap & attrs, XMLWriter & doc,
 }
 
 
+static void printPatternAsXML(Pattern pat, XMLWriter & doc)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    if (matchVarPat(pat, name))
+        doc.writeEmptyElement("varpat", singletonAttrs("name", aterm2String(name)));
+    else if (matchAttrsPat(pat, formals, ellipsis)) {
+        XMLOpenElement _(doc, "attrspat");
+        for (ATermIterator i(formals); i; ++i) {
+            Expr name; ATerm dummy;
+            if (!matchFormal(*i, name, dummy)) abort();
+            doc.writeEmptyElement("attr", singletonAttrs("name", aterm2String(name)));
+        }
+        if (ellipsis == eTrue) doc.writeEmptyElement("ellipsis");
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        XMLOpenElement _(doc, "at");
+        printPatternAsXML(pat1, doc);
+        printPatternAsXML(pat2, doc);
+    }
+}
+
+
 static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
     ExprSet & drvsSeen)
 {
@@ -45,8 +72,8 @@ static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
     string s;
     ATerm s2;
     int i;
-    ATermList as, es, formals;
-    ATerm body, pos;
+    ATermList as, es;
+    ATerm pat, body, pos;
 
     checkInterrupt();
 
@@ -107,22 +134,9 @@ static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
             printTermAsXML(*i, doc, context, drvsSeen);
     }
 
-    else if (matchFunction(e, formals, body, pos)) {
+    else if (matchFunction(e, pat, body, pos)) {
         XMLOpenElement _(doc, "function");
-        
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ValidValues valids; ATerm dummy;
-            if (!matchFormal(*i, name, valids, dummy)) abort();
-            XMLOpenElement _(doc, "arg", singletonAttrs("name", aterm2String(name)));
-
-            ATermList valids2;
-            if (matchValidValues(valids, valids2)) {
-                for (ATermIterator j(valids2); j; ++j) {
-                    XMLOpenElement _(doc, "value");
-                    printTermAsXML(*j, doc, context, drvsSeen);
-                }
-            }
-        }
+        printPatternAsXML(pat, doc);
     }
 
     else

src/libexpr/get-drvs.cc

@@ -50,31 +50,55 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
         Expr e = evalExpr(state, i->value);
         string s;
         PathSet context;
-        if (matchStr(e, s, context))
-            meta[aterm2String(i->key)] = s;
-        /* For future compatibility, ignore attribute values that are
-           not strings. */
+        MetaValue value;
+        int n;
+        ATermList es;
+        if (matchStr(e, s, context)) {
+            value.type = MetaValue::tpString;
+            value.stringValue = s;
+            meta[aterm2String(i->key)] = value;
+        } else if (matchInt(e, n)) {
+            value.type = MetaValue::tpInt;
+            value.intValue = n;
+            meta[aterm2String(i->key)] = value;
+        } else if (matchList(e, es)) {
+            value.type = MetaValue::tpStrings;
+            for (ATermIterator j(es); j; ++j)
+                value.stringValues.push_back(evalStringNoCtx(state, *j));
+            meta[aterm2String(i->key)] = value;
+        }
     }
 
     return meta;
 }
 
 
-string DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
+MetaValue DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
 {
     /* !!! evaluates all meta attributes => inefficient */
-    MetaInfo meta = queryMetaInfo(state);
-    MetaInfo::iterator i = meta.find(name);
-    return i == meta.end() ? "" : i->second;
+    return queryMetaInfo(state)[name];
 }
 
 
 void DrvInfo::setMetaInfo(const MetaInfo & meta)
 {
     ATermMap metaAttrs;
-    for (MetaInfo::const_iterator i = meta.begin(); i != meta.end(); ++i)
-        metaAttrs.set(toATerm(i->first),
-            makeAttrRHS(makeStr(i->second), makeNoPos()));
+    foreach (MetaInfo::const_iterator, i, meta) {
+        Expr e;
+        switch (i->second.type) {
+            case MetaValue::tpInt: e = makeInt(i->second.intValue); break;
+            case MetaValue::tpString: e = makeStr(i->second.stringValue); break;
+            case MetaValue::tpStrings: {
+                ATermList es = ATempty;
+                foreach (Strings::const_iterator, j, i->second.stringValues)
+                    es = ATinsert(es, makeStr(*j));
+                e = makeList(ATreverse(es));
+                break;
+            }
+            default: abort();
+        }
+        metaAttrs.set(toATerm(i->first), makeAttrRHS(e, makeNoPos()));
+    }
     attrs->set(toATerm("meta"), makeAttrs(metaAttrs));
 }
 
@@ -102,7 +126,7 @@ static bool getDerivation(EvalState & state, Expr e,
 
         boost::shared_ptr<ATermMap> attrs(new ATermMap());
         queryAllAttrs(e, *attrs, false);
-    
+        
         Expr a = attrs->get(toATerm("type"));
         if (!a || evalStringNoCtx(state, a) != "derivation") return true;
 
@@ -173,24 +197,30 @@ static void getDerivations(EvalState & state, Expr e,
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        Expr e2 = drvMap.get(toATerm("_combineChannels"));
-        bool combineChannels = e2 && evalBool(state, e2);
-        
-        for (ATermMap::const_iterator i = drvMap.begin(); i != drvMap.end(); ++i) {
-            startNest(nest, lvlDebug,
-                format("evaluating attribute `%1%'") % aterm2String(i->key));
-            string pathPrefix2 = addToPath(pathPrefix, aterm2String(i->key));
-            if (combineChannels) {
-                if (((string) aterm2String(i->key)) != "_combineChannels")
-                    getDerivations(state, i->value, pathPrefix2, autoArgs, drvs, doneExprs);
-            }
-            else if (getDerivation(state, i->value, pathPrefix2, drvs, doneExprs)) {
+        bool combineChannels = drvMap.get(toATerm("_combineChannels"));
+
+        /* Consider the attributes in sorted order to get more
+           deterministic behaviour in nix-env operations (e.g. when
+           there are names clashes between derivations, the derivation
+           bound to the attribute with the "lower" name should take
+           precedence). */
+        typedef std::map<string, Expr> AttrsSorted;
+        AttrsSorted attrsSorted;
+        foreach (ATermMap::const_iterator, i, drvMap)
+            attrsSorted[aterm2String(i->key)] = i->value;
+
+        foreach (AttrsSorted::iterator, i, attrsSorted) {
+            startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first);
+            string pathPrefix2 = addToPath(pathPrefix, i->first);
+            if (combineChannels)
+                getDerivations(state, i->second, pathPrefix2, autoArgs, drvs, doneExprs);
+            else if (getDerivation(state, i->second, pathPrefix2, drvs, doneExprs)) {
                 /* If the value of this attribute is itself an
                    attribute set, should we recurse into it?  => Only
                    if it has a `recurseForDerivations = true'
                    attribute. */
                 ATermList es;
-                Expr e = evalExpr(state, i->value);
+                Expr e = evalExpr(state, i->second), e2;
                 if (matchAttrs(e, es)) {
                     ATermMap attrs(ATgetLength(es));
                     queryAllAttrs(e, attrs, false);

src/libexpr/get-drvs.hh

@@ -12,7 +12,16 @@
 namespace nix {
 
 
-typedef std::map<string, string> MetaInfo;
+struct MetaValue
+{
+    enum { tpNone, tpString, tpStrings, tpInt } type;
+    string stringValue;
+    Strings stringValues;
+    int intValue;
+};
+
+
+typedef std::map<string, MetaValue> MetaInfo;
 
 
 struct DrvInfo
@@ -34,7 +43,7 @@ public:
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
     MetaInfo queryMetaInfo(EvalState & state) const;
-    string queryMetaInfo(EvalState & state, const string & name) const;
+    MetaValue queryMetaInfo(EvalState & state, const string & name) const;
 
     void setDrvPath(const string & s)
     {

src/libexpr/lexer.l

@@ -95,6 +95,7 @@ let         { return LET; }
 in          { return IN; }
 rec         { return REC; }
 inherit     { return INHERIT; }
+\.\.\.      { return ELLIPSIS; }
 
 \=\=        { return EQ; }
 \!\=        { return NEQ; }

src/libexpr/names.cc

@@ -92,25 +92,6 @@ int compareVersions(const string & v1, const string & v2)
 }
 
 
-static void testCompareVersions()
-{
-#define TEST(v1, v2, n) assert( \
-    compareVersions(v1, v2) == n && compareVersions(v2, v1) == -n)
-    TEST("1.0", "2.3", -1);
-    TEST("2.1", "2.3", -1);
-    TEST("2.3", "2.3", 0);
-    TEST("2.5", "2.3", 1);
-    TEST("3.1", "2.3", 1);
-    TEST("2.3.1", "2.3", 1);
-    TEST("2.3.1", "2.3a", 1);
-    TEST("2.3pre1", "2.3", -1);
-    TEST("2.3pre3", "2.3pre12", -1);
-    TEST("2.3a", "2.3c", -1);
-    TEST("2.3pre1", "2.3c", -1);
-    TEST("2.3pre1", "2.3q", -1);
-}
-
-
 DrvNames drvNamesFromArgs(const Strings & opArgs)
 {
     DrvNames result;

src/libexpr/nix.sdf

@@ -1,10 +1,13 @@
+%% Note: this SDF grammar is no longer used in the Nix expression
+%% parser and may not be up to date.
+
 definition
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Top level syntax.
 
-module Nix
+module Main
 imports Nix-Exprs Nix-Layout
 
 
@@ -38,6 +41,7 @@ exports
     "with" Expr ";" Expr -> Expr {cons("With")}
 
     "rec" "{" Bind* "}" -> Expr {cons("Rec")}
+    "let" Bind* "in" Expr -> Expr {cons("Let")}
     "let" "{" Bind* "}" -> Expr {cons("LetRec")}
     "{" Bind* "}" -> Expr {cons("Attrs")}
 
@@ -102,7 +106,8 @@ exports
 
     [0-9]+ -> Int
 
-    "\"" ~[\n\"]* "\"" -> Str
+    "\"" (~[\"\\] | ("\\" ~[]) )* "\"" -> Str
+    "''" (~[\"\\] | ("\\" ~[]) )* "''" -> Str
 
     [a-zA-Z0-9\.\_\-\+]* ("/"[a-zA-Z0-9\.\_\-\+]+)+ -> Path
 

src/libexpr/nixexpr-ast.def

@@ -3,8 +3,7 @@ init initNixExprHelpers
 Pos | string int int | Pos |
 NoPos | | Pos |
 
-Function | ATermList Expr Pos | Expr |
-Function1 | string Expr Pos | Expr |
+Function | Pattern Expr Pos | Expr |
 Assert | Expr Expr Pos | Expr |
 With | Expr Expr Pos | Expr |
 If | Expr Expr Expr | Expr |
@@ -67,25 +66,27 @@ PrimOp | int ATermBlob ATermList | Expr |
 Attrs | ATermList | Expr |
 Closed | Expr | Expr |
 Rec | ATermList ATermList | Expr |
-Bool | ATerm | Expr |
+Bool | ATermBool | Expr |
 Null | | Expr |
 
 Bind | string Expr Pos | ATerm |
+BindAttrPath | ATermList Expr Pos | ATerm | # desugared during parsing
 Bind | string Expr | ATerm | ObsoleteBind
 Inherit | Expr ATermList Pos | ATerm |
 
 Scope | | Expr |
 
-Formal | string ValidValues DefaultValue | ATerm |
+VarPat | string | Pattern |
+AttrsPat | ATermList ATermBool | Pattern | # bool = `...'
+AtPat | Pattern Pattern | Pattern |
 
-ValidValues | ATermList | ValidValues |
-UnrestrictedValues | | ValidValues |
+Formal | string DefaultValue | ATerm |
 
 DefaultValue | Expr | DefaultValue |
 NoDefaultValue | | DefaultValue |
 
-True | | ATerm |
-False | | ATerm |
+True | | ATermBool |
+False | | ATermBool |
 
 PrimOpDef | int ATermBlob | ATerm |
 
@@ -93,3 +94,4 @@ AttrRHS | Expr Pos | ATerm |
 
 eTrue = makeBool(makeTrue())
 eFalse = makeBool(makeFalse())
+sOverrides = toATerm("__overrides")

src/libexpr/nixexpr.cc

@@ -6,6 +6,8 @@
 #include "nixexpr-ast.hh"
 #include "nixexpr-ast.cc"
 
+#include <cstdlib>
+
 
 namespace nix {
     
@@ -17,7 +19,7 @@ string showPos(ATerm pos)
     if (matchNoPos(pos)) return "undefined position";
     if (!matchPos(pos, path, line, column))
         throw badTerm("position expected", pos);
-    return (format("`%1%', line %2%") % aterm2String(path) % line).str();
+    return (format("`%1%:%2%:%3%'") % aterm2String(path) % line % column).str();
 }
     
 
@@ -108,6 +110,31 @@ Expr makeAttrs(const ATermMap & attrs)
 }
 
 
+static void varsBoundByPattern(ATermMap & map, Pattern pat)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    /* Use makeRemoved() so that it can be used directly in
+       substitute(). */
+    if (matchVarPat(pat, name))
+        map.set(name, makeRemoved());
+    else if (matchAttrsPat(pat, formals, ellipsis)) { 
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm d1;
+            if (!matchFormal(*i, name, d1)) abort();
+            map.set(name, makeRemoved());
+        }
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        varsBoundByPattern(map, pat1);
+        varsBoundByPattern(map, pat2);
+    }
+    else abort();
+}
+
+
 Expr substitute(const Substitution & subs, Expr e)
 {
     checkInterrupt();
@@ -133,27 +160,17 @@ Expr substitute(const Substitution & subs, Expr e)
 
     /* In case of a function, filter out all variables bound by this
        function. */
-    ATermList formals;
+    Pattern pat;
     ATerm body;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermMap map(ATgetLength(formals));
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            map.set(name, makeRemoved());
-        }
+    if (matchFunction(e, pat, body, pos)) {
+        ATermMap map(16);
+        varsBoundByPattern(map, pat);
         Substitution subs2(&subs, &map);
         return makeFunction(
-            (ATermList) substitute(subs2, (ATerm) formals),
+            (Pattern) substitute(subs2, (Expr) pat),
             substitute(subs2, body), pos);
     }
 
-    if (matchFunction1(e, name, body, pos)) {
-        ATermMap map(1);
-        map.set(name, makeRemoved());
-        return makeFunction1(name, substitute(Substitution(&subs, &map), body), pos);
-    }
-        
     /* Idem for a mutually recursive attribute set. */
     ATermList rbnds, nrbnds;
     if (matchRec(e, rbnds, nrbnds)) {
@@ -211,9 +228,9 @@ static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
     done.insert(e);
     
     ATerm name, pos, value;
-    ATermList formals;
     ATerm with, body;
     ATermList rbnds, nrbnds;
+    Pattern pat;
 
     /* Closed terms don't have free variables, so we don't have to
        check by definition. */
@@ -225,28 +242,11 @@ static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
                 % aterm2String(name));
     }
 
-    else if (matchFunction(e, formals, body, pos)) {
+    else if (matchFunction(e, pat, body, pos)) {
         ATermMap defs2(defs);
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            defs2.set(name, (ATerm) ATempty);
-        }
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm valids, deflt;
-            set<Expr> done2;
-            if (!matchFormal(*i, name, valids, deflt)) abort();
-            checkVarDefs2(done, defs, valids);
-            checkVarDefs2(done2, defs2, deflt);
-        }
-        set<Expr> done2;
-        checkVarDefs2(done2, defs2, body);
-    }
-        
-    else if (matchFunction1(e, name, body, pos)) {
-        ATermMap defs2(defs);
-        defs2.set(name, (ATerm) ATempty);
+        varsBoundByPattern(defs2, pat);
         set<Expr> done2;
+        checkVarDefs2(done2, defs2, pat);
         checkVarDefs2(done2, defs2, body);
     }
         
@@ -360,17 +360,17 @@ Expr makeStr(const string & s, const PathSet & context)
 
 string showType(Expr e)
 {
-    ATerm t1, t2, t3;
+    ATerm t1, t2;
     ATermList l1;
     ATermBlob b1;
     int i1;
+    Pattern p1;
     if (matchStr(e, t1, l1)) return "a string";
     if (matchPath(e, t1)) return "a path";
     if (matchNull(e)) return "null";
     if (matchInt(e, i1)) return "an integer";
     if (matchBool(e, t1)) return "a boolean";
-    if (matchFunction(e, l1, t1, t2)) return "a function";
-    if (matchFunction1(e, t1, t2, t3)) return "a function";
+    if (matchFunction(e, p1, t1, t2)) return "a function";
     if (matchAttrs(e, l1)) return "an attribute set";
     if (matchList(e, l1)) return "a list";
     if (matchPrimOp(e, i1, b1, l1)) return "a partially applied built-in function";

src/libexpr/nixexpr.hh

@@ -11,6 +11,7 @@ namespace nix {
 
 
 MakeError(EvalError, Error)
+MakeError(ParseError, Error)
 MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
@@ -21,11 +22,10 @@ MakeError(TypeError, EvalError)
    property of the ATerm library allows us to implement caching of
    normals forms efficiently. */
 typedef ATerm Expr;
-
 typedef ATerm DefaultValue;
-typedef ATerm ValidValues;
-
 typedef ATerm Pos;
+typedef ATerm Pattern;
+typedef ATerm ATermBool;
 
 
 /* A STL vector of ATerms.  Should be used with great care since it's

src/libexpr/normal-forms.cc

@@ -0,0 +1,64 @@
+#include "eval.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "serialise.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+namespace nix {
+
+
+void EvalState::loadNormalForms()
+{
+    //    nixCacheFile = getEnv("NIX_CACHE_FILE", nixStateDir + "/reduce-cache");
+    nixCacheFile = getEnv("NIX_CACHE_FILE");
+    string loadFlag = getEnv("NIX_CACHE_FILE_LOAD", "1");
+
+    if(nixCacheFile == "" || loadFlag == "")
+        return;
+    if(!pathExists(nixCacheFile))
+        return;
+
+    printMsg(lvlInfo, format("Load cache: ..."));
+    try {
+        int fd = open(nixCacheFile.c_str(), O_RDONLY);
+        if (fd == -1)
+            throw SysError(format("opening file `%1%'") % nixCacheFile);
+        AutoCloseFD auto_fd = fd;
+
+        FdSource source = fd;
+        normalForms = readATermMap(source);
+    } catch (Error & e) {
+        e.addPrefix(format("Cannot load cached reduce operations from %1%:\n") % nixCacheFile);
+        throw;
+    }
+    printMsg(lvlInfo, format("Load cache: end"));
+}
+
+void EvalState::saveNormalForms()
+{
+    string saveFlag = getEnv("NIX_CACHE_FILE_SAVE", "");
+
+    if(nixCacheFile == "" || saveFlag == "")
+        return;
+
+    printMsg(lvlInfo, format("Save cache: ..."));
+    try {
+        int fd = open(nixCacheFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
+        if (fd == -1)
+            throw SysError(format("opening file `%1%'") % nixCacheFile);
+        AutoCloseFD auto_fd = fd;
+
+        FdSink sink = fd;
+        writeATermMap(normalForms, sink);
+    } catch (Error & e) {
+        e.addPrefix(format("Cannot save cached reduce operations to %1%:\n") % nixCacheFile);
+        throw;
+    }
+    printMsg(lvlInfo, format("Save cache: end"));
+}
+
+
+}

src/libexpr/parser.y

@@ -25,6 +25,7 @@
     
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
+#define YYSTYPE YYSTYPE // workaround a bug in Bison 2.4
 
 #include "nixexpr.hh"
 #include "nixexpr-ast.hh"
@@ -43,28 +44,128 @@ struct ParseData
     Path path;
     string error;
 };
+ 
 
-
-static Expr fixAttrs(int recursive, ATermList as)
+static string showAttrPath(ATermList attrPath)
 {
-    ATermList bs = ATempty, cs = ATempty;
-    ATermList * is = recursive ? &cs : &bs;
+    string s;
+    for (ATermIterator i(attrPath); i; ++i) {
+        if (!s.empty()) s += '.';
+        s += aterm2String(*i);
+    }
+    return s;
+}
+ 
+
+struct Tree
+{
+    Expr leaf; ATerm pos; bool recursive;
+    typedef std::map<ATerm, Tree> Children;
+    Children children;
+    Tree() { leaf = 0; recursive = true; }
+};
+
+
+static ATermList buildAttrs(const Tree & t, ATermList & nonrec)
+{
+    ATermList res = ATempty;
+    for (Tree::Children::const_reverse_iterator i = t.children.rbegin();
+         i != t.children.rend(); ++i)
+        if (!i->second.recursive)
+            nonrec = ATinsert(nonrec, makeBind(i->first, i->second.leaf, i->second.pos));
+        else
+            res = ATinsert(res, i->second.leaf
+                ? makeBind(i->first, i->second.leaf, i->second.pos)
+                : makeBind(i->first, makeAttrs(buildAttrs(i->second, nonrec)), makeNoPos()));
+    return res;
+}
+ 
+
+static Expr fixAttrs(bool recursive, ATermList as)
+{
+    Tree attrs;
+
     for (ATermIterator i(as); i; ++i) {
-        ATermList names;
-        Expr src;
-        ATerm pos;
+        ATermList names, attrPath; Expr src, e; ATerm name, pos;
+
         if (matchInherit(*i, src, names, pos)) {
             bool fromScope = matchScope(src);
             for (ATermIterator j(names); j; ++j) {
-                Expr rhs = fromScope ? makeVar(*j) : makeSelect(src, *j);
-                *is = ATinsert(*is, makeBind(*j, rhs, pos));
+                if (attrs.children.find(*j) != attrs.children.end()) 
+                    throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
+                        % showAttrPath(ATmakeList1(*j)) % showPos(pos));
+                Tree & t(attrs.children[*j]);
+                t.leaf = fromScope ? makeVar(*j) : makeSelect(src, *j);
+                t.pos = pos;
+                if (recursive && fromScope) t.recursive = false;
             }
-        } else bs = ATinsert(bs, *i);
+        }
+
+        else if (matchBindAttrPath(*i, attrPath, e, pos)) {
+
+            Tree * t(&attrs);
+            
+            for (ATermIterator j(attrPath); j; ) {
+                name = *j; ++j;
+                if (t->leaf) throw ParseError(format("attribute set containing `%1%' at %2% already defined at %3%")
+                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
+                t = &(t->children[name]);
+            }
+
+            if (t->leaf)
+                throw ParseError(format("duplicate definition of attribute `%1%' at %2% and %3%")
+                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
+            if (!t->children.empty())
+                throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
+                    % showAttrPath(attrPath) % showPos(pos));
+
+            t->leaf = e; t->pos = pos;
+        }
+
+        else abort(); /* can't happen */
     }
-    if (recursive)
-        return makeRec(bs, cs);
-    else
-        return makeAttrs(bs);
+
+    ATermList nonrec = ATempty;
+    ATermList rec = buildAttrs(attrs, nonrec);
+        
+    return recursive ? makeRec(rec, nonrec) : makeAttrs(rec);
+}
+
+
+static void checkPatternVars(ATerm pos, ATermMap & map, Pattern pat)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    if (matchVarPat(pat, name)) {
+        if (map.get(name))
+            throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+                % aterm2String(name) % showPos(pos));
+        map.set(name, name);
+    }
+    else if (matchAttrsPat(pat, formals, ellipsis)) { 
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm d1;
+            if (!matchFormal(*i, name, d1)) abort();
+            if (map.get(name))
+                throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+                    % aterm2String(name) % showPos(pos));
+            map.set(name, name);
+        }
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        checkPatternVars(pos, map, pat1);
+        checkPatternVars(pos, map, pat2);
+    }
+    else abort();
+}
+
+
+static void checkPatternVars(ATerm pos, Pattern pat)
+{
+    ATermMap map;
+    checkPatternVars(pos, map, pat);
 }
 
 
@@ -146,7 +247,7 @@ static Expr stripIndentation(ATermList es)
         /* Remove the last line if it is empty and consists only of
            spaces. */
         if (n == 1) {
-            unsigned int p = s2.find_last_of('\n');
+            string::size_type p = s2.find_last_of('\n');
             if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos)
                 s2 = string(s2, 0, p + 1);
         }
@@ -208,15 +309,22 @@ static void freeAndUnprotect(void * p)
 %union {
   ATerm t;
   ATermList ts;
+  struct {
+    ATermList formals;
+    bool ellipsis;
+  } formals;
 }
 
 %type <t> start expr expr_function expr_if expr_op
 %type <t> expr_app expr_select expr_simple bind inheritsrc formal
-%type <ts> binds ids expr_list formals string_parts ind_string_parts
+%type <t> pattern pattern2
+%type <ts> binds ids attrpath expr_list string_parts ind_string_parts
+%type <formals> formals
 %token <t> ID INT STR IND_STR PATH URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
+%token ELLIPSIS
 
 %nonassoc IMPL
 %left OR
@@ -236,16 +344,14 @@ start: expr { data->result = $1; };
 expr: expr_function;
 
 expr_function
-  : '{' formals '}' ':' expr_function
-    { $$ = makeFunction($2, $5, CUR_POS); }
-  | ID ':' expr_function
-    { $$ = makeFunction1($1, $3, CUR_POS); }
+  : pattern ':' expr_function
+    { checkPatternVars(CUR_POS, $1); $$ = makeFunction($1, $3, CUR_POS); }
   | ASSERT expr ';' expr_function
     { $$ = makeAssert($2, $4, CUR_POS); }
   | WITH expr ';' expr_function
     { $$ = makeWith($2, $4, CUR_POS); }
   | LET binds IN expr_function
-    { $$ = makeSelect(fixAttrs(1, ATinsert($2, makeBind(toATerm("<let-body>"), $4, CUR_POS))), toATerm("<let-body>")); }
+    { $$ = makeSelect(fixAttrs(true, ATinsert($2, makeBindAttrPath(ATmakeList1(toATerm("<let-body>")), $4, CUR_POS))), toATerm("<let-body>")); }
   | expr_if
   ;
 
@@ -300,12 +406,12 @@ expr_simple
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
   | LET '{' binds '}'
-    { $$ = makeSelect(fixAttrs(1, $3), toATerm("body")); }
+    { $$ = makeSelect(fixAttrs(true, $3), toATerm("body")); }
   | REC '{' binds '}'
-    { $$ = fixAttrs(1, $3); }
+    { $$ = fixAttrs(true, $3); }
   | '{' binds '}'
-    { $$ = fixAttrs(0, $2); }
-  | '[' expr_list ']' { $$ = makeList($2); }
+    { $$ = fixAttrs(false, $2); }
+  | '[' expr_list ']' { $$ = makeList(ATreverse($2)); }
   ;
 
 string_parts
@@ -320,14 +426,24 @@ ind_string_parts
   | { $$ = ATempty; }
   ;
 
+pattern
+  : pattern2 '@' pattern { $$ = makeAtPat($1, $3); }
+  | pattern2
+  ;
+
+pattern2
+  : ID { $$ = makeVarPat($1); }
+  | '{' formals '}' { $$ = makeAttrsPat($2.formals, $2.ellipsis ? eTrue : eFalse); }
+  ;
+
 binds
   : binds bind { $$ = ATinsert($1, $2); }
   | { $$ = ATempty; }
   ;
 
 bind
-  : ID '=' expr ';'
-    { $$ = makeBind($1, $3, CUR_POS); }
+  : attrpath '=' expr ';'
+    { $$ = makeBindAttrPath(ATreverse($1), $3, CUR_POS); }
   | INHERIT inheritsrc ids ';'
     { $$ = makeInherit($2, $3, CUR_POS); }
   ;
@@ -339,24 +455,30 @@ inheritsrc
 
 ids: ids ID { $$ = ATinsert($1, $2); } | { $$ = ATempty; };
 
+attrpath
+  : attrpath '.' ID { $$ = ATinsert($1, $3); }
+  | ID { $$ = ATmakeList1($1); }
+  ;
+
 expr_list
-  : expr_select expr_list { $$ = ATinsert($2, $1); }
-    /* yes, this is right-recursive, but it doesn't matter since
-       otherwise we would need ATreverse which requires unbounded
-       stack space */
+  : expr_list expr_select { $$ = ATinsert($1, $2); }
   | { $$ = ATempty; }
   ;
 
 formals
-  : formal ',' formals { $$ = ATinsert($3, $1); } /* idem - right recursive */
-  | formal { $$ = ATinsert(ATempty, $1); }
-  | { $$ = ATempty; }
+  : formal ',' formals /* !!! right recursive */
+    { $$.formals = ATinsert($3.formals, $1); $$.ellipsis = $3.ellipsis; }
+  | formal
+    { $$.formals = ATinsert(ATempty, $1); $$.ellipsis = false; }
+  |
+    { $$.formals = ATempty; $$.ellipsis = false; }
+  | ELLIPSIS
+    { $$.formals = ATempty; $$.ellipsis = true; }
   ;
 
 formal
-  : ID { $$ = makeFormal($1, makeUnrestrictedValues(), makeNoDefaultValue()); }
-  | ID ':' '[' expr_list ']' { $$ = makeFormal($1, makeValidValues($4), makeNoDefaultValue()); }
-  | ID '?' expr { $$ = makeFormal($1, makeUnrestrictedValues(), makeDefaultValue($3)); }
+  : ID { $$ = makeFormal($1, makeNoDefaultValue()); }
+  | ID '?' expr { $$ = makeFormal($1, makeDefaultValue($3)); }
   ;
   
 %%
@@ -373,63 +495,6 @@ formal
 namespace nix {
       
 
-static void checkAttrs(ATermMap & names, ATermList bnds)
-{
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm name;
-        Expr e;
-        ATerm pos;
-        if (!matchBind(*i, name, e, pos)) abort(); /* can't happen */
-        if (names.get(name))
-            throw EvalError(format("duplicate attribute `%1%' at %2%")
-                % aterm2String(name) % showPos(pos));
-        names.set(name, name);
-    }
-}
-
-
-static void checkAttrSets(ATerm e)
-{
-    ATermList formals;
-    ATerm body, pos;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermMap names(ATgetLength(formals));
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm name;
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            if (names.get(name))
-                throw EvalError(format("duplicate formal function argument `%1%' at %2%")
-                    % aterm2String(name) % showPos(pos));
-            names.set(name, name);
-        }
-    }
-
-    ATermList bnds;
-    if (matchAttrs(e, bnds)) {
-        ATermMap names(ATgetLength(bnds));
-        checkAttrs(names, bnds);
-    }
-    
-    ATermList rbnds, nrbnds;
-    if (matchRec(e, rbnds, nrbnds)) {
-        ATermMap names(ATgetLength(rbnds) + ATgetLength(nrbnds));
-        checkAttrs(names, rbnds);
-        checkAttrs(names, nrbnds);
-    }
-    
-    if (ATgetType(e) == AT_APPL) {
-        int arity = ATgetArity(ATgetAFun(e));
-        for (int i = 0; i < arity; ++i)
-            checkAttrSets(ATgetArgument(e, i));
-    }
-
-    else if (ATgetType(e) == AT_LIST)
-        for (ATermIterator i((ATermList) e); i; ++i)
-            checkAttrSets(*i);
-}
-
-
 static Expr parse(EvalState & state,
     const char * text, const Path & path,
     const Path & basePath)
@@ -444,16 +509,14 @@ static Expr parse(EvalState & state,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
     
-    if (res) throw EvalError(data.error);
+    if (res) throw ParseError(data.error);
 
     try {
         checkVarDefs(state.primOps, data.result);
     } catch (Error & e) {
-        throw EvalError(format("%1%, in `%2%'") % e.msg() % path);
+        throw ParseError(format("%1%, in `%2%'") % e.msg() % path);
     }
     
-    checkAttrSets(data.result);
-
     return data.result;
 }
 

src/libexpr/primops.cc

@@ -7,6 +7,7 @@
 #include "expr-to-xml.hh"
 #include "nixexpr-ast.hh"
 #include "parser.hh"
+#include "names.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -73,12 +74,14 @@ static Expr prim_null(EvalState & state, const ATermVector & args)
    platforms. */
 static Expr prim_currentSystem(EvalState & state, const ATermVector & args)
 {
+    state.nrDepthAfterReset = 0;
     return makeStr(thisSystem);
 }
 
 
 static Expr prim_currentTime(EvalState & state, const ATermVector & args)
 {
+    state.nrDepthAfterReset = 0;
     return ATmake("Int(<int>)", time(0));
 }
 
@@ -119,67 +122,34 @@ static Expr prim_isNull(EvalState & state, const ATermVector & args)
 static Expr prim_isFunction(EvalState & state, const ATermVector & args)
 {
     Expr e = evalExpr(state, args[0]);
-    ATermList formals;
-    ATerm name, body, pos;
-    return makeBool(
-        matchFunction(e, formals, body, pos) ||
-        matchFunction1(e, name, body, pos));
+    Pattern pat;
+    ATerm body, pos;
+    return makeBool(matchFunction(e, pat, body, pos));
 }
 
-
-static Path findDependency(Path dir, string dep)
+/* Determine whether the argument is an Int. */
+static Expr prim_isInt(EvalState & state, const ATermVector & args)
 {
-    if (dep[0] == '/') throw EvalError(
-        format("illegal absolute dependency `%1%'") % dep);
-
-    Path p = canonPath(dir + "/" + dep);
-
-    if (pathExists(p))
-        return p;
-    else
-        return "";
+    int i;
+    return makeBool(matchInt(evalExpr(state, args[0]), i));
 }
 
-
-/* Make path `p' relative to directory `pivot'.  E.g.,
-   relativise("/a/b/c", "a/b/x/y") => "../x/y".  Both input paths
-   should be in absolute canonical form. */
-static string relativise(Path pivot, Path p)
+/* Determine whether the argument is an String. */
+static Expr prim_isString(EvalState & state, const ATermVector & args)
 {
-    assert(pivot.size() > 0 && pivot[0] == '/');
-    assert(p.size() > 0 && p[0] == '/');
-        
-    if (pivot == p) return ".";
-
-    /* `p' is in `pivot'? */
-    Path pivot2 = pivot + "/";
-    if (p.substr(0, pivot2.size()) == pivot2) {
-        return p.substr(pivot2.size());
-    }
-
-    /* Otherwise, `p' is in a parent of `pivot'.  Find up till which
-       path component `p' and `pivot' match, and add an appropriate
-       number of `..' components. */
-    string::size_type i = 1;
-    while (1) {
-        string::size_type j = pivot.find('/', i);
-        if (j == string::npos) break;
-        j++;
-        if (pivot.substr(0, j) != p.substr(0, j)) break;
-        i = j;
-    }
-
-    string prefix;
-    unsigned int slashes = count(pivot.begin() + i, pivot.end(), '/') + 1;
-    while (slashes--) {
-        prefix += "../";
-    }
-
-    return prefix + p.substr(i);
+    string s;
+    PathSet l;
+    return makeBool(matchStr(evalExpr(state, args[0]), s, l));
 }
 
+/* Determine whether the argument is an Bool. */
+static Expr prim_isBool(EvalState & state, const ATermVector & args)
+{
+    ATermBool b;
+    return makeBool(matchBool(evalExpr(state, args[0]), b));
+}
 
-static Expr prim_dependencyClosure(EvalState & state, const ATermVector & args)
+static Expr prim_genericClosure(EvalState & state, const ATermVector & args)
 {
     startNest(nest, lvlDebug, "finding dependencies");
 
@@ -190,87 +160,40 @@ static Expr prim_dependencyClosure(EvalState & state, const ATermVector & args)
     if (!startSet) throw EvalError("attribute `startSet' required");
     ATermList startSet2 = evalList(state, startSet);
 
-    Path pivot;
-    PathSet workSet;
-    for (ATermIterator i(startSet2); i; ++i) {
-        PathSet context; /* !!! what to do? */
-        Path p = coerceToPath(state, *i, context);
-        workSet.insert(p);
-        pivot = dirOf(p);
-    }
+    set<Expr> workSet; // !!! gc roots
+    for (ATermIterator i(startSet2); i; ++i) workSet.insert(*i);
 
-    /* Get the search path. */
-    PathSet searchPath;
-    Expr e = queryAttr(attrs, "searchPath");
-    if (e) {
-        ATermList list = evalList(state, e);
-        for (ATermIterator i(list); i; ++i) {
-            PathSet context; /* !!! what to do? */
-            Path p = coerceToPath(state, *i, context);
-            searchPath.insert(p);
-        }
-    }
-
-    Expr scanner = queryAttr(attrs, "scanner");
-    if (!scanner) throw EvalError("attribute `scanner' required");
+    /* Get the operator. */
+    Expr op = queryAttr(attrs, "operator");
+    if (!op) throw EvalError("attribute `operator' required");
     
-    /* Construct the dependency closure by querying the dependency of
-       each path in `workSet', adding the dependencies to
-       `workSet'. */
-    PathSet doneSet;
+    /* Construct the closure by applying the operator to element of
+       `workSet', adding the result to `workSet', continuing until
+       no new elements are found. */
+    ATermList res = ATempty;
+    set<Expr> doneKeys; // !!! gc roots
     while (!workSet.empty()) {
-	Path path = *(workSet.begin());
-	workSet.erase(path);
+	Expr e = *(workSet.begin());
+	workSet.erase(e);
 
-	if (doneSet.find(path) != doneSet.end()) continue;
-        doneSet.insert(path);
+        e = strictEvalExpr(state, e);
 
-        try {
-            
-            /* Call the `scanner' function with `path' as argument. */
-            debug(format("finding dependencies in `%1%'") % path);
-            ATermList deps = evalList(state, makeCall(scanner, makeStr(path)));
+        Expr key = queryAttr(e, "key");
+        if (!key) throw EvalError("attribute `key' required");
 
-            /* Try to find the dependencies relative to the `path'. */
-            for (ATermIterator i(deps); i; ++i) {
-                string s = evalStringNoCtx(state, *i);
-                
-                Path dep = findDependency(dirOf(path), s);
+	if (doneKeys.find(key) != doneKeys.end()) continue;
+        doneKeys.insert(key);
+        res = ATinsert(res, e);
+        
+        /* Call the `operator' function with `e' as argument. */
+        ATermList res = evalList(state, makeCall(op, e));
 
-                if (dep == "") {
-                    for (PathSet::iterator j = searchPath.begin();
-                         j != searchPath.end(); ++j)
-                    {
-                        dep = findDependency(*j, s);
-                        if (dep != "") break;
-                    }
-                }
-                
-                if (dep == "")
-                    debug(format("did NOT find dependency `%1%'") % s);
-                else {
-                    debug(format("found dependency `%1%'") % dep);
-                    workSet.insert(dep);
-                }
-            }
-
-        } catch (Error & e) {
-            e.addPrefix(format("while finding dependencies in `%1%':\n")
-                % path);
-            throw;
-        }
+        /* Try to find the dependencies relative to the `path'. */
+        for (ATermIterator i(res); i; ++i)
+            workSet.insert(evalExpr(state, *i));
     }
 
-    /* Return a list of the dependencies we've just found. */
-    ATermList deps = ATempty;
-    for (PathSet::iterator i = doneSet.begin(); i != doneSet.end(); ++i) {
-        deps = ATinsert(deps, makeStr(relativise(pivot, *i)));
-        deps = ATinsert(deps, makeStr(*i));
-    }
-
-    debug(format("dependency list is `%1%'") % makeList(deps));
-    
-    return makeList(deps);
+    return makeList(res);
 }
 
 
@@ -285,15 +208,29 @@ static Expr prim_abort(EvalState & state, const ATermVector & args)
 static Expr prim_throw(EvalState & state, const ATermVector & args)
 {
     PathSet context;
-    throw ThrownError(format("user-thrown exception: `%1%'") %
+    throw ThrownError(format("user-thrown exception: %1%") %
         evalString(state, args[0], context));
 }
 
 
+static Expr prim_addErrorContext(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    try {
+        return evalExpr(state, args[1]);
+    } catch (Error & e) {
+        e.addPrefix(format("%1%\n") %
+            evalString(state, args[0], context));
+        throw;
+    }
+}
+
+
 /* Return an environment variable.  Use with care. */
 static Expr prim_getEnv(EvalState & state, const ATermVector & args)
 {
     string name = evalStringNoCtx(state, args[0]);
+    state.nrDepthAfterReset = 0;
     return makeStr(getEnv(name));
 }
 
@@ -310,20 +247,19 @@ static Expr prim_trace(EvalState & state, const ATermVector & args)
 }
 
 
-static Expr prim_relativise(EvalState & state, const ATermVector & args)
-{
-    PathSet context; /* !!! what to do? */
-    Path pivot = coerceToPath(state, args[0], context);
-    Path path = coerceToPath(state, args[1], context);
-    return makeStr(relativise(pivot, path));
-}
-
-
 /*************************************************************
  * Derivations
  *************************************************************/
 
 
+static bool isFixedOutput(const Derivation & drv)
+{
+    return drv.outputs.size() == 1 &&
+        drv.outputs.begin()->first == "out" &&
+        drv.outputs.begin()->second.hash != "";
+}
+
+
 /* Returns the hash of a derivation modulo fixed-output
    subderivations.  A fixed-output derivation is a derivation with one
    output (`out') for which an expected hash and hash algorithm are
@@ -336,35 +272,29 @@ static Expr prim_relativise(EvalState & state, const ATermVector & args)
    function, we do not want to rebuild everything depending on it
    (after all, (the hash of) the file being downloaded is unchanged).
    So the *output paths* should not change.  On the other hand, the
-   *derivation store expression paths* should change to reflect the
-   new dependency graph.
+   *derivation paths* should change to reflect the new dependency
+   graph.
 
    That's what this function does: it returns a hash which is just the
-   of the derivation ATerm, except that any input store expression
+   hash of the derivation ATerm, except that any input derivation
    paths have been replaced by the result of a recursive call to this
-   function, and that for fixed-output derivations we return
-   (basically) its outputHash. */
+   function, and that for fixed-output derivations we return a hash of
+   its output path. */
 static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 {
     /* Return a fixed hash for fixed-output derivations. */
-    if (drv.outputs.size() == 1) {
+    if (isFixedOutput(drv)) {
         DerivationOutputs::const_iterator i = drv.outputs.begin();
-        if (i->first == "out" &&
-            i->second.hash != "")
-        {
-            return hashString(htSHA256, "fixed:out:"
-                + i->second.hashAlgo + ":"
-                + i->second.hash + ":"
-                + i->second.path);
-        }
+        return hashString(htSHA256, "fixed:out:"
+            + i->second.hashAlgo + ":"
+            + i->second.hash + ":"
+            + i->second.path);
     }
 
     /* For other derivations, replace the inputs paths with recursive
        calls to this function.*/
     DerivationInputs inputs2;
-    for (DerivationInputs::iterator i = drv.inputDrvs.begin();
-         i != drv.inputDrvs.end(); ++i)
-    {
+    foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) {
         Hash h = state.drvHashes[i->first];
         if (h.type == htUnknown) {
             Derivation drv2 = derivationFromPath(i->first);
@@ -389,6 +319,7 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
 {
     startNest(nest, lvlVomit, "evaluating derivation");
+    state.nrDepthAfterReset = 0;
 
     ATermMap attrs;
     queryAllAttrs(evalExpr(state, args[0]), attrs, true);
@@ -413,8 +344,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
     
     PathSet context;
 
-    string outputHash;
-    string outputHashAlgo;
+    string outputHash, outputHashAlgo;
     bool outputHashRecursive = false;
 
     for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i) {
@@ -473,13 +403,32 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
-    for (PathSet::iterator i = context.begin(); i != context.end(); ++i) {
-        debug(format("derivation uses `%1%'") % *i);
-        assert(isStorePath(*i));
-        if (isDerivation(*i))
-            drv.inputDrvs[*i] = singleton<StringSet>("out");
+    foreach (PathSet::iterator, i, context) {
+        Path path = *i;
+        
+        /* Paths marked with `=' denote that the path of a derivation
+           is explicitly passed to the builder.  Since that allows the
+           builder to gain access to every path in the dependency
+           graph of the derivation (including all outputs), all paths
+           in the graph must be added to this derivation's list of
+           inputs to ensure that they are available when the builder
+           runs. */
+        if (path.at(0) == '=') {
+            path = string(path, 1);
+            PathSet refs; computeFSClosure(path, refs);
+            foreach (PathSet::iterator, j, refs) {
+                drv.inputSrcs.insert(*j);
+                if (isDerivation(*j))
+                    drv.inputDrvs[*j] = singleton<StringSet>("out");
+            }
+        }
+        
+        debug(format("derivation uses `%1%'") % path);
+        assert(isStorePath(path));
+        if (isDerivation(path))
+            drv.inputDrvs[path] = singleton<StringSet>("out");
         else
-            drv.inputSrcs.insert(*i);
+            drv.inputSrcs.insert(path);
     }
             
     /* Do we have all required attributes? */
@@ -489,6 +438,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
         throw EvalError("required attribute `system' missing");
 
     /* If an output hash was given, check it. */
+    Path outPath;
     if (outputHash == "")
         outputHashAlgo = "";
     else {
@@ -507,6 +457,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
                 % outputHash % outputHashAlgo);
         string s = outputHash;
         outputHash = printHash(h);
+        outPath = makeFixedOutputPath(outputHashRecursive, ht, h, drvName);
         if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
     }
 
@@ -522,13 +473,12 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
        have an empty value.  This ensures that changes in the set of
        output names do get reflected in the hash. */
     drv.env["out"] = "";
-    drv.outputs["out"] =
-        DerivationOutput("", outputHashAlgo, outputHash);
+    drv.outputs["out"] = DerivationOutput("", outputHashAlgo, outputHash);
         
     /* Use the masked derivation expression to compute the output
        path. */
-    Path outPath = makeStorePath("output:out",
-        hashDerivationModulo(state, drv), drvName);
+    if (outPath == "")
+        outPath = makeStorePath("output:out", hashDerivationModulo(state, drv), drvName);
 
     /* Construct the final derivation store expression. */
     drv.env["out"] = outPath;
@@ -551,7 +501,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
     outAttrs.set(toATerm("outPath"),
         makeAttrRHS(makeStr(outPath, singleton<PathSet>(drvPath)), makeNoPos()));
     outAttrs.set(toATerm("drvPath"),
-        makeAttrRHS(makeStr(drvPath, singleton<PathSet>(drvPath)), makeNoPos()));
+        makeAttrRHS(makeStr(drvPath, singleton<PathSet>("=" + drvPath)), makeNoPos()));
 
     return makeAttrs(outAttrs);
 }
@@ -591,12 +541,36 @@ static Expr prim_toPath(EvalState & state, const ATermVector & args)
 }
 
 
+/* Allow a valid store path to be used in an expression.  This is
+   useful in some generated expressions such as in nix-push, which
+   generates a call to a function with an already existing store path
+   as argument.  You don't want to use `toPath' here because it copies
+   the path to the Nix store, which yields a copy like
+   /nix/store/newhash-oldhash-oldname.  In the past, `toPath' had
+   special case behaviour for store paths, but that created weird
+   corner cases. */
+static Expr prim_storePath(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    Path path = canonPath(coerceToPath(state, args[0], context));
+    state.nrDepthAfterReset = 0;
+    if (!isInStore(path))
+        throw EvalError(format("path `%1%' is not in the Nix store") % path);
+    Path path2 = toStorePath(path);
+    if (!store->isValidPath(path2))
+        throw EvalError(format("store path `%1%' is not valid") % path2);
+    context.insert(path2);
+    return makeStr(path, context);
+}
+
+
 static Expr prim_pathExists(EvalState & state, const ATermVector & args)
 {
     PathSet context;
     Path path = coerceToPath(state, args[0], context);
     if (!context.empty())
         throw EvalError(format("string `%1%' cannot refer to other paths") % path);
+    state.nrDepthAfterReset = 0;
     return makeBool(pathExists(path));
 }
 
@@ -630,6 +604,7 @@ static Expr prim_readFile(EvalState & state, const ATermVector & args)
     Path path = coerceToPath(state, args[0], context);
     if (!context.empty())
         throw EvalError(format("string `%1%' cannot refer to other paths") % path);
+    state.nrDepthAfterReset = 0;
     return makeStr(readFile(path));
 }
 
@@ -662,11 +637,14 @@ static Expr prim_toFile(EvalState & state, const ATermVector & args)
     PathSet refs;
 
     for (PathSet::iterator i = context.begin(); i != context.end(); ++i) {
-        if (isDerivation(*i))
+        Path path = *i;
+        if (path.at(0) == '=') path = string(path, 1);
+        if (isDerivation(path))
             throw EvalError(format("in `toFile': the file `%1%' cannot refer to derivation outputs") % name);
-        refs.insert(*i);
+        refs.insert(path);
     }
     
+    state.nrDepthAfterReset = 0;
     Path storePath = readOnlyMode
         ? computeStorePathForText(name, contents, refs)
         : store->addTextToStore(name, contents, refs);
@@ -719,9 +697,10 @@ static Expr prim_filterSource(EvalState & state, const ATermVector & args)
 
     FilterFromExpr filter(state, args[0]);
 
+    state.nrDepthAfterReset = 0;
     Path dstPath = readOnlyMode
-        ? computeStorePathForPath(path, false, false, "", filter).first
-        : store->addToStore(path, false, false, "", filter);
+        ? computeStorePathForPath(path, true, htSHA256, filter).first
+        : store->addToStore(path, true, htSHA256, filter);
 
     return makeStr(dstPath, singleton<PathSet>(dstPath));
 }
@@ -817,7 +796,7 @@ static Expr prim_removeAttrs(EvalState & state, const ATermVector & args)
 }
 
 
-/* Determine whether the argument is a list. */
+/* Determine whether the argument is an attribute set. */
 static Expr prim_isAttrs(EvalState & state, const ATermVector & args)
 {
     ATermList list;
@@ -873,6 +852,14 @@ static Expr prim_map(EvalState & state, const ATermVector & args)
 }
 
 
+/* Return the length of a list.  This is an O(1) time operation. */
+static Expr prim_length(EvalState & state, const ATermVector & args)
+{
+    ATermList list = evalList(state, args[0]);
+    return makeInt(ATgetLength(list));
+}
+
+
 /*************************************************************
  * Integer arithmetic
  *************************************************************/
@@ -894,6 +881,23 @@ static Expr prim_sub(EvalState & state, const ATermVector & args)
 }
 
 
+static Expr prim_mul(EvalState & state, const ATermVector & args)
+{
+    int i1 = evalInt(state, args[0]);
+    int i2 = evalInt(state, args[1]);
+    return makeInt(i1 * i2);
+}
+
+
+static Expr prim_div(EvalState & state, const ATermVector & args)
+{
+    int i1 = evalInt(state, args[0]);
+    int i2 = evalInt(state, args[1]);
+    if (i2 == 0) throw EvalError("division by zero");
+    return makeInt(i1 / i2);
+}
+
+
 static Expr prim_lessThan(EvalState & state, const ATermVector & args)
 {
     int i1 = evalInt(state, args[0]);
@@ -950,23 +954,54 @@ static Expr prim_unsafeDiscardStringContext(EvalState & state, const ATermVector
     return makeStr(s, PathSet());
 }
 
-/* Expression serialization/deserialization */ 
 
-static Expr prim_ExprToString ( EvalState & state, const ATermVector & args)
+/* Expression serialization/deserialization */
+
+
+static Expr prim_exprToString(EvalState & state, const ATermVector & args)
 {
-	return makeStr ( atPrint ( evalExpr ( state, args [ 0 ] ) ) );
+    /* !!! this disregards context */
+    return makeStr(atPrint(evalExpr(state, args[0])));
 }
 
-static Expr prim_StringToExpr ( EvalState & state, const ATermVector & args)
+
+static Expr prim_stringToExpr(EvalState & state, const ATermVector & args)
 {
-	string s;
-	PathSet l;
-	if (! matchStr ( evalExpr ( state, args[0] ), s, l )) {
-		throw EvalError("__stringToExpr needs string argument!");
-	}
-	return ATreadFromString(s.c_str());
+    /* !!! this can introduce arbitrary garbage terms in the
+       evaluator! */;
+    string s;
+    PathSet l;
+    if (!matchStr(evalExpr(state, args[0]), s, l))
+        throw EvalError("stringToExpr needs string argument!");
+    return ATreadFromString(s.c_str());
 }
 
+
+/*************************************************************
+ * Versions
+ *************************************************************/
+
+
+static Expr prim_parseDrvName(EvalState & state, const ATermVector & args)
+{
+    string name = evalStringNoCtx(state, args[0]);
+    DrvName parsed(name);
+    ATermMap attrs(2);
+    attrs.set(toATerm("name"), makeAttrRHS(makeStr(parsed.name), makeNoPos()));
+    attrs.set(toATerm("version"), makeAttrRHS(makeStr(parsed.version), makeNoPos()));
+    return makeAttrs(attrs);
+}
+
+
+static Expr prim_compareVersions(EvalState & state, const ATermVector & args)
+{
+    string version1 = evalStringNoCtx(state, args[0]);
+    string version2 = evalStringNoCtx(state, args[1]);
+    int d = compareVersions(version1, version2);
+    return makeInt(d);
+}
+
+
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -987,17 +1022,20 @@ void EvalState::addPrimOps()
     addPrimOp("import", 1, prim_import);
     addPrimOp("isNull", 1, prim_isNull);
     addPrimOp("__isFunction", 1, prim_isFunction);
-    addPrimOp("dependencyClosure", 1, prim_dependencyClosure);
+    addPrimOp("__isString", 1, prim_isString);
+    addPrimOp("__isInt", 1, prim_isInt);
+    addPrimOp("__isBool", 1, prim_isBool);
+    addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("throw", 1, prim_throw);
+    addPrimOp("__addErrorContext", 2, prim_addErrorContext);
     addPrimOp("__getEnv", 1, prim_getEnv);
     addPrimOp("__trace", 2, prim_trace);
+
     
     // Expr <-> String
-    addPrimOp("__exprToString", 1, prim_ExprToString);
-    addPrimOp("__stringToExpr", 1, prim_StringToExpr);
-
-    addPrimOp("relativise", 2, prim_relativise);
+    addPrimOp("__exprToString", 1, prim_exprToString);
+    addPrimOp("__stringToExpr", 1, prim_stringToExpr);
 
     // Derivations
     addPrimOp("derivation!", 1, prim_derivationStrict);
@@ -1005,6 +1043,7 @@ void EvalState::addPrimOps()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
+    addPrimOp("__storePath", 1, prim_storePath);
     addPrimOp("__pathExists", 1, prim_pathExists);
     addPrimOp("baseNameOf", 1, prim_baseNameOf);
     addPrimOp("dirOf", 1, prim_dirOf);
@@ -1028,10 +1067,13 @@ void EvalState::addPrimOps()
     addPrimOp("__head", 1, prim_head);
     addPrimOp("__tail", 1, prim_tail);
     addPrimOp("map", 2, prim_map);
+    addPrimOp("__length", 1, prim_length);
 
     // Integer arithmetic
     addPrimOp("__add", 2, prim_add);
     addPrimOp("__sub", 2, prim_sub);
+    addPrimOp("__mul", 2, prim_mul);
+    addPrimOp("__div", 2, prim_div);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation
@@ -1039,7 +1081,10 @@ void EvalState::addPrimOps()
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    
+
+    // Versions
+    addPrimOp("__parseDrvName", 1, prim_parseDrvName);
+    addPrimOp("__compareVersions", 2, prim_compareVersions);
 }
 
 

src/libmain/Makefile.am

@@ -1,6 +1,10 @@
 pkglib_LTLIBRARIES = libmain.la
 
-libmain_la_SOURCES = shared.cc shared.hh
+libmain_la_SOURCES = shared.cc
+
+libmain_la_LIBADD = ../libstore/libstore.la
+
+pkginclude_HEADERS = shared.hh
 
 AM_CXXFLAGS = \
  -DNIX_STORE_DIR=\"$(storedir)\" \

src/libmain/shared.cc

@@ -4,6 +4,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "misc.hh"
 
 #include <iostream>
 #include <cctype>
@@ -49,6 +50,34 @@ void printGCWarning()
 }
 
 
+void printMissing(const PathSet & paths)
+{
+    unsigned long long downloadSize;
+    PathSet willBuild, willSubstitute, unknown;
+    queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize);
+
+    if (!willBuild.empty()) {
+        printMsg(lvlInfo, format("the following derivations will be built:"));
+        foreach (PathSet::iterator, i, willBuild)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+
+    if (!willSubstitute.empty()) {
+        printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") %
+            (downloadSize / (1024.0 * 1024.0)));
+        foreach (PathSet::iterator, i, willSubstitute)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+
+    if (!unknown.empty()) {
+        printMsg(lvlInfo, format("don't know how to build the following paths%1%:")
+            % (readOnlyMode ? " (may be caused by read-only store access)" : ""));
+        foreach (PathSet::iterator, i, unknown)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+}
+
+
 static void setLogType(string lt)
 {
     if (lt == "pretty") logType = ltPretty;
@@ -58,27 +87,18 @@ static void setLogType(string lt)
 }
 
 
-static unsigned int getIntArg(const string & opt,
+unsigned long long getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
     if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
-    int n;
+    long long n;
     if (!string2Int(*i, n) || n < 0)
         throw UsageError(format("`%1%' requires a non-negative integer") % opt);
     return n;
 }
 
 
-struct RemoveTempRoots 
-{
-    ~RemoveTempRoots()
-    {
-        removeTempRoots();
-    }
-};
-
-
 void initDerivationsHelpers();
 
 
@@ -99,6 +119,15 @@ static void closeStore()
 }
 
 
+RemoveTempRoots::~RemoveTempRoots()
+{
+    removeTempRoots();
+}
+
+
+static bool showTrace = false;
+
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
@@ -114,9 +143,10 @@ static void initAndRun(int argc, char * * argv)
     nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
-    if (subs == "default")
-        substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl");
-    else
+    if (subs == "default") {
+        substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+        substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+    } else
         substituters = tokenizeString(subs, ":");
 
     /* Get some settings from the configuration file. */
@@ -192,6 +222,8 @@ static void initAndRun(int argc, char * * argv)
             ; /* !!! obsolete - remove eventually */
         else if (arg == "--no-build-output" || arg == "-Q")
             buildVerbosity = lvlVomit;
+        else if (arg == "--print-build-trace")
+            printBuildTrace = true;
         else if (arg == "--help") {
             printHelp();
             return;
@@ -214,12 +246,21 @@ static void initAndRun(int argc, char * * argv)
             maxSilentTime = getIntArg(arg, i, args.end());
         else if (arg == "--no-build-hook")
             useBuildHook = false;
+        else if (arg == "--show-trace")
+            showTrace = true;
+        else if (arg == "--option") {
+            ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
+            string name = *i;
+            ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
+            string value = *i;
+            overrideSetting(name, tokenizeString(value));
+        }
         else remaining.push_back(arg);
     }
 
     /* Automatically clean up the temporary roots file when we
        exit. */
-    RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
+    RemoveTempRoots removeTempRoots __attribute__((unused));
 
     /* Make sure that the database gets closed properly, even if
        terminate() is called (which happens sometimes due to bugs in
@@ -329,7 +370,9 @@ int main(int argc, char * * argv)
             % e.what() % programId);
         return 1;
     } catch (BaseError & e) {
-        printMsg(lvlError, format("error: %1%") % e.msg());
+        printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg());
+        if (e.prefix() != "" && !showTrace)
+            printMsg(lvlError, "(use `--show-trace' to show detailed location information)");
         return 1;
     } catch (std::exception & e) {
         printMsg(lvlError, format("error: %1%") % e.what());

src/libmain/shared.hh

@@ -26,6 +26,11 @@ namespace nix {
 Path makeRootName(const Path & gcRoot, int & counter);
 void printGCWarning();
 
+void printMissing(const PathSet & paths);
+
+unsigned long long getIntArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end);
+
 /* Whether we're running setuid. */
 extern bool setuidMode;
 
@@ -33,6 +38,11 @@ extern volatile ::sig_atomic_t blockInt;
 
 MakeError(UsageError, nix::Error);
 
+struct RemoveTempRoots 
+{
+    ~RemoveTempRoots();    
+};
+
 }
 
 

src/libstore/derivations.cc

@@ -2,6 +2,7 @@
 #include "store-api.hh"
 #include "aterm.hh"
 #include "globals.hh"
+#include "util.hh"
 
 #include "derivations-ast.hh"
 #include "derivations-ast.cc"
@@ -20,8 +21,7 @@ Path writeDerivation(const Derivation & drv, const string & name)
 {
     PathSet references;
     references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
-    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
-         i != drv.inputDrvs.end(); ++i)
+    foreach (DerivationInputs::const_iterator, i, drv.inputDrvs)
         references.insert(i->first);
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
@@ -163,9 +163,7 @@ ATerm unparseDerivation(const Derivation & drv)
 
 bool isDerivation(const string & fileName)
 {
-    return
-        fileName.size() >= drvExtension.size() &&
-        string(fileName, fileName.size() - drvExtension.size()) == drvExtension;
+    return hasSuffix(fileName, drvExtension);
 }
 
  

src/libstore/gc.cc

@@ -5,6 +5,9 @@
 
 #include <boost/shared_ptr.hpp>
 
+#include <functional>
+#include <queue>
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <errno.h>
@@ -123,7 +126,11 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
         createSymlink(gcRoot, storePath, false);
     }
 
-    /* Check that the root can be found by the garbage collector. */
+    /* Check that the root can be found by the garbage collector.
+       !!! This can be very slow on machines that have many roots.
+       Instead of reading all the roots, it would be more efficient to
+       check if the root is in a directory in or linked from the
+       gcroots directory. */
     if (queryBoolSetting("gc-check-reachability", true)) {
         Roots roots = store->findRoots();
         if (roots.find(gcRoot) == roots.end())
@@ -240,9 +247,7 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
     Strings tempRootFiles = readDirectory(
         (format("%1%/%2%") % nixStateDir % tempRootsDir).str());
 
-    for (Strings::iterator i = tempRootFiles.begin();
-         i != tempRootFiles.end(); ++i)
-    {
+    foreach (Strings::iterator, i, tempRootFiles) {
         Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
 
         debug(format("reading temporary root file `%1%'") % path);
@@ -321,7 +326,7 @@ static void findRoots(const Path & path, bool recurseSymlinks,
 
         if (S_ISDIR(st.st_mode)) {
             Strings names = readDirectory(path);
-            for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+            foreach (Strings::iterator, i, names)
                 findRoots(path + "/" + *i, recurseSymlinks, deleteStale, roots);
         }
 
@@ -392,7 +397,7 @@ static void addAdditionalRoots(PathSet & roots)
 
     Strings paths = tokenizeString(result, "\n");
     
-    for (Strings::iterator i = paths.begin(); i != paths.end(); ++i) {
+    foreach (Strings::iterator, i, paths) {
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
             if (roots.find(path) == roots.end() && store->isValidPath(path)) {
@@ -414,8 +419,7 @@ static void dfsVisit(const PathSet & paths, const Path & path,
     if (store->isValidPath(path))
         store->queryReferences(path, references);
     
-    for (PathSet::iterator i = references.begin();
-         i != references.end(); ++i)
+    foreach (PathSet::iterator, i, references)
         /* Don't traverse into paths that don't exist.  That can
            happen due to substitutes for non-existent paths. */
         if (*i != path && paths.find(*i) != paths.end())
@@ -429,18 +433,146 @@ Paths topoSortPaths(const PathSet & paths)
 {
     Paths sorted;
     PathSet visited;
-    for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i)
+    foreach (PathSet::const_iterator, i, paths)
         dfsVisit(paths, *i, visited, sorted);
     return sorted;
 }
 
 
-void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
-    bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
+static time_t lastFileAccessTime(const Path & path)
 {
-    result.clear();
-    bytesFreed = 0;
+    checkInterrupt();
+    
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1)
+        throw SysError(format("statting `%1%'") % path);
 
+    if (S_ISDIR(st.st_mode)) {
+        time_t last = 0;
+	Strings names = readDirectory(path);
+	foreach (Strings::iterator, i, names) {
+            time_t t = lastFileAccessTime(path + "/" + *i);
+            if (t > last) last = t;
+        }
+        return last;
+    }
+
+    else if (S_ISLNK(st.st_mode)) return 0;
+
+    else return st.st_atime;
+}
+
+
+struct GCLimitReached { };
+
+
+void LocalStore::gcPath(const GCOptions & options, GCResults & results, 
+    const Path & path)
+{
+    results.paths.insert(path);
+
+    if (!pathExists(path)) return;
+                
+    /* Okay, it's safe to delete. */
+    unsigned long long bytesFreed, blocksFreed;
+    deleteFromStore(path, bytesFreed, blocksFreed);
+    results.bytesFreed += bytesFreed;
+    results.blocksFreed += blocksFreed;
+
+    if (options.maxFreed && results.bytesFreed > options.maxFreed) {
+        printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % options.maxFreed);
+        throw GCLimitReached();
+    }
+
+    if (options.maxLinks) {
+        struct stat st;
+        if (stat(nixStore.c_str(), &st) == -1)
+            throw SysError(format("statting `%1%'") % nixStore);
+        if (st.st_nlink < options.maxLinks) {
+            printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % options.maxLinks);
+            throw GCLimitReached();
+        }
+    }
+}
+
+
+void LocalStore::gcPathRecursive(const GCOptions & options,
+    GCResults & results, PathSet & done, const Path & path)
+{
+    if (done.find(path) != done.end()) return;
+    done.insert(path);
+
+    startNest(nest, lvlDebug, format("looking at `%1%'") % path);
+        
+    /* Delete all the referrers first.  They must be garbage too,
+       since if they were live, then the current path would also be
+       live.  Note that deleteFromStore() below still makes sure that
+       the referrer set has become empty, just in case.  (However that
+       doesn't guard against deleting top-level paths that are only
+       reachable from GC roots.) */
+    PathSet referrers;
+    if (isValidPath(path))
+        queryReferrers(path, referrers);
+    foreach (PathSet::iterator, i, referrers)
+        if (*i != path) gcPathRecursive(options, results, done, *i);
+
+    printMsg(lvlInfo, format("deleting `%1%'") % path);
+            
+    gcPath(options, results, path);
+}
+
+
+struct CachingAtimeComparator : public std::binary_function<Path, Path, bool> 
+{
+    std::map<Path, time_t> cache;
+
+    time_t lookup(const Path & p)
+    {
+        std::map<Path, time_t>::iterator i = cache.find(p);
+        if (i != cache.end()) return i->second;
+        debug(format("computing atime of `%1%'") % p);
+        cache[p] = lastFileAccessTime(p);
+        assert(cache.find(p) != cache.end());
+        return cache[p];
+    }
+        
+    bool operator () (const Path & p1, const Path & p2)
+    {
+        return lookup(p2) < lookup(p1);
+    }
+};
+
+
+static string showTime(const string & format, time_t t)
+{
+    char s[128];
+    strftime(s, sizeof s, format.c_str(), localtime(&t));
+    return string(s);
+}
+
+
+static bool isLive(const Path & path, const PathSet & livePaths,
+    const PathSet & tempRoots, const PathSet & tempRootsClosed)
+{
+    if (livePaths.find(path) != livePaths.end() ||
+        tempRootsClosed.find(path) != tempRootsClosed.end()) return true;
+
+    /* A lock file belonging to a path that we're building right
+       now isn't garbage. */
+    if (hasSuffix(path, ".lock") && tempRoots.find(string(path, 0, path.size() - 5)) != tempRoots.end())
+        return true;
+
+    /* Don't delete .chroot directories for derivations that are
+       currently being built. */
+    if (hasSuffix(path, ".chroot") && tempRoots.find(string(path, 0, path.size() - 7)) != tempRoots.end())
+        return true;
+
+    return false;
+}
+
+
+void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
     bool gcKeepOutputs =
         queryBoolSetting("gc-keep-outputs", false);
     bool gcKeepDerivations =
@@ -455,47 +587,45 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);
+    printMsg(lvlError, format("finding garbage collector roots..."));
+    Roots rootMap = options.ignoreLiveness ? Roots() : nix::findRoots(true);
 
     PathSet roots;
-    for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i)
-        roots.insert(i->second);
+    foreach (Roots::iterator, i, rootMap) roots.insert(i->second);
 
     /* Add additional roots returned by the program specified by the
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
-    if (!ignoreLiveness)
+    if (!options.ignoreLiveness)
         addAdditionalRoots(roots);
 
-    if (action == gcReturnRoots) {
-        result = roots;
+    if (options.action == GCOptions::gcReturnRoots) {
+        results.paths = roots;
         return;
     }
 
     /* Determine the live paths which is just the closure of the
        roots under the `references' relation. */
+    printMsg(lvlError, format("computing live paths..."));
     PathSet livePaths;
-    for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
+    foreach (PathSet::const_iterator, i, roots)
         computeFSClosure(canonPath(*i), livePaths);
 
     if (gcKeepDerivations) {
-        for (PathSet::iterator i = livePaths.begin();
-             i != livePaths.end(); ++i)
-        {
+        foreach (PathSet::iterator, i, livePaths) {
             /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
-            Path deriver = store->queryDeriver(*i);
-            if (deriver != "" && store->isValidPath(deriver))
+            Path deriver = queryDeriver(*i);
+            if (deriver != "" && isValidPath(deriver))
                 computeFSClosure(deriver, livePaths);
         }
     }
 
     if (gcKeepOutputs) {
         /* Hmz, identical to storePathRequisites in nix-store. */
-        for (PathSet::iterator i = livePaths.begin();
-             i != livePaths.end(); ++i)
+        foreach (PathSet::iterator, i, livePaths)
             if (isDerivation(*i)) {
                 Derivation drv = derivationFromPath(*i);
 
@@ -505,15 +635,14 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
 		    gcLevel = defaultGcLevel;
 		
 		if (gcLevel >= gcKeepOutputsThreshold)    
-		    for (DerivationOutputs::iterator j = drv.outputs.begin();
-                         j != drv.outputs.end(); ++j)
-			if (store->isValidPath(j->second.path))
+		    foreach (DerivationOutputs::iterator, j, drv.outputs)
+			if (isValidPath(j->second.path))
 			    computeFSClosure(j->second.path, livePaths);
             }
     }
 
-    if (action == gcReturnLive) {
-        result = livePaths;
+    if (options.action == GCOptions::gcReturnLive) {
+        results.paths = livePaths;
         return;
     }
 
@@ -531,109 +660,129 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
        (and computeFSClosure() assumes that the presence of a path
        means that it has already been closed). */
     PathSet tempRootsClosed;
-    for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
-        if (store->isValidPath(*i))
+    foreach (PathSet::iterator, i, tempRoots)
+        if (isValidPath(*i))
             computeFSClosure(*i, tempRootsClosed);
         else
             tempRootsClosed.insert(*i);
 
-    /* For testing - see tests/gc-concurrent.sh. */
-    if (getenv("NIX_DEBUG_GC_WAIT"))
-        sleep(2);
-    
     /* After this point the set of roots or temporary roots cannot
        increase, since we hold locks on everything.  So everything
        that is not currently in in `livePaths' or `tempRootsClosed'
        can be deleted. */
     
     /* Read the Nix store directory to find all currently existing
-       paths. */
-    PathSet storePathSet;
-    if (action != gcDeleteSpecific) {
+       paths and filter out all live paths. */
+    printMsg(lvlError, format("reading the Nix store..."));
+    PathSet storePaths;
+    
+    if (options.action != GCOptions::gcDeleteSpecific) {
         Paths entries = readDirectory(nixStore);
-        for (Paths::iterator i = entries.begin(); i != entries.end(); ++i)
-            storePathSet.insert(canonPath(nixStore + "/" + *i));
-    } else {
-        for (PathSet::iterator i = pathsToDelete.begin();
-             i != pathsToDelete.end(); ++i)
-        {
-            assertStorePath(*i);
-            storePathSet.insert(*i);
+        foreach (Paths::iterator, i, entries) {
+            Path path = canonPath(nixStore + "/" + *i);
+            if (!isLive(path, livePaths, tempRoots, tempRootsClosed)) storePaths.insert(path);
         }
     }
 
-    /* Topologically sort them under the `referrers' relation.  That
-       is, a < b iff a is in referrers(b).  This gives us the order in
-       which things can be deleted safely. */
-    /* !!! when we have multiple output paths per derivation, this
-       will not work anymore because we get cycles. */
-    Paths storePaths = topoSortPaths(storePathSet);
-
-    /* Try to delete store paths in the topologically sorted order. */
-    for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
-
-        debug(format("considering deletion of `%1%'") % *i);
-        
-        if (livePaths.find(*i) != livePaths.end()) {
-            if (action == gcDeleteSpecific)
+    else {
+        foreach (PathSet::iterator, i, options.pathsToDelete) {
+            assertStorePath(*i);
+            storePaths.insert(*i);
+            if (isLive(*i, livePaths, tempRoots, tempRootsClosed))
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
-            debug(format("live path `%1%'") % *i);
-            continue;
+        }
+    }
+
+    if (options.action == GCOptions::gcReturnDead) {
+        results.paths.insert(storePaths.begin(), storePaths.end());
+        return;
+    }
+
+    /* Delete all dead store paths (or until one of the stop
+       conditions is reached). */
+
+    PathSet done;
+    try {
+
+        if (!options.useAtime) {
+            /* Delete the paths, respecting the partial ordering
+               determined by the references graph. */
+            printMsg(lvlError, format("deleting garbage..."));
+            foreach (PathSet::iterator, i, storePaths)
+                gcPathRecursive(options, results, done, *i);
         }
 
-        if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
-            debug(format("temporary root `%1%'") % *i);
-            continue;
-        }
+        else {
 
-        debug(format("dead path `%1%'") % *i);
-        result.insert(*i);
+            /* Delete in order of ascending last access time, still
+               maintaining the partial ordering of the reference
+               graph.  Note that we can't use a topological sort for
+               this because that takes time O(V+E), and in this case
+               E=O(V^2) (i.e. the graph is dense because of the edges
+               due to the atime ordering).  So instead we put all
+               deletable paths in a priority queue (ordered by atime),
+               and after deleting a path, add additional paths that
+               have become deletable to the priority queue. */
 
-        /* If just returning the set of dead paths, we also return the
-           space that would be freed if we deleted them. */
-        if (action == gcReturnDead)
-            bytesFreed += computePathSize(*i);
+            CachingAtimeComparator atimeComp;
 
-        if (action == gcDeleteDead || action == gcDeleteSpecific) {
+            /* Create a priority queue that orders paths by ascending
+               atime.  This is why C++ needs type inferencing... */
+            std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> > prioQueue =
+                std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> >(binary_function_ref_adapter<CachingAtimeComparator>(&atimeComp));
 
-#ifndef __CYGWIN__
-            AutoCloseFD fdLock;
+           /* Initially put the paths that are invalid or have no
+              referrers into the priority queue. */
+            printMsg(lvlError, format("finding deletable paths..."));
+            foreach (PathSet::iterator, i, storePaths) {
+                checkInterrupt();
+                /* We can safely delete a path if it's invalid or
+                   it has no referrers.  Note that all the invalid
+                   paths will be deleted in the first round. */
+                if (isValidPath(*i)) {
+                    if (queryReferrersNoSelf(*i).empty()) prioQueue.push(*i);
+                } else prioQueue.push(*i);
+            }
 
-            /* Only delete a lock file if we can acquire a write lock
-               on it.  That means that it's either stale, or the
-               process that created it hasn't locked it yet.  In the
-               latter case the other process will detect that we
-               deleted the lock, and retry (see pathlocks.cc). */
-            if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
-                fdLock = openLockFile(*i, false);
-                if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
-                    debug(format("skipping active lock `%1%'") % *i);
+            debug(format("%1% initially deletable paths") % prioQueue.size());
+
+            /* Now delete everything in the order of the priority
+               queue until nothing is left. */
+            printMsg(lvlError, format("deleting garbage..."));
+            while (!prioQueue.empty()) {
+                checkInterrupt();
+                Path path = prioQueue.top(); prioQueue.pop();
+
+                if (options.maxAtime != (time_t) -1 &&
+                    atimeComp.lookup(path) > options.maxAtime)
                     continue;
+                
+                printMsg(lvlInfo, format("deleting `%1%' (last accessed %2%)") % path % showTime("%F %H:%M:%S", atimeComp.lookup(path)));
+
+                PathSet references;
+                if (isValidPath(path)) references = queryReferencesNoSelf(path);
+
+                gcPath(options, results, path);
+
+                /* For each reference of the current path, see if the
+                   reference has now become deletable (i.e. is in the
+                   set of dead paths and has no referrers left).  If
+                   so add it to the priority queue. */
+                foreach (PathSet::iterator, i, references) {
+                    if (storePaths.find(*i) != storePaths.end() &&
+                        queryReferrersNoSelf(*i).empty())
+                    {
+                        debug(format("path `%1%' has become deletable") % *i);
+                        prioQueue.push(*i);
+                    }
                 }
             }
-#endif
-
-            if (!pathExists(*i)) continue;
-                
-            printMsg(lvlInfo, format("deleting `%1%'") % *i);
             
-            /* Okay, it's safe to delete. */
-            try {
-                unsigned long long freed;
-                deleteFromStore(*i, freed);
-                bytesFreed += freed;
-            } catch (PathInUse & e) {
-                printMsg(lvlError, format("warning: %1%") % e.msg());
-            }
-
-#ifndef __CYGWIN__
-            if (fdLock != -1)
-                /* Write token to stale (deleted) lock file. */
-                writeFull(fdLock, (const unsigned char *) "d", 1);
-#endif
         }
+        
+    } catch (GCLimitReached & e) {
     }
 }
 
- 
+
 }

src/libstore/globals.cc

@@ -13,6 +13,7 @@ string nixDataDir = "/UNINIT";
 string nixLogDir = "/UNINIT";
 string nixStateDir = "/UNINIT";
 string nixDBPath = "/UNINIT";
+string nixCacheFile = "/UNINIT";
 string nixConfDir = "/UNINIT";
 string nixLibexecDir = "/UNINIT";
 string nixBinDir = "/UNINIT";
@@ -24,15 +25,19 @@ Verbosity buildVerbosity = lvlInfo;
 unsigned int maxBuildJobs = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
-unsigned int maxSilentTime = 0;
+time_t maxSilentTime = 0;
 Paths substituters;
 bool useBuildHook = true;
+bool printBuildTrace = false;
 
 
 static bool settingsRead = false;
 
 static std::map<string, Strings> settings;
 
+/* Overriden settings. */
+std::map<string, Strings> settingsCmdline;
+
 
 string & at(Strings & ss, unsigned int n)
 {
@@ -72,6 +77,8 @@ static void readSettings()
         advance(i, 2);
         settings[name] = Strings(i, tokens.end());
     };
+
+    settings.insert(settingsCmdline.begin(), settingsCmdline.end());
     
     settingsRead = true;
 }
@@ -116,5 +123,19 @@ unsigned int queryIntSetting(const string & name, unsigned int def)
     return n;
 }
 
+
+void overrideSetting(const string & name, const Strings & value)
+{
+    if (settingsRead) settings[name] = value;
+    settingsCmdline[name] = value;
+}
+
+
+void reloadSettings()
+{
+    settingsRead = false;
+    settings.clear();
+}
+
  
 }

src/libstore/globals.hh

@@ -24,6 +24,9 @@ extern string nixStateDir;
 /* nixDBPath is the path name of our Berkeley DB environment. */
 extern string nixDBPath;
 
+/* nixCacheFile is the path name of the normal form cache. */
+extern string nixCacheFile;
+
 /* nixConfDir is the directory where configuration files are
    stored. */
 extern string nixConfDir;
@@ -65,7 +68,7 @@ extern string thisSystem;
 /* The maximum time in seconds that a builer can go without producing
    any output on stdout/stderr before it is killed.  0 means
    infinity. */
-extern unsigned int maxSilentTime;
+extern time_t maxSilentTime;
 
 /* The substituters.  There are programs that can somehow realise a
    store path without building, e.g., by downloading it or copying it
@@ -76,6 +79,22 @@ extern Paths substituters;
    users want to disable this from the command-line. */
 extern bool useBuildHook;
 
+/* Whether buildDerivations() should print out lines on stderr in a
+   fixed format to allow its progress to be monitored.  Each line
+   starts with a "@".  The following are defined:
+
+   @ build-started <drvpath> <outpath> <system> <logfile>
+   @ build-failed <drvpath> <outpath> <exitcode> <error text>
+   @ build-succeeded <drvpath> <outpath>
+   @ substituter-started <outpath> <substituter>
+   @ substituter-failed <outpath> <exitcode> <error text>
+   @ substituter-succeeded <outpath>
+
+   Best combined with --no-build-output, otherwise stderr might
+   conceivably contain lines in this format printed by the builders.
+*/
+extern bool printBuildTrace;
+
 
 Strings querySetting(const string & name, const Strings & def);
 
@@ -85,6 +104,10 @@ bool queryBoolSetting(const string & name, bool def);
 
 unsigned int queryIntSetting(const string & name, unsigned int def);
 
+void overrideSetting(const string & name, const Strings & value);
+
+void reloadSettings();
+
     
 }
 

src/libstore/local-store.cc

@@ -15,6 +15,7 @@
 #include <unistd.h>
 #include <utime.h>
 #include <fcntl.h>
+#include <errno.h>
 
 
 namespace nix {
@@ -46,10 +47,18 @@ LocalStore::LocalStore()
     
     if (readOnlyMode) return;
 
+    createDirs(nixStore);
+    
     checkStoreNotSymlink();
 
-    Path globalLockPath = nixDBPath + "/big-lock";
-    globalLock = openLockFile(globalLockPath.c_str(), true);
+    try {
+        Path globalLockPath = nixDBPath + "/big-lock";
+        globalLock = openLockFile(globalLockPath.c_str(), true);
+    } catch (SysError & e) {
+        if (e.errNo != EACCES) throw;
+        readOnlyMode = true;
+        return;
+    }
     
     if (!lockFile(globalLock, ltRead, false)) {
         printMsg(lvlError, "waiting for the big Nix store lock...");
@@ -58,9 +67,7 @@ LocalStore::LocalStore()
 
     createDirs(nixDBPath + "/info");
     createDirs(nixDBPath + "/referrer");
-
-    //printMsg(lvlTalkative, "cannot access Nix database; continuing anyway");
-    //readOnlyMode = true;
+    createDirs(nixDBPath + "/failed");
 
     int curSchema = getSchema();
     if (curSchema > nixSchemaVersion)
@@ -79,6 +86,13 @@ LocalStore::~LocalStore()
 {
     try {
         flushDelayedUpdates();
+
+        foreach (RunningSubstituters::iterator, i, runningSubstituters) {
+            i->second.to.close();
+            i->second.from.close();
+            i->second.pid.wait(true);
+        }
+                
     } catch (...) {
         ignoreException();
     }
@@ -97,23 +111,6 @@ int LocalStore::getSchema()
 }
 
 
-void copyPath(const Path & src, const Path & dst, PathFilter & filter)
-{
-    debug(format("copying `%1%' to `%2%'") % src % dst);
-
-    /* Dump an archive of the path `src' into a string buffer, then
-       restore the archive to `dst'.  This is not a very good method
-       for very large paths, but `copyPath' is mainly used for small
-       files. */ 
-
-    StringSink sink;
-    dumpPath(src, sink, filter);
-
-    StringSource source(sink.s);
-    restorePath(dst, source);
-}
-
-
 void canonicalisePathMetaData(const Path & path, bool recurse)
 {
     checkInterrupt();
@@ -156,7 +153,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
         if (st.st_mtime != 0) {
             struct utimbuf utimbuf;
             utimbuf.actime = st.st_atime;
-            utimbuf.modtime = 0;
+            utimbuf.modtime = 1; /* 1 second into the epoch */
             if (utime(path.c_str(), &utimbuf) == -1) 
                 throw SysError(format("changing modification time of `%1%'") % path);
         }
@@ -165,7 +162,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
 
     if (recurse && S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+	foreach (Strings::iterator, i, names)
 	    canonicalisePathMetaData(path + "/" + *i, true);
     }
 }
@@ -202,6 +199,13 @@ static Path referrersFileFor(const Path & path)
 }
 
 
+static Path failedFileFor(const Path & path)
+{
+    string baseName = baseNameOf(path);
+    return (format("%1%/failed/%2%") % nixDBPath % baseName).str();
+}
+
+
 static Path tmpFileForAtomicUpdate(const Path & path)
 {
     return (format("%1%/.%2%.%3%") % dirOf(path) % getpid() % baseNameOf(path)).str();
@@ -321,6 +325,8 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi
             appendReferrer(*i, info.path, false);
     }
 
+    assert(info.hash.type == htSHA256);
+
     string s = (format(
         "Hash: sha256:%1%\n"
         "References: %2%\n"
@@ -339,6 +345,20 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi
 }
 
 
+void LocalStore::registerFailedPath(const Path & path)
+{
+    /* Write an empty file in the .../failed directory to denote the
+       failure of the builder for `path'. */
+    writeFile(failedFileFor(path), "");
+}
+
+
+bool LocalStore::hasPathFailed(const Path & path)
+{
+    return pathExists(failedFileFor(path));
+}
+
+
 Hash parseHashField(const Path & path, const string & s)
 {
     string::size_type colon = s.find(':');
@@ -353,7 +373,7 @@ Hash parseHashField(const Path & path, const string & s)
 }
 
 
-ValidPathInfo LocalStore::queryPathInfo(const Path & path)
+ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
 {
     ValidPathInfo res;
     res.path = path;
@@ -363,8 +383,6 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
     std::map<Path, ValidPathInfo>::iterator lookup = pathInfoCache.find(path);
     if (lookup != pathInfoCache.end()) return lookup->second;
     
-    //printMsg(lvlError, "queryPathInfo: " + path);
-    
     /* Read the info file. */
     Path infoFile = infoFileFor(path);
     if (!pathExists(infoFile))
@@ -374,8 +392,8 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
     /* Parse it. */
     Strings lines = tokenizeString(info, "\n");
 
-    for (Strings::iterator i = lines.begin(); i != lines.end(); ++i) {
-        unsigned int p = i->find(':');
+    foreach (Strings::iterator, i, lines) {
+        string::size_type p = i->find(':');
         if (p == string::npos) continue; /* bad line */
         string name(*i, 0, p);
         string value(*i, p + 2);
@@ -385,7 +403,12 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
         } else if (name == "Deriver") {
             res.deriver = value;
         } else if (name == "Hash") {
-            res.hash = parseHashField(path, value);
+            try {
+                res.hash = parseHashField(path, value);
+            } catch (Error & e) {
+                if (!ignoreErrors) throw;
+                printMsg(lvlError, format("cannot parse hash field in `%1%': %2%") % infoFile % e.msg());
+            }
         } else if (name == "Registered-At") {
             int n = 0;
             string2Int(value, n);
@@ -399,6 +422,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
 
 bool LocalStore::isValidPath(const Path & path)
 {
+    /* Files in the info directory starting with a `.' are temporary
+       files. */
+    if (baseNameOf(path).at(0) == '.') return false;
     return pathExists(infoFileFor(path));
 }
 
@@ -407,8 +433,8 @@ PathSet LocalStore::queryValidPaths()
 {
     PathSet paths;
     Strings entries = readDirectory(nixDBPath + "/info");
-    for (Strings::iterator i = entries.begin(); i != entries.end(); ++i) 
-        paths.insert(nixStore + "/" + *i);
+    foreach (Strings::iterator, i, entries)
+        if (i->at(0) != '.') paths.insert(nixStore + "/" + *i);
     return paths;
 }
 
@@ -442,7 +468,7 @@ bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers)
 
     Paths refs = tokenizeString(readFile(fd), " ");
 
-    for (Paths::iterator i = refs.begin(); i != refs.end(); ++i)
+    foreach (Paths::iterator, i, refs)
         /* Referrers can be invalid (see registerValidPath() for the
            invariant), so we only return one if it is valid. */
         if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false;
@@ -463,33 +489,100 @@ Path LocalStore::queryDeriver(const Path & path)
 }
 
 
-PathSet LocalStore::querySubstitutablePaths()
+void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
 {
-    if (!substitutablePathsLoaded) {
-        for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) {
-            debug(format("running `%1%' to find out substitutable paths") % *i);
-            Strings args;
-            args.push_back("--query-paths");
-            Strings ss = tokenizeString(runProgram(*i, false, args), "\n");
-            for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) {
-                if (!isStorePath(*j))
-                    throw Error(format("`%1%' returned a bad substitutable path `%2%'")
-                        % *i % *j);
-                substitutablePaths.insert(*j);
-            }
+    if (run.pid != -1) return;
+    
+    debug(format("starting substituter program `%1%'") % substituter);
+
+    Pipe toPipe, fromPipe;
+            
+    toPipe.create();
+    fromPipe.create();
+
+    run.pid = fork();
+            
+    switch (run.pid) {
+
+    case -1:
+        throw SysError("unable to fork");
+
+    case 0: /* child */
+        try {
+            fromPipe.readSide.close();
+            toPipe.writeSide.close();
+            if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
+                throw SysError("dupping stdin");
+            if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
+                throw SysError("dupping stdout");
+            closeMostFDs(set<int>());
+            execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
+            throw SysError(format("executing `%1%'") % substituter);
+        } catch (std::exception & e) {
+            std::cerr << "error: " << e.what() << std::endl;
         }
-        substitutablePathsLoaded = true;
+        quickExit(1);
     }
 
-    return substitutablePaths;
+    /* Parent. */
+    
+    run.to = toPipe.writeSide.borrow();
+    run.from = fromPipe.readSide.borrow();
+}
+
+
+template<class T> T getIntLine(int fd)
+{
+    string s = readLine(fd);
+    T res;
+    if (!string2Int(s, res)) throw Error("integer expected from stream");
+    return res;
 }
 
 
 bool LocalStore::hasSubstitutes(const Path & path)
 {
-    if (!substitutablePathsLoaded)
-        querySubstitutablePaths(); 
-    return substitutablePaths.find(path) != substitutablePaths.end();
+    foreach (Paths::iterator, i, substituters) {
+        RunningSubstituter & run(runningSubstituters[*i]);
+        startSubstituter(*i, run);
+        writeLine(run.to, "have\n" + path);
+        if (getIntLine<int>(run.from)) return true;
+    }
+
+    return false;
+}
+
+
+bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
+    const Path & path, SubstitutablePathInfo & info)
+{
+    RunningSubstituter & run(runningSubstituters[substituter]);
+    startSubstituter(substituter, run);
+
+    writeLine(run.to, "info\n" + path);
+
+    if (!getIntLine<int>(run.from)) return false;
+    
+    info.deriver = readLine(run.from);
+    if (info.deriver != "") assertStorePath(info.deriver);
+    int nrRefs = getIntLine<int>(run.from);
+    while (nrRefs--) {
+        Path p = readLine(run.from);
+        assertStorePath(p);
+        info.references.insert(p);
+    }
+    info.downloadSize = getIntLine<long long>(run.from);
+    
+    return true;
+}
+
+
+bool LocalStore::querySubstitutablePathInfo(const Path & path,
+    SubstitutablePathInfo & info)
+{
+    foreach (Paths::iterator, i, substituters)
+        if (querySubstitutablePathInfo(*i, path, info)) return true;
+    return false;
 }
 
 
@@ -507,8 +600,7 @@ static void dfsVisit(std::map<Path, ValidPathInfo> & infos,
 
     ValidPathInfo & info(infos[path]);
     
-    for (PathSet::iterator i = info.references.begin();
-         i != info.references.end(); ++i)
+    foreach (PathSet::iterator, i, info.references)
         if (infos.find(*i) != infos.end())
             dfsVisit(infos, *i, visited, sorted);
 
@@ -523,15 +615,15 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
     /* Sort the paths topologically under the references relation, so
        that if path A is referenced by B, then A is registered before
        B. */
-    for (ValidPathInfos::const_iterator i = infos.begin(); i != infos.end(); ++i)
+    foreach (ValidPathInfos::const_iterator, i, infos)
         infosMap[i->path] = *i;
 
     PathSet visited;
     Paths sorted;
-    for (ValidPathInfos::const_iterator i = infos.begin(); i != infos.end(); ++i)
+    foreach (ValidPathInfos::const_iterator, i, infos)
         dfsVisit(infosMap, i->path, visited, sorted);
 
-    for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i)
+    foreach (Paths::iterator, i, sorted)
         registerValidPath(infosMap[*i]);
 }
 
@@ -582,16 +674,12 @@ void LocalStore::invalidatePath(const Path & path)
 }
 
 
-Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
-    bool recursive, string hashAlgo, PathFilter & filter)
+Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
+    bool recursive, HashType hashAlgo)
 {
-    Path srcPath(absPath(_srcPath));
-    debug(format("adding `%1%' to the store") % srcPath);
+    Hash h = hashString(hashAlgo, dump);
 
-    std::pair<Path, Hash> pr =
-        computeStorePathForPath(srcPath, fixed, recursive, hashAlgo, filter);
-    Path & dstPath(pr.first);
-    Hash & h(pr.second);
+    Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
 
     addTempRoot(dstPath);
 
@@ -606,16 +694,22 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
 
             if (pathExists(dstPath)) deletePathWrapped(dstPath);
 
-            copyPath(srcPath, dstPath, filter);
-
-            Hash h2 = hashPath(htSHA256, dstPath, filter);
-            if (h != h2)
-                throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
-                    % srcPath % dstPath % printHash(h) % printHash(h2));
+            if (recursive) {
+                StringSource source(dump);
+                restorePath(dstPath, source);
+            } else
+                writeStringToFile(dstPath, dump);
 
             canonicalisePathMetaData(dstPath);
-            
-            registerValidPath(dstPath, h, PathSet(), "");
+
+            /* Register the SHA-256 hash of the NAR serialisation of
+               the path in the database.  We may just have computed it
+               above (if called with recursive == true and hashAlgo ==
+               sha256); otherwise, compute it here. */
+            registerValidPath(dstPath,
+                (recursive && hashAlgo == htSHA256) ? h :
+                (recursive ? hashString(htSHA256, dump) : hashPath(htSHA256, dstPath)),
+                PathSet(), "");
         }
 
         outputLock.setDeletion(true);
@@ -625,10 +719,29 @@ Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
 }
 
 
-Path LocalStore::addTextToStore(const string & suffix, const string & s,
+Path LocalStore::addToStore(const Path & _srcPath,
+    bool recursive, HashType hashAlgo, PathFilter & filter)
+{
+    Path srcPath(absPath(_srcPath));
+    debug(format("adding `%1%' to the store") % srcPath);
+
+    /* Read the whole path into memory. This is not a very scalable
+       method for very large paths, but `copyPath' is mainly used for
+       small files. */
+    StringSink sink;
+    if (recursive) 
+        dumpPath(srcPath, sink, filter);
+    else
+        sink.s = readFile(srcPath);
+
+    return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo);
+}
+
+
+Path LocalStore::addTextToStore(const string & name, const string & s,
     const PathSet & references)
 {
-    Path dstPath = computeStorePathForText(suffix, s, references);
+    Path dstPath = computeStorePathForText(name, s, references);
     
     addTempRoot(dstPath);
 
@@ -766,7 +879,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
        store path follows the archive data proper), and besides, we
        don't know yet whether the signature is valid. */
     Path tmpDir = createTempDir(nixStore);
-    AutoDelete delTmp(tmpDir);
+    AutoDelete delTmp(tmpDir); /* !!! could be GC'ed! */
     Path unpacked = tmpDir + "/unpacked";
 
     restorePath(unpacked, hashAndReadSource);
@@ -825,7 +938,14 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 
     if (!isValidPath(dstPath)) {
 
-        PathLocks outputLock(singleton<PathSet, Path>(dstPath));
+        PathLocks outputLock;
+
+        /* Lock the output path.  But don't lock if we're being called
+           from a build hook (whose parent process already acquired a
+           lock on this path). */
+        Strings locksHeld = tokenizeString(getEnv("NIX_HELD_LOCKS"));
+        if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end())
+            outputLock.lockPaths(singleton<PathSet, Path>(dstPath));
 
         if (!isValidPath(dstPath)) {
 
@@ -851,7 +971,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 }
 
 
-void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed)
+void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed,
+    unsigned long long & blocksFreed)
 {
     bytesFreed = 0;
 
@@ -871,7 +992,7 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr
         invalidatePath(path);
     }
 
-    deletePathWrapped(path, bytesFreed);
+    deletePathWrapped(path, bytesFreed, blocksFreed);
 }
 
 
@@ -882,7 +1003,7 @@ void LocalStore::verifyStore(bool checkContents)
 
     PathSet validPaths2 = queryValidPaths(), validPaths;
     
-    for (PathSet::iterator i = validPaths2.begin(); i != validPaths2.end(); ++i) {
+    foreach (PathSet::iterator, i, validPaths2) {
         checkInterrupt();
         if (!isStorePath(*i)) {
             printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i);
@@ -900,26 +1021,25 @@ void LocalStore::verifyStore(bool checkContents)
 
     std::map<Path, PathSet> referrersCache;
     
-    for (PathSet::iterator i = validPaths.begin(); i != validPaths.end(); ++i) {
+    foreach (PathSet::iterator, i, validPaths) {
         bool update = false;
-        ValidPathInfo info = queryPathInfo(*i);
+        ValidPathInfo info = queryPathInfo(*i, true);
 
         /* Check the references: each reference should be valid, and
            it should have a matching referrer. */
-        for (PathSet::iterator j = info.references.begin();
-             j != info.references.end(); ++j)
-        {
-            if (referrersCache.find(*j) == referrersCache.end())
-                queryReferrers(*j, referrersCache[*j]);
-            if (referrersCache[*j].find(*i) == referrersCache[*j].end()) {
-                printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'")
-                    % *j % *i);
-                appendReferrer(*j, *i, true);
-            }
+        foreach (PathSet::iterator, j, info.references) {
             if (validPaths.find(*j) == validPaths.end()) {
                 printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'")
                     % *i % *j);
                 /* nothing we can do about it... */
+            } else {
+                if (referrersCache.find(*j) == referrersCache.end())
+                    queryReferrers(*j, referrersCache[*j]);
+                if (referrersCache[*j].find(*i) == referrersCache[*j].end()) {
+                    printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'")
+                        % *j % *i);
+                    appendReferrer(*j, *i, true);
+                }
             }
         }
 
@@ -931,7 +1051,11 @@ void LocalStore::verifyStore(bool checkContents)
         }
 
         /* Check the content hash (optionally - slow). */
-        if (checkContents) {
+        if (info.hash.hashSize == 0) {
+            printMsg(lvlError, format("re-hashing `%1%'") % *i);
+            info.hash = hashPath(htSHA256, *i);
+            update = true;
+        } else if (checkContents) {
             debug(format("checking contents of `%1%'") % *i);
             Hash current = hashPath(info.hash.type, *i);
             if (current != info.hash) {
@@ -953,10 +1077,12 @@ void LocalStore::verifyStore(bool checkContents)
     std::map<Path, PathSet> referencesCache;
     
     Strings entries = readDirectory(nixDBPath + "/referrer");
-    for (Strings::iterator i = entries.begin(); i != entries.end(); ++i) {
+    foreach (Strings::iterator, i, entries) {
         Path from = nixStore + "/" + *i;
         
         if (validPaths.find(from) == validPaths.end()) {
+            /* !!! This removes lock files as well.  Need to check
+               whether that's okay. */
             printMsg(lvlError, format("removing referrers file for invalid `%1%'") % from);
             Path p = referrersFileFor(from);
             if (unlink(p.c_str()) == -1)
@@ -975,7 +1101,7 @@ void LocalStore::verifyStore(bool checkContents)
 
         /* Each referrer should have a matching reference. */
         PathSet referrersNew;
-        for (PathSet::iterator j = referrers.begin(); j != referrers.end(); ++j) {
+        foreach (PathSet::iterator, j, referrers) {
             if (referencesCache.find(*j) == referencesCache.end())
                 queryReferences(*j, referencesCache[*j]);
             if (referencesCache[*j].find(from) == referencesCache[*j].end()) {

src/libstore/local-store.hh

@@ -25,19 +25,30 @@ struct OptimiseStats
     unsigned long sameContents;
     unsigned long filesLinked;
     unsigned long long bytesFreed;
+    unsigned long long blocksFreed;
     OptimiseStats()
     {
         totalFiles = sameContents = filesLinked = 0;
-        bytesFreed = 0;
+        bytesFreed = blocksFreed = 0;
     }
 };
 
 
+struct RunningSubstituter
+{
+    Pid pid;
+    AutoCloseFD to, from;
+};
+
+
 class LocalStore : public StoreAPI
 {
 private:
     bool substitutablePathsLoaded;
     PathSet substitutablePaths;
+
+    typedef std::map<Path, RunningSubstituter> RunningSubstituters;
+    RunningSubstituters runningSubstituters;
     
 public:
 
@@ -64,12 +75,25 @@ public:
     PathSet querySubstitutablePaths();
     
     bool hasSubstitutes(const Path & path);
+
+    bool querySubstitutablePathInfo(const Path & path,
+        SubstitutablePathInfo & info);
     
-    Path addToStore(const Path & srcPath, bool fixed = false,
-        bool recursive = false, string hashAlgo = "",
+    bool querySubstitutablePathInfo(const Path & substituter,
+        const Path & path, SubstitutablePathInfo & info);
+    
+    Path addToStore(const Path & srcPath,
+        bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter);
 
-    Path addTextToStore(const string & suffix, const string & s,
+    /* Like addToStore(), but the contents of the path are contained
+       in `dump', which is either a NAR serialisation (if recursive ==
+       true) or simply the contents of a regular file (if recursive ==
+       false). */
+    Path addToStoreFromDump(const string & dump, const string & name,
+        bool recursive = true, HashType hashAlgo = htSHA256);
+
+    Path addTextToStore(const string & name, const string & s,
         const PathSet & references);
 
     void exportPath(const Path & path, bool sign,
@@ -89,11 +113,11 @@ public:
 
     Roots findRoots();
 
-    void collectGarbage(GCAction action, const PathSet & pathsToDelete,
-        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
+    void collectGarbage(const GCOptions & options, GCResults & results);
 
     /* Delete a path from the Nix store. */
-    void deleteFromStore(const Path & path, unsigned long long & bytesFreed);
+    void deleteFromStore(const Path & path, unsigned long long & bytesFreed,
+        unsigned long long & blocksFreed);
     
     /* Optimise the disk space usage of the Nix store by hard-linking
        files with the same contents. */
@@ -113,6 +137,13 @@ public:
 
     void registerValidPaths(const ValidPathInfos & infos);
 
+    /* Register that the build of a derivation with output `path' has
+       failed. */
+    void registerFailedPath(const Path & path);
+
+    /* Query whether `path' previously failed to build. */
+    bool hasPathFailed(const Path & path);
+
 private:
 
     Path schemaPath;
@@ -131,7 +162,7 @@ private:
 
     void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false);
 
-    ValidPathInfo queryPathInfo(const Path & path);
+    ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false);
 
     void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
 
@@ -143,6 +174,14 @@ private:
     
     void upgradeStore12();
 
+    void gcPath(const GCOptions & options, GCResults & results,
+        const Path & path);
+
+    void gcPathRecursive(const GCOptions & options,
+        GCResults & results, PathSet & done, const Path & path);
+
+    void startSubstituter(const Path & substituter,
+        RunningSubstituter & runningSubstituter);
 };
 
 
@@ -151,8 +190,8 @@ void copyPath(const Path & src, const Path & dst);
 
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
-   - the last modification date on each file is set to 0 (i.e.,
-     00:00:00 1/1/1970 UTC)
+   - the last modification date on each file is set to 1 (i.e.,
+     00:00:01 1/1/1970 UTC)
    - the permissions are set of 444 or 555 (i.e., read-only with or
      without execute permission; setuid bits etc. are cleared)
    - the owner and group are set to the Nix user and group, if we're
@@ -175,7 +214,7 @@ void getOwnership(const Path & path);
 /* Like deletePath(), but changes the ownership of `path' using the
    setuid wrapper if necessary (and possible). */
 void deletePathWrapped(const Path & path,
-    unsigned long long & bytesFreed);
+    unsigned long long & bytesFreed, unsigned long long & blocksFreed);
 
 void deletePathWrapped(const Path & path);
  

src/libstore/misc.cc

@@ -1,5 +1,6 @@
 #include "misc.hh"
 #include "store-api.hh"
+#include "local-store.hh"
 
 #include <aterm2.h>
 
@@ -29,24 +30,25 @@ void computeFSClosure(const Path & storePath,
     else
         store->queryReferences(storePath, references);
 
-    for (PathSet::iterator i = references.begin();
-         i != references.end(); ++i)
+    foreach (PathSet::iterator, i, references)
         computeFSClosure(*i, paths, flipDirection);
 }
 
 
 Path findOutput(const Derivation & drv, string id)
 {
-    for (DerivationOutputs::const_iterator i = drv.outputs.begin();
-         i != drv.outputs.end(); ++i)
+    foreach (DerivationOutputs::const_iterator, i, drv.outputs)
         if (i->first == id) return i->second.path;
     throw Error(format("derivation has no output `%1%'") % id);
 }
 
 
 void queryMissing(const PathSet & targets,
-    PathSet & willBuild, PathSet & willSubstitute)
+    PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    unsigned long long & downloadSize)
 {
+    downloadSize = 0;
+    
     PathSet todo(targets.begin(), targets.end()), done;
 
     while (!todo.empty()) {
@@ -56,33 +58,36 @@ void queryMissing(const PathSet & targets,
         done.insert(p);
 
         if (isDerivation(p)) {
-            if (!store->isValidPath(p)) continue;
+            if (!store->isValidPath(p)) {
+                unknown.insert(p);
+                continue;
+            }
             Derivation drv = derivationFromPath(p);
 
             bool mustBuild = false;
-            for (DerivationOutputs::iterator i = drv.outputs.begin();
-                 i != drv.outputs.end(); ++i)
+            foreach (DerivationOutputs::iterator, i, drv.outputs)
                 if (!store->isValidPath(i->second.path) && !store->hasSubstitutes(i->second.path))
                     mustBuild = true;
 
             if (mustBuild) {
                 willBuild.insert(p);
                 todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
-                for (DerivationInputs::iterator i = drv.inputDrvs.begin();
-                     i != drv.inputDrvs.end(); ++i)
+                foreach (DerivationInputs::iterator, i, drv.inputDrvs)
                     todo.insert(i->first);
             } else 
-                for (DerivationOutputs::iterator i = drv.outputs.begin();
-                     i != drv.outputs.end(); ++i)
+                foreach (DerivationOutputs::iterator, i, drv.outputs)
                     todo.insert(i->second.path);
         }
 
         else {
             if (store->isValidPath(p)) continue;
-            if (store->hasSubstitutes(p))
+            SubstitutablePathInfo info;
+            if (store->querySubstitutablePathInfo(p, info)) {
                 willSubstitute.insert(p);
-            // XXX call the substituters
-            // store->queryReferences(p, todo);
+                downloadSize += info.downloadSize;
+                todo.insert(info.references.begin(), info.references.end());
+            } else
+                unknown.insert(p);
         }
     }
 }

src/libstore/misc.hh

@@ -29,7 +29,8 @@ Path findOutput(const Derivation & drv, string id);
    derivations that will be built, and the set of output paths that
    will be substituted. */
 void queryMissing(const PathSet & targets,
-    PathSet & willBuild, PathSet & willSubstitute);
+    PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
+    unsigned long long & downloadSize);
 
 
 }

src/libstore/optimise-store.cc

@@ -4,6 +4,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <errno.h>
 
 
 namespace nix {
@@ -22,6 +23,21 @@ static void makeWritable(const Path & path)
 }
 
 
+struct MakeReadOnly
+{
+    Path path;
+    MakeReadOnly(const Path & path) : path(path) { }
+    ~MakeReadOnly()
+    {
+        try {
+            if (path != "") canonicalisePathMetaData(path, false);
+        } catch (...) {
+            ignoreException();
+        }
+    }
+};
+
+
 static void hashAndLink(bool dryRun, HashToPath & hashToPath,
     OptimiseStats & stats, const Path & path)
 {
@@ -82,30 +98,40 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath,
                mess with  its permissions). */
             bool mustToggle = !isStorePath(path);
             if (mustToggle) makeWritable(dirOf(path));
+            
+            /* When we're done, make the directory read-only again and
+               reset its timestamp back to 0. */
+            MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
         
-            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1)
+            if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) {
+                if (errno == EMLINK) {
+                    /* Too many links to the same file (>= 32000 on
+                       most file systems).  This is likely to happen
+                       with empty files.  Just start over, creating
+                       links to the current file. */
+                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
+                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
+                    return;
+                }
                 throw SysError(format("cannot link `%1%' to `%2%'")
                     % tempLink % prevPath.first);
+            }
 
             /* Atomically replace the old file with the new hard link. */
             if (rename(tempLink.c_str(), path.c_str()) == -1)
                 throw SysError(format("cannot rename `%1%' to `%2%'")
                     % tempLink % path);
-
-            /* Make the directory read-only again and reset its
-               timestamp back to 0. */
-            if (mustToggle) canonicalisePathMetaData(dirOf(path), false);
-            
         } else
             printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
         
         stats.filesLinked++;
         stats.bytesFreed += st.st_size;
+        stats.blocksFreed += st.st_blocks;
     }
 
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectory(path);
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+	foreach (Strings::iterator, i, names)
 	    hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
     }
 }
@@ -117,7 +143,7 @@ void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
 
     PathSet paths = queryValidPaths();
 
-    for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) {
+    foreach (PathSet::iterator, i, paths) {
         addTempRoot(*i);
         if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
         startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);

src/libstore/pathlocks.cc

@@ -2,6 +2,7 @@
 #include "util.hh"
 
 #include <cerrno>
+#include <cstdlib>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -140,9 +141,9 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg)
 }
 
 
-void PathLocks::lockPaths(const PathSet & _paths, const string & waitMsg)
+bool PathLocks::lockPaths(const PathSet & _paths,
+    const string & waitMsg, bool wait)
 {
-    /* May be called only once! */
     assert(fds.empty());
     
     /* Note that `fds' is built incrementally so that the destructor
@@ -154,7 +155,7 @@ void PathLocks::lockPaths(const PathSet & _paths, const string & waitMsg)
     paths.sort();
     
     /* Acquire the lock for each path. */
-    for (Paths::iterator i = paths.begin(); i != paths.end(); i++) {
+    foreach (Paths::iterator, i, paths) {
         checkInterrupt();
         Path path = *i;
         Path lockPath = path + ".lock";
@@ -173,8 +174,15 @@ void PathLocks::lockPaths(const PathSet & _paths, const string & waitMsg)
 
             /* Acquire an exclusive lock. */
             if (!lockFile(fd, ltWrite, false)) {
-                if (waitMsg != "") printMsg(lvlError, waitMsg);
-                lockFile(fd, ltWrite, true);
+                if (wait) {
+                    if (waitMsg != "") printMsg(lvlError, waitMsg);
+                    lockFile(fd, ltWrite, true);
+                } else {
+                    /* Failed to lock this path; release all other
+                       locks. */
+                    unlock();
+                    return false;
+                }
             }
 
             debug(format("lock acquired on `%1%'") % lockPath);
@@ -198,12 +206,20 @@ void PathLocks::lockPaths(const PathSet & _paths, const string & waitMsg)
         fds.push_back(FDPair(fd.borrow(), lockPath));
         lockedPaths.insert(lockPath);
     }
+
+    return true;
 }
 
 
 PathLocks::~PathLocks()
 {
-    for (list<FDPair>::iterator i = fds.begin(); i != fds.end(); i++) {
+    unlock();
+}
+
+
+void PathLocks::unlock()
+{
+    foreach (list<FDPair>::iterator, i, fds) {
         if (deletePaths) deleteLockFilePreClose(i->second, i->first);
 
         lockedPaths.erase(i->second);
@@ -215,6 +231,8 @@ PathLocks::~PathLocks()
 
         debug(format("lock released on `%1%'") % i->second);
     }
+
+    fds.clear();
 }
 
 

src/libstore/pathlocks.hh

@@ -17,7 +17,7 @@ int openLockFile(const Path & path, bool create);
 void deleteLockFilePreClose(const Path & path, int fd);
 void deleteLockFilePostClose(const Path & path);
 
-typedef enum LockType { ltRead, ltWrite, ltNone };
+enum LockType { ltRead, ltWrite, ltNone };
 
 bool lockFile(int fd, LockType lockType, bool wait);
 
@@ -33,9 +33,11 @@ public:
     PathLocks();
     PathLocks(const PathSet & paths,
         const string & waitMsg = "");
-    void lockPaths(const PathSet & _paths,
-        const string & waitMsg = "");
+    bool lockPaths(const PathSet & _paths,
+        const string & waitMsg = "",
+        bool wait = true);
     ~PathLocks();
+    void unlock();
     void setDeletion(bool deletePaths);
 };
 

src/libstore/references.cc

@@ -1,15 +1,10 @@
 #include "references.hh"
 #include "hash.hh"
 #include "util.hh"
+#include "archive.hh"
 
-#include <cerrno>
 #include <map>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <dirent.h>
-#include <fcntl.h>
+#include <cstdlib>
 
 
 namespace nix {
@@ -18,8 +13,8 @@ namespace nix {
 static unsigned int refLength = 32; /* characters */
 
 
-static void search(size_t len, const unsigned char * s,
-    StringSet & ids, StringSet & seen)
+static void search(const unsigned char * s, unsigned int len, 
+    StringSet & hashes, StringSet & seen)
 {
     static bool initialised = false;
     static bool isBase32[256];
@@ -41,93 +36,60 @@ static void search(size_t len, const unsigned char * s,
             }
         if (!match) continue;
         string ref((const char *) s + i, refLength);
-        if (ids.find(ref) != ids.end()) {
+        if (hashes.find(ref) != hashes.end()) {
             debug(format("found reference to `%1%' at offset `%2%'")
                   % ref % i);
             seen.insert(ref);
-            ids.erase(ref);
+            hashes.erase(ref);
         }
         ++i;
     }
 }
 
 
-void checkPath(const string & path,
-    StringSet & ids, StringSet & seen)
+struct RefScanSink : Sink
 {
-    checkInterrupt();
+    HashSink hashSink;
+    StringSet hashes;
+    StringSet seen;
+
+    string tail;
+
+    RefScanSink() : hashSink(htSHA256) { }
     
-    debug(format("checking `%1%'") % path);
+    void operator () (const unsigned char * data, unsigned int len);
+};
 
-    struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError(format("getting attributes of path `%1%'") % path);
 
-    if (S_ISDIR(st.st_mode)) {
-        Strings names = readDirectory(path);
-	for (Strings::iterator i = names.begin(); i != names.end(); i++) {
-            search(i->size(), (const unsigned char *) i->c_str(), ids, seen);
-            checkPath(path + "/" + *i, ids, seen);
-        }
-    }
+void RefScanSink::operator () (const unsigned char * data, unsigned int len)
+{
+    hashSink(data, len);
 
-    else if (S_ISREG(st.st_mode)) {
-        
-        AutoCloseFD fd = open(path.c_str(), O_RDONLY);
-        if (fd == -1) throw SysError(format("opening file `%1%'") % path);
+    /* It's possible that a reference spans the previous and current
+       fragment, so search in the concatenation of the tail of the
+       previous fragment and the start of the current fragment. */
+    string s = tail + string((const char *) data, len > refLength ? refLength : len);
+    search((const unsigned char *) s.c_str(), s.size(), hashes, seen);
 
-        size_t bufSize = 1024 * 1024;
-        assert(refLength <= bufSize);
-        unsigned char * buf = new unsigned char[bufSize];
+    search(data, len, hashes, seen);
 
-        size_t left = st.st_size;
-        bool firstBlock = true;
-        
-        while (left > 0) {
-            checkInterrupt();
-            
-            size_t read = left > bufSize ? bufSize : left;
-            size_t copiedBytes = 0;
-
-            if (!firstBlock) {
-                /* Move the last (refLength - 1) bytes from the last
-                   block to the start of the buffer to deal with
-                   references that cross block boundaries. */
-                copiedBytes = refLength - 1;
-                if (read + copiedBytes > bufSize)
-                    read -= copiedBytes;
-                memcpy(buf, buf + (bufSize - copiedBytes), copiedBytes);
-            }
-            firstBlock = false;
-
-            readFull(fd, buf + copiedBytes, read);
-            left -= read;
-
-            search(copiedBytes + read, buf, ids, seen);
-        }
-        
-        delete[] buf; /* !!! autodelete */
-    }
-    
-    else if (S_ISLNK(st.st_mode)) {
-        string target = readLink(path);
-        search(target.size(), (const unsigned char *) target.c_str(), ids, seen);
-    }
-    
-    else throw Error(format("unknown file type: %1%") % path);
+    unsigned int tailLen = len <= refLength ? len : refLength;
+    tail =
+        string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
+        string((const char *) data + len - tailLen, tailLen);
 }
 
 
-PathSet scanForReferences(const string & path, const PathSet & paths)
+PathSet scanForReferences(const string & path,
+    const PathSet & refs, Hash & hash)
 {
+    RefScanSink sink;
     std::map<string, Path> backMap;
-    StringSet ids;
-    StringSet seen;
 
     /* For efficiency (and a higher hit rate), just search for the
        hash part of the file name.  (This assumes that all references
        have the form `HASH-bla'). */
-    for (PathSet::const_iterator i = paths.begin(); i != paths.end(); i++) {
+    foreach (PathSet::const_iterator, i, refs) {
         string baseName = baseNameOf(*i);
         string::size_type pos = baseName.find('-');
         if (pos == string::npos)
@@ -136,21 +98,25 @@ PathSet scanForReferences(const string & path, const PathSet & paths)
         assert(s.size() == refLength);
         assert(backMap.find(s) == backMap.end());
         // parseHash(htSHA256, s);
-        ids.insert(s);
+        sink.hashes.insert(s);
         backMap[s] = *i;
     }
 
-    checkPath(path, ids, seen);
+    /* Look for the hashes in the NAR dump of the path. */
+    dumpPath(path, sink);
 
+    /* Map the hashes found back to their store paths. */
     PathSet found;
-    for (StringSet::iterator i = seen.begin(); i != seen.end(); i++) {
+    foreach (StringSet::iterator, i, sink.seen) {
         std::map<string, Path>::iterator j;
         if ((j = backMap.find(*i)) == backMap.end()) abort();
         found.insert(j->second);
     }
 
+    hash = sink.hashSink.finish();
+        
     return found;
 }
 
- 
+
 }

src/libstore/references.hh

@@ -2,10 +2,12 @@
 #define __REFERENCES_H
 
 #include "types.hh"
+#include "hash.hh"
 
 namespace nix {
 
-PathSet scanForReferences(const Path & path, const PathSet & refs);
+PathSet scanForReferences(const Path & path, const PathSet & refs,
+    Hash & hash);
     
 }
 

src/libstore/remote-store.cc

@@ -29,14 +29,22 @@ Path readStorePath(Source & from)
 PathSet readStorePaths(Source & from)
 {
     PathSet paths = readStringSet(from);
-    for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
-        assertStorePath(*i);
+    foreach (PathSet::iterator, i, paths) assertStorePath(*i);
     return paths;
 }
 
 
 RemoteStore::RemoteStore()
 {
+    initialised = false;
+}
+
+
+void RemoteStore::openConnection()
+{
+    if (initialised) return;
+    initialised = true;
+
     string remoteMode = getEnv("NIX_REMOTE");
 
     if (remoteMode == "slave")
@@ -64,8 +72,8 @@ RemoteStore::RemoteStore()
             throw Error("Nix daemon protocol version not supported");
         writeInt(PROTOCOL_VERSION, to);
         processStderr();
-
-    } catch (Error & e) {
+    }
+    catch (Error & e) {
         throw Error(format("cannot start worker (%1%)")
             % e.msg());
     }
@@ -137,14 +145,26 @@ void RemoteStore::connectToDaemon()
 
     string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
 
+    /* Urgh, sockaddr_un allows path names of only 108 characters.  So
+       chdir to the socket directory so that we can pass a relative
+       path name.  !!! this is probably a bad idea in multi-threaded
+       applications... */
+    AutoCloseFD fdPrevDir = open(".", O_RDONLY);
+    if (fdPrevDir == -1) throw SysError("couldn't open current directory");
+    chdir(dirOf(socketPath).c_str()); 
+    Path socketPathRel = "./" + baseNameOf(socketPath);
+    
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
-    if (socketPath.size() >= sizeof(addr.sun_path))
-        throw Error(format("socket path `%1%' is too long") % socketPath);
-    strcpy(addr.sun_path, socketPath.c_str());
+    if (socketPathRel.size() >= sizeof(addr.sun_path))
+        throw Error(format("socket path `%1%' is too long") % socketPathRel);
+    strcpy(addr.sun_path, socketPathRel.c_str());
     
     if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1)
         throw SysError(format("cannot connect to daemon at `%1%'") % socketPath);
+
+    if (fchdir(fdPrevDir) == -1)
+        throw SysError("couldn't change back to previous directory");
 }
 
 
@@ -171,12 +191,18 @@ void RemoteStore::setOptions()
     writeInt(maxSilentTime, to);
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 2)
         writeInt(useBuildHook, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 4) {
+        writeInt(buildVerbosity, to);
+        writeInt(logType, to);
+        writeInt(printBuildTrace, to);
+    }
     processStderr();
 }
 
 
 bool RemoteStore::isValidPath(const Path & path)
 {
+    openConnection();
     writeInt(wopIsValidPath, to);
     writeString(path, to);
     processStderr();
@@ -187,12 +213,14 @@ bool RemoteStore::isValidPath(const Path & path)
 
 PathSet RemoteStore::queryValidPaths()
 {
+    openConnection();
     throw Error("not implemented");
 }
 
 
 bool RemoteStore::hasSubstitutes(const Path & path)
 {
+    openConnection();
     writeInt(wopHasSubstitutes, to);
     writeString(path, to);
     processStderr();
@@ -201,8 +229,27 @@ bool RemoteStore::hasSubstitutes(const Path & path)
 }
 
 
+bool RemoteStore::querySubstitutablePathInfo(const Path & path,
+    SubstitutablePathInfo & info)
+{
+    openConnection();
+    if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false;
+    writeInt(wopQuerySubstitutablePathInfo, to);
+    writeString(path, to);
+    processStderr();
+    unsigned int reply = readInt(from);
+    if (reply == 0) return false;
+    info.deriver = readString(from);
+    if (info.deriver != "") assertStorePath(info.deriver);
+    info.references = readStorePaths(from);
+    info.downloadSize = readLongLong(from);
+    return true;
+}
+
+
 Hash RemoteStore::queryPathHash(const Path & path)
 {
+    openConnection();
     writeInt(wopQueryPathHash, to);
     writeString(path, to);
     processStderr();
@@ -214,6 +261,7 @@ Hash RemoteStore::queryPathHash(const Path & path)
 void RemoteStore::queryReferences(const Path & path,
     PathSet & references)
 {
+    openConnection();
     writeInt(wopQueryReferences, to);
     writeString(path, to);
     processStderr();
@@ -225,6 +273,7 @@ void RemoteStore::queryReferences(const Path & path,
 void RemoteStore::queryReferrers(const Path & path,
     PathSet & referrers)
 {
+    openConnection();
     writeInt(wopQueryReferrers, to);
     writeString(path, to);
     processStderr();
@@ -235,6 +284,7 @@ void RemoteStore::queryReferrers(const Path & path,
 
 Path RemoteStore::queryDeriver(const Path & path)
 {
+    openConnection();
     writeInt(wopQueryDeriver, to);
     writeString(path, to);
     processStderr();
@@ -244,33 +294,31 @@ Path RemoteStore::queryDeriver(const Path & path)
 }
 
 
-PathSet RemoteStore::querySubstitutablePaths()
-{
-    throw Error("not implemented");
-}
-
-
-Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
-    bool recursive, string hashAlgo, PathFilter & filter)
+Path RemoteStore::addToStore(const Path & _srcPath,
+    bool recursive, HashType hashAlgo, PathFilter & filter)
 {
+    openConnection();
+    
     Path srcPath(absPath(_srcPath));
     
     writeInt(wopAddToStore, to);
     writeString(baseNameOf(srcPath), to);
-    writeInt(fixed ? 1 : 0, to);
+    /* backwards compatibility hack */
+    writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to);
     writeInt(recursive ? 1 : 0, to);
-    writeString(hashAlgo, to);
+    writeString(printHashType(hashAlgo), to);
     dumpPath(srcPath, to, filter);
     processStderr();
     return readStorePath(from);
 }
 
 
-Path RemoteStore::addTextToStore(const string & suffix, const string & s,
+Path RemoteStore::addTextToStore(const string & name, const string & s,
     const PathSet & references)
 {
+    openConnection();
     writeInt(wopAddTextToStore, to);
-    writeString(suffix, to);
+    writeString(name, to);
     writeString(s, to);
     writeStringSet(references, to);
     
@@ -282,6 +330,7 @@ Path RemoteStore::addTextToStore(const string & suffix, const string & s,
 void RemoteStore::exportPath(const Path & path, bool sign,
     Sink & sink)
 {
+    openConnection();
     writeInt(wopExportPath, to);
     writeString(path, to);
     writeInt(sign ? 1 : 0, to);
@@ -292,10 +341,10 @@ void RemoteStore::exportPath(const Path & path, bool sign,
 
 Path RemoteStore::importPath(bool requireSignature, Source & source)
 {
+    openConnection();
     writeInt(wopImportPath, to);
     /* We ignore requireSignature, since the worker forces it to true
-       anyway. */
-    
+       anyway. */    
     processStderr(0, &source);
     return readStorePath(from);
 }
@@ -303,6 +352,7 @@ Path RemoteStore::importPath(bool requireSignature, Source & source)
 
 void RemoteStore::buildDerivations(const PathSet & drvPaths)
 {
+    openConnection();
     writeInt(wopBuildDerivations, to);
     writeStringSet(drvPaths, to);
     processStderr();
@@ -312,6 +362,7 @@ void RemoteStore::buildDerivations(const PathSet & drvPaths)
 
 void RemoteStore::ensurePath(const Path & path)
 {
+    openConnection();
     writeInt(wopEnsurePath, to);
     writeString(path, to);
     processStderr();
@@ -321,6 +372,7 @@ void RemoteStore::ensurePath(const Path & path)
 
 void RemoteStore::addTempRoot(const Path & path)
 {
+    openConnection();
     writeInt(wopAddTempRoot, to);
     writeString(path, to);
     processStderr();
@@ -330,6 +382,7 @@ void RemoteStore::addTempRoot(const Path & path)
 
 void RemoteStore::addIndirectRoot(const Path & path)
 {
+    openConnection();
     writeInt(wopAddIndirectRoot, to);
     writeString(path, to);
     processStderr();
@@ -339,6 +392,7 @@ void RemoteStore::addIndirectRoot(const Path & path)
 
 void RemoteStore::syncWithGC()
 {
+    openConnection();
     writeInt(wopSyncWithGC, to);
     processStderr();
     readInt(from);
@@ -347,6 +401,7 @@ void RemoteStore::syncWithGC()
 
 Roots RemoteStore::findRoots()
 {
+    openConnection();
     writeInt(wopFindRoots, to);
     processStderr();
     unsigned int count = readInt(from);
@@ -360,24 +415,26 @@ Roots RemoteStore::findRoots()
 }
 
 
-void RemoteStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
-    bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
+void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    result.clear();
-    bytesFreed = 0;
+    openConnection();
+    
     writeInt(wopCollectGarbage, to);
-    writeInt(action, to);
-    writeStringSet(pathsToDelete, to);
-    writeInt(ignoreLiveness, to);
+    writeInt(options.action, to);
+    writeStringSet(options.pathsToDelete, to);
+    writeInt(options.ignoreLiveness, to);
+    writeLongLong(options.maxFreed, to);
+    writeInt(options.maxLinks, to);
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) {
+        writeInt(options.useAtime, to);
+        writeInt(options.maxAtime, to);
+    }
     
     processStderr();
     
-    result = readStringSet(from);
-
-    /* Ugh - NAR integers are 64 bits, but read/writeInt() aren't. */
-    unsigned int lo = readInt(from);
-    unsigned int hi = readInt(from);
-    bytesFreed = (((unsigned long long) hi) << 32) | lo;
+    results.paths = readStringSet(from);
+    results.bytesFreed = readLongLong(from);
+    results.blocksFreed = readLongLong(from);
 }
 
 

src/libstore/remote-store.hh

@@ -37,15 +37,16 @@ public:
 
     Path queryDeriver(const Path & path);
     
-    PathSet querySubstitutablePaths();
-    
     bool hasSubstitutes(const Path & path);
     
-    Path addToStore(const Path & srcPath, bool fixed = false,
-        bool recursive = false, string hashAlgo = "",
+    bool querySubstitutablePathInfo(const Path & path,
+        SubstitutablePathInfo & info);
+    
+    Path addToStore(const Path & srcPath,
+        bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter);
 
-    Path addTextToStore(const string & suffix, const string & s,
+    Path addTextToStore(const string & name, const string & s,
         const PathSet & references);
 
     void exportPath(const Path & path, bool sign,
@@ -65,8 +66,7 @@ public:
     
     Roots findRoots();
 
-    void collectGarbage(GCAction action, const PathSet & pathsToDelete,
-        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
+    void collectGarbage(const GCOptions & options, GCResults & results);
     
 private:
     AutoCloseFD fdSocket;
@@ -74,6 +74,9 @@ private:
     FdSource from;
     Pid child;
     unsigned int daemonVersion;
+    bool initialised;
+
+    void openConnection();
 
     void processStderr(Sink * sink = 0, Source * source = 0);
 

src/libstore/store-api.cc

@@ -2,14 +2,20 @@
 #include "globals.hh"
 #include "util.hh"
 
+#include <limits.h>
+
 
 namespace nix {
 
 
-bool StoreAPI::hasSubstitutes(const Path & path)
+GCOptions::GCOptions()
 {
-    PathSet paths = querySubstitutablePaths();
-    return paths.find(path) != paths.end();
+    action = gcDeleteDead;
+    ignoreLiveness = false;
+    maxFreed = 0;
+    maxLinks = 0;
+    useAtime = false;
+    maxAtime = (time_t) -1;
 }
 
 
@@ -75,7 +81,7 @@ void checkStoreName(const string & name)
        reasons (e.g., "." and ".."). */
     if (string(name, 0, 1) == ".")
         throw Error(format("illegal name: `%1%'") % name);
-    for (string::const_iterator i = name.begin(); i != name.end(); ++i)
+    foreach (string::const_iterator, i, name)
         if (!((*i >= 'A' && *i <= 'Z') ||
               (*i >= 'a' && *i <= 'z') ||
               (*i >= '0' && *i <= '9') ||
@@ -87,55 +93,113 @@ void checkStoreName(const string & name)
 }
 
 
+/* Store paths have the following form:
+
+   <store>/<h>-<name>
+
+   where
+
+   <store> = the location of the Nix store, usually /nix/store
+   
+   <name> = a human readable name for the path, typically obtained
+     from the name attribute of the derivation, or the name of the
+     source file from which the store path is created
+     
+   <h> = base-32 representation of the first 160 bits of a SHA-256
+     hash of <s>; the hash part of the store name
+     
+   <s> = the string "<type>:sha256:<h2>:<store>:<name>";
+     note that it includes the location of the store as well as the
+     name to make sure that changes to either of those are reflected
+     in the hash (e.g. you won't get /nix/store/<h>-name1 and
+     /nix/store/<h>-name2 with equal hash parts).
+     
+   <type> = one of:
+     "text:<r1>:<r2>:...<rN>"
+       for plain text files written to the store using
+       addTextToStore(); <r1> ... <rN> are the references of the
+       path.
+     "source"
+       for paths copied to the store using addToStore() when recursive
+       = true and hashAlgo = "sha256"
+     "output:out"
+       for either the outputs created by derivations, OR paths copied
+       to the store using addToStore() with recursive != true or
+       hashAlgo != "sha256" (in that case "source" is used; it's
+       silly, but it's done that way for compatibility).
+
+   <h2> = base-16 representation of a SHA-256 hash of:
+     if <type> = "text:...":
+       the string written to the resulting store path
+     if <type> = "source":
+       the serialisation of the path from which this store path is
+       copied, as returned by hashPath()
+     if <type> = "output:out":
+       for non-fixed derivation outputs:
+         the derivation (see hashDerivationModulo() in
+         primops.cc)
+       for paths copied by addToStore() or produced by fixed-output
+       derivations:
+         the string "fixed:out:<rec><algo>:<hash>:", where
+           <rec> = "r:" for recursive (path) hashes, or "" or flat
+             (file) hashes
+           <algo> = "md5", "sha1" or "sha256"
+           <hash> = base-16 representation of the path or flat hash of
+             the contents of the path (or expected contents of the
+             path for fixed-output derivations)
+
+   It would have been nicer to handle fixed-output derivations under
+   "source", e.g. have something like "source:<rec><algo>", but we're
+   stuck with this for now...
+
+   The main reason for this way of computing names is to prevent name
+   collisions (for security).  For instance, it shouldn't be feasible
+   to come up with a derivation whose output path collides with the
+   path for a copied source.  The former would have a <s> starting with
+   "output:out:", while the latter would have a <2> starting with
+   "source:".
+*/
+
+
 Path makeStorePath(const string & type,
-    const Hash & hash, const string & suffix)
+    const Hash & hash, const string & name)
 {
     /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
     string s = type + ":sha256:" + printHash(hash) + ":"
-        + nixStore + ":" + suffix;
+        + nixStore + ":" + name;
 
-    checkStoreName(suffix);
+    checkStoreName(name);
 
     return nixStore + "/"
         + printHash32(compressHash(hashString(htSHA256, s), 20))
-        + "-" + suffix;
+        + "-" + name;
 }
 
 
 Path makeFixedOutputPath(bool recursive,
-    string hashAlgo, Hash hash, string name)
+    HashType hashAlgo, Hash hash, string name)
 {
-    /* !!! copy/paste from primops.cc */
-    Hash h = hashString(htSHA256, "fixed:out:"
-        + (recursive ? (string) "r:" : "") + hashAlgo + ":"
-        + printHash(hash) + ":"
-        + "");
-    return makeStorePath("output:out", h, name);
+    return hashAlgo == htSHA256 && recursive
+        ? makeStorePath("source", hash, name)
+        : makeStorePath("output:out", hashString(htSHA256,
+                "fixed:out:" + (recursive ? (string) "r:" : "") +
+                printHashType(hashAlgo) + ":" + printHash(hash) + ":"),
+            name);
 }
 
 
 std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
-    bool fixed, bool recursive, string hashAlgo, PathFilter & filter)
+    bool recursive, HashType hashAlgo, PathFilter & filter)
 {
-    Hash h = hashPath(htSHA256, srcPath, filter);
-
-    string baseName = baseNameOf(srcPath);
-
-    Path dstPath;
-    
-    if (fixed) {
-        HashType ht(parseHashType(hashAlgo));
-        Hash h2 = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath);
-        dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
-    }
-        
-    else dstPath = makeStorePath("source", h, baseName);
-
+    HashType ht(hashAlgo);
+    Hash h = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath);
+    string name = baseNameOf(srcPath);
+    Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
     return std::pair<Path, Hash>(dstPath, h);
 }
 
 
-Path computeStorePathForText(const string & suffix, const string & s,
+Path computeStorePathForText(const string & name, const string & s,
     const PathSet & references)
 {
     Hash hash = hashString(htSHA256, s);
@@ -143,11 +207,11 @@ Path computeStorePathForText(const string & suffix, const string & s,
        hacky, but we can't put them in `s' since that would be
        ambiguous. */
     string type = "text";
-    for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i) {
+    foreach (PathSet::const_iterator, i, references) {
         type += ":";
         type += *i;
     }
-    return makeStorePath(type, hash, suffix);
+    return makeStorePath(type, hash, name);
 }
 
 
@@ -159,7 +223,7 @@ string makeValidityRegistration(const PathSet & paths,
 {
     string s = "";
     
-    for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) {
+    foreach (PathSet::iterator, i, paths) {
         s += *i + "\n";
 
         if (showHash)
@@ -173,8 +237,7 @@ string makeValidityRegistration(const PathSet & paths,
 
         s += (format("%1%\n") % references.size()).str();
             
-        for (PathSet::iterator j = references.begin();
-             j != references.end(); ++j)
+        foreach (PathSet::iterator, j, references)
             s += *j + "\n";
     }
 
@@ -208,9 +271,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven)
 string showPaths(const PathSet & paths)
 {
     string s;
-    for (PathSet::const_iterator i = paths.begin();
-         i != paths.end(); ++i)
-    {
+    foreach (PathSet::const_iterator, i, paths) {
         if (s.size() != 0) s += ", ";
         s += "`" + *i + "'";
     }

src/libstore/store-api.hh

@@ -16,14 +16,101 @@ namespace nix {
 typedef std::map<Path, Path> Roots;
 
 
-/* Garbage collector operation. */
-typedef enum {
-    gcReturnRoots,
-    gcReturnLive,
-    gcReturnDead,
-    gcDeleteDead,
-    gcDeleteSpecific,
-} GCAction;
+
+
+struct GCOptions
+{
+    /* Garbage collector operation:
+
+       - `gcReturnRoots': find and return the set of roots for the
+         garbage collector.  These are the store paths symlinked to in
+         the `gcroots' directory.
+
+       - `gcReturnLive': return the set of paths reachable from
+         (i.e. in the closure of) the roots.
+
+       - `gcReturnDead': return the set of paths not reachable from
+         the roots.
+
+       - `gcDeleteDead': actually delete the latter set.
+
+       - `gcDeleteSpecific': delete the paths listed in
+          `pathsToDelete', insofar as they are not reachable.
+    */
+    typedef enum {
+        gcReturnRoots,
+        gcReturnLive,
+        gcReturnDead,
+        gcDeleteDead,
+        gcDeleteSpecific,
+    } GCAction;
+
+    GCAction action;
+
+    /* If `ignoreLiveness' is set, then reachability from the roots is
+       ignored (dangerous!).  However, the paths must still be
+       unreferenced *within* the store (i.e., there can be no other
+       store paths that depend on them). */
+    bool ignoreLiveness;
+
+    /* For `gcDeleteSpecific', the paths to delete. */
+    PathSet pathsToDelete;
+
+    /* Stop after at least `maxFreed' bytes have been freed.  0 means
+       no limit. */
+    unsigned long long maxFreed;
+
+    /* Stop after the number of hard links to the Nix store directory
+       has dropped below `maxLinks'. */
+    unsigned int maxLinks;
+
+    /* Delete paths in order of ascending last access time.  I.e.,
+       prefer deleting unrecently used paths.  Useful in conjunction
+       with `maxFreed' and `maxLinks' (or manual interruption).  The
+       access time of a path is defined as the highest atime of any
+       non-directory, non-symlink file under that path.  Directories
+       and symlinks are ignored because their atimes are frequently
+       mass-updated, e.g. by `locate'.  Note that optimiseStore()
+       somewhat reduces the usefulness of this option: it hard-links
+       regular files and symlink together, giving them a "shared"
+       atime. */
+    bool useAtime;
+
+    /* Do not delete paths newer than `maxAtime'.  -1 means no age
+       limit. */
+    time_t maxAtime;
+
+    GCOptions();
+};
+
+
+struct GCResults 
+{
+    /* Depending on the action, the GC roots, or the paths that would
+       be or have been deleted. */
+    PathSet paths;
+
+    /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
+       number of bytes that would be or was freed. */
+    unsigned long long bytesFreed;
+
+    /* The number of file system blocks that would be or was freed. */
+    unsigned long long blocksFreed;
+
+    GCResults()
+    {
+        bytesFreed = 0;
+        blocksFreed = 0;
+    }
+};
+
+
+struct SubstitutablePathInfo
+{
+    Path deriver;
+    PathSet references;
+    unsigned long long downloadSize; /* 0 = unknown or inapplicable */
+};
 
 
 class StoreAPI 
@@ -46,21 +133,40 @@ public:
     virtual void queryReferences(const Path & path,
         PathSet & references) = 0;
 
+    /* Like queryReferences, but with self-references filtered out. */
+    PathSet queryReferencesNoSelf(const Path & path)
+    {
+        PathSet res;
+        queryReferences(path, res);
+        res.erase(path);
+        return res;
+    }
+
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
     virtual void queryReferrers(const Path & path,
         PathSet & referrers) = 0;
 
+    /* Like queryReferrers, but with self-references filtered out. */
+    PathSet queryReferrersNoSelf(const Path & path)
+    {
+        PathSet res;
+        queryReferrers(path, res);
+        res.erase(path);
+        return res;
+    }
+
     /* Query the deriver of a store path.  Return the empty string if
        no deriver has been set. */
     virtual Path queryDeriver(const Path & path) = 0;
 
-    /* Query the set of substitutable paths. */
-    virtual PathSet querySubstitutablePaths() = 0;
+    /* Query whether a path has substitutes. */
+    virtual bool hasSubstitutes(const Path & path) = 0;
 
-    /* More efficient variant if we just want to know if a path has
-       substitutes. */
-    virtual bool hasSubstitutes(const Path & path);
+    /* Query the references, deriver and download size of a
+       substitutable path. */
+    virtual bool querySubstitutablePathInfo(const Path & path,
+        SubstitutablePathInfo & info) = 0;
     
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
@@ -68,13 +174,13 @@ public:
        derivation is pre-loaded into the Nix store.  The function
        object `filter' can be used to exclude files (see
        libutil/archive.hh). */
-    virtual Path addToStore(const Path & srcPath, bool fixed = false,
-        bool recursive = false, string hashAlgo = "",
+    virtual Path addToStore(const Path & srcPath,
+        bool recursive = true, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter) = 0;
 
     /* Like addToStore, but the contents written to the output path is
        a regular file containing the given string. */
-    virtual Path addTextToStore(const string & suffix, const string & s,
+    virtual Path addTextToStore(const string & name, const string & s,
         const PathSet & references) = 0;
 
     /* Export a store path, that is, create a NAR dump of the store
@@ -137,33 +243,8 @@ public:
        outside of the Nix store that point to `storePath'.  */
     virtual Roots findRoots() = 0;
 
-    /* Depending on `action', this function does the following:
-
-       - `gcReturnRoots': find and return the set of roots for the
-         garbage collector.  These are the store paths symlinked to in
-         the `gcroots' directory.
-
-       - `gcReturnLive': return the set of paths reachable from
-         (i.e. in the closure of) the roots.
-
-       - `gcReturnDead': return the set of paths not reachable from
-         the roots.
-
-       - `gcDeleteDead': actually delete the latter set.
-
-       - `gcDeleteSpecific': delete the paths listed in
-         `pathsToDelete', insofar as they are not reachable.
-
-       If `ignoreLiveness' is set, then reachability from the roots is
-       ignored (dangerous!).  However, the paths must still be
-       unreferenced *within* the store (i.e., there can be no other
-       store paths that depend on them).
-
-       For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
-       number of bytes that would be or was freed is returned in
-       `bytesFreed'. */
-    virtual void collectGarbage(GCAction action, const PathSet & pathsToDelete,
-        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed) = 0;
+    /* Perform a garbage collection. */
+    virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
 };
 
 
@@ -194,10 +275,10 @@ Path followLinksToStorePath(const Path & path);
 
 /* Constructs a unique store path name. */
 Path makeStorePath(const string & type,
-    const Hash & hash, const string & suffix);
+    const Hash & hash, const string & name);
     
 Path makeFixedOutputPath(bool recursive,
-    string hashAlgo, Hash hash, string name);
+    HashType hashAlgo, Hash hash, string name);
 
 
 /* This is the preparatory part of addToStore() and addToStoreFixed();
@@ -205,7 +286,7 @@ Path makeFixedOutputPath(bool recursive,
    Returns the store path and the cryptographic hash of the
    contents of srcPath. */
 std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
-    bool fixed = false, bool recursive = false, string hashAlgo = "",
+    bool recursive = true, HashType hashAlgo = htSHA256,
     PathFilter & filter = defaultPathFilter);
 
 /* Preparatory part of addTextToStore().
@@ -222,7 +303,7 @@ std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
    simply yield a different store path, so other users wouldn't be
    affected), but it has some backwards compatibility issues (the
    hashing scheme changes), so I'm not doing that for now. */
-Path computeStorePathForText(const string & suffix, const string & s,
+Path computeStorePathForText(const string & name, const string & s,
     const PathSet & references);
 
 

src/libstore/upgrade-schema.cc

@@ -73,7 +73,7 @@ void LocalStore::upgradeStore12()
     Paths paths;
     nixDB.enumTable(noTxn, dbValidPaths, paths);
     
-    for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
+    foreach (Paths::iterator, i, paths) {
         ValidPathInfo info;
         info.path = *i;
         

src/libstore/worker-protocol.hh

@@ -8,31 +8,32 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x102
+#define PROTOCOL_VERSION 0x105
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
 
 typedef enum {
     wopQuit = 0,
-    wopIsValidPath,
+    wopIsValidPath = 1,
     wopHasSubstitutes = 3,
-    wopQueryPathHash,
-    wopQueryReferences,
-    wopQueryReferrers,
-    wopAddToStore,
-    wopAddTextToStore,
-    wopBuildDerivations,
-    wopEnsurePath,
-    wopAddTempRoot,
-    wopAddIndirectRoot,
-    wopSyncWithGC,
-    wopFindRoots,
-    wopCollectGarbage,
-    wopExportPath,
-    wopImportPath,
-    wopQueryDeriver,
-    wopSetOptions,
+    wopQueryPathHash = 4,
+    wopQueryReferences = 5,
+    wopQueryReferrers = 6,
+    wopAddToStore = 7,
+    wopAddTextToStore = 8,
+    wopBuildDerivations = 9,
+    wopEnsurePath = 10,
+    wopAddTempRoot = 11,
+    wopAddIndirectRoot = 12,
+    wopSyncWithGC = 13,
+    wopFindRoots = 14,
+    wopExportPath = 16,
+    wopImportPath = 17,
+    wopQueryDeriver = 18,
+    wopSetOptions = 19,
+    wopCollectGarbage = 20,
+    wopQuerySubstitutablePathInfo = 21,
 } WorkerOp;
 
 

src/libutil/archive.cc

@@ -2,6 +2,7 @@
 #include <algorithm>
 #include <vector>
 
+#define _XOPEN_SOURCE 600
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -11,6 +12,8 @@
 #include "archive.hh"
 #include "util.hh"
 
+#include "config.h"
+
 
 namespace nix {
 
@@ -47,17 +50,17 @@ static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter)
 }
 
 
-static void dumpContents(const Path & path, unsigned int size, 
+static void dumpContents(const Path & path, size_t size, 
     Sink & sink)
 {
     writeString("contents", sink);
-    writeInt(size, sink);
+    writeLongLong(size, sink);
 
     AutoCloseFD fd = open(path.c_str(), O_RDONLY);
     if (fd == -1) throw SysError(format("opening file `%1%'") % path);
     
     unsigned char buf[65536];
-    unsigned int left = size;
+    size_t left = size;
 
     while (left > 0) {
         size_t n = left > sizeof(buf) ? sizeof(buf) : left;
@@ -85,7 +88,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
             writeString("executable", sink);
             writeString("", sink);
         }
-        dumpContents(path, st.st_size, sink);
+        dumpContents(path, (size_t) st.st_size, sink);
     } 
 
     else if (S_ISDIR(st.st_mode)) {
@@ -101,7 +104,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
         writeString(readLink(path), sink);
     }
 
-    else throw Error("unknown file type: " + path);
+    else throw Error(format("file `%1%' has an unknown type") % path);
 
     writeString(")", sink);
 }
@@ -114,9 +117,9 @@ void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
 }
 
 
-static Error badArchive(string s)
+static SerialisationError badArchive(string s)
 {
-    return Error("bad archive: " + s);
+    return SerialisationError("bad archive: " + s);
 }
 
 
@@ -129,10 +132,11 @@ static void skipGeneric(Source & source)
 }
 
 
-static void restore(const Path & path, Source & source);
+static void parse(ParseSink & sink, Source & source, const Path & path);
 
 
-static void restoreEntry(const Path & path, Source & source)
+
+static void parseEntry(ParseSink & sink, Source & source, const Path & path)
 {
     string s, name;
 
@@ -150,7 +154,7 @@ static void restoreEntry(const Path & path, Source & source)
             name = readString(source);
         } else if (s == "node") {
             if (s == "") throw badArchive("entry name missing");
-            restore(path + "/" + name, source);
+            parse(sink, source, path + "/" + name);
         } else {
             throw badArchive("unknown field " + s);
             skipGeneric(source);
@@ -159,26 +163,31 @@ static void restoreEntry(const Path & path, Source & source)
 }
 
 
-static void restoreContents(int fd, const Path & path, Source & source)
+static void parseContents(ParseSink & sink, Source & source, const Path & path)
 {
-    unsigned int size = readInt(source);
-    unsigned int left = size;
+    unsigned long long size = readLongLong(source);
+    
+    sink.preallocateContents(size);
+
+    unsigned long long left = size;
     unsigned char buf[65536];
 
     while (left) {
         checkInterrupt();
         unsigned int n = sizeof(buf);
-        if (n > left) n = left;
+        if ((unsigned long long) n > left) n = left;
         source(buf, n);
-        writeFull(fd, buf, n);
+        sink.receiveContents(buf, n);
         left -= n;
     }
 
+    sink.finalizeContents(size);
+
     readPadding(size, source);
 }
 
 
-static void restore(const Path & path, Source & source)
+static void parse(ParseSink & sink, Source & source, const Path & path)
 {
     string s;
 
@@ -186,7 +195,6 @@ static void restore(const Path & path, Source & source)
     if (s != "(") throw badArchive("expected open tag");
 
     enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
-    AutoCloseFD fd;
 
     while (1) {
         checkInterrupt();
@@ -204,15 +212,12 @@ static void restore(const Path & path, Source & source)
 
             if (t == "regular") {
                 type = tpRegular;
-                fd = open(path.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666);
-                if (fd == -1)
-                    throw SysError("creating file " + path);
+                sink.createRegularFile(path);
             }
 
             else if (t == "directory") {
+                sink.createDirectory(path);
                 type = tpDirectory;
-                if (mkdir(path.c_str(), 0777) == -1)
-                    throw SysError("creating directory " + path);
             }
 
             else if (t == "symlink") {
@@ -224,42 +229,109 @@ static void restore(const Path & path, Source & source)
         }
 
         else if (s == "contents" && type == tpRegular) {
-            restoreContents(fd, path, source);
+            parseContents(sink, source, path);
         }
 
         else if (s == "executable" && type == tpRegular) {
             readString(source);
-            struct stat st;
-            if (fstat(fd, &st) == -1)
-                throw SysError("fstat");
-            if (fchmod(fd, st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
-                throw SysError("fchmod");
+            sink.isExecutable();
         }
 
         else if (s == "entry" && type == tpDirectory) {
-            restoreEntry(path, source);
+            parseEntry(sink, source, path);
         }
 
         else if (s == "target" && type == tpSymlink) {
             string target = readString(source);
-            if (symlink(target.c_str(), path.c_str()) == -1)
-                throw SysError("creating symlink " + path);
+            sink.createSymlink(path, target);
         }
 
         else {
             throw badArchive("unknown field " + s);
             skipGeneric(source);
         }
-        
     }
 }
 
 
+void parseDump(ParseSink & sink, Source & source)
+{
+    string version;    
+    try {
+        version = readString(source);
+    } catch (SerialisationError & e) {
+        /* This generally means the integer at the start couldn't be
+           decoded.  Ignore and throw the exception below. */
+    }
+    if (version != archiveVersion1)
+        throw badArchive("input doesn't look like a Nix archive");
+    parse(sink, source, "");
+}
+
+
+struct RestoreSink : ParseSink
+{
+    Path dstPath;
+    AutoCloseFD fd;
+
+    void createDirectory(const Path & path)
+    {
+        Path p = dstPath + path;
+        if (mkdir(p.c_str(), 0777) == -1)
+            throw SysError(format("creating directory `%1%'") % p);
+    };
+
+    void createRegularFile(const Path & path)
+    {
+        Path p = dstPath + path;
+        fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666);
+        if (fd == -1) throw SysError(format("creating file `%1%'") % p);
+    }
+
+    void isExecutable()
+    {
+        struct stat st;
+        if (fstat(fd, &st) == -1)
+            throw SysError("fstat");
+        if (fchmod(fd, st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
+            throw SysError("fchmod");
+    }
+
+    void preallocateContents(unsigned long long len)
+    {
+#if HAVE_POSIX_FALLOCATE
+        if (len) {
+            errno = posix_fallocate(fd, 0, len);
+            if (errno) throw SysError(format("preallocating file of %1% bytes") % len);
+        }
+#endif
+    }
+
+    void receiveContents(unsigned char * data, unsigned int len)
+    {
+        writeFull(fd, data, len);
+    }
+
+    void finalizeContents(unsigned long long size)
+    {
+        errno = ftruncate(fd, size);
+        if (errno) throw SysError(format("truncating file to its allocated length of %1% bytes") % size);
+    }
+
+    void createSymlink(const Path & path, const string & target)
+    {
+        Path p = dstPath + path;
+        if (symlink(target.c_str(), p.c_str()) == -1)
+            throw SysError(format("creating symlink `%1%'") % p);
+    }
+};
+
+ 
 void restorePath(const Path & path, Source & source)
 {
-    if (readString(source) != archiveVersion1)
-        throw badArchive("expected Nix archive");
-    restore(path, source);
+    RestoreSink sink;
+    sink.dstPath = path;
+    parseDump(sink, source);
 }
 
  

src/libutil/archive.hh

@@ -56,6 +56,21 @@ extern PathFilter defaultPathFilter;
 void dumpPath(const Path & path, Sink & sink,
     PathFilter & filter = defaultPathFilter);
 
+struct ParseSink
+{
+    virtual void createDirectory(const Path & path) { };
+    
+    virtual void createRegularFile(const Path & path) { };
+    virtual void isExecutable() { };
+    virtual void preallocateContents(unsigned long long size) { };
+    virtual void receiveContents(unsigned char * data, unsigned int len) { };
+    virtual void finalizeContents(unsigned long long size) { };
+
+    virtual void createSymlink(const Path & path, const string & target) { };
+};
+    
+void parseDump(ParseSink & sink, Source & source);
+
 void restorePath(const Path & path, Source & source);
 
  

src/libutil/aterm-map.cc

@@ -215,7 +215,7 @@ void ATermMap::remove(ATerm key)
 }
 
 
-unsigned int ATermMap::size()
+unsigned int ATermMap::size() const
 {
     return count; /* STL nomenclature */
 }

src/libutil/aterm-map.hh

@@ -59,7 +59,7 @@ public:
 
     void remove(ATerm key);
 
-    unsigned int size();
+    unsigned int size() const;
 
     struct const_iterator
     {

src/libutil/aterm.cc

@@ -1,5 +1,7 @@
 #include "aterm.hh"
 
+#include <cstring>
+
 using std::string;
 
 

src/libutil/hash.cc

@@ -1,6 +1,7 @@
 #include "config.h"
 
 #include <iostream>
+#include <cstring>
 
 #ifdef HAVE_OPENSSL
 #include <openssl/md5.h>
@@ -334,4 +335,13 @@ HashType parseHashType(const string & s)
 }
 
  
+string printHashType(HashType ht)
+{
+    if (ht == htMD5) return "md5";
+    else if (ht == htSHA1) return "sha1";
+    else if (ht == htSHA256) return "sha256";
+    else throw Error("cannot print unknown hash type");
+}
+
+ 
 }

src/libutil/hash.hh

@@ -82,8 +82,11 @@ Hash compressHash(const Hash & hash, unsigned int newSize);
 /* Parse a string representing a hash type. */
 HashType parseHashType(const string & s);
 
+/* And the reverse. */
+string printHashType(HashType ht);
 
-typedef union Ctx;
+
+union Ctx;
 
 class HashSink : public Sink
 {

src/libutil/serialise.cc

@@ -1,6 +1,8 @@
 #include "serialise.hh"
 #include "util.hh"
+#include "aterm.hh"
 
+#include <cstring>
 
 namespace nix {
 
@@ -39,6 +41,21 @@ void writeInt(unsigned int n, Sink & sink)
 }
 
 
+void writeLongLong(unsigned long long n, Sink & sink)
+{
+    unsigned char buf[8];
+    buf[0] = n & 0xff;
+    buf[1] = (n >> 8) & 0xff;
+    buf[2] = (n >> 16) & 0xff;
+    buf[3] = (n >> 24) & 0xff;
+    buf[4] = (n >> 32) & 0xff;
+    buf[5] = (n >> 40) & 0xff;
+    buf[6] = (n >> 48) & 0xff;
+    buf[7] = (n >> 56) & 0xff;
+    sink(buf, sizeof(buf));
+}
+
+
 void writeString(const string & s, Sink & sink)
 {
     unsigned int len = s.length();
@@ -56,6 +73,67 @@ void writeStringSet(const StringSet & ss, Sink & sink)
 }
 
 
+void writeATerm(ATerm t, Sink & sink)
+{
+    int len;
+    unsigned char *buf = (unsigned char *) ATwriteToBinaryString(t, &len);
+    AutoDeleteArray<unsigned char> d(buf);
+    writeInt(len, sink);
+    sink(buf, len);
+}
+
+
+/* convert the ATermMap to a list of couple because many terms are shared
+   between the keys and between the values.  Thus the BAF stored by
+   writeATerm consume less memory space.  The list of couples is saved
+   inside a tree structure of /treedepth/ height because the serialiasation
+   of ATerm cause a tail recurssion on list tails. */
+void writeATermMap(const ATermMap & tm, Sink & sink)
+{
+    const unsigned int treedepth = 5;
+    const unsigned int maxarity = 128; // 2 < maxarity < MAX_ARITY (= 255)
+    const unsigned int bufsize = treedepth * maxarity;
+
+    AFun map = ATmakeAFun("map", 2, ATfalse);
+    AFun node = ATmakeAFun("node", maxarity, ATfalse);
+    ATerm empty = (ATerm) ATmakeAppl0(ATmakeAFun("empty", 0, ATfalse));
+
+    unsigned int c[treedepth];
+    ATerm *buf = new ATerm[bufsize];
+    AutoDeleteArray<ATerm> d(buf);
+
+    memset(buf, 0, bufsize * sizeof(ATerm));
+    ATprotectArray(buf, bufsize);
+
+    for (unsigned int j = 0; j < treedepth; j++)
+        c[j] = 0;
+
+    for (ATermMap::const_iterator i = tm.begin(); i != tm.end(); ++i) {
+         unsigned int depth = treedepth - 1;
+         ATerm term = (ATerm) ATmakeAppl2(map, i->key, i->value);
+         buf[depth * maxarity + c[depth]++] = term;
+         while (c[depth] % maxarity == 0) {
+             c[depth] = 0;
+             term = (ATerm) ATmakeApplArray(node, &buf[depth * maxarity]);
+             depth--;
+             buf[depth * maxarity + c[depth]++] = term;
+         }
+    }
+
+    unsigned int depth = treedepth;
+    ATerm last_node = empty;
+    while (depth--) {
+        buf[depth * maxarity + c[depth]++] = last_node;
+        while (c[depth] % maxarity)
+            buf[depth * maxarity + c[depth]++] = empty;
+        last_node = (ATerm) ATmakeApplArray(node, &buf[depth * maxarity]);
+    }
+
+    writeATerm(last_node, sink);
+    ATunprotectArray(buf);
+}
+
+
 void readPadding(unsigned int len, Source & source)
 {
     if (len % 8) {
@@ -63,7 +141,7 @@ void readPadding(unsigned int len, Source & source)
         unsigned int n = 8 - (len % 8);
         source(zero, n);
         for (unsigned int i = 0; i < n; i++)
-            if (zero[i]) throw Error("non-zero padding");
+            if (zero[i]) throw SerialisationError("non-zero padding");
     }
 }
 
@@ -73,7 +151,7 @@ unsigned int readInt(Source & source)
     unsigned char buf[8];
     source(buf, sizeof(buf));
     if (buf[4] || buf[5] || buf[6] || buf[7])
-        throw Error("implementation cannot deal with > 32-bit integers");
+        throw SerialisationError("implementation cannot deal with > 32-bit integers");
     return
         buf[0] |
         (buf[1] << 8) |
@@ -82,6 +160,22 @@ unsigned int readInt(Source & source)
 }
 
 
+unsigned long long readLongLong(Source & source)
+{
+    unsigned char buf[8];
+    source(buf, sizeof(buf));
+    return
+        ((unsigned long long) buf[0]) |
+        ((unsigned long long) buf[1] << 8) |
+        ((unsigned long long) buf[2] << 16) |
+        ((unsigned long long) buf[3] << 24) |
+        ((unsigned long long) buf[4] << 32) |
+        ((unsigned long long) buf[5] << 40) |
+        ((unsigned long long) buf[6] << 48) |
+        ((unsigned long long) buf[7] << 56);
+}
+
+
 string readString(Source & source)
 {
     unsigned int len = readInt(source);
@@ -103,4 +197,48 @@ StringSet readStringSet(Source & source)
 }
 
 
+ATerm readATerm(Source & source)
+{
+    unsigned int len = readInt(source);
+    unsigned char * buf = new unsigned char[len];
+    AutoDeleteArray<unsigned char> d(buf);
+    source(buf, len);
+    ATerm t = ATreadFromBinaryString((char *) buf, len);
+    if (t == 0)
+        throw SerialisationError("cannot read a valid ATerm.");
+    return t;
+}
+
+
+static void recursiveInitATermMap(ATermMap &tm, bool &stop, ATermAppl node)
+{
+    const unsigned int arity = ATgetArity(ATgetAFun(node));
+    ATerm key, value;
+
+    switch (arity) {
+        case 0:
+            stop = true;
+            return;
+        case 2:
+            key = ATgetArgument(node, 0);
+            value = ATgetArgument(node, 1);
+            tm.set(key, value);
+            return;
+        default:
+            for (unsigned int i = 0; i < arity && !stop; i++)
+              recursiveInitATermMap(tm, stop, (ATermAppl) ATgetArgument(node, i));
+            return;
+    }
+}
+
+ATermMap readATermMap(Source & source)
+{
+    ATermMap tm;
+    bool stop = false;
+
+    recursiveInitATermMap(tm, stop, (ATermAppl) readATerm(source));
+    return tm;
+}
+
+
 }

src/libutil/serialise.hh

@@ -2,7 +2,7 @@
 #define __SERIALISE_H
 
 #include "types.hh"
-
+#include "aterm-map.hh"
 
 namespace nix {
 
@@ -35,7 +35,7 @@ struct FdSink : Sink
 
     FdSink()
     {
-        fd = 0;
+        fd = -1;
     }
     
     FdSink(int fd) 
@@ -54,7 +54,7 @@ struct FdSource : Source
 
     FdSource()
     {
-        fd = 0;
+        fd = -1;
     }
     
     FdSource(int fd) 
@@ -80,9 +80,9 @@ struct StringSink : Sink
 /* A source that reads data from a string. */
 struct StringSource : Source
 {
-    string & s;
+    const string & s;
     unsigned int pos;
-    StringSource(string & _s) : s(_s), pos(0) { }
+    StringSource(const string & _s) : s(_s), pos(0) { }
     virtual void operator () (unsigned char * data, unsigned int len)
     {
         s.copy((char *) data, len, pos);
@@ -95,13 +95,22 @@ struct StringSource : Source
 
 void writePadding(unsigned int len, Sink & sink);
 void writeInt(unsigned int n, Sink & sink);
+void writeLongLong(unsigned long long n, Sink & sink);
 void writeString(const string & s, Sink & sink);
 void writeStringSet(const StringSet & ss, Sink & sink);
+void writeATerm(ATerm t, Sink & sink);
+void writeATermMap(const ATermMap & tm, Sink & sink);
 
 void readPadding(unsigned int len, Source & source);
 unsigned int readInt(Source & source);
+unsigned long long readLongLong(Source & source);
 string readString(Source & source);
 StringSet readStringSet(Source & source);
+ATerm readATerm(Source & source);
+ATermMap readATermMap(Source & source);
+
+
+MakeError(SerialisationError, Error)
 
 
 }

src/libutil/types.hh

@@ -24,12 +24,14 @@ using boost::format;
 class BaseError : public std::exception 
 {
 protected:
+    string prefix_; // used for location traces etc.
     string err;
 public:
     BaseError(const format & f);
     ~BaseError() throw () { };
     const char * what() const throw () { return err.c_str(); }
     const string & msg() const throw () { return err; }
+    const string & prefix() const throw () { return prefix_; }
     BaseError & addPrefix(const format & f);
 };