* Document Boehm GC support.

--enable-gc is given.

.gitignore

@@ -0,0 +1,262 @@
+# START "git svn show-ignore"
+
+# /
+/Makefile
+/Makefile.in
+/aclocal.m4
+/autom4te.cache
+/config.*
+/configure
+/nix.spec
+/stamp-h1
+/svn-revision
+/NEWS
+/libtool
+
+# /config/
+/config/config.guess
+/config/config.sub
+/config/depcomp
+/config/install-sh
+/config/missing
+/config/mkinstalldirs
+/config/ltmain.sh
+
+# /corepkgs/
+/corepkgs/Makefile
+/corepkgs/Makefile.in
+
+# /corepkgs/buildenv/
+/corepkgs/buildenv/Makefile.in
+/corepkgs/buildenv/Makefile
+/corepkgs/buildenv/builder.pl
+
+# /corepkgs/channels/
+/corepkgs/channels/Makefile.in
+/corepkgs/channels/Makefile
+/corepkgs/channels/unpack.sh
+
+# /corepkgs/nar/
+/corepkgs/nar/Makefile
+/corepkgs/nar/Makefile.in
+/corepkgs/nar/nar.sh
+/corepkgs/nar/unnar.sh
+
+# /doc/
+/doc/Makefile
+/doc/Makefile.in
+
+# /doc/manual/
+/doc/manual/Makefile
+/doc/manual/Makefile.in
+/doc/manual/manual.html
+/doc/manual/manual.is-valid
+/doc/manual/*.1
+/doc/manual/*.8
+/doc/manual/images
+/doc/manual/version.txt
+/doc/manual/NEWS.html
+/doc/manual/NEWS.txt
+
+# /externals/
+/externals/Makefile
+/externals/Makefile.in
+/externals/aterm-*
+/externals/have-aterm
+/externals/build-aterm
+/externals/inst-aterm
+/externals/bzip2-*
+/externals/have-bzip2
+/externals/build-bzip2
+/externals/inst-bzip2
+
+# /make/examples/aterm/
+/make/examples/aterm/result*
+
+# /make/examples/aterm/aterm/
+/make/examples/aterm/aterm/*
+
+# /make/examples/aterm/test/
+/make/examples/aterm/test/*
+
+# /misc/
+/misc/Makefile.in
+/misc/Makefile
+
+# /misc/emacs/
+/misc/emacs/Makefile.in
+/misc/emacs/Makefile
+
+# /scripts/
+/scripts/Makefile
+/scripts/Makefile.in
+/scripts/nix-profile.sh
+/scripts/nix-pull
+/scripts/nix-push
+/scripts/nix-switch
+/scripts/nix-collect-garbage
+/scripts/nix-prefetch-url
+/scripts/nix-install-package
+/scripts/nix-channel
+/scripts/nix-build
+/scripts/nix-copy-closure
+/scripts/readmanifest.pm
+/scripts/readconfig.pm
+/scripts/download-using-manifests.pl
+/scripts/copy-from-other-stores.pl
+/scripts/generate-patches.pl
+/scripts/find-runtime-roots.pl
+/scripts/build-remote.pl
+
+# /src/
+/src/Makefile
+/src/Makefile.in
+
+# /src/bin2c/
+/src/bin2c/Makefile.in
+/src/bin2c/Makefile
+/src/bin2c/bin2c
+/src/bin2c/.deps
+/src/bin2c/.libs
+
+# /src/boost/
+/src/boost/Makefile
+/src/boost/Makefile.in
+
+# /src/boost/format/
+/src/boost/format/Makefile
+/src/boost/format/Makefile.in
+/src/boost/format/.deps
+/src/boost/format/libformat.a
+/src/boost/format/.libs
+
+# /src/bsdiff-4.3/
+/src/bsdiff-4.3/Makefile
+/src/bsdiff-4.3/Makefile.in
+/src/bsdiff-4.3/bsdiff
+/src/bsdiff-4.3/bspatch
+/src/bsdiff-4.3/.deps
+/src/bsdiff-4.3/.libs
+
+# /src/libexpr/
+/src/libexpr/Makefile
+/src/libexpr/Makefile.in
+/src/libexpr/.deps
+/src/libexpr/libexpr.a
+/src/libexpr/lexer-tab.cc
+/src/libexpr/lexer-tab.hh
+/src/libexpr/parser-tab.cc
+/src/libexpr/parser-tab.hh
+/src/libexpr/parser-tab.output
+/src/libexpr/nixexpr-ast.hh
+/src/libexpr/nixexpr-ast.cc
+/src/libexpr/.libs
+/src/libexpr/nix.tbl
+
+# /src/libmain/
+/src/libmain/Makefile
+/src/libmain/Makefile.in
+/src/libmain/.deps
+/src/libmain/libmain.a
+/src/libmain/.libs
+
+# /src/libstore/
+/src/libstore/Makefile
+/src/libstore/Makefile.in
+/src/libstore/.deps
+/src/libstore/libstore.a
+/src/libstore/derivations-ast.cc
+/src/libstore/derivations-ast.hh
+/src/libstore/.libs
+
+# /src/libutil/
+/src/libutil/Makefile
+/src/libutil/Makefile.in
+/src/libutil/.deps
+/src/libutil/libutil.a
+/src/libutil/.libs
+
+# /src/nix-env/
+/src/nix-env/Makefile.in
+/src/nix-env/Makefile
+/src/nix-env/.deps
+/src/nix-env/nix-env
+/src/nix-env/help.txt.hh
+/src/nix-env/.libs
+
+# /src/nix-hash/
+/src/nix-hash/Makefile
+/src/nix-hash/Makefile.in
+/src/nix-hash/.deps
+/src/nix-hash/.libs
+/src/nix-hash/nix-hash
+/src/nix-hash/help.txt.hh
+
+# /src/nix-instantiate/
+/src/nix-instantiate/Makefile.in
+/src/nix-instantiate/Makefile
+/src/nix-instantiate/.deps
+/src/nix-instantiate/nix-instantiate
+/src/nix-instantiate/help.txt.hh
+/src/nix-instantiate/.libs
+
+# /src/nix-log2xml/
+/src/nix-log2xml/Makefile.in
+/src/nix-log2xml/Makefile
+/src/nix-log2xml/.deps
+/src/nix-log2xml/nix-log2xml
+/src/nix-log2xml/test*.*
+/src/nix-log2xml/.libs
+/src/nix-log2xml/*.log
+/src/nix-log2xml/*.xml
+/src/nix-log2xml/*.html
+
+# /src/nix-setuid-helper/
+/src/nix-setuid-helper/Makefile.in
+/src/nix-setuid-helper/Makefile
+/src/nix-setuid-helper/.deps
+/src/nix-setuid-helper/nix-setuid-helper
+/src/nix-setuid-helper/help.txt.hh
+/src/nix-setuid-helper/.libs
+
+# /src/nix-store/
+/src/nix-store/Makefile
+/src/nix-store/Makefile.in
+/src/nix-store/.deps
+/src/nix-store/help.txt.hh
+/src/nix-store/nix-store
+/src/nix-store/.libs
+
+# /src/nix-worker/
+/src/nix-worker/Makefile.in
+/src/nix-worker/Makefile
+/src/nix-worker/.deps
+/src/nix-worker/nix-worker
+/src/nix-worker/help.txt.hh
+/src/nix-worker/.libs
+
+# /tests/
+/tests/Makefile
+/tests/Makefile.in
+/tests/test-tmp
+/tests/config.nix
+/tests/common.sh
+/tests/dummy
+
+# /tests/lang/
+/tests/lang/*.out
+/tests/lang/*.out.xml
+/tests/lang/*.ast
+
+# END "git svn show-ignore"
+
+*.lo
+*.la
+*.o
+*~
+
+# GNU Global
+GPATH
+GRTAGS
+GSYMS
+GTAGS

Makefile.am

@@ -2,6 +2,8 @@ SUBDIRS = externals src scripts corepkgs doc misc tests
 EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
   nix.conf.example NEWS version
 
+pkginclude_HEADERS = config.h
+
 include ./substitute.mk
 
 nix.spec: nix.spec.in
@@ -29,11 +31,9 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
-	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
-	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
-	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
+	-$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
 

configure.ac

@@ -10,8 +10,8 @@ AC_CANONICAL_HOST
 
 # Construct a Nix system name (like "i686-linux").
 AC_MSG_CHECKING([for the canonical Nix system name])
-cpu_name=$(uname -p | tr 'A-Z ' 'a-z_')
-machine_name=$(uname -m | tr 'A-Z ' 'a-z_')
+cpu_name=$(uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
+machine_name=$(uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
 
 case $machine_name in
     i*86)
@@ -30,7 +30,7 @@ case $machine_name in
         ;;
 esac
 
-sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
+sys_name=$(uname -s | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
 
 case $sys_name in
     cygwin*)
@@ -55,7 +55,7 @@ test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 # except that it requires the ATerm "lib" directory to be in $PATH, as
 # Windows doesn't have anything like an RPATH embedded in executable.
 # Since this is kind of annoying, we use static libraries for now.
-    
+
 AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
   [produce statically linked binaries]),
   static_nix=$enableval, static_nix=no)
@@ -63,23 +63,38 @@ AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
 if test "$sys_name" = cygwin; then
    static_nix=yes
 fi
-  
+
 if test "$static_nix" = yes; then
     AC_DISABLE_SHARED
     AC_ENABLE_STATIC
 fi
 
-  
+
 # Windows-specific stuff.
 if test "$sys_name" = "cygwin"; then
     # We cannot delete open files.
     AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.])
 fi
 
+# Solaris-specific stuff.
+if test "$sys_name" = "sunos"; then
+    # Solaris requires -lsocket -lnsl for network functions
+    ADDITIONAL_NETWORK_LIBS="-lsocket -lnsl"
+    AC_SUBST(ADDITIONAL_NETWORK_LIBS)
+fi
 
 AC_PROG_CC
 AC_PROG_CXX
 
+# To build programs to be run in the build machine
+if test "$CC_FOR_BUILD" = ""; then
+    if test "$cross_compiling" = "yes"; then
+        AC_CHECK_PROGS(CC_FOR_BUILD, gcc cc)
+    else
+        CC_FOR_BUILD="$CC"
+    fi
+fi
+AC_SUBST([CC_FOR_BUILD])
 
 # We are going to use libtool.
 AC_DISABLE_STATIC
@@ -88,8 +103,7 @@ AC_PROG_LIBTOOL
 
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
-CFLAGS="-D_FILE_OFFSET_BITS=64 $CFLAGS"
-CXXFLAGS="-D_FILE_OFFSET_BITS=64 $CXXFLAGS"
+AC_SYS_LARGEFILE
 
 
 # Check for pubsetbuf.
@@ -122,11 +136,22 @@ AC_CHECK_HEADERS([locale], [], [], [])
 AC_LANG_POP(C++)
 
 
+# Check for <err.h>.
+AC_CHECK_HEADER([err.h], [], [bsddiff_compat_include="-Icompat-include"])
+AC_SUBST([bsddiff_compat_include])
+
+
 # Check whether we have the personality() syscall, which allows us to
 # do i686-linux builds on x86_64-linux machines.
 AC_CHECK_HEADERS([sys/personality.h])
 
 
+# Check for tr1/unordered_set.
+AC_LANG_PUSH(C++)
+AC_CHECK_HEADERS([tr1/unordered_set], [], [], [])
+AC_LANG_POP(C++)
+
+
 AC_DEFUN([NEED_PROG],
 [
 AC_PATH_PROG($1, $2)
@@ -140,11 +165,11 @@ NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
-AC_PATH_PROG(jing, jing, false) # needed because xmllint --relaxng seems broken
 AC_PATH_PROG(w3m, w3m, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 NEED_PROG(perl, perl)
+NEED_PROG(sed, sed)
 NEED_PROG(tar, tar)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(dblatex, dblatex)
@@ -189,48 +214,6 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
   storedir=$withval, storedir='/nix/store')
 AC_SUBST(storedir)
 
-AC_ARG_ENABLE(old-db-compat, AC_HELP_STRING([--disable-old-db-compat],
-  [disable support for converting from old Berkeley DB-based Nix stores]),
-  old_db_compat=$enableval, old_db_compat=yes)
-AM_CONDITIONAL(OLD_DB_COMPAT, test "$old_db_compat" = "yes")
-
-AC_ARG_WITH(bdb, AC_HELP_STRING([--with-bdb=PATH],
-  [prefix of Berkeley DB (for Nix <= 0.11 compatibility)]),
-  bdb=$withval, bdb=)
-AM_CONDITIONAL(HAVE_BDB, test -n "$bdb")
-if test -z "$bdb"; then
-  bdb_lib='-L${top_builddir}/externals/inst-bdb/lib -ldb_cxx'
-  bdb_include='-I${top_builddir}/externals/inst-bdb/include'
-else
-  bdb_lib="-L$bdb/lib -ldb_cxx"
-  bdb_include="-I$bdb/include"
-fi
-if test "$old_db_compat" = "no"; then
-  bdb_lib=
-  bdb_include=
-else
-  AC_DEFINE(OLD_DB_COMPAT, 1, [Whether to support converting from old Berkeley DB-based Nix stores.])
-fi
-AC_SUBST(bdb_lib)
-AC_SUBST(bdb_include)
-
-AC_ARG_WITH(aterm, AC_HELP_STRING([--with-aterm=PATH],
-  [prefix of CWI ATerm library]),
-  aterm=$withval, aterm=)
-AM_CONDITIONAL(HAVE_ATERM, test -n "$aterm")
-if test -z "$aterm"; then
-  aterm_lib='-L${top_builddir}/externals/inst-aterm/lib -lATerm'
-  aterm_include='-I${top_builddir}/externals/inst-aterm/include'
-  aterm_bin='${top_builddir}/externals/inst-aterm/bin'
-else
-  aterm_lib="-L$aterm/lib -lATerm"
-  aterm_include="-I$aterm/include"
-  aterm_bin="$aterm/bin"
-fi
-AC_SUBST(aterm_lib)
-AC_SUBST(aterm_include)
-AC_SUBST(aterm_bin)
-
 AC_ARG_WITH(openssl, AC_HELP_STRING([--with-openssl=PATH],
   [prefix of the OpenSSL library]),
   openssl=$withval, openssl=)
@@ -260,14 +243,24 @@ else
   bzip2_include="-I$bzip2/include"
   bzip2_bin="$bzip2/bin"
   bzip2_bin_test="$bzip2/bin"
-fi   
+fi
 AC_SUBST(bzip2_lib)
 AC_SUBST(bzip2_include)
 AC_SUBST(bzip2_bin)
 AC_SUBST(bzip2_bin_test)
 
 
-AC_CHECK_LIB(pthread, pthread_mutex_init)
+# Whether to use the Boehm garbage collector.
+AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
+  [enable garbage collection in the Nix expression evaluator (requires Boehm GC)]),
+  gc=$enableval, gc=)
+if test -n "$gc"; then
+  PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
+  boehmgc_lib="-L$boehmgc/lib -lgc"
+  CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
+  AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
+fi
+AC_SUBST(boehmgc_lib)
 
 
 AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
@@ -285,7 +278,7 @@ AC_CHECK_FUNCS([strsignal])
 AC_CHECK_FUNCS([posix_fallocate])
 
 
-# This is needed if ATerm, Berkeley DB or bzip2 are static libraries,
+# This is needed if ATerm or bzip2 are static libraries,
 # and the Nix libraries are dynamic.
 if test "$(uname)" = "Darwin"; then
     LDFLAGS="-all_load $LDFLAGS"

corepkgs/buildenv/builder.pl.in

@@ -29,10 +29,18 @@ sub createLinks {
         $baseName =~ s/^.*\///g; # strip directory
         my $dstFile = "$dstDir/$baseName";
 
+        # The files below are special-cased so that they don't show up
+        # in user profiles, either because they are useless, or
+        # because they would cause pointless collisions (e.g., each
+        # Python package brings its own
+        # `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
         # Urgh, hacky...
-	if ($srcFile =~ /\/propagated-build-inputs$/ ||
+        if ($srcFile =~ /\/propagated-build-inputs$/ ||
             $srcFile =~ /\/nix-support$/ ||
             $srcFile =~ /\/perllocal.pod$/ ||
+            $srcFile =~ /\/easy-install.pth$/ ||
+            $srcFile =~ /\/site.py$/ ||
+            $srcFile =~ /\/site.pyc$/ ||
             $srcFile =~ /\/info\/dir$/ ||
             $srcFile =~ /\/log$/)
         {
@@ -160,4 +168,4 @@ while (scalar(keys %postponed) > 0) {
 print STDERR "created $symlinks symlinks in user environment\n";
 
 
-symlink($ENV{"manifest"}, "$out/manifest") or die "cannot create manifest";
+symlink($ENV{"manifest"}, "$out/manifest.nix") or die "cannot create manifest";

corepkgs/channels/unpack.sh.in

@@ -1,5 +1,8 @@
 #! @shell@ -e
 
+# Cygwin compatibility hack: bunzip2 expects cygwin.dll in $PATH.
+export PATH=@coreutils@
+
 @coreutils@/mkdir $out
 @coreutils@/mkdir $out/tmp
 cd $out/tmp

doc/manual/Makefile.am

@@ -6,7 +6,12 @@ XSLTPROC = $(xsltproc) $(xmlflags) \
  --param xref.with.number.and.title 1 \
  --param toc.section.depth 3 \
  --param admon.style \'\' \
- --param callout.graphics.extension \'.gif\'
+ --param callout.graphics.extension \'.gif\' \
+ --param contrib.inline.enabled 0
+
+dblatex_opts = \
+ -P doc.collab.show=0 \
+ -P latex.output.revhistory=0
 
 # Note: we use GIF for now, since the PNGs shipped with Docbook aren't
 # transparent.
@@ -29,13 +34,9 @@ MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  conf-file.xml release-notes.xml \
  style.css images
 
+# Note: RelaxNG validation requires xmllint >= 2.7.4.
 manual.is-valid: $(MANUAL_SRCS) version.txt
-#	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --relaxng $(docbookrng)/docbook.rng -
-	if test "$(jing)" != "false"; then \
-		$(XMLLINT) --xinclude $< | $(jing) $(docbookrng)/docbook.rng /dev/fd/0; \
-	else \
-		echo "Not validating."; \
-	fi
+	$(XMLLINT) --noout --nonet --xinclude --noxincludenode --relaxng $(docbookrng)/docbook.rng $<
 	touch $@
 
 version.txt:
@@ -50,7 +51,7 @@ manual.html: $(MANUAL_SRCS) manual.is-valid images
 
 manual.pdf: $(MANUAL_SRCS) manual.is-valid images
 	if test "$(dblatex)" != ""; then \
-		$(dblatex) manual.xml; \
+		$(dblatex) $(dblatex_opts) manual.xml; \
 	else \
 		echo "Please install dblatex and rerun configure."; \
 		exit 1; \

doc/manual/builtins.xml

@@ -324,6 +324,16 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.intersectAttrs</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return an attribute set consisting of the
+    attributes in the set <replaceable>e2</replaceable> that also
+    exist in the set <replaceable>e1</replaceable>.</para></listitem>
+
+  </varlistentry>
+        
+    
   <varlistentry><term><function>builtins.isAttrs</function>
   <replaceable>e</replaceable></term>
 

doc/manual/conf-file.xml

@@ -97,6 +97,25 @@ env-keep-derivations = false
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-build-cores"><term><literal>build-cores</literal></term>
+
+    <listitem><para>Sets the value of the
+    <envar>NIX_BUILD_CORES</envar> environment variable in the
+    invocation of builders.  Builders can use this variable at their
+    discretion to control the maximum amount of parallelism.  For
+    instance, in Nixpkgs, if the derivation attribute
+    <varname>enableParallelBuilding</varname> is set to
+    <literal>true</literal>, the builder passes the
+    <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+    It can be overriden using the <option
+    linkend='opt-cores'>--cores</option> command line switch and
+    defaults to <literal>1</literal>.  The value <literal>0</literal>
+    means that the builder should use all available CPU cores in the
+    system.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-build-max-silent-time"><term><literal>build-max-silent-time</literal></term>
 
     <listitem>
@@ -233,7 +252,17 @@ build-use-chroot = /dev /proc /bin</programlisting>
     <filename>configure</filename> at build time.</para></listitem>
 
   </varlistentry>
-  
+
+
+  <varlistentry><term><literal>fsync-metadata</literal></term>
+
+    <listitem><para>If set to <literal>true</literal>, changes to the
+    Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
+    synchronously flushed to disk.  This improves robustness in case
+    of system crashes, but reduces performance.  The default is
+    <literal>false</literal>.</para></listitem>
+
+  </varlistentry>
     
 </variablelist>
 

doc/manual/env-common.xml

@@ -271,6 +271,17 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 </varlistentry>
 
+
+<varlistentry><term><envar>GC_INITIAL_HEAP_SIZE</envar></term>
+
+  <listitem><para>If Nix has been configured to use the Boehm garbage
+  collector, this variable sets the initial size of the heap in bytes.
+  It defaults to 384 MiB.  Setting it to a low value reduces memory
+  consumption, but will increase runtime due to the overhead of
+  garbage collection.</para></listitem>
+
+</varlistentry>
+
     
 </variablelist>
 

doc/manual/installation.xml

@@ -96,23 +96,21 @@ ubiquitous 2.5.4a won't.  Note that these are only required if you
 modify the parser or when you are building from the Subversion
 repository.</para>
 
-<para>Nix uses CWI's ATerm library and the bzip2 compressor (including
-the bzip2 library).  These are included in the Nix source
-distribution.  If you build from the Subversion repository, you must
-download them yourself and place them in the
-<filename>externals/</filename> directory.  See
+<para>Nix uses the bzip2 compressor (including the bzip2 library).  It
+is included in the Nix source distribution.  If you build from the
+Subversion repository, you must download it yourself and place it in
+the <filename>externals/</filename> directory.  See
 <filename>externals/Makefile.am</filename> for the precise URLs of
-these packages.  Alternatively, if you already have them installed,
-you can use <command>configure</command>'s
-<option>--with-aterm</option> and <option>--with-bzip2</option>
+this packages.  Alternatively, if you already have it installed, you
+can use <command>configure</command>'s <option>--with-bzip2</option>
 options to point to their respective locations.</para>
 
-<para>If you want to be able to upgrade Nix stores from before version
-0.12pre12020, you need Sleepycat's Berkeley DB version version 4.5.
-(Other versions may not have compatible database formats.).  Berkeley
-DB 4.5 is included in the Nix source distribution.  If you do not need
-this ability, you can build Nix with the
-<option>--disable-old-db-compat</option> configure option.</para>
+<para>Nix can optionally use the <link
+xlink:href="http://www.hpl.hp.com/personal/Hans_Boehm/gc/">Boehm
+garbage collector</link> to reduce the evaluator’s memory consumption.
+To enable it, install <literal>pkgconfig</literal> and the Boehm
+garbage collector, and pass the flag <option>--enable-gc</option> to
+<command>configure</command>.</para>
 
 </section>
 
@@ -133,23 +131,32 @@ $ make install</screen>
 preceded by the command:
 
 <screen>
-$ ./bootstrap</screen>
+$ ./bootstrap.sh</screen>
 
 </para>
 
 <para>The installation path can be specified by passing the
 <option>--prefix=<replaceable>prefix</replaceable></option> to
 <command>configure</command>.  The default installation directory is
-<filename>/nix</filename>.  You can change this to any location you
-like.  You must have write permission to the
+<filename>/usr/local</filename>.  You can change this to any location
+you like.  You must have write permission to the
 <replaceable>prefix</replaceable> path.</para>
 
-<warning><para>It is best <emphasis>not</emphasis> to change the
-installation prefix from its default, since doing so makes it
-impossible to use pre-built binaries from the standard Nixpkgs
-channels.</para></warning>
+<para>Nix keeps its <emphasis>store</emphasis> (the place where
+packages are stored) in <filename>/nix/store</filename> by default.
+This can be changed using
+<option>--with-store-dir=<replaceable>path</replaceable></option>.</para>
 
-<para>If you want to rebuilt the documentation, pass the full path to
+<warning><para>It is best <emphasis>not</emphasis> to change the Nix
+store from its default, since doing so makes it impossible to use
+pre-built binaries from the standard Nixpkgs channels — that is, all
+packages will need to be built from source.</para></warning>
+
+<para>Nix keeps state (such as its database and log files) in
+<filename>/nix/var</filename> by default.  This can be changed using
+<option>--localstatedir=<replaceable>path</replaceable></option>.</para>
+
+<para>If you want to rebuild the documentation, pass the full path to
 the DocBook RELAX NG schemas and to the DocBook XSL stylesheets using
 the
 <option>--with-docbook-rng=<replaceable>path</replaceable></option>
@@ -160,27 +167,26 @@ options.</para>
 </section>
 
 
-<section><title>Installing from RPMs</title>
+<section><title>Installing a binary distribution</title>
 
-<para>RPM packages of Nix can be downloaded from <link
-xlink:href="http://nixos.org/" />.  These RPMs should work for most
-fairly recent releases of SuSE and Red Hat Linux.  They have been
-known to work work on SuSE Linux 8.1 and 9.0, and Red Hat 9.0.  In
-fact, it should work on any RPM-based Linux distribution based on
-<literal>glibc</literal> 2.3 or later.</para>
-
-<para>Once downloaded, the RPMs can be installed or upgraded using
-<command>rpm -U</command>.  For example,
+<para>RPM and Deb packages of Nix for a number of different versions
+of Fedora, openSUSE, Debian and Ubuntu can be downloaded from <link
+xlink:href="http://nixos.org/" />.  Once downloaded, the RPMs can be
+installed or upgraded using <command>rpm -U</command>.  For example,
 
 <screen>
-$ rpm -U nix-0.5pre664-1.i386.rpm</screen>
+$ rpm -U nix-0.13pre18104-1.i386.rpm</screen>
+
+Likewise, for a Deb package:
+
+<screen>
+$ dpkg -i nix_0.13pre18104-1_amd64.deb</screen>
 
 </para>
 
-<para>The RPMs install into the directory <filename>/nix</filename>.
-Nix can be uninstalled using <command>rpm -e nix</command>.  After
-this it will be necessary to manually remove the Nix store and other
-auxiliary data:
+<para>Nix can be uninstalled using <command>rpm -e nix</command> or
+<command>dpkg -r nix</command>.  After this you should manually remove
+the Nix store and other auxiliary data, if desired:
 
 <screen>
 $ rm -rf /nix/store
@@ -191,6 +197,7 @@ $ rm -rf /nix/var</screen>
 </section>
 
 
+<!-- TODO: should be updated
 <section><title>Upgrading Nix through Nix</title>
 
 <para>You can install the latest stable version of Nix through Nix
@@ -203,6 +210,7 @@ installation</link> by clicking on the package links at <link
 xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para>
 
 </section>
+-->
 
 
 <section><title>Security</title>

doc/manual/introduction.xml

@@ -113,7 +113,7 @@ $ nix-env --rollback
 
 <simplesect><title>Garbage collection</title>
 
-<para>When you install a package like this…
+<para>When you uninstall a package like this…
 
 <screen>
 $ nix-env --uninstall firefox
@@ -320,7 +320,7 @@ overview of NixOS is given in the HotOS XI paper <citetitle
 xlink:href="http://www.st.ewi.tudelft.nl/~dolstra/pubs/hotos-final.pdf">Purely
 Functional System Configuration Management</citetitle>.  The Nix
 homepage has <link
-xlink:href="http://nix.cs.uu.nl/docs/papers.html">an up-to-date list
+xlink:href="http://nixos.org/docs/papers.html">an up-to-date list
 of Nix-related papers</link>.</para>
 
 <para>Nix is the subject of Eelco Dolstra’s PhD thesis <citetitle

doc/manual/manual.xml

@@ -17,6 +17,7 @@
         <orgname>Delft University of Technology</orgname>
         <orgdiv>Department of Software Technology</orgdiv>
       </affiliation>
+      <contrib>Author</contrib>
     </author>
 
     <copyright>
@@ -25,10 +26,12 @@
       <year>2006</year>
       <year>2007</year>
       <year>2008</year>
+      <year>2009</year>
+      <year>2010</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>November 2008</date>
+    <date>August 2010</date>
     
   </info>
 
@@ -49,64 +52,25 @@
     
     <section>
       <title>Main commands</title>
-      <section xml:id="sec-nix-env">
-        <title>nix-env</title>
-        <xi:include href="nix-env.xml" />
-      </section>
-      <section xml:id="sec-nix-instantiate">
-        <title>nix-instantiate</title>
-        <xi:include href="nix-instantiate.xml" />
-      </section>
-      <section xml:id="sec-nix-store">
-        <title>nix-store</title>
-        <xi:include href="nix-store.xml" />
-      </section>
+      <xi:include href="nix-env.xml" />
+      <xi:include href="nix-instantiate.xml" />
+      <xi:include href="nix-store.xml" />
     </section>
     
     <section>
       <title>Utilities</title>
-      <section xml:id="sec-nix-build">
-        <title>nix-build</title>
-        <xi:include href="nix-build.xml" />
-      </section>
-      <section xml:id="sec-nix-channel">
-        <title>nix-channel</title>
-        <xi:include href="nix-channel.xml" />
-      </section>
-      <section xml:id="sec-nix-collect-garbage">
-        <title>nix-collect-garbage</title>
-        <xi:include href="nix-collect-garbage.xml" />
-      </section>
-      <section xml:id="sec-nix-copy-closure">
-        <title>nix-copy-closure</title>
-        <xi:include href="nix-copy-closure.xml" />
-      </section>
-      <section xml:id="sec-nix-hash">
-        <title>nix-hash</title>
-        <xi:include href="nix-hash.xml" />
-      </section>
-      <section xml:id="sec-nix-install-package">
-        <title>nix-install-package</title>
-        <xi:include href="nix-install-package.xml" />
-      </section>
-      <section xml:id="sec-nix-prefetch-url">
-        <title>nix-prefetch-url</title>
-        <xi:include href="nix-prefetch-url.xml" />
-      </section>
-      <section xml:id="sec-nix-pull">
-        <title>nix-pull</title>
-        <xi:include href="nix-pull.xml" />
-      </section>
-      <section xml:id="sec-nix-push">
-        <title>nix-push</title>
-        <xi:include href="nix-push.xml" />
-      </section>
-      <section xml:id="sec-nix-worker">
-        <title>nix-worker</title>
-        <xi:include href="nix-worker.xml" />
-      </section>
+      <xi:include href="nix-build.xml" />
+      <xi:include href="nix-channel.xml" />
+      <xi:include href="nix-collect-garbage.xml" />
+      <xi:include href="nix-copy-closure.xml" />
+      <xi:include href="nix-hash.xml" />
+      <xi:include href="nix-install-package.xml" />
+      <xi:include href="nix-prefetch-url.xml" />
+      <xi:include href="nix-pull.xml" />
+      <xi:include href="nix-push.xml" />
+      <xi:include href="nix-worker.xml" />
     </section>
-    
+
   </appendix>
 
   <xi:include href="troubleshooting.xml" />

doc/manual/nix-build.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-build">
 
 <refmeta>
   <refentrytitle>nix-build</refentrytitle>

doc/manual/nix-channel.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-channel">
   
 <refmeta>
   <refentrytitle>nix-channel</refentrytitle>

doc/manual/nix-collect-garbage.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-collect-garbage">
   
 <refmeta>
   <refentrytitle>nix-collect-garbage</refentrytitle>

doc/manual/nix-copy-closure.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-copy-closure">
 
 <refmeta>
   <refentrytitle>nix-copy-closure</refentrytitle>

doc/manual/nix-env.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-env">
   
 <refmeta>
   <refentrytitle>nix-env</refentrytitle>

doc/manual/nix-hash.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-hash">
   
 <refmeta>
   <refentrytitle>nix-hash</refentrytitle>

doc/manual/nix-install-package.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-install-package">
   
 <refmeta>
   <refentrytitle>nix-install-package</refentrytitle>

doc/manual/nix-instantiate.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-instantiate">
   
 <refmeta>
   <refentrytitle>nix-instantiate</refentrytitle>

doc/manual/nix-lang-ref.xml

@@ -178,100 +178,5 @@
     </productionset>
     
   </sect1>
-  
-
-
-  <sect1>
-    <title>Semantics</title>
-
-
-    
-    <sect2>
-      <title>Built-in functions</title>
-
-      <para>
-        The Nix language provides the following built-in function
-        (<quote>primops</quote>):
-      </para>
-
-      <variablelist>
-
-        <varlistentry>
-          <term><function>import</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield a path value.  The Nix expression
-              stored at this path in the file system is then read,
-              parsed, and evaluated.  Returns the result of the
-              evaluation of the Nix expression just read.
-            </para>
-
-            <para>
-              Example: <literal>import ./foo.nix</literal> evaluates
-              the expression stored in <filename>foo.nix</filename>
-              (in the directory containing the expression in which the
-              <function>import</function> occurs).
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>derivation</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield an attribute set.  [...]
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>baseNameOf</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield a string value, and returns a string
-              representing its <emphasis>base name</emphasis>.  This
-              is the substring following the last path separator
-              (<literal>/</literal>).
-            </para>
-
-            <para>
-              Example: <literal>baseNameOf "/foo/bar"</literal>
-              returns <literal>"bar"</literal>, and
-              <literal>baseNameOf "/foo/bar/"</literal> returns
-              <literal>""</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>toString</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>
-              and coerces it into a string, if possible.  Only
-              strings, paths, and URIs can be so coerced.
-            </para>
-
-            <para>
-              Example: <literal>toString
-              http://www.cs.uu.nl/</literal> returns
-              <literal>"http://www.cs.uu.nl/"</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-            
-      </variablelist>
-
-    </sect2>
-
-  </sect1>
-  
 
 </appendix>

doc/manual/nix-prefetch-url.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-prefetch-url">
   
 <refmeta>
   <refentrytitle>nix-prefetch-url</refentrytitle>

doc/manual/nix-pull.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-pull">
 
 <refmeta>
   <refentrytitle>nix-pull</refentrytitle>

doc/manual/nix-push.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-push">
 
 <refmeta>
   <refentrytitle>nix-push</refentrytitle>
@@ -119,7 +120,7 @@ dependencies used in the build, such as compilers).</para>
 dependencies, we can do:
       
 <screen>
-$ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
+$ nix-push <replaceable>urls</replaceable> $(nix-store -r $(nix-instantiate foo.nix))</screen>
     
 </para>
 

doc/manual/nix-store.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-store">
 
 <refmeta>
   <refentrytitle>nix-store</refentrytitle>
@@ -212,8 +213,6 @@ linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
   <arg><option>--max-links</option> <replaceable>nrlinks</replaceable></arg>
-  <arg><option>--max-atime</option> <replaceable>atime</replaceable></arg>
-  <arg><option>--use-atime</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -291,42 +290,6 @@ options control what gets deleted and in what order:
     
   </varlistentry>
 
-  <varlistentry><term><option>--max-atime</option> <replaceable>atime</replaceable></term>
-  
-    <listitem><para>Only delete a store path if its last-accessed time
-    is less than <replaceable>atime</replaceable>.  This allows you to
-    garbage-collect only packages that haven’t been used recently.
-    The time is expressed as the number of seconds in the Unix epoch,
-    i.e., since 1970-01-01 00:00:00 UTC.  An easy way to convert to
-    this format is <literal>date +%s -d "<replaceable>date
-    specification</replaceable>"</literal>.</para>
-
-    <para>For directories, the last-accessed time is the highest
-    last-accessed time of any regular file in the directory (or in any
-    of its subdirectories).  That is, the <literal>atime</literal>
-    field maintained by the filesystem is ignored for directories.
-    This is because operations such as rebuilding the
-    <command>locate</command> database tend to update the
-    <literal>atime</literal> values of all directories, so they’re not
-    a useful indicator of whether a package was recently used.</para>
-
-    <para>Note that <command>nix-store --optimise</command> reads all
-    regular files in the Nix store, and so causes all last-accessed
-    times to be set to the present time.  This makes
-    <option>--max-atime</option> ineffective (for a while at
-    least).</para></listitem>
-    
-  </varlistentry>
-
-  <varlistentry><term><option>--use-atime</option></term>
-
-    <listitem><para>Delete store paths in order of ascending
-    last-accessed time.  This is useful in conjunction with the other
-    options to delete only the least recently used
-    packages.</para></listitem>
-
-  </varlistentry>
-  
 </variablelist>
 
 </para>
@@ -357,13 +320,6 @@ deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
 
 </para>
 
-<para>To delete unreachable paths not accessed in the last two months:
-
-<screen>
-$ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
-
-</para>
-
 <para>To delete at least 100 MiBs of unreachable paths:
 
 <screen>
@@ -386,7 +342,7 @@ $ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
 
 <cmdsynopsis>
   <command>nix-store</command>
-  <arg choice='plain'><option>--gc</option></arg>
+  <arg choice='plain'><option>--delete</option></arg>
   <arg><option>--ignore-liveness</option></arg>
   <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
 </cmdsynopsis>
@@ -448,6 +404,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
     <arg choice='plain'><option>--tree</option></arg>
     <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
     <arg choice='plain'><option>--hash</option></arg>
+    <arg choice='plain'><option>--roots</option></arg>
   </group>
   <arg><option>--use-output</option></arg>
   <arg><option>-u</option></arg>
@@ -630,12 +587,20 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--hash</option></term>
   
     <listitem><para>Prints the SHA-256 hash of the contents of the
-    store path <replaceable>paths</replaceable>.  Since the hash is
+    store paths <replaceable>paths</replaceable>.  Since the hash is
     stored in the Nix database, this is a fast
     operation.</para></listitem>
 
   </varlistentry>
 
+  <varlistentry><term><option>--roots</option></term>
+  
+    <listitem><para>Prints the garbage collector roots that point,
+    directly or indirectly, at the store paths
+    <replaceable>paths</replaceable>.</para></listitem>
+
+  </varlistentry>
+
 </variablelist>
 
 </refsection>
@@ -713,6 +678,18 @@ $ gv graph.ps</screen>
 
 </para>
 
+<para>Show every garbage collector root that points to a store path
+that depends on <command>svn</command>:
+
+<screen>
+$ nix-store -q --roots $(which svn)
+/nix/var/nix/profiles/default-81-link
+/nix/var/nix/profiles/default-82-link
+/nix/var/nix/profiles/per-user/eelco/profile-97-link
+</screen>
+
+</para>
+
 </refsection>
 
 

doc/manual/nix-worker.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-worker">
 
 <refmeta>
   <refentrytitle>nix-worker</refentrytitle>

doc/manual/opt-common-syn.xml

@@ -13,6 +13,10 @@
   </group>
   <replaceable>number</replaceable>
 </arg>
+<arg>
+  <arg><option>--cores</option></arg>
+  <replaceable>number</replaceable>
+</arg>
 <arg>
   <arg><option>--max-silent-time</option></arg>
   <replaceable>number</replaceable>

doc/manual/opt-common.xml

@@ -98,7 +98,25 @@
   linkend='conf-build-max-jobs'><literal>build-max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
-  exploit I/O latency.  </para></listitem>
+  exploit I/O latency.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry xml:id="opt-cores"><term><option>--cores</option></term>
+
+  <listitem><para>Sets the value of the <envar>NIX_BUILD_CORES</envar>
+  environment variable in the invocation of builders.  Builders can
+  use this variable at their discretion to control the maximum amount
+  of parallelism.  For instance, in Nixpkgs, if the derivation
+  attribute <varname>enableParallelBuilding</varname> is set to
+  <literal>true</literal>, the builder passes the
+  <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+  It defaults to the value of the <link
+  linkend='conf-build-cores'><literal>build-cores</literal></link>
+  configuration setting, if set, or <literal>1</literal> otherwise.
+  The value <literal>0</literal> means that the builder should use all
+  available CPU cores in the system.</para></listitem>
   
 </varlistentry>
 

doc/manual/package-management.xml

@@ -38,7 +38,7 @@ to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
 expressions based on it, or completely new ones.)  You can download
 the latest version from <link
-xlink:href='http://nixos.org/releases/full-index-nixpkgs.html' />.</para>
+xlink:href='http://nixos.org/nixpkgs/download.html' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available packages in the release:
@@ -331,7 +331,7 @@ default profile, respectively.  If the profile doesn’t exist, it will
 be created automatically.  You should be careful about storing a
 profile in another location than the <filename>profiles</filename>
 directory, since otherwise it might not be used as a root of the
-garbage collector (see section <xref linkend='sec-garbage-collection'
+garbage collector (see <xref linkend='sec-garbage-collection'
 />).</para>
 
 <para>All <command>nix-env</command> operations work on the profile
@@ -496,7 +496,7 @@ available in the subscribed channels.</para>
 
 <para>Often, when you want to install a specific package (e.g., from
 the <link
-xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/">Nix
+xlink:href="http://nixos.org/nixpkgs/">Nix
 Packages collection</link>), subscribing to a channel is a bit
 cumbersome.  And channels don’t help you at all if you want to install
 an older version of a package than the one provided by the current
@@ -507,19 +507,16 @@ click on it, and it will be installed with all the necessary
 dependencies.</para>
 
 <para>For instance, you can go to <link
-xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/" /> —
-or to any older release of Nix Packages — and click on any link for
-the individual packages for your platform (say, <link
-xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.10pre6622/pkgs/subversion-1.4.0-i686-linux.nixpkg'><literal>subversion-1.4.0</literal>
-for <literal>i686-linux</literal></link>).  The first time you do
-this, your browser will ask what to do with
-<literal>application/nix-package</literal> files.  You should open
-them with <filename>/nix/bin/nix-install-package</filename>.  This
-will open a window that asks you to confirm that you want to install
-the package.  When you answer <literal>Y</literal>, the package and
-all its dependencies will be installed.  This is a binary deployment
-mechanism — you get packages pre-compiled for the selected platform
-type.</para>
+xlink:href="http://hydra.nixos.org/jobset/nixpkgs/trunk/channel/latest"
+/> and click on any link for the individual packages for your
+platform.  The first time you do this, your browser will ask what to
+do with <literal>application/nix-package</literal> files.  You should
+open them with <filename>/nix/bin/nix-install-package</filename>.
+This will open a window that asks you to confirm that you want to
+install the package.  When you answer <literal>Y</literal>, the
+package and all its dependencies will be installed.  This is a binary
+deployment mechanism — you get packages pre-compiled for the selected
+platform type.</para>
 
 <para>You can also install <literal>application/nix-package</literal>
 files from the command line directly.  See <xref

doc/manual/quick-start.xml

@@ -1,5 +1,6 @@
 <chapter xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink">
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xml:id="chap-quick-start">
 
 <title>Quick Start</title>
 
@@ -10,9 +11,9 @@ to the following chapters.</para>
 
 <orderedlist>
 
-<listitem><para>Download a source tarball or RPM from <link
-xlink:href='http://nixos.org/'/>.  Build source
-distributions using the regular sequence:
+<listitem><para>Download a source tarball, RPM or Deb from <link
+xlink:href='http://nixos.org/'/>.  Build source distributions using
+the regular sequence:
         
 <screen>
 $ tar xvfj nix-<replaceable>version</replaceable>.tar.bz2
@@ -20,13 +21,21 @@ $ ./configure
 $ make
 $ make install <lineannotation>(as root)</lineannotation></screen>
 
-This will install Nix in <filename>/nix</filename>.  You shouldn't
-change the prefix if at all possible since that will make it
-impossible to use pre-built binaries from the Nixpkgs channel and
-other channels.  Alternatively, you could grab an RPM if you're on an
-RPM-based system.  You should also add
-<filename>/nix/etc/profile.d/nix.sh</filename> to your
-<filename>~/.bashrc</filename> (or some other login
+This will install the Nix binaries in <filename>/usr/local</filename>
+and keep the Nix store and other state in <filename>/nix</filename>.
+You can change the former by specifying
+<option>--prefix=<replaceable>path</replaceable></option>.  The
+location of the store can be changed using
+<option>--with-store-dir=<replaceable>path</replaceable></option>.
+However, you shouldn't change the store location, if at all possible,
+since that will make it impossible to use pre-built binaries from the
+Nixpkgs channel and other channels.  The location of the state can be
+changed using
+<option>--localstatedir=<replaceable>path</replaceable>.</option></para></listitem>
+
+<listitem><para>You should add
+<filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
+to your <filename>~/.bashrc</filename> (or some other login
 file).</para></listitem>
 
 <listitem><para>Subscribe to the Nix Packages channel.
@@ -99,7 +108,7 @@ numbers).</para></listitem>
 
 <listitem><para>You can also install specific packages directly from
 your web browser.  For instance, you can go to <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/" />
+xlink:href="http://hydra.nixos.org/jobset/nixpkgs/trunk/channel/latest" />
 and click on any link for the individual packages for your platform.
 Associate <literal>application/nix-package</literal> with the program
 <filename>/nix/bin/nix-install-package</filename>.  A window should

doc/manual/release-notes.xml

@@ -8,7 +8,137 @@
 
 <!--==================================================================-->
 
-<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (April NN,
+<section xml:id="ssec-relnotes-1.0"><title>Release 1.0 (TBA)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix can now optionally use the Boehm garbage collector.
+    This significantly reduces the Nix evaluator’s memory footprint,
+    especially when evaluating large NixOS system configurations.  It
+    can be enabled using the <option>--enable-gc</option> configure
+    option.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.16"><title>Release 0.16 (August 17, 2010)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The Nix expression evaluator is now much faster in most
+    cases: typically, <link
+    xlink:href="http://www.mail-archive.com/nix-dev@cs.uu.nl/msg04113.html">3
+    to 8 times compared to the old implementation</link>.  It also
+    uses less memory.  It no longer depends on the ATerm
+    library.</para>
+  </listitem>
+
+  <listitem>
+    <para>
+      Support for configurable parallelism inside builders.  Build
+      scripts have always had the ability to perform multiple build
+      actions in parallel (for instance, by running <command>make -j
+      2</command>), but this was not desirable because the number of
+      actions to be performed in parallel was not configurable.  Nix
+      now has an option <option>--cores
+      <replaceable>N</replaceable></option> as well as a configuration
+      setting <varname>build-cores =
+      <replaceable>N</replaceable></varname> that causes the
+      environment variable <envar>NIX_BUILD_CORES</envar> to be set to
+      <replaceable>N</replaceable> when the builder is invoked.  The
+      builder can use this at its discretion to perform a parallel
+      build, e.g., by calling <command>make -j
+      <replaceable>N</replaceable></command>.  In Nixpkgs, this can be
+      enabled on a per-package basis by setting the derivation
+      attribute <varname>enableParallelBuilding</varname> to
+      <literal>true</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-store -q</command> now supports XML output
+    through the <option>--xml</option> flag.</para>
+  </listitem>
+
+  <listitem>
+    <para>Several bug fixes.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.15"><title>Release 0.15 (March 17, 2010)</title>
+
+<para>This is a bug-fix release.  Among other things, it fixes
+building on Mac OS X (Snow Leopard), and improves the contents of
+<filename>/etc/passwd</filename> and <filename>/etc/group</filename>
+in <literal>chroot</literal> builds.</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.14"><title>Release 0.14 (February 4, 2010)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The garbage collector now starts deleting garbage much
+    faster than before.  It no longer determines liveness of all paths
+    in the store, but does so on demand.</para>
+  </listitem>
+
+  <listitem>
+    <para>Added a new operation, <command>nix-store --query
+    --roots</command>, that shows the garbage collector roots that
+    directly or indirectly point to the given store paths.</para>
+  </listitem>
+
+  <listitem>
+    <para>Removed support for converting Berkeley DB-based Nix
+    databases to the new schema.</para>
+  </listitem>
+
+  <listitem>
+    <para>Removed the <option>--use-atime</option> and
+    <option>--max-atime</option> garbage collector options.  They were
+    not very useful in practice.</para>
+  </listitem>
+
+  <listitem>
+    <para>On Windows, Nix now requires Cygwin 1.7.x.</para>
+  </listitem>
+
+  <listitem>
+    <para>A few bug fixes.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (November 5,
 2009)</title>
 
 <para>This is primarily a bug fix release.  It has some new
@@ -85,7 +215,25 @@ features:</para>
     add a string to stack traces — useful for debugging),
     <varname>builtins.isBool</varname>,
     <varname>builtins.isString</varname>,
-    <varname>builtins.isInt</varname>.</para>
+    <varname>builtins.isInt</varname>,
+    <varname>builtins.intersectAttrs</varname>.</para>
+  </listitem>
+
+  <listitem>
+    <para>OpenSolaris support (Sander van der Burg).</para>
+  </listitem>
+
+  <listitem>
+    <para>Stack traces are no longer displayed unless the
+    <option>--show-trace</option> option is used.</para>
+  </listitem>
+
+  <listitem>
+    <para>The scoping rules for <literal>inherit
+    (<replaceable>e</replaceable>) ...</literal> in recursive
+    attribute sets have changed.  The expression
+    <replaceable>e</replaceable> can now refer to the attributes
+    defined in the containing set.</para>
   </listitem>
 
 </itemizedlist>

doc/manual/writing-nix-expressions.xml

@@ -1106,7 +1106,7 @@ used in the Nix expression for Subversion.</para>
     incompatibility might occur.</para>
   </callout>
 
-  <callout arearefs='ex-subversion-nix-co-2'>
+  <callout arearefs='ex-subversion-nix-co-3'>
     <para>This assertion says that in order for Subversion to have SSL
     support (so that it can access <literal>https</literal> URLs), an
     OpenSSL library must be passed.  Additionally, it says that

externals/Makefile.am

@@ -1,78 +1,3 @@
-# Berkeley DB
-
-DB = db-4.5.20
-
-if OLD_DB_COMPAT
-
-$(DB).tar.gz:
-	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.5.20 from"
-	@echo "  http://download-east.oracle.com/berkeley-db/db-4.5.20.tar.gz"
-	@echo "and place it in the externals/ directory."
-	false
-
-$(DB): $(DB).tar.gz
-	gunzip < $(srcdir)/$(DB).tar.gz | tar xvf -
-	(cd $(DB) && $(patch) -p1) < $(srcdir)/bdb-cygwin.patch
-
-have-db:
-	$(MAKE) $(DB)
-	touch have-db
-
-if HAVE_BDB
-build-db:
-else
-build-db: have-db
-	(pfx=`pwd` && \
-	cd $(DB)/build_unix && \
-	CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" \
-	 ../dist/configure --prefix=$$pfx/inst-bdb \
-	 --enable-cxx --disable-shared --disable-cryptography \
-         --disable-replication --disable-verify && \
-	$(MAKE) && \
-	$(MAKE) install_include install_lib)
-	touch build-db
-endif
-
-else
-
-build-db:
-
-endif
-
-
-# CWI ATerm
-
-ATERM = aterm-2.4.2-fixes-r2
-
-$(ATERM).tar.bz2:
-	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.4.2-fixes-r2 from"
-	@echo "  http://nixos.org/tarballs/aterm-2.4.2-fixes-r2.tar.bz2"
-	@echo "and place it in the externals/ directory."
-	false
-
-$(ATERM): $(ATERM).tar.bz2
-	bunzip2 < $(srcdir)/$(ATERM).tar.bz2 | tar xvf -
-
-have-aterm:
-	$(MAKE) $(ATERM)
-	touch have-aterm
-
-if HAVE_ATERM
-build-aterm:
-else
-build-aterm: have-aterm
-	(pfx=`pwd` && \
-	cd $(ATERM) && \
-	CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm \
-	  --disable-shared --enable-static && \
-	$(MAKE) && \
-	$(MAKE) install)
-	touch build-aterm
-endif
-
-
 # bzip2
 
 BZIP2 = bzip2-1.0.5
@@ -107,11 +32,10 @@ install:
 endif
 
 
-all: build-db build-aterm build-bzip2
+all: build-bzip2
 
-EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.bz2 $(BZIP2).tar.gz \
-  bdb-cygwin.patch
+EXTRA_DIST = $(BZIP2).tar.gz
 
 ext-clean:
-	$(RM) -f have-db build-db have-aterm build-aterm have-bzip2 build-bzip2
-	$(RM) -rf $(DB) $(ATERM) $(BZIP2)
+	$(RM) -f have-bzip2 build-bzip2
+	$(RM) -rf $(BZIP2)

externals/bdb-cygwin.patch

@@ -1,22 +0,0 @@
-diff -rc db-4.5.20-orig/os/os_flock.c db-4.5.20/os/os_flock.c
-*** db-4.5.20-orig/os/os_flock.c	2006-10-13 12:36:12.000000000 +0200
---- db-4.5.20/os/os_flock.c	2006-10-13 12:40:11.000000000 +0200
-***************
-*** 30,35 ****
---- 30,44 ----
-  
-  	DB_ASSERT(dbenv, F_ISSET(fhp, DB_FH_OPENED) && fhp->fd != -1);
-  
-+ #ifdef __CYGWIN__
-+ 	/*
-+ 	 * Windows file locking interferes with read/write operations, so we
-+ 	 * map the ranges to an area past the end of the file.
-+ 	 */
-+ 	DB_ASSERT(dbenv, offset < (off_t) 1 << 62);
-+ 	offset += (off_t) 1 << 62;
-+ #endif
-+ 
-  	fl.l_start = offset;
-  	fl.l_len = 1;
-  	fl.l_type = acquire ? F_WRLCK : F_UNLCK;
-Only in db-4.5.20/os: os_flock.c~

nix.conf.example

@@ -59,6 +59,17 @@
 #build-max-jobs = 1
 
 
+### Option `build-cores'
+#
+# This option defines the number of CPU cores to utilize in parallel
+# within a build job, i.e. by passing an appropriate `-jN' flag to GNU
+# Make. The default is 1, meaning that parallel building within jobs
+# is disabled.  Passing the special value `0' causes Nix to try and
+# auto-detect the number of available cores on the local host.  This
+# setting can be overridden using the `--cores' command line switch.
+#build-cores = 1
+
+
 ### Option `build-max-silent-time'
 #
 # This option defines the maximum number of seconds that a builder can

release.nix

@@ -1,4 +1,7 @@
-{ nixpkgs ? ../nixpkgs }:
+{ nixpkgs ? ../nixpkgs
+, nix ? { outPath = ./.; rev = 1234; }
+, officialRelease ? false
+}:
 
 let
 
@@ -6,10 +9,6 @@ let
 
 
     tarball =
-      { nix ? {outPath = ./.; rev = 1234;}
-      , officialRelease ? false
-      }:
-
       with import nixpkgs {};
 
       releaseTools.sourceTarball {
@@ -18,7 +17,10 @@ let
         src = nix;
         inherit officialRelease;
 
-        buildInputs = [curl bison flex2533 perl libxml2 libxslt w3m bzip2 jing_tools tetex dblatex];
+        buildInputs =
+          [ curl bison24 flex2535 perl libxml2 libxslt w3m bzip2
+            tetex dblatex nukeReferences pkgconfig
+          ];
 
         configureFlags = ''
           --with-docbook-rng=${docbook5}/xml/rng/docbook
@@ -26,27 +28,13 @@ let
           --with-xml-flags=--nonet
         '';
 
-        # Include the BDB, ATerm and Bzip2 tarballs in the distribution.
+        # Include the Bzip2 tarball in the distribution.
         preConfigure = ''
-          stripHash ${db45.src}
-          # Remove unnecessary stuff from the Berkeley DB tarball.
-          ( mkdir bdb-temp
-            cd bdb-temp
-            tar xfz ${db45.src}
-            cd *
-            rm -rf docs test tcl perl libdb_java java rpc_server build_vxworks \
-              examples_java examples_c examples_cxx dist/tags
-            mkdir test
-            touch test/include.tcl
-            cd ..
-            tar cvfz ../externals/$strippedName *
-          )
-
-          stripHash ${aterm242fixes.src}
-          cp -pv ${aterm242fixes.src} externals/$strippedName
-
           stripHash ${bzip2.src}
           cp -pv ${bzip2.src} externals/$strippedName
+
+          # TeX needs a writable font cache.
+          export VARTEXFONTS=$TMPDIR/texfonts
         '';
 
         preDist = ''
@@ -54,6 +42,14 @@ let
           
           make -C doc/manual manual.pdf prefix=$out
           cp doc/manual/manual.pdf $out/manual.pdf
+
+          # The PDF containes filenames of included graphics (see
+          # http://www.tug.org/pipermail/pdftex/2007-August/007290.html).
+          # This causes a retained dependency on dblatex, which Hydra
+          # doesn't like (the output of the tarball job is distributed
+          # to Windows and Macs, so there should be no Linux binaries
+          # in the closure).
+          nuke-refs $out/manual.pdf
           
           echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
           echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products
@@ -63,65 +59,40 @@ let
 
 
     build =
-      { tarball ? jobs.tarball {}
-      , system ? "i686-linux"
-      }:
+      { system ? "i686-linux" }:
 
-      with import nixpkgs {inherit system;};
+      with import nixpkgs { inherit system; };
 
       releaseTools.nixBuild {
         name = "nix";
         src = tarball;
 
-        buildInputs = [curl perl bzip2 openssl];
+        buildInputs = [ curl perl bzip2 openssl pkgconfig boehmgc ];
 
         configureFlags = ''
           --disable-init-state
-          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --with-bzip2=${bzip2}
+          --enable-gc
         '';
       };
 
-      
-    static =
-      { tarball ? jobs.tarball {}
-      , system ? "i686-linux"
-      }:
 
-      with import nixpkgs {inherit system;};
-
-      releaseTools.binaryTarball {
-        name = "nix-static-tarball";
-        src = tarball;
-
-        buildInputs = [curl perl bzip2];
-
-        configureFlags = ''
-          --disable-init-state
-          --disable-old-db-compat --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
-          --enable-static-nix
-        '';
-      };
-
-      
     coverage =
-      { tarball ? jobs.tarball {}
-      }:
-
-      with import nixpkgs {};
+      with import nixpkgs { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
         src = tarball;
 
-        buildInputs = [
-          curl perl bzip2 openssl
-          # These are for "make check" only:
-          graphviz libxml2 libxslt
-        ];
+        buildInputs =
+          [ curl perl bzip2 openssl
+            # These are for "make check" only:
+            graphviz libxml2 libxslt
+          ];
 
         configureFlags = ''
           --disable-init-state --disable-shared
-          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --with-bzip2=${bzip2}
         '';
 
         lcovFilter = ["*/boost/*" "*-tab.*"];
@@ -133,22 +104,32 @@ let
       };
 
       
-    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 20;
-    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 50;
-    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 50;
-    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 40;
-    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 40;
+    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 10;
+    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 20;
+    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 20;
+    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 30;
+    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 30;
+    rpm_fedora11i386 = makeRPM_i686 (diskImages: diskImages.fedora11i386) 40;
+    rpm_fedora11x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora11x86_64) 40;
+    rpm_fedora12i386 = makeRPM_i686 (diskImages: diskImages.fedora12i386) 50;
+    rpm_fedora12x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora12x86_64) 50;
     rpm_opensuse103i386 = makeRPM_i686 (diskImages: diskImages.opensuse103i386) 40;
+    rpm_opensuse110i386 = makeRPM_i686 (diskImages: diskImages.opensuse110i386) 50;
+    rpm_opensuse110x86_64 = makeRPM_x86_64 (diskImages: diskImages.opensuse110x86_64) 50;
 
     
     deb_debian40i386 = makeDeb_i686 (diskImages: diskImages.debian40i386) 40;
     deb_debian40x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian40x86_64) 40;
-    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 30;
-    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 30;
-    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 50;
-    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 50;
-    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 40;
-    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 40;
+    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 50;
+    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 50;
+    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 20;
+    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 20;
+    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 30;
+    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 30;
+    deb_ubuntu904i386 = makeDeb_i686 (diskImages: diskImages.ubuntu904i386) 40;
+    deb_ubuntu904x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu904x86_64) 40;
+    deb_ubuntu910i386 = makeDeb_i686 (diskImages: diskImages.ubuntu910i386) 50;
+    deb_ubuntu910x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu910x86_64) 50;
 
 
   };
@@ -159,17 +140,15 @@ let
 
   makeRPM = 
     system: diskImageFun: prio:
-    { tarball ? jobs.tarball {}
-    }:
 
-    with import nixpkgs {inherit system;};
+    with import nixpkgs { inherit system; };
 
     releaseTools.rpmBuild rec {
-      name = "nix-rpm-${diskImage.name}";
-      src = tarball;
+      name = "nix-rpm";
+      src = jobs.tarball;
       diskImage = diskImageFun vmTools.diskImages;
       memSize = 1024;
-      meta = { schedulingPriority = toString prio; };
+      meta = { schedulingPriority = prio; };
     };
 
 
@@ -178,18 +157,17 @@ let
   
   makeDeb =
     system: diskImageFun: prio:
-    { tarball ? jobs.tarball {}
-    }:
 
-    with import nixpkgs {inherit system;};
+    with import nixpkgs { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";
-      src = tarball;
+      src = jobs.tarball;
       diskImage = diskImageFun vmTools.diskImages;
       memSize = 1024;
-      meta = { schedulingPriority = toString prio; };
+      meta = { schedulingPriority = prio; };
       configureFlags = "--sysconfdir=/etc";
+      debRequires = [ "curl" ];
     };
 
 

scripts/Makefile.am

@@ -15,6 +15,7 @@ install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) ssh.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
@@ -31,6 +32,7 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-channel.in \
   readmanifest.pm.in \
   readconfig.pm.in \
+  ssh.pm \
   nix-build.in \
   download-using-manifests.pl.in \
   copy-from-other-stores.pl.in \

scripts/build-remote.pl.in

@@ -1,9 +1,10 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
 
-use strict;
 use Fcntl ':flock';
 use English '-no_match_vars';
 use IO::Handle;
+use ssh qw/sshOpts openSSHConnection/;
+
 
 # General operation:
 #
@@ -22,6 +23,14 @@ use IO::Handle;
 # The nice thing about this scheme is that if we die prematurely, the
 # locks are released automatically.
 
+
+# Make sure that we don't get any SSH passphrase or host key popups -
+# if there is any problem it should fail, not do something
+# interactive.
+$ENV{"DISPLAY"} = "";
+$ENV{"SSH_ASKPASS"} = "";
+
+
 my $loadIncreased = 0;
 
 my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV;
@@ -47,24 +56,22 @@ decline if !defined $conf || ! -e $conf;
 my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
 
 
-# Otherwise find a willing remote machine.
-my @machines;
-my %curJobs;
-
-
 # Read the list of machines.
+my @machines;
 open CONF, "< $conf" or die;
 
 while (<CONF>) {
     chomp;
     s/\#.*$//g;
     next if /^\s*$/;
-    /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die;
+    /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)(\s+([0-9\.]+))?\s*$/ or die;
     push @machines,
         { hostName => $1
-        , systemType => $2
+        , systemTypes => [split(/,/, $2)]
         , sshKeys => $3
         , maxJobs => $4
+        , speedFactor => 1.0 * ($6 || 1)
+        , enabled => 1
         };
 }
 
@@ -77,50 +84,105 @@ open MAINLOCK, ">>$mainLock" or die;
 flock(MAINLOCK, LOCK_EX) or die;
 
 
-# Find a suitable system.
-my $rightType = 0;
-my $machine;
-my $slotLock;
-LOOP: foreach my $cur (@machines) {
-    if ($neededSystem eq $cur->{systemType}
-        || ($neededSystem eq "i686-linux" && $cur->{systemType} eq "x86_64-linux"))
-    {
-        $rightType = 1;
+sub openSlotLock {
+    my ($machine, $slot) = @_;
+    my $slotLockFn = "$currentLoad/" . (join '+', @{$machine->{systemTypes}}) . "-" . $machine->{hostName} . "-$slot";
+    my $slotLock = new IO::Handle;
+    open $slotLock, ">>$slotLockFn" or die;
+    return $slotLock;
+}
+    
 
-        # We have a machine of the right type.  Try to get a lock on
-        # one of the machine's lock files.
-        my $slot = 0;
-        while ($slot < $cur->{maxJobs}) {
-            my $slotLockFn = "$currentLoad/" . $cur->{systemType} . "-" . $cur->{hostName} . "-$slot";
-            $slotLock = new IO::Handle;
-            open $slotLock, ">>$slotLockFn" or die;
-            if (flock($slotLock, LOCK_EX | LOCK_NB)) {
-                utime undef, undef, $slotLock;
-                $machine = $cur;
-                last LOOP;
+my $hostName;
+my $slotLock;
+
+while (1) {
+    
+    # Find all machine that can execute this build, i.e., that support
+    # builds for the given platform and are not at their job limit.
+    my $rightType = 0;
+    my @available = ();
+    LOOP: foreach my $cur (@machines) {
+        if ($cur->{enabled} && grep { $neededSystem eq $_ } @{$cur->{systemTypes}}) {
+            $rightType = 1;
+
+            # We have a machine of the right type.  Determine the load on
+            # the machine.
+            my $slot = 0;
+            my $load = 0;
+            my $free;
+            while ($slot < $cur->{maxJobs}) {
+                my $slotLock = openSlotLock($cur, $slot);
+                if (flock($slotLock, LOCK_EX | LOCK_NB)) {
+                    $free = $slot unless defined $free;
+                    flock($slotLock, LOCK_UN) or die;
+                } else {
+                    $load++;
+                }
+                close $slotLock;
+                $slot++;
             }
-            close $slotLock;
-            $slot++;
+
+            push @available, { machine => $cur, load => $load, free => $free }
+            if $load < $cur->{maxJobs};
         }
     }
-}
 
-close MAINLOCK;
-
-
-# Didn't find one?  Then decline or postpone.
-if (!defined $machine) {
-    # Postpone if we have a machine of the right type, except if the
-    # local system can and wants to do the build.
-    if ($rightType && !$canBuildLocally) {
-        sendReply "postpone";
-        exit 0;
-    } else {
-        decline;
+    if (defined $ENV{NIX_DEBUG_HOOK}) {
+        print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n"
+            foreach @available;
     }
+
+
+    # Didn't find any available machine?  Then decline or postpone.
+    if (scalar @available == 0) {
+        # Postpone if we have a machine of the right type, except if the
+        # local system can and wants to do the build.
+        if ($rightType && !$canBuildLocally) {
+            sendReply "postpone";
+            exit 0;
+        } else {
+            decline;
+        }
+    }
+
+
+    # Prioritise the available machines as follows:
+    # - First by load divided by speed factor, rounded to the nearest
+    #   integer.  This causes fast machines to be preferred over slow
+    #   machines with similar loads.
+    # - Then by speed factor.
+    # - Finally by load.
+    sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); }
+    @available = sort
+        { lf($a) <=> lf($b)
+              || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor}
+              || $a->{load} <=> $b->{load}
+        } @available;
+
+
+    # Select the best available machine and lock a free slot.
+    my $selected = $available[0]; 
+    my $machine = $selected->{machine};
+
+    $slotLock = openSlotLock($machine, $selected->{free});
+    flock($slotLock, LOCK_EX | LOCK_NB) or die;
+    utime undef, undef, $slotLock;
+
+    close MAINLOCK;
+
+
+    # Connect to the selected machine.
+    @sshOpts = ("-i", $machine->{sshKeys}, "-x");
+    $hostName = $machine->{hostName};
+    last if openSSHConnection $hostName;
+    
+    warn "unable to open SSH connection to $hostName, trying other available machines...\n";
+    $machine->{enabled} = 0;
 }
 
-# Yes we did, accept.
+
+# Tell Nix we've accepted the build.
 sendReply "accept";
 my $x = <STDIN>;
 chomp $x;
@@ -130,43 +192,9 @@ if ($x ne "okay") {
 }
 
 
-# Do the actual job.
-my $hostName = $machine->{hostName};
+# Do the actual build.
 print STDERR "building `$drvPath' on `$hostName'\n";
 
-# Make sure that we don't get any SSH passphrase or host key popups -
-# if there is any problem it should fail, not do something
-# interactive.
-$ENV{"DISPLAY"} = "";
-$ENV{"SSH_PASSWORD_FILE="} = "";
-$ENV{"SSH_ASKPASS="} = "";
-
-my $sshOpts = "-i " . $machine->{sshKeys} . " -x";
-
-# Hack to support Cygwin: if we login without a password, we don't
-# have exactly the same rights as when we do.  This causes the
-# Microsoft C compiler to fail with certain flags:
-#
-#   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676
-#
-# So as a workaround, we pass a verbatim password.  ssh tries to makes
-# this very hard; the trick is to make it call SSH_ASKPASS to get the
-# password.  (It only calls this command when there is no controlling
-# terminal, but Nix ensures that is is the case.  When doing this
-# manually, use setsid(1).)
-if ($machine->{sshKeys} =~ /^password:/) {
-    my $passwordFile = $machine->{sshKeys};
-    $passwordFile =~ s/^password://;
-    $sshOpts = "ssh -x";
-    $ENV{"SSH_PASSWORD_FILE"} = $passwordFile;
-    $ENV{"SSH_ASKPASS"} = "/tmp/writepass";
-
-    open WRITEPASS, ">/tmp/writepass" or die;
-    print WRITEPASS "#! /bin/sh\ncat \"\$SSH_PASSWORD_FILE\"";
-    close WRITEPASS;
-    chmod 0755, "/tmp/writepass" or die;
-}
-
 my $inputs = `cat inputs`; die if ($? != 0);
 $inputs =~ s/\n/ /g;
 
@@ -178,22 +206,22 @@ print "copying inputs...\n";
 my $maybeSign = "";
 $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
 
-system("NIX_SSHOPTS=\"$sshOpts\" @bindir@/nix-copy-closure --gzip $hostName $maybeSign $drvPath $inputs") == 0
+system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath $inputs") == 0
     or die "cannot copy inputs to $hostName: $?";
 
 print "building...\n";
 
-my $buildFlags = "--max-silent-time $maxSilentTime";
+my $buildFlags = "--max-silent-time $maxSilentTime --fallback";
 
 # `-tt' forces allocation of a pseudo-terminal.  This is required to
 # make the remote nix-store process receive a signal when the
 # connection dies.  Without it, the remote process might continue to
 # run indefinitely (that is, until it next tries to write to
 # stdout/stderr).
-if (system("ssh -tt $sshOpts $hostName 'nix-store --realise -K $buildFlags $drvPath > /dev/null'") != 0) {
+if (system("ssh $hostName @sshOpts -tt 'nix-store -r $drvPath $buildFlags > /dev/null'") != 0) {
     # If we couldn't run ssh or there was an ssh problem (indicated by
     # exit code 255), then we return exit code 1; otherwise we assume
-    # that the builder failed, which we indicated to Nix using exit
+    # that the builder failed, which we indicate to Nix using exit
     # code 100.  It's important to distinguish between the two because
     # the first is a transient failure and the latter is permanent.
     my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100;
@@ -207,6 +235,6 @@ foreach my $output (split '\n', $outputs) {
     my $maybeSignRemote = "";
     $maybeSignRemote = "--sign" if $UID != 0;
     
-    system("ssh $sshOpts $hostName 'nix-store --export $maybeSignRemote $output | gzip' | gunzip | @bindir@/nix-store --import > /dev/null") == 0
+    system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output' | @bindir@/nix-store --import > /dev/null") == 0
 	or die "cannot copy $output from $hostName: $?";
 }

scripts/download-using-manifests.pl.in

@@ -73,7 +73,7 @@ if ($ARGV[0] eq "--query") {
 }
 
 elsif ($ARGV[0] ne "--substitute") {
-    die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n";
+    die;
 }
 
 
@@ -85,8 +85,6 @@ my $targetPath = $ARGV[1];
 my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-chdir $tmpDir or die "cannot change to `$tmpDir': $!";
-
 my $tmpNar = "$tmpDir/nar";
 my $tmpNar2 = "$tmpDir/nar2";
 

scripts/generate-patches.pl.in

@@ -20,11 +20,11 @@ die unless scalar @ARGV == 5;
 
 my $hashAlgo = "sha256";
 
-my $cacheDir = $ARGV[0];
+my $narDir = $ARGV[0];
 my $patchesDir = $ARGV[1];
 my $patchesURL = $ARGV[2];
-my $srcDir = $ARGV[3];
-my $dstDir = $ARGV[4];
+my $srcManifest = $ARGV[3];
+my $dstManifest = $ARGV[4];
 
 my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
@@ -41,10 +41,10 @@ my %dstNarFiles;
 my %dstLocalPaths;
 my %dstPatches;
 
-readManifest "$srcDir/MANIFEST",
+readManifest "$srcManifest",
     \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
 
-readManifest "$dstDir/MANIFEST",
+readManifest "$dstManifest",
     \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
 
 
@@ -55,8 +55,7 @@ sub findOutputPaths {
     
     foreach my $p (keys %{$narFiles}) {
 
-        # Ignore store expressions.
-        next if ($p =~ /\.store$/);
+        # Ignore derivations.
         next if ($p =~ /\.drv$/);
         
         # Ignore builders (too much ambiguity -- they're all called
@@ -65,7 +64,7 @@ sub findOutputPaths {
         next if ($p =~ /\.patch$/);
         
         # Don't bother including tar files etc.
-        next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
+        next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/);
 
         $outPaths{$p} = 1;
     }
@@ -85,6 +84,7 @@ sub getNameVersion {
     $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
     my $name = $1;
     my $version = $2;
+    return undef unless defined $name && defined $version;
     $name =~ s/^-//;
     $version =~ s/^-//;
     return ($name, $version);
@@ -112,14 +112,14 @@ sub getNarBz2 {
     my $storePath = shift;
     
     my $narFileList = $$narFiles{$storePath};
-    die "missing store expression $storePath" unless defined $narFileList;
+    die "missing path $storePath" unless defined $narFileList;
 
     my $narFile = @{$narFileList}[0];
     die unless defined $narFile;
 
     $narFile->{url} =~ /\/([^\/]+)$/;
     die unless defined $1;
-    return "$cacheDir/$1";
+    return "$narDir/$1";
 }
 
 
@@ -213,6 +213,7 @@ foreach my $p (keys %dstOutPaths) {
     # this path.
 
     (my $name, my $version) = getNameVersion $p;
+    next unless defined $name && defined $version;
 
     my @closest = ();
     my $closestVersion;
@@ -222,6 +223,8 @@ foreach my $p (keys %dstOutPaths) {
 
     foreach my $q (keys %srcOutPaths) {
         (my $name2, my $version2) = getNameVersion $q;
+	next unless defined $name2 && defined $version2;
+
         if ($name eq $name2) {
 
             # If the sizes differ too much, then skip.  This
@@ -241,11 +244,11 @@ foreach my $p (keys %dstOutPaths) {
             # If the numbers of weighted uses differ too much, then
             # skip.  This disambiguates between, e.g., the bootstrap
             # GCC and the final GCC in Nixpkgs.
-            my $srcUses = computeUses \%srcNarFiles, $q;
-            my $dstUses = computeUses \%dstNarFiles, $p;
-            $ratio = $srcUses / $dstUses;
-            $ratio = 1 / $ratio if $ratio < 1;
-            print "  USE $srcUses $dstUses $ratio $q\n";
+#            my $srcUses = computeUses \%srcNarFiles, $q;
+#            my $dstUses = computeUses \%dstNarFiles, $p;
+#            $ratio = $srcUses / $dstUses;
+#            $ratio = 1 / $ratio if $ratio < 1;
+#            print "  USE $srcUses $dstUses $ratio $q\n";
             
 #            if ($ratio >= 2) {
 #                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
@@ -288,6 +291,11 @@ foreach my $p (keys %dstOutPaths) {
         my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
         my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
 
+        if (! -f $srcNarBz2) {
+            warn "patch source archive $srcNarBz2 is missing\n";
+            next;
+        }
+
         system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
             or die "cannot unpack $srcNarBz2";
 
@@ -404,5 +412,5 @@ do {
 
 
 # Rewrite the manifest of the destination (with the new patches).
-writeManifest "$dstDir/MANIFEST",
+writeManifest "${dstManifest}",
     \%dstNarFiles, \%dstPatches;

scripts/nix-build.in

@@ -12,6 +12,7 @@ my $outLink;
 my $drvLink;
 
 my $dryRun = 0;
+my $verbose = 0;
 
 my @instArgs = ();
 my @buildArgs = ();
@@ -101,7 +102,7 @@ EOF
         $n += 2;
     }
 
-    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type") {
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @buildArgs, ($arg, $ARGV[$n]);
@@ -112,6 +113,16 @@ EOF
         $dryRun = 1;
     }
     
+    elsif ($arg eq "--show-trace") {
+        push @instArgs, $arg;
+    }
+    
+    elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") {
+        push @buildArgs, $arg;
+        push @instArgs, $arg;
+        $verbose = 1;
+    }
+    
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -149,7 +160,7 @@ foreach my $expr (@exprs) {
 
     foreach my $drvPath (@drvPaths) {
         my $target = readlink $drvPath or die "cannot read symlink `$drvPath'";
-        print STDERR "store derivation is $target\n";
+        print STDERR "derivation is $target\n" if $verbose;
     }
 
     # Build.

scripts/nix-channel.in

@@ -78,6 +78,9 @@ sub removeChannel {
 sub update {
     readChannels;
 
+    # Create the manifests directory if it doesn't exist.
+    mkdir "$stateDir/manifests", 0755 unless -e "$stateDir/manifests";
+
     # Do we have write permission to the manifests directory?  If not,
     # then just skip pulling the manifest and just download the Nix
     # expressions.  If the user is a non-privileged user in a
@@ -85,11 +88,6 @@ sub update {
     # source.
     if (-W "$stateDir/manifests") {
 
-        # Remove all the old manifests.
-        for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
-            unlink $manifest or die "cannot remove `$manifest': $!";
-        }
-
         # Pull cache manifests.
         foreach my $url (@channels) {
             #print "pulling cache manifest from `$url'\n";

scripts/nix-copy-closure.in

@@ -1,4 +1,6 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
+
+use ssh;
 
 my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
@@ -14,12 +16,11 @@ EOF
 
 # Get the target host.
 my $sshHost;
-my @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
 
 my $sign = 0;
 
-my $compressor = "cat";
-my $decompressor = "cat";
+my $compressor = "";
+my $decompressor = "";
 
 my $toMode = 1;
 
@@ -34,8 +35,8 @@ while (@ARGV) {
         $sign = 1;
     }
     elsif ($arg eq "--gzip") {
-        $compressor = "gzip";
-        $decompressor = "gunzip";
+        $compressor = "| gzip";
+        $decompressor = "gunzip |";
     }
     elsif ($arg eq "--from") {
         $toMode = 0;
@@ -52,6 +53,9 @@ while (@ARGV) {
 }
 
 
+openSSHConnection $sshHost or die "$0: unable to start SSH\n";
+
+
 if ($toMode) { # Copy TO the remote machine.
 
     my @allStorePaths;
@@ -69,11 +73,10 @@ if ($toMode) { # Copy TO the remote machine.
 
 
     # Ask the remote host which paths are invalid.
-    open(READ, "ssh @sshOpts $sshHost nix-store --check-validity --print-invalid @allStorePaths|");
+    open(READ, "ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
     my @missing = ();
     while (<READ>) {
         chomp;
-        print STDERR "target machine needs $_\n";
         push @missing, $_;
     }
     close READ or die;
@@ -81,9 +84,11 @@ if ($toMode) { # Copy TO the remote machine.
 
     # Export the store paths and import them on the remote machine.
     if (scalar @missing > 0) {
+        print STDERR "copying these missing paths:\n";
+        print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("nix-store --export $extraOpts @missing | $compressor | ssh @sshOpts $sshHost '$decompressor | nix-store --import'") == 0
+        system("nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
             or die "copying store paths to remote machine `$sshHost' failed: $?";
     }
 
@@ -114,7 +119,6 @@ else { # Copy FROM the remote machine.
     my @missing = ();
     while (<READ>) {
         chomp;
-        print STDERR "local machine needs $_\n";
         push @missing, $_;
     }
     close READ or die;
@@ -122,10 +126,12 @@ else { # Copy FROM the remote machine.
 
     # Export the store paths on the remote machine and import them on locally.
     if (scalar @missing > 0) {
+        print STDERR "copying these missing paths:\n";
+        print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("ssh @sshOpts $sshHost 'nix-store --export $extraOpts @missing | $compressor' | $decompressor | @bindir@/nix-store --import") == 0
-            or die "copying store paths to remote machine `$sshHost' failed: $?";
+        system("ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0
+            or die "copying store paths from remote machine `$sshHost' failed: $?";
     }
 
 }

scripts/nix-prefetch-url.in

@@ -21,7 +21,10 @@ if test -z "$url"; then
     exit 1
 fi
 
-name=$(basename "$url")
+# Handle escaped characters in the URI.  `+', `=' and `?' are the only
+# characters that are valid in Nix store path names but have a special
+# meaning in URIs.
+name=$(basename "$url" | @sed@ -e 's/%2b/+/g' -e 's/%3d/=/g' -e 's/%3f/\?/g')
 if test -z "$name"; then echo "invalid url"; exit 1; fi
 
 

scripts/nix-pull.in

@@ -18,6 +18,12 @@ my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "$stateDir/manifests");
 umask 0022;
 
 
+# Create the manifests directory if it doesn't exist.
+if (! -e $manifestDir) {
+    mkdir $manifestDir, 0755 or die "cannot create directory `$manifestDir'";
+}
+
+
 # Process the URLs specified on the command line.
 my %narFiles;
 my %localPaths;
@@ -45,7 +51,7 @@ sub processURL {
 
     # First see if a bzipped manifest is available.
     if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) {
-        print "obtaining list of Nix archives at `$url.bz2'...\n";
+        print "fetching list of Nix archives at `$url.bz2'...\n";
         my $bzipped = downloadFile "$url.bz2";
 
         $manifest = "$tmpDir/MANIFEST";
@@ -86,11 +92,31 @@ sub processURL {
     my $hash = `$binDir/nix-hash --flat '$manifest'`
         or die "cannot hash `$manifest'";
     chomp $hash;
+
+    my $urlFile = "$manifestDir/$baseName-$hash.url";
+    open URL, ">$urlFile" or die "cannot create `$urlFile'";
+    print URL "$url";
+    close URL;
     
     my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest";
-    
-    system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
+
+    unlink $finalPath if -e $finalPath;
+        
+    symlink("$manifest", "$finalPath")
         or die "cannot link `$finalPath to `$manifest'";
+
+    # Delete all old manifests downloaded from this URL.
+    for my $urlFile2 (glob "$manifestDir/*.url") {
+        next if $urlFile eq $urlFile2;
+        open URL, "<$urlFile2" or die;
+        my $url2 = <URL>;
+        chomp $url2;
+        close URL;
+        next unless $url eq $url2;
+        my $base = $urlFile2; $base =~ s/.url$//;
+        unlink "${base}.url";
+        unlink "${base}.nixmanifest";
+    }
 }
 
 while (@ARGV) {

scripts/ssh.pm

@@ -0,0 +1,47 @@
+use strict;
+use File::Temp qw(tempdir);
+
+our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
+
+my $sshStarted = 0;
+my $sshHost;
+
+# Open a master SSH connection to `host', unless there already is a
+# running master connection (as determined by `-O check').
+sub openSSHConnection {
+    my ($host) = @_;
+    die if $sshStarted;
+    $sshHost = $host;
+    return 1 if system("ssh $sshHost @sshOpts -O check 2> /dev/null") == 0;
+
+    my $tmpDir = tempdir("nix-ssh.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+        or die "cannot create a temporary directory";
+    
+    push @sshOpts, "-S", "$tmpDir/control";
+
+    # Start the master.  We can't use the `-f' flag (fork into
+    # background after establishing the connection) because then the
+    # child continues to run if we are killed.  So instead make SSH
+    # print "started" when it has established the connection, and wait
+    # until we see that.
+    open SSH, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die;
+    while (<SSH>) {
+        chomp;
+        last if /started/;
+    }
+    
+    $sshStarted = 1;
+    return 1;
+}
+
+# Tell the master SSH client to exit.
+sub closeSSHConnection {
+    if ($sshStarted) {
+        system("ssh $sshHost @sshOpts -O exit 2> /dev/null") == 0
+            or warn "unable to stop SSH master: $?";
+    }
+}
+
+END { my $saved = $?; closeSSHConnection; $? = $saved; }
+
+return 1;

src/Makefile.am

@@ -1,5 +1,3 @@
 SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
  libexpr nix-instantiate nix-env nix-worker nix-setuid-helper \
  nix-log2xml bsdiff-4.3
-
-EXTRA_DIST = aterm-helper.pl

src/aterm-helper.pl

@@ -1,178 +0,0 @@
-#! /usr/bin/perl -w
-
-# This program generates C/C++ code for efficiently manipulating
-# ATerms.  It generates functions to build and match ATerms according
-# to a set of constructor definitions defined in a file read from
-# standard input.  A constructor is defined by a line with the
-# following format:
-#
-#   SYM | ARGS | TYPE | FUN?
-#
-# where SYM is the name of the constructor, ARGS is a
-# whitespace-separated list of argument types, TYPE is the type of the
-# resulting ATerm (which should be `ATerm' or a type synonym for
-# `ATerm'), and the optional FUN is used to construct the names of the
-# build and match functions (it defaults to SYM; overriding it is
-# useful if there are overloaded constructors, e.g., with different
-# arities).  Note that SYM may be empty.
-#
-# A line of the form
-#
-#   VAR = EXPR
-#
-# causes a ATerm variable to be generated that is initialised to the
-# value EXPR.
-#
-# Finally, a line of the form
-#
-#   init NAME
-#
-# causes the initialisation function to be called `NAME'.  This
-# function must be called before any of the build/match functions or
-# the generated variables are used.
-
-die if scalar @ARGV != 2;
-
-my $syms = "";
-my $init = "";
-my $initFun = "init";
-
-open HEADER, ">$ARGV[0]";
-open IMPL, ">$ARGV[1]";
-
-print HEADER "#include <aterm2.h>\n";
-print HEADER "#ifdef __cplusplus\n";
-print HEADER "namespace nix {\n";
-print HEADER "#endif\n\n\n";
-print IMPL "namespace nix {\n";
-
-while (<STDIN>) {
-    s/\#.*//;
-    next if (/^\s*$/);
-    
-    if (/^\s*(\w*)\s*\|([^\|]*)\|\s*(\w+)\s*\|\s*(\w+)?/) {
-        my $const = $1;
-        my @types = split ' ', $2;
-        my $result = $3;
-        my $funname = $4;
-        $funname = $const unless defined $funname;
-
-        my $formals = "";
-        my $formals2 = "";
-        my $args = "";
-        my $unpack = "";
-        my $n = 1;
-        foreach my $type (@types) {
-            my $realType = $type;
-            $args .= ", ";
-            if ($type eq "string") {
-#                $args .= "(ATerm) ATmakeAppl0(ATmakeAFun((char *) e$n, 0, ATtrue))";
-#                $type = "const char *";
-                $type = "ATerm";
-                $args .= "e$n";
-                # !!! in the matcher, we should check that the
-                # argument is a string (i.e., a nullary application).
-            } elsif ($type eq "int") {
-                $args .= "(ATerm) ATmakeInt(e$n)";
-            } elsif ($type eq "ATermList" || $type eq "ATermBlob") {
-                $args .= "(ATerm) e$n";
-            } else {
-                $args .= "e$n";
-            }
-            $formals .= ", " if $formals ne "";
-            $formals .= "$type e$n";
-            $formals2 .= ", ";
-            $formals2 .= "$type & e$n";
-            my $m = $n - 1;
-            # !!! more checks here
-            if ($type eq "int") {
-                $unpack .= "    e$n = ATgetInt((ATermInt) ATgetArgument(e, $m));\n";
-            } elsif ($type eq "ATermList") {
-                $unpack .= "    e$n = (ATermList) ATgetArgument(e, $m);\n";
-            } elsif ($type eq "ATermBlob") {
-                $unpack .= "    e$n = (ATermBlob) ATgetArgument(e, $m);\n";
-            } elsif ($realType eq "string") {
-                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
-                $unpack .= "    if (ATgetType(e$n) != AT_APPL) return false;\n";
-            } else {
-                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
-            }
-            $n++;
-        }
-
-        my $arity = scalar @types;
-
-        print HEADER "extern AFun sym$funname;\n\n";
-        
-        print IMPL "AFun sym$funname = 0;\n";
-        
-        if ($arity == 0) {
-            print HEADER "extern ATerm const$funname;\n\n";
-            print IMPL "ATerm const$funname = 0;\n";
-        }
-        
-        print HEADER "static inline $result make$funname($formals) __attribute__ ((pure, nothrow));\n";
-        print HEADER "static inline $result make$funname($formals) {\n";
-        if ($arity == 0) {
-            print HEADER "    return const$funname;\n";
-        }
-        elsif ($arity <= 6) {
-            print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
-        } else {
-            $args =~ s/^,//;
-            print HEADER "    ATerm array[$arity] = {$args};\n";
-            print HEADER "    return (ATerm) ATmakeApplArray(sym$funname, array);\n";
-        }
-        print HEADER "}\n\n";
-
-        print HEADER "#ifdef __cplusplus\n";
-        print HEADER "static inline bool match$funname(ATerm e$formals2) {\n";
-        print HEADER "    if (ATgetType(e) != AT_APPL || (AFun) ATgetAFun(e) != sym$funname) return false;\n";
-        print HEADER "$unpack";
-        print HEADER "    return true;\n";
-        print HEADER "}\n";
-        print HEADER "#endif\n\n\n";
-
-        $init .= "    sym$funname = ATmakeAFun(\"$const\", $arity, ATfalse);\n";
-        $init .= "    ATprotectAFun(sym$funname);\n";
-        if ($arity == 0) {
-            $init .= "    const$funname = (ATerm) ATmakeAppl0(sym$funname);\n";
-            $init .= "    ATprotect(&const$funname);\n";
-        }
-    }
-
-    elsif (/^\s*(\w+)\s*=\s*(.*)$/) {
-        my $name = $1;
-        my $value = $2;
-        print HEADER "extern ATerm $name;\n";
-        print IMPL "ATerm $name = 0;\n";
-        $init .= "    $name = $value;\n";
-    }
-
-    elsif (/^\s*init\s+(\w+)\s*$/) {
-        $initFun = $1;
-    }
-
-    else {
-        die "bad line: `$_'";
-    }
-}
-
-print HEADER "void $initFun();\n\n";
-
-print HEADER "static inline const char * aterm2String(ATerm t) {\n";
-print HEADER "    return (const char *) ATgetName(ATgetAFun(t));\n";
-print HEADER "}\n\n";
-
-print IMPL "\n";
-print IMPL "void $initFun() {\n";
-print IMPL "$init";
-print IMPL "}\n";
-
-print HEADER "#ifdef __cplusplus\n";
-print HEADER "}\n";
-print HEADER "#endif\n\n\n";
-print IMPL "}\n";
-
-close HEADER;
-close IMPL;

src/bin2c/Makefile.am

@@ -1,3 +1,6 @@
 noinst_PROGRAMS = bin2c
 
 bin2c_SOURCES = bin2c.c
+
+bin2c$(EXEEXT): bin2c.c
+	$(CC_FOR_BUILD) $(CFLAGS_FOR_BUILD) -o bin2c bin2c.c

src/bsdiff-4.3/Makefile.am

@@ -1,3 +1,5 @@
+EXTRA_DIST = compat-include
+
 libexec_PROGRAMS = bsdiff bspatch
 
 bsdiff_SOURCES = bsdiff.c
@@ -8,4 +10,4 @@ bspatch_SOURCES = bspatch.c
 
 bspatch_LDADD = ${bzip2_lib}
 
-AM_CFLAGS = -O3 ${bzip2_include}
+AM_CFLAGS = -O3 ${bzip2_include} ${bsddiff_compat_include}

src/bsdiff-4.3/compat-include/err.h

@@ -0,0 +1,12 @@
+/* Simulate BSD's <err.h> functionality. */
+
+#ifndef COMPAT_ERR_H_INCLUDED
+#define COMPAT_ERR_H_INCLUDED 1
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define err(rc,...)  do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
+#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
+
+#endif

src/libexpr/Makefile.am

@@ -2,27 +2,25 @@ pkglib_LTLIBRARIES = libexpr.la
 
 libexpr_la_SOURCES = \
  nixexpr.cc eval.cc primops.cc lexer-tab.cc parser-tab.cc \
- get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc \
+ get-drvs.cc attr-path.cc value-to-xml.cc common-opts.cc \
  names.cc
 
 pkginclude_HEADERS = \
  nixexpr.hh eval.hh parser.hh lexer-tab.hh parser-tab.hh \
- get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh \
- names.hh nixexpr-ast.hh
+ get-drvs.hh attr-path.hh value-to-xml.hh common-opts.hh \
+ names.hh symbol-table.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
- ../boost/format/libformat.la
+ ../boost/format/libformat.la @boehmgc_lib@
 
-BUILT_SOURCES = nixexpr-ast.cc nixexpr-ast.hh \
+BUILT_SOURCES = \
  parser-tab.hh lexer-tab.hh parser-tab.cc lexer-tab.cc
 
-EXTRA_DIST = lexer.l parser.y nixexpr-ast.def nixexpr-ast.cc
+EXTRA_DIST = lexer.l parser.y
 
 AM_CXXFLAGS = \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} \
+ -I$(srcdir)/.. \
  -I$(srcdir)/../libutil -I$(srcdir)/../libstore
-AM_CFLAGS = \
- ${aterm_include}
 
 
 # Parser generation.
@@ -34,15 +32,6 @@ lexer-tab.cc lexer-tab.hh: lexer.l
 	$(flex) --outfile lexer-tab.cc --header-file=lexer-tab.hh $(srcdir)/lexer.l 
 
 
-# ATerm helper function generation.
-
-nixexpr-ast.cc nixexpr-ast.hh: ../aterm-helper.pl nixexpr-ast.def
-	$(perl) $(srcdir)/../aterm-helper.pl nixexpr-ast.hh nixexpr-ast.cc < $(srcdir)/nixexpr-ast.def
-
-
-CLEANFILES =
-
-
 # SDF stuff (not built by default).
 nix.tbl: nix.sdf
 	sdf2table -m Nix -s -i nix.sdf -o nix.tbl

src/libexpr/attr-path.cc

@@ -1,23 +1,13 @@
 #include "attr-path.hh"
-#include "nixexpr-ast.hh"
 #include "util.hh"
 
 
 namespace nix {
 
 
-bool isAttrs(EvalState & state, Expr e, ATermMap & attrs)
-{
-    e = evalExpr(state, e);
-    ATermList dummy;
-    if (!matchAttrs(e, dummy)) return false;
-    queryAllAttrs(e, attrs, false);
-    return true;
-}
-
-
-Expr findAlongAttrPath(EvalState & state, const string & attrPath,
-    const ATermMap & autoArgs, Expr e)
+// !!! Shouldn't we return a pointer to a Value?
+void findAlongAttrPath(EvalState & state, const string & attrPath,
+    Bindings & autoArgs, Expr * e, Value & v)
 {
     Strings tokens = tokenizeString(attrPath, ".");
 
@@ -25,8 +15,10 @@ Expr findAlongAttrPath(EvalState & state, const string & attrPath,
         Error(format("attribute selection path `%1%' does not match expression") % attrPath);
 
     string curPath;
+
+    state.mkThunk_(v, e);
     
-    for (Strings::iterator i = tokens.begin(); i != tokens.end(); ++i) {
+    foreach (Strings::iterator, i, tokens) {
 
         if (!curPath.empty()) curPath += ".";
         curPath += *i;
@@ -38,7 +30,10 @@ Expr findAlongAttrPath(EvalState & state, const string & attrPath,
         if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
-        e = evalExpr(state, autoCallFunction(evalExpr(state, e), autoArgs));
+        Value vTmp;
+        state.autoCallFunction(autoArgs, v, vTmp);
+        v = vTmp;
+        state.forceValue(v);
 
         /* It should evaluate to either an attribute set or an
            expression, according to what is specified in the
@@ -46,36 +41,31 @@ Expr findAlongAttrPath(EvalState & state, const string & attrPath,
 
         if (apType == apAttr) {
 
-            ATermMap attrs;
-
-            if (!isAttrs(state, e, attrs))
+            if (v.type != tAttrs)
                 throw TypeError(
                     format("the expression selected by the selection path `%1%' should be an attribute set but is %2%")
-                    % curPath % showType(e));
-                
-            e = attrs.get(toATerm(attr));
-            if (!e)
-                throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % curPath);
+                    % curPath % showType(v));
 
+            Bindings::iterator a = v.attrs->find(state.symbols.create(attr));
+            if (a == v.attrs->end())
+                throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % curPath);
+            v = *a->value;
         }
 
         else if (apType == apIndex) {
 
-            ATermList es;
-            if (!matchList(e, es))
+            if (v.type != tList)
                 throw TypeError(
                     format("the expression selected by the selection path `%1%' should be a list but is %2%")
-                    % curPath % showType(e));
+                    % curPath % showType(v));
 
-            e = ATelementAt(es, attrIndex);
-            if (!e)
-                throw Error(format("list index %1% in selection path `%2%' not found") % attrIndex % curPath);
-            
+            if (attrIndex >= v.list.length)
+                throw Error(format("list index %1% in selection path `%2%' is out of range") % attrIndex % curPath);
+
+            v = *v.list.elems[attrIndex];
         }
         
     }
-    
-    return e;
 }
 
  

src/libexpr/attr-path.hh

@@ -1,17 +1,17 @@
 #ifndef __ATTR_PATH_H
 #define __ATTR_PATH_H
 
+#include "eval.hh"
+
 #include <string>
 #include <map>
 
-#include "eval.hh"
-
 
 namespace nix {
 
     
-Expr findAlongAttrPath(EvalState & state, const string & attrPath,
-    const ATermMap & autoArgs, Expr e);
+void findAlongAttrPath(EvalState & state, const string & attrPath,
+    Bindings & autoArgs, Expr * e, Value & v);
 
     
 }

src/libexpr/common-opts.cc

@@ -9,7 +9,7 @@ namespace nix {
 
 bool parseOptionArg(const string & arg, Strings::iterator & i,
     const Strings::iterator & argsEnd, EvalState & state,
-    ATermMap & autoArgs)
+    Bindings & autoArgs)
 {
     if (arg != "--arg" && arg != "--argstr") return false;
 
@@ -19,12 +19,18 @@ bool parseOptionArg(const string & arg, Strings::iterator & i,
     string name = *i++;
     if (i == argsEnd) throw error;
     string value = *i++;
-    
-    Expr e = arg == "--arg"
-        ? parseExprFromString(state, value, absPath("."))
-        : makeStr(value);
-    autoArgs.set(toATerm(name), e);
-    
+
+    /* !!! check for duplicates! */
+    Value * v = state.allocValue();
+    autoArgs.push_back(Attr(state.symbols.create(name), v));
+
+    if (arg == "--arg")
+        state.mkThunk_(*v, parseExprFromString(state, value, absPath(".")));
+    else
+        mkString(*v, value);
+
+    autoArgs.sort(); // !!! inefficient
+
     return true;
 }
 

src/libexpr/common-opts.hh

@@ -9,7 +9,7 @@ namespace nix {
 /* Some common option parsing between nix-env and nix-instantiate. */
 bool parseOptionArg(const string & arg, Strings::iterator & i,
     const Strings::iterator & argsEnd, EvalState & state,
-    ATermMap & autoArgs);
+    Bindings & autoArgs);
     
 }
 

src/libexpr/eval.hh

@@ -1,19 +1,193 @@
 #ifndef __EVAL_H
 #define __EVAL_H
 
+#include "nixexpr.hh"
+#include "symbol-table.hh"
+#include "hash.hh"
+
 #include <map>
 
-#include "aterm.hh"
-#include "nixexpr.hh"
+#if HAVE_BOEHMGC
+#include <gc/gc_allocator.h>
+#endif
 
 
 namespace nix {
 
 
-class Hash;
-    
+class EvalState;
+struct Env;
+struct Value;
+struct Attr;
+
+
+/* Attribute sets are represented as a vector of attributes, sorted by
+   symbol (i.e. pointer to the attribute name in the symbol table). */
+#if HAVE_BOEHMGC
+typedef std::vector<Attr, gc_allocator<Attr> > BindingsBase;
+#else
+typedef std::vector<Attr> BindingsBase;
+#endif
+
+
+class Bindings : public BindingsBase
+{
+public:
+    iterator find(const Symbol & name);
+    void sort();
+};
+
+
+typedef enum {
+    tInt = 1,
+    tBool,
+    tString,
+    tPath,
+    tNull,
+    tAttrs,
+    tList,
+    tThunk,
+    tApp,
+    tLambda,
+    tBlackhole,
+    tPrimOp,
+    tPrimOpApp,
+} ValueType;
+
+
+typedef void (* PrimOpFun) (EvalState & state, Value * * args, Value & v);
+
+
+struct PrimOp
+{
+    PrimOpFun fun;
+    unsigned int arity;
+    Symbol name;
+    PrimOp(PrimOpFun fun, unsigned int arity, Symbol name)
+        : fun(fun), arity(arity), name(name) { }
+};
+
+
+struct Value
+{
+    ValueType type;
+    union 
+    {
+        int integer;
+        bool boolean;
+        
+        /* Strings in the evaluator carry a so-called `context' (the
+           ATermList) which is a list of strings representing store
+           paths.  This is to allow users to write things like
+
+             "--with-freetype2-library=" + freetype + "/lib"
+
+           where `freetype' is a derivation (or a source to be copied
+           to the store).  If we just concatenated the strings without
+           keeping track of the referenced store paths, then if the
+           string is used as a derivation attribute, the derivation
+           will not have the correct dependencies in its inputDrvs and
+           inputSrcs.
+
+           The semantics of the context is as follows: when a string
+           with context C is used as a derivation attribute, then the
+           derivations in C will be added to the inputDrvs of the
+           derivation, and the other store paths in C will be added to
+           the inputSrcs of the derivations.
+
+           For canonicity, the store paths should be in sorted order. */
+        struct {
+            const char * s;
+            const char * * context; // must be in sorted order
+        } string;
+        
+        const char * path;
+        Bindings * attrs;
+        struct {
+            unsigned int length;
+            Value * * elems;
+        } list;
+        struct {
+            Env * env;
+            Expr * expr;
+        } thunk;
+        struct {
+            Value * left, * right;
+        } app;
+        struct {
+            Env * env;
+            ExprLambda * fun;
+        } lambda;
+        PrimOp * primOp;
+        struct {
+            Value * left, * right;
+        } primOpApp;
+    };
+};
+
+
+struct Env
+{
+    Env * up;
+    unsigned int prevWith; // nr of levels up to next `with' environment
+    Value * values[0];
+};
+
+
+struct Attr
+{
+    Symbol name;
+    Value * value;
+    Pos * pos;
+    Attr(Symbol name, Value * value, Pos * pos = &noPos)
+        : name(name), value(value), pos(pos) { };
+    Attr() : pos(&noPos) { };
+    bool operator < (const Attr & a) const
+    {
+        return name < a.name;
+    }
+};
+
+
+/* After overwriting an app node, be sure to clear pointers in the
+   Value to ensure that the target isn't kept alive unnecessarily. */
+static inline void clearValue(Value & v)
+{
+    v.app.right = 0;
+}
+
+
+static inline void mkInt(Value & v, int n)
+{
+    clearValue(v);
+    v.type = tInt;
+    v.integer = n;
+}
+
+
+static inline void mkBool(Value & v, bool b)
+{
+    clearValue(v);
+    v.type = tBool;
+    v.boolean = b;
+}
+
+
+static inline void mkApp(Value & v, Value & left, Value & right)
+{
+    v.type = tApp;
+    v.app.left = &left;
+    v.app.right = &right;
+}
+
+
+void mkString(Value & v, const char * s);
+void mkString(Value & v, const string & s, const PathSet & context = PathSet());
+void mkPath(Value & v, const char * s);
+
+void copyContext(const Value & v, PathSet & context);
+
 
-typedef std::map<Path, PathSet> DrvRoots;
 typedef std::map<Path, Hash> DrvHashes;
 
 /* Cache for calls to addToStore(); maps source paths to the store
@@ -22,75 +196,160 @@ typedef std::map<Path, Path> SrcToStore;
 
 struct EvalState;
 
-/* Note: using a ATermVector is safe here, since when we call a primop
-   we also have an ATermList on the stack. */
-typedef Expr (* PrimOp) (EvalState &, const ATermVector & args);
+
+std::ostream & operator << (std::ostream & str, const Value & v);
 
 
-struct EvalState 
+class EvalState 
 {
-    ATermMap normalForms;
-    ATermMap primOps;
-    DrvRoots drvRoots;
+public:
     DrvHashes drvHashes; /* normalised derivation hashes */
-    SrcToStore srcToStore; 
 
-    unsigned int nrEvaluated;
-    unsigned int nrCached;
+    SymbolTable symbols;
+
+    const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName,
+        sSystem, sOverrides;
+
+private:
+    SrcToStore srcToStore; 
 
     bool allowUnsafeEquality;
 
-    EvalState();
+    std::map<Path, Expr *> parseTrees;
+
+public:
+    
+    EvalState();
+    ~EvalState();
+
+    /* Evaluate an expression read from the given file to normal
+       form. */
+    void evalFile(const Path & path, Value & v);
+
+    /* Evaluate an expression to normal form, storing the result in
+       value `v'. */
+    void eval(Expr * e, Value & v);
+    void eval(Env & env, Expr * e, Value & v);
+
+    /* Evaluation the expression, then verify that it has the expected
+       type. */
+    bool evalBool(Env & env, Expr * e);
+    void evalAttrs(Env & env, Expr * e, Value & v);
+
+    /* If `v' is a thunk, enter it and overwrite `v' with the result
+       of the evaluation of the thunk.  If `v' is a delayed function
+       application, call the function and overwrite `v' with the
+       result.  Otherwise, this is a no-op. */
+    void forceValue(Value & v);
+
+    /* Force a value, then recursively force list elements and
+       attributes. */
+    void strictForceValue(Value & v);
+
+    /* Force `v', and then verify that it has the expected type. */
+    int forceInt(Value & v);
+    bool forceBool(Value & v);
+    void forceAttrs(Value & v);
+    void forceList(Value & v);
+    void forceFunction(Value & v); // either lambda or primop
+    string forceString(Value & v);
+    string forceString(Value & v, PathSet & context);
+    string forceStringNoCtx(Value & v);
+
+    /* Return true iff the value `v' denotes a derivation (i.e. a
+       set with attribute `type = "derivation"'). */
+    bool isDerivation(Value & v);
+
+    /* String coercion.  Converts strings, paths and derivations to a
+       string.  If `coerceMore' is set, also converts nulls, integers,
+       booleans and lists to a string.  If `copyToStore' is set,
+       referenced paths are copied to the Nix store as a side effect.q */
+    string coerceToString(Value & v, PathSet & context,
+        bool coerceMore = false, bool copyToStore = true);
+
+    /* Path coercion.  Converts strings, paths and derivations to a
+       path.  The result is guaranteed to be a canonicalised, absolute
+       path.  Nothing is copied to the store. */
+    Path coerceToPath(Value & v, PathSet & context);
+
+private:
+
+    /* The base environment, containing the builtin functions and
+       values. */
+    Env & baseEnv;
+
+    unsigned int baseEnvDispl;
+
+public:
+    
+    /* The same, but used during parsing to resolve variables. */
+    StaticEnv staticBaseEnv; // !!! should be private
+
+private:
+    
+    void createBaseEnv();
+    
+    void addConstant(const string & name, Value & v);
 
-    void addPrimOps();
     void addPrimOp(const string & name,
-        unsigned int arity, PrimOp primOp);
+        unsigned int arity, PrimOpFun primOp);
+
+    Value * lookupVar(Env * env, const VarRef & var);
+    
+    friend class ExprVar;
+    friend class ExprAttrs;
+    friend class ExprLet;
+
+public:
+    
+    /* Do a deep equality test between two values.  That is, list
+       elements and attributes are compared recursively. */
+    bool eqValues(Value & v1, Value & v2);
+
+    void callFunction(Value & fun, Value & arg, Value & v);
+
+    /* Automatically call a function for which each argument has a
+       default value or has a binding in the `args' map. */
+    void autoCallFunction(Bindings & args, Value & fun, Value & res);
+    
+    /* Allocation primitives. */
+    Value * allocValue();
+    Env & allocEnv(unsigned int size);
+
+    Value * allocAttr(Value & vAttrs, const Symbol & name);
+
+    void mkList(Value & v, unsigned int length);
+    void mkAttrs(Value & v, unsigned int expected);
+    void mkThunk_(Value & v, Expr * expr);
+
+    Value * maybeThunk(Env & env, Expr * expr);
+    
+    /* Print statistics. */
+    void printStats();
+
+private:
+    
+    unsigned long nrEnvs;
+    unsigned long nrValuesInEnvs;
+    unsigned long nrValues;
+    unsigned long nrListElems;
+    unsigned long nrEvaluated;
+    unsigned long nrAttrsets;
+    unsigned long nrOpUpdates;
+    unsigned long nrOpUpdateValuesCopied;
+    unsigned int recursionDepth;
+    unsigned int maxRecursionDepth;
+    char * deepestStack; /* for measuring stack usage */
+    
+    friend class RecursionCounter;
+    friend class ExprOpUpdate;
 };
 
 
-/* Evaluate an expression to normal form. */
-Expr evalExpr(EvalState & state, Expr e);
+/* Return a string representing the type of the value `v'. */
+string showType(const Value & v);
 
-/* Evaluate an expression read from the given file to normal form. */
-Expr evalFile(EvalState & state, const Path & path);
 
-/* Evaluate an expression, and recursively evaluate list elements and
-   attributes.  If `canonicalise' is true, we remove things like
-   position information and make sure that attribute sets are in
-   sorded order. */
-Expr strictEvalExpr(EvalState & state, Expr e);
-
-/* Specific results. */
-string evalString(EvalState & state, Expr e, PathSet & context);
-string evalStringNoCtx(EvalState & state, Expr e);
-int evalInt(EvalState & state, Expr e);
-bool evalBool(EvalState & state, Expr e);
-ATermList evalList(EvalState & state, Expr e);
-
-/* Flatten nested lists into a single list (or expand a singleton into
-   a list). */
-ATermList flattenList(EvalState & state, Expr e);
-
-/* String coercion.  Converts strings, paths and derivations to a
-   string.  If `coerceMore' is set, also converts nulls, integers,
-   booleans and lists to a string. */
-string coerceToString(EvalState & state, Expr e, PathSet & context,
-    bool coerceMore = false, bool copyToStore = true);
-
-/* Path coercion.  Converts strings, paths and derivations to a path.
-   The result is guaranteed to be an canonicalised, absolute path.
-   Nothing is copied to the store. */
-Path coerceToPath(EvalState & state, Expr e, PathSet & context);
-
-/* Automatically call a function for which each argument has a default
-   value or has a binding in the `args' map.  Note: result is a call,
-   not a normal form; it should be evaluated by calling evalExpr(). */
-Expr autoCallFunction(Expr e, const ATermMap & args);
-
-/* Print statistics. */
-void printEvalStats(EvalState & state);
-
- 
 }
 
 

src/libexpr/expr-to-xml.cc

@@ -1,156 +0,0 @@
-#include "expr-to-xml.hh"
-#include "xml-writer.hh"
-#include "nixexpr-ast.hh"
-#include "aterm.hh"
-#include "util.hh"
-
-#include <cstdlib>
-
-
-namespace nix {
-
-    
-static XMLAttrs singletonAttrs(const string & name, const string & value)
-{
-    XMLAttrs attrs;
-    attrs[name] = value;
-    return attrs;
-}
-
-
-/* set<Expr> is safe because all the expressions are also reachable
-   from the stack, therefore can't be garbage-collected. */
-typedef set<Expr> ExprSet;
-
-
-static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
-    ExprSet & drvsSeen);
-
-
-static void showAttrs(const ATermMap & attrs, XMLWriter & doc,
-    PathSet & context, ExprSet & drvsSeen)
-{
-    StringSet names;
-    for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i)
-        names.insert(aterm2String(i->key));
-    for (StringSet::iterator i = names.begin(); i != names.end(); ++i) {
-        XMLOpenElement _(doc, "attr", singletonAttrs("name", *i));
-        printTermAsXML(attrs.get(toATerm(*i)), doc, context, drvsSeen);
-    }
-}
-
-
-static void printPatternAsXML(Pattern pat, XMLWriter & doc)
-{
-    ATerm name;
-    ATermList formals;
-    Pattern pat1, pat2;
-    ATermBool ellipsis;
-    if (matchVarPat(pat, name))
-        doc.writeEmptyElement("varpat", singletonAttrs("name", aterm2String(name)));
-    else if (matchAttrsPat(pat, formals, ellipsis)) {
-        XMLOpenElement _(doc, "attrspat");
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ATerm dummy;
-            if (!matchFormal(*i, name, dummy)) abort();
-            doc.writeEmptyElement("attr", singletonAttrs("name", aterm2String(name)));
-        }
-        if (ellipsis == eTrue) doc.writeEmptyElement("ellipsis");
-    }
-    else if (matchAtPat(pat, pat1, pat2)) {
-        XMLOpenElement _(doc, "at");
-        printPatternAsXML(pat1, doc);
-        printPatternAsXML(pat2, doc);
-    }
-}
-
-
-static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
-    ExprSet & drvsSeen)
-{
-    XMLAttrs attrs;
-    string s;
-    ATerm s2;
-    int i;
-    ATermList as, es;
-    ATerm pat, body, pos;
-
-    checkInterrupt();
-
-    if (matchStr(e, s, context)) /* !!! show the context? */
-        doc.writeEmptyElement("string", singletonAttrs("value", s));
-
-    else if (matchPath(e, s2))
-        doc.writeEmptyElement("path", singletonAttrs("value", aterm2String(s2)));
-
-    else if (matchNull(e))
-        doc.writeEmptyElement("null");
-
-    else if (matchInt(e, i))
-        doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % i).str()));
-
-    else if (e == eTrue)
-        doc.writeEmptyElement("bool", singletonAttrs("value", "true"));
-
-    else if (e == eFalse)
-        doc.writeEmptyElement("bool", singletonAttrs("value", "false"));
-
-    else if (matchAttrs(e, as)) {
-        ATermMap attrs;
-        queryAllAttrs(e, attrs);
-
-        Expr a = attrs.get(toATerm("type"));
-        if (a && matchStr(a, s, context) && s == "derivation") {
-
-            XMLAttrs xmlAttrs;
-            Path outPath, drvPath;
-            
-            a = attrs.get(toATerm("drvPath"));
-            if (matchStr(a, drvPath, context))
-                xmlAttrs["drvPath"] = drvPath;
-        
-            a = attrs.get(toATerm("outPath"));
-            if (matchStr(a, outPath, context))
-                xmlAttrs["outPath"] = outPath;
-        
-            XMLOpenElement _(doc, "derivation", xmlAttrs);
-
-            if (drvsSeen.find(e) == drvsSeen.end()) {
-                drvsSeen.insert(e);
-                showAttrs(attrs, doc, context, drvsSeen);
-            } else
-                doc.writeEmptyElement("repeated");
-        }
-
-        else {
-            XMLOpenElement _(doc, "attrs");
-            showAttrs(attrs, doc, context, drvsSeen);
-        }
-    }
-
-    else if (matchList(e, es)) {
-        XMLOpenElement _(doc, "list");
-        for (ATermIterator i(es); i; ++i)
-            printTermAsXML(*i, doc, context, drvsSeen);
-    }
-
-    else if (matchFunction(e, pat, body, pos)) {
-        XMLOpenElement _(doc, "function");
-        printPatternAsXML(pat, doc);
-    }
-
-    else
-        doc.writeEmptyElement("unevaluated");
-}
-
-
-void printTermAsXML(Expr e, std::ostream & out, PathSet & context)
-{
-    XMLWriter doc(true, out);
-    XMLOpenElement root(doc, "expr");
-    ExprSet drvsSeen;    
-    printTermAsXML(e, doc, context, drvsSeen);
-}
-
- 
-}

src/libexpr/expr-to-xml.hh

@@ -1,16 +0,0 @@
-#ifndef __EXPR_TO_XML_H
-#define __EXPR_TO_XML_H
-
-#include <string>
-#include <map>
-
-#include "nixexpr.hh"
-#include "aterm.hh"
-
-namespace nix {
-
-void printTermAsXML(Expr e, std::ostream & out, PathSet & context);
-    
-}
-
-#endif /* !__EXPR_TO_XML_H */

src/libexpr/get-drvs.cc

@@ -1,5 +1,4 @@
 #include "get-drvs.hh"
-#include "nixexpr-ast.hh"
 #include "util.hh"
 
 
@@ -8,17 +7,10 @@ namespace nix {
 
 string DrvInfo::queryDrvPath(EvalState & state) const
 {
-    if (drvPath == "") {
-        Expr a = attrs->get(toATerm("drvPath"));
-
-        /* Backwards compatibility hack with user environments made by
-           Nix <= 0.10: these contain illegal Path("") expressions. */
-        ATerm t;
-        if (a && matchPath(evalExpr(state, a), t))
-            return aterm2String(t);
-        
+    if (drvPath == "" && attrs) {
+        Bindings::iterator i = attrs->find(state.sDrvPath);
         PathSet context;
-        (string &) drvPath = a ? coerceToPath(state, a, context) : "";
+        (string &) drvPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : "";
     }
     return drvPath;
 }
@@ -26,11 +18,10 @@ string DrvInfo::queryDrvPath(EvalState & state) const
 
 string DrvInfo::queryOutPath(EvalState & state) const
 {
-    if (outPath == "") {
-        Expr a = attrs->get(toATerm("outPath"));
-        if (!a) throw TypeError("output path missing");
+    if (outPath == "" && attrs) {
+        Bindings::iterator i = attrs->find(state.sOutPath);
         PathSet context;
-        (string &) outPath = coerceToPath(state, a, context);
+        (string &) outPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : "";
     }
     return outPath;
 }
@@ -38,35 +29,30 @@ string DrvInfo::queryOutPath(EvalState & state) const
 
 MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
 {
-    MetaInfo meta;
+    if (metaInfoRead) return meta;
     
-    Expr a = attrs->get(toATerm("meta"));
-    if (!a) return meta; /* fine, empty meta information */
+    (bool &) metaInfoRead = true;
+    
+    Bindings::iterator a = attrs->find(state.sMeta);
+    if (a == attrs->end()) return meta; /* fine, empty meta information */
 
-    ATermMap attrs2;
-    queryAllAttrs(evalExpr(state, a), attrs2);
+    state.forceAttrs(*a->value);
 
-    for (ATermMap::const_iterator i = attrs2.begin(); i != attrs2.end(); ++i) {
-        Expr e = evalExpr(state, i->value);
-        string s;
-        PathSet context;
+    foreach (Bindings::iterator, i, *a->value->attrs) {
         MetaValue value;
-        int n;
-        ATermList es;
-        if (matchStr(e, s, context)) {
+        state.forceValue(*i->value);
+        if (i->value->type == tString) {
             value.type = MetaValue::tpString;
-            value.stringValue = s;
-            meta[aterm2String(i->key)] = value;
-        } else if (matchInt(e, n)) {
+            value.stringValue = i->value->string.s;
+        } else if (i->value->type == tInt) {
             value.type = MetaValue::tpInt;
-            value.intValue = n;
-            meta[aterm2String(i->key)] = value;
-        } else if (matchList(e, es)) {
+            value.intValue = i->value->integer;
+        } else if (i->value->type == tList) {
             value.type = MetaValue::tpStrings;
-            for (ATermIterator j(es); j; ++j)
-                value.stringValues.push_back(evalStringNoCtx(state, *j));
-            meta[aterm2String(i->key)] = value;
-        }
+            for (unsigned int j = 0; j < i->value->list.length; ++j)
+                value.stringValues.push_back(state.forceStringNoCtx(*i->value->list.elems[j]));
+        } else continue;
+        ((MetaInfo &) meta)[i->name] = value;
     }
 
     return meta;
@@ -82,73 +68,46 @@ MetaValue DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
 
 void DrvInfo::setMetaInfo(const MetaInfo & meta)
 {
-    ATermMap metaAttrs;
-    foreach (MetaInfo::const_iterator, i, meta) {
-        Expr e;
-        switch (i->second.type) {
-            case MetaValue::tpInt: e = makeInt(i->second.intValue); break;
-            case MetaValue::tpString: e = makeStr(i->second.stringValue); break;
-            case MetaValue::tpStrings: {
-                ATermList es = ATempty;
-                foreach (Strings::const_iterator, j, i->second.stringValues)
-                    es = ATinsert(es, makeStr(*j));
-                e = makeList(ATreverse(es));
-                break;
-            }
-            default: abort();
-        }
-        metaAttrs.set(toATerm(i->first), makeAttrRHS(e, makeNoPos()));
-    }
-    attrs->set(toATerm("meta"), makeAttrs(metaAttrs));
+    metaInfoRead = true;
+    this->meta = meta;
 }
 
 
-/* Cache for already evaluated derivations.  Usually putting ATerms in
-   a STL container is unsafe (they're not scanning for GC roots), but
-   here it doesn't matter; everything in this set is reachable from
-   the stack as well. */
-typedef set<Expr> Exprs;
+/* Cache for already considered attrsets. */
+typedef set<Bindings *> Done;
 
 
-/* Evaluate expression `e'.  If it evaluates to an attribute set of
-   type `derivation', then put information about it in `drvs' (unless
-   it's already in `doneExprs').  The result boolean indicates whether
-   it makes sense for the caller to recursively search for derivations
-   in `e'. */
-static bool getDerivation(EvalState & state, Expr e,
-    const string & attrPath, DrvInfos & drvs, Exprs & doneExprs)
+/* Evaluate value `v'.  If it evaluates to an attribute set of type
+   `derivation', then put information about it in `drvs' (unless it's
+   already in `doneExprs').  The result boolean indicates whether it
+   makes sense for the caller to recursively search for derivations in
+   `v'. */
+static bool getDerivation(EvalState & state, Value & v,
+    const string & attrPath, DrvInfos & drvs, Done & done)
 {
     try {
-        
-        ATermList es;
-        e = evalExpr(state, e);
-        if (!matchAttrs(e, es)) return true;
-
-        boost::shared_ptr<ATermMap> attrs(new ATermMap());
-        queryAllAttrs(e, *attrs, false);
-        
-        Expr a = attrs->get(toATerm("type"));
-        if (!a || evalStringNoCtx(state, a) != "derivation") return true;
+        state.forceValue(v);
+        if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., an attribute set like
            `rec { x = derivation {...}; y = x;}'. */
-        if (doneExprs.find(e) != doneExprs.end()) return false;
-        doneExprs.insert(e);
+        if (done.find(v.attrs) != done.end()) return false;
+        done.insert(v.attrs);
 
         DrvInfo drv;
     
-        a = attrs->get(toATerm("name"));
+        Bindings::iterator i = v.attrs->find(state.sName);
         /* !!! We really would like to have a decent back trace here. */
-        if (!a) throw TypeError("derivation name missing");
-        drv.name = evalStringNoCtx(state, a);
+        if (i == v.attrs->end()) throw TypeError("derivation name missing");
+        drv.name = state.forceStringNoCtx(*i->value);
 
-        a = attrs->get(toATerm("system"));
-        if (!a)
+        Bindings::iterator i2 = v.attrs->find(state.sSystem);
+        if (i2 == v.attrs->end())
             drv.system = "unknown";
         else
-            drv.system = evalStringNoCtx(state, a);
+            drv.system = state.forceStringNoCtx(*i2->value);
 
-        drv.attrs = attrs;
+        drv.attrs = v.attrs;
 
         drv.attrPath = attrPath;
 
@@ -161,11 +120,11 @@ static bool getDerivation(EvalState & state, Expr e,
 }
 
 
-bool getDerivation(EvalState & state, Expr e, DrvInfo & drv)
+bool getDerivation(EvalState & state, Value & v, DrvInfo & drv)
 {
-    Exprs doneExprs;
+    Done done;
     DrvInfos drvs;
-    getDerivation(state, e, "", drvs, doneExprs);
+    getDerivation(state, v, "", drvs, done);
     if (drvs.size() != 1) return false;
     drv = drvs.front();
     return true;
@@ -178,83 +137,73 @@ static string addToPath(const string & s1, const string & s2)
 }
 
 
-static void getDerivations(EvalState & state, Expr e,
-    const string & pathPrefix, const ATermMap & autoArgs,
-    DrvInfos & drvs, Exprs & doneExprs)
+static void getDerivations(EvalState & state, Value & vIn,
+    const string & pathPrefix, Bindings & autoArgs,
+    DrvInfos & drvs, Done & done)
 {
-    e = evalExpr(state, autoCallFunction(evalExpr(state, e), autoArgs));
-
+    Value v;
+    state.autoCallFunction(autoArgs, vIn, v);
+    
     /* Process the expression. */
-    ATermList es;
     DrvInfo drv;
 
-    if (!getDerivation(state, e, pathPrefix, drvs, doneExprs))
-        return;
+    if (!getDerivation(state, v, pathPrefix, drvs, done)) ;
 
-    if (matchAttrs(e, es)) {
-        ATermMap drvMap(ATgetLength(es));
-        queryAllAttrs(e, drvMap);
+    else if (v.type == tAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = drvMap.get(toATerm("_combineChannels"));
+        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        typedef std::map<string, Expr> AttrsSorted;
-        AttrsSorted attrsSorted;
-        foreach (ATermMap::const_iterator, i, drvMap)
-            attrsSorted[aterm2String(i->key)] = i->value;
+        typedef std::map<string, Symbol> SortedSymbols;
+        SortedSymbols attrs;
+        foreach (Bindings::iterator, i, *v.attrs)
+            attrs.insert(std::pair<string, Symbol>(i->name, i->name));
 
-        foreach (AttrsSorted::iterator, i, attrsSorted) {
+        foreach (SortedSymbols::iterator, i, attrs) {
             startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first);
             string pathPrefix2 = addToPath(pathPrefix, i->first);
+            Value & v2(*v.attrs->find(i->second)->value);
             if (combineChannels)
-                getDerivations(state, i->second, pathPrefix2, autoArgs, drvs, doneExprs);
-            else if (getDerivation(state, i->second, pathPrefix2, drvs, doneExprs)) {
+                getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done);
+            else if (getDerivation(state, v2, pathPrefix2, drvs, done)) {
                 /* If the value of this attribute is itself an
                    attribute set, should we recurse into it?  => Only
                    if it has a `recurseForDerivations = true'
                    attribute. */
-                ATermList es;
-                Expr e = evalExpr(state, i->second), e2;
-                if (matchAttrs(e, es)) {
-                    ATermMap attrs(ATgetLength(es));
-                    queryAllAttrs(e, attrs, false);
-                    if (((e2 = attrs.get(toATerm("recurseForDerivations")))
-                            && evalBool(state, e2)))
-                        getDerivations(state, e, pathPrefix2, autoArgs, drvs, doneExprs);
+                if (v2.type == tAttrs) {
+                    Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations"));
+                    if (j != v2.attrs->end() && state.forceBool(*j->value))
+                        getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done);
                 }
             }
         }
-        
-        return;
     }
 
-    if (matchList(e, es)) {
-        int n = 0;
-        for (ATermIterator i(es); i; ++i, ++n) {
+    else if (v.type == tList) {
+        for (unsigned int n = 0; n < v.list.length; ++n) {
             startNest(nest, lvlDebug,
                 format("evaluating list element"));
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *i, pathPrefix2, drvs, doneExprs))
-                getDerivations(state, *i, pathPrefix2, autoArgs, drvs, doneExprs);
+            if (getDerivation(state, *v.list.elems[n], pathPrefix2, drvs, done))
+                getDerivations(state, *v.list.elems[n], pathPrefix2, autoArgs, drvs, done);
         }
-        return;
     }
 
-    throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
+    else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
 }
 
 
-void getDerivations(EvalState & state, Expr e, const string & pathPrefix,
-    const ATermMap & autoArgs, DrvInfos & drvs)
+void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+    Bindings & autoArgs, DrvInfos & drvs)
 {
-    Exprs doneExprs;
-    getDerivations(state, e, pathPrefix, autoArgs, drvs, doneExprs);
+    Done done;
+    getDerivations(state, v, pathPrefix, autoArgs, drvs, done);
 }
 
  

src/libexpr/get-drvs.hh

@@ -1,13 +1,13 @@
 #ifndef __GET_DRVS_H
 #define __GET_DRVS_H
 
+#include "eval.hh"
+
 #include <string>
 #include <map>
 
 #include <boost/shared_ptr.hpp>
 
-#include "eval.hh"
-
 
 namespace nix {
 
@@ -29,16 +29,19 @@ struct DrvInfo
 private:
     string drvPath;
     string outPath;
+
+    bool metaInfoRead;
+    MetaInfo meta;
     
 public:
     string name;
     string attrPath; /* path towards the derivation */
     string system;
 
-    /* !!! these should really be hidden, and setMetaInfo() should
-       make a copy since the ATermMap can be shared between multiple
-       DrvInfos. */
-    boost::shared_ptr<ATermMap> attrs;
+    /* !!! make this private */
+    Bindings * attrs;
+
+    DrvInfo() : metaInfoRead(false), attrs(0) { };
 
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
@@ -62,13 +65,12 @@ public:
 typedef list<DrvInfo> DrvInfos;
 
 
-/* Evaluate expression `e'.  If it evaluates to a derivation, store
-   information about the derivation in `drv' and return true.
-   Otherwise, return false. */
-bool getDerivation(EvalState & state, Expr e, DrvInfo & drv);
+/* If value `v' denotes a derivation, store information about the
+   derivation in `drv' and return true.  Otherwise, return false. */
+bool getDerivation(EvalState & state, Value & v, DrvInfo & drv);
 
-void getDerivations(EvalState & state, Expr e, const string & pathPrefix,
-    const ATermMap & autoArgs, DrvInfos & drvs);
+void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+    Bindings & autoArgs, DrvInfos & drvs);
 
  
 }

src/libexpr/lexer.l

@@ -8,9 +8,7 @@
 
 
 %{
-#include "aterm.hh"
 #include "nixexpr.hh"
-#include "nixexpr-ast.hh"
 #define BISON_HEADER_HACK
 #include "parser-tab.hh"
 
@@ -21,13 +19,16 @@ namespace nix {
     
 static void initLoc(YYLTYPE * loc)
 {
-    loc->first_line = 1;
-    loc->first_column = 1;
+    loc->first_line = loc->last_line = 1;
+    loc->first_column = loc->last_column = 1;
 }
 
     
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
+    loc->first_line = loc->last_line;
+    loc->first_column = loc->last_column;
+
     while (len--) {
        switch (*s++) {
        case '\r':
@@ -35,17 +36,17 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
                s++;
            /* fall through */
        case '\n': 
-           ++loc->first_line;
-           loc->first_column = 1;
+           ++loc->last_line;
+           loc->last_column = 1;
            break;
        default:
-           ++loc->first_column;
+           ++loc->last_column;
        }
     }
 }
 
 
-static Expr unescapeStr(const char * s)
+static Expr * unescapeStr(SymbolTable & symbols, const char * s)
 {
     string t;
     char c;
@@ -65,7 +66,7 @@ static Expr unescapeStr(const char * s)
         }
         else t += c;
     }
-    return makeStr(toATerm(t), ATempty);
+    return new ExprString(symbols.create(t));
 }
 
  
@@ -105,19 +106,20 @@ inherit     { return INHERIT; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
 
-{ID}        { yylval->t = toATerm(yytext); return ID; /* !!! alloc */ }
+{ID}        { yylval->id = strdup(yytext); return ID; }
 {INT}       { int n = atoi(yytext); /* !!! overflow */
-              yylval->t = ATmake("<int>", n);
+              yylval->n = n;
               return INT;
             }
 
 \"          { BEGIN(STRING); return '"'; }
 <STRING>([^\$\"\\]|\$[^\{\"]|\\.)+ {
-/* !!! Not quite right: we want a follow restriction on "$", it
-   shouldn't be followed by a "{".  Right now "$\"" will be consumed
-   as part of a string, rather than a "$" followed by the string
-   terminator.  Disallow "$\"" for now. */
-              yylval->t = unescapeStr(yytext); /* !!! alloc */ 
+              /* !!! Not quite right: we want a follow restriction on
+                 "$", it shouldn't be followed by a "{".  Right now
+                 "$\"" will be consumed as part of a string, rather
+                 than a "$" followed by the string terminator.
+                 Disallow "$\"" for now. */
+              yylval->e = unescapeStr(data->symbols, yytext);
               return STR;
             }
 <STRING>\$\{  { BEGIN(INITIAL); return DOLLAR_CURLY; }
@@ -126,31 +128,31 @@ inherit     { return INHERIT; }
 
 \'\'(\ *\n)?     { BEGIN(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
-                   yylval->t = makeIndStr(toATerm(yytext));
+                   yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
                  }
 <IND_STRING>\'\'\$ {
-                   yylval->t = makeIndStr(toATerm("$"));
+                   yylval->e = new ExprIndStr("$");
                    return IND_STR;
                  }
 <IND_STRING>\'\'\' {
-                   yylval->t = makeIndStr(toATerm("''"));
+                   yylval->e = new ExprIndStr("''");
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\. {
-                   yylval->t = unescapeStr(yytext + 2);
+                   yylval->e = unescapeStr(data->symbols, yytext + 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { BEGIN(INITIAL); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
-                   yylval->t = makeIndStr(toATerm("'"));
+                   yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
 <IND_STRING>.    return yytext[0]; /* just in case: shouldn't be reached */
 
-{PATH}      { yylval->t = toATerm(yytext); return PATH; /* !!! alloc */ }
-{URI}       { yylval->t = toATerm(yytext); return URI; /* !!! alloc */ }
+{PATH}      { yylval->path = strdup(yytext); return PATH; }
+{URI}       { yylval->uri = strdup(yytext); return URI; }
 
 [ \t\r\n]+    /* eat up whitespace */
 \#[^\r\n]*    /* single-line comments */

src/libexpr/nixexpr-ast.def

@@ -1,97 +0,0 @@
-init initNixExprHelpers
-
-Pos | string int int | Pos |
-NoPos | | Pos |
-
-Function | Pattern Expr Pos | Expr |
-Assert | Expr Expr Pos | Expr |
-With | Expr Expr Pos | Expr |
-If | Expr Expr Expr | Expr |
-OpNot | Expr | Expr |
-OpEq | Expr Expr | Expr |
-OpNEq | Expr Expr | Expr |
-OpAnd | Expr Expr | Expr |
-OpOr | Expr Expr | Expr |
-OpImpl | Expr Expr | Expr |
-OpUpdate | Expr Expr | Expr |
-SubPath | Expr Expr | Expr |
-OpHasAttr | Expr string | Expr |
-OpPlus | Expr Expr | Expr |
-OpConcat | Expr Expr | Expr |
-ConcatStrings | ATermList | Expr |
-Call | Expr Expr | Expr |
-Select | Expr string | Expr |
-Var | string | Expr |
-Int | int | Expr |
-
-# Strings in the evaluator carry a so-called `context' (the ATermList)
-# which is a list of strings representing store paths.  This is to
-# allow users to write things like
-#
-#   "--with-freetype2-library=" + freetype + "/lib"
-#
-# where `freetype' is a derivation (or a source to be copied to the
-# store).  If we just concatenated the strings without keeping track
-# of the referenced store paths, then if the string is used as a
-# derivation attribute, the derivation will not have the correct
-# dependencies in its inputDrvs and inputSrcs.
-#
-# The semantics of the context is as follows: when a string with
-# context C is used as a derivation attribute, then the derivations in
-# C will be added to the inputDrvs of the derivation, and the other
-# store paths in C will be added to the inputSrcs of the derivations.
-#
-# For canonicity, the store paths should be in sorted order.
-Str | string ATermList | Expr |
-Str | string | Expr | ObsoleteStr
-
-# Internal to the parser, doesn't occur in ASTs.
-IndStr | string | Expr |
-
-# A path is a reference to a file system object that is to be copied
-# to the Nix store when used as a derivation attribute.  When it is
-# concatenated to a string (i.e., `str + path'), it is also copied and
-# the resulting store path is concatenated to the string (with the
-# store path in the context).  If a string or path is concatenated to
-# a path (i.e., `path + str' or `path + path'), the result is a new
-# path (if the right-hand side is a string, the context must be
-# empty).
-Path | string | Expr |
-
-List | ATermList | Expr |
-BlackHole | | Expr |
-Undefined | | Expr |
-Removed | | Expr |
-PrimOp | int ATermBlob ATermList | Expr |
-Attrs | ATermList | Expr |
-Closed | Expr | Expr |
-Rec | ATermList ATermList | Expr |
-Bool | ATermBool | Expr |
-Null | | Expr |
-
-Bind | string Expr Pos | ATerm |
-BindAttrPath | ATermList Expr Pos | ATerm | # desugared during parsing
-Bind | string Expr | ATerm | ObsoleteBind
-Inherit | Expr ATermList Pos | ATerm |
-
-Scope | | Expr |
-
-VarPat | string | Pattern |
-AttrsPat | ATermList ATermBool | Pattern | # bool = `...'
-AtPat | Pattern Pattern | Pattern |
-
-Formal | string DefaultValue | ATerm |
-
-DefaultValue | Expr | DefaultValue |
-NoDefaultValue | | DefaultValue |
-
-True | | ATermBool |
-False | | ATermBool |
-
-PrimOpDef | int ATermBlob | ATerm |
-
-AttrRHS | Expr Pos | ATerm |
-
-eTrue = makeBool(makeTrue())
-eFalse = makeBool(makeFalse())
-sOverrides = toATerm("__overrides")

src/libexpr/nixexpr.cc

@@ -1,407 +1,314 @@
 #include "nixexpr.hh"
 #include "derivations.hh"
 #include "util.hh"
-#include "aterm.hh"
-
-#include "nixexpr-ast.hh"
-#include "nixexpr-ast.cc"
 
 #include <cstdlib>
 
 
 namespace nix {
-    
 
-string showPos(ATerm pos)
+
+/* Displaying abstract syntax trees. */
+
+std::ostream & operator << (std::ostream & str, Expr & e)
 {
-    ATerm path;
-    int line, column;
-    if (matchNoPos(pos)) return "undefined position";
-    if (!matchPos(pos, path, line, column))
-        throw badTerm("position expected", pos);
-    return (format("`%1%:%2%:%3%'") % aterm2String(path) % line % column).str();
+    e.show(str);
+    return str;
 }
-    
 
-ATerm bottomupRewrite(TermFun & f, ATerm e)
+void Expr::show(std::ostream & str)
 {
-    checkInterrupt();
+    abort();
+}
 
-    if (ATgetType(e) == AT_APPL) {
-        AFun fun = ATgetAFun(e);
-        int arity = ATgetArity(fun);
-        ATerm args[arity];
+void ExprInt::show(std::ostream & str)
+{
+    str << n;
+}
 
-        for (int i = 0; i < arity; ++i)
-            args[i] = bottomupRewrite(f, ATgetArgument(e, i));
+void ExprString::show(std::ostream & str)
+{
+    str << "\"" << s << "\""; // !!! escaping
+}
+
+void ExprPath::show(std::ostream & str)
+{
+    str << s;
+}
+
+void ExprVar::show(std::ostream & str)
+{
+    str << info.name;
+}
+
+void ExprSelect::show(std::ostream & str)
+{
+    str << "(" << *e << ")." << name;
+}
+
+void ExprOpHasAttr::show(std::ostream & str)
+{
+    str << "(" << *e << ") ? " << name;
+}
+
+void ExprAttrs::show(std::ostream & str)
+{
+    if (recursive) str << "rec ";
+    str << "{ ";
+    foreach (AttrDefs::iterator, i, attrs)
+        if (i->second.inherited)
+            str << "inherit " << i->first << " " << "; ";
+        else
+            str << i->first << " = " << *i->second.e << "; ";
+    str << "}";
+}
+
+void ExprList::show(std::ostream & str)
+{
+    str << "[ ";
+    foreach (vector<Expr *>::iterator, i, elems)
+        str << "(" << **i << ") ";
+    str << "]";
+}
+
+void ExprLambda::show(std::ostream & str)
+{
+    str << "(";
+    if (matchAttrs) {
+        str << "{ ";
+        bool first = true;
+        foreach (Formals::Formals_::iterator, i, formals->formals) {
+            if (first) first = false; else str << ", ";
+            str << i->name;
+            if (i->def) str << " ? " << *i->def;
+        }
+        str << " }";
+        if (!arg.empty()) str << " @ ";
+    }
+    if (!arg.empty()) str << arg;
+    str << ": " << *body << ")";
+}
+
+void ExprLet::show(std::ostream & str)
+{
+    str << "let ";
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        if (i->second.inherited)
+            str << "inherit " << i->first << "; ";
+        else
+            str << i->first << " = " << *i->second.e << "; ";
+    str << "in " << *body;
+}
+
+void ExprWith::show(std::ostream & str)
+{
+    str << "with " << *attrs << "; " << *body;
+}
+
+void ExprIf::show(std::ostream & str)
+{
+    str << "if " << *cond << " then " << *then << " else " << *else_;
+}
+
+void ExprAssert::show(std::ostream & str)
+{
+    str << "assert " << *cond << "; " << *body;
+}
+
+void ExprOpNot::show(std::ostream & str)
+{
+    str << "! " << *e;
+}
+
+void ExprConcatStrings::show(std::ostream & str)
+{
+    bool first = true;
+    foreach (vector<Expr *>::iterator, i, *es) {
+        if (first) first = false; else str << " + ";
+        str << **i;
+    }
+}
+
+
+std::ostream & operator << (std::ostream & str, const Pos & pos)
+{
+    if (!pos.line)
+        str << "undefined position";
+    else
+        str << (format("`%1%:%2%:%3%'") % pos.file % pos.line % pos.column).str();
+    return str;
+}
+
+
+Pos noPos;
+
+
+/* Computing levels/displacements for variables. */
+
+void Expr::bindVars(const StaticEnv & env)
+{
+    abort();
+}
+
+void ExprInt::bindVars(const StaticEnv & env)
+{
+}
+
+void ExprString::bindVars(const StaticEnv & env)
+{
+}
+
+void ExprPath::bindVars(const StaticEnv & env)
+{
+}
+
+void VarRef::bind(const StaticEnv & env)
+{
+    /* Check whether the variable appears in the environment.  If so,
+       set its level and displacement. */
+    const StaticEnv * curEnv;
+    unsigned int level;
+    int withLevel = -1;
+    for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) {
+        if (curEnv->isWith) {
+            if (withLevel == -1) withLevel = level;
+        } else {
+            StaticEnv::Vars::const_iterator i = curEnv->vars.find(name);
+            if (i != curEnv->vars.end()) {
+                fromWith = false;
+                this->level = level;
+                displ = i->second;
+                return;
+            }
+        }
+    }
+
+    /* Otherwise, the variable must be obtained from the nearest
+       enclosing `with'.  If there is no `with', then we can issue an
+       "undefined variable" error now. */
+    if (withLevel == -1) throw EvalError(format("undefined variable `%1%'") % name);
+
+    fromWith = true;
+    this->level = withLevel;
+}
+
+void ExprVar::bindVars(const StaticEnv & env)
+{
+    info.bind(env);
+}
+
+void ExprSelect::bindVars(const StaticEnv & env)
+{
+    e->bindVars(env);
+}
+
+void ExprOpHasAttr::bindVars(const StaticEnv & env)
+{
+    e->bindVars(env);
+}
+
+void ExprAttrs::bindVars(const StaticEnv & env)
+{
+    if (recursive) {
+        StaticEnv newEnv(false, &env);
+    
+        unsigned int displ = 0;
+        foreach (AttrDefs::iterator, i, attrs)
+            newEnv.vars[i->first] = i->second.displ = displ++;
         
-        e = (ATerm) ATmakeApplArray(fun, args);
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited) i->second.var.bind(env);
+            else i->second.e->bindVars(newEnv);
     }
 
-    else if (ATgetType(e) == AT_LIST) {
-        ATermList in = (ATermList) e;
-        ATermList out = ATempty;
-
-        for (ATermIterator i(in); i; ++i)
-            out = ATinsert(out, bottomupRewrite(f, *i));
-
-        e = (ATerm) ATreverse(out);
-    }
-
-    return f(e);
+    else
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited) i->second.var.bind(env);
+            else i->second.e->bindVars(env);
 }
 
-
-void queryAllAttrs(Expr e, ATermMap & attrs, bool withPos)
+void ExprList::bindVars(const StaticEnv & env)
 {
-    ATermList bnds;
-    if (!matchAttrs(e, bnds))
-        throw TypeError(format("value is %1% while an attribute set was expected") % showType(e));
-
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm name;
-        Expr e;
-        ATerm pos;
-        if (!matchBind(*i, name, e, pos)) abort(); /* can't happen */
-        attrs.set(name, withPos ? makeAttrRHS(e, pos) : e);
-    }
+    foreach (vector<Expr *>::iterator, i, elems)
+        (*i)->bindVars(env);
 }
 
-
-Expr queryAttr(Expr e, const string & name)
+void ExprLambda::bindVars(const StaticEnv & env)
 {
-    ATerm dummy;
-    return queryAttr(e, name, dummy);
-}
-
-
-Expr queryAttr(Expr e, const string & name, ATerm & pos)
-{
-    ATermList bnds;
-    if (!matchAttrs(e, bnds))
-        throw TypeError(format("value is %1% while an attribute set was expected") % showType(e));
-
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm name2, pos2;
-        Expr e;
-        if (!matchBind(*i, name2, e, pos2))
-            abort(); /* can't happen */
-        if (aterm2String(name2) == name) {
-            pos = pos2;
-            return e;
-        }
-    }
-
-    return 0;
-}
-
-
-Expr makeAttrs(const ATermMap & attrs)
-{
-    ATermList bnds = ATempty;
-    for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i) {
-        Expr e;
-        ATerm pos;
-        if (!matchAttrRHS(i->value, e, pos))
-            abort(); /* can't happen */
-        bnds = ATinsert(bnds, makeBind(i->key, e, pos));
-    }
-    return makeAttrs(bnds);
-}
-
-
-static void varsBoundByPattern(ATermMap & map, Pattern pat)
-{
-    ATerm name;
-    ATermList formals;
-    Pattern pat1, pat2;
-    ATermBool ellipsis;
-    /* Use makeRemoved() so that it can be used directly in
-       substitute(). */
-    if (matchVarPat(pat, name))
-        map.set(name, makeRemoved());
-    else if (matchAttrsPat(pat, formals, ellipsis)) { 
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1;
-            if (!matchFormal(*i, name, d1)) abort();
-            map.set(name, makeRemoved());
-        }
-    }
-    else if (matchAtPat(pat, pat1, pat2)) {
-        varsBoundByPattern(map, pat1);
-        varsBoundByPattern(map, pat2);
-    }
-    else abort();
-}
-
-
-Expr substitute(const Substitution & subs, Expr e)
-{
-    checkInterrupt();
-
-    //if (subs.size() == 0) return e;
-
-    ATerm name, pos, e2;
-
-    /* As an optimisation, don't substitute in subterms known to be
-       closed. */
-    if (matchClosed(e, e2)) return e;
-
-    if (matchVar(e, name)) {
-        Expr sub = subs.lookup(name);
-        if (sub == makeRemoved()) sub = 0;
-        Expr wrapped;
-        /* Add a "closed" wrapper around terms that aren't already
-           closed.  The check is necessary to prevent repeated
-           wrapping, e.g., closed(closed(closed(...))), which kills
-           caching. */
-        return sub ? (matchClosed(sub, wrapped) ? sub : makeClosed(sub)) : e;
-    }
-
-    /* In case of a function, filter out all variables bound by this
-       function. */
-    Pattern pat;
-    ATerm body;
-    if (matchFunction(e, pat, body, pos)) {
-        ATermMap map(16);
-        varsBoundByPattern(map, pat);
-        Substitution subs2(&subs, &map);
-        return makeFunction(
-            (Pattern) substitute(subs2, (Expr) pat),
-            substitute(subs2, body), pos);
-    }
-
-    /* Idem for a mutually recursive attribute set. */
-    ATermList rbnds, nrbnds;
-    if (matchRec(e, rbnds, nrbnds)) {
-        ATermMap map(ATgetLength(rbnds) + ATgetLength(nrbnds));
-        for (ATermIterator i(rbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
-        for (ATermIterator i(nrbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
-        return makeRec(
-            (ATermList) substitute(Substitution(&subs, &map), (ATerm) rbnds),
-            (ATermList) substitute(subs, (ATerm) nrbnds));
-    }
-
-    if (ATgetType(e) == AT_APPL) {
-        AFun fun = ATgetAFun(e);
-        int arity = ATgetArity(fun);
-        ATerm args[arity];
-        bool changed = false;
-
-        for (int i = 0; i < arity; ++i) {
-            ATerm arg = ATgetArgument(e, i);
-            args[i] = substitute(subs, arg);
-            if (args[i] != arg) changed = true;
-        }
-        
-        return changed ? (ATerm) ATmakeApplArray(fun, args) : e;
-    }
-
-    if (ATgetType(e) == AT_LIST) {
-        unsigned int len = ATgetLength((ATermList) e);
-        ATerm es[len];
-        ATermIterator i((ATermList) e);
-        for (unsigned int j = 0; i; ++i, ++j)
-            es[j] = substitute(subs, *i);
-        ATermList out = ATempty;
-        for (unsigned int j = len; j; --j)
-            out = ATinsert(out, es[j - 1]);
-        return (ATerm) out;
-    }
-
-    return e;
-}
-
-
-/* We use memoisation to prevent exponential complexity on heavily
-   shared ATerms (remember, an ATerm is a graph, not a tree!).  Note
-   that using an STL set is fine here wrt to ATerm garbage collection
-   since all the ATerms in the set are already reachable from
-   somewhere else. */
-static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
-{
-    if (done.find(e) != done.end()) return;
-    done.insert(e);
+    StaticEnv newEnv(false, &env);
     
-    ATerm name, pos, value;
-    ATerm with, body;
-    ATermList rbnds, nrbnds;
-    Pattern pat;
-
-    /* Closed terms don't have free variables, so we don't have to
-       check by definition. */
-    if (matchClosed(e, value)) return;
+    unsigned int displ = 0;
     
-    else if (matchVar(e, name)) {
-        if (!defs.get(name))
-            throw EvalError(format("undefined variable `%1%'")
-                % aterm2String(name));
+    if (!arg.empty()) newEnv.vars[arg] = displ++;
+
+    if (matchAttrs) {
+        foreach (Formals::Formals_::iterator, i, formals->formals)
+            newEnv.vars[i->name] = displ++;
+
+        foreach (Formals::Formals_::iterator, i, formals->formals)
+            if (i->def) i->def->bindVars(newEnv);
     }
 
-    else if (matchFunction(e, pat, body, pos)) {
-        ATermMap defs2(defs);
-        varsBoundByPattern(defs2, pat);
-        set<Expr> done2;
-        checkVarDefs2(done2, defs2, pat);
-        checkVarDefs2(done2, defs2, body);
-    }
-        
-    else if (matchRec(e, rbnds, nrbnds)) {
-        checkVarDefs2(done, defs, (ATerm) nrbnds);
-        ATermMap defs2(defs);
-        for (ATermIterator i(rbnds); i; ++i) {
-            if (!matchBind(*i, name, value, pos)) abort(); /* can't happen */
-            defs2.set(name, (ATerm) ATempty);
-        }
-        for (ATermIterator i(nrbnds); i; ++i) {
-            if (!matchBind(*i, name, value, pos)) abort(); /* can't happen */
-            defs2.set(name, (ATerm) ATempty);
-        }
-        set<Expr> done2;
-        checkVarDefs2(done2, defs2, (ATerm) rbnds);
-    }
+    body->bindVars(newEnv);
+}
 
-    else if (matchWith(e, with, body, pos)) {
-        /* We can't check the body without evaluating the definitions
-           (which is an arbitrary expression), so we don't do that
-           here but only when actually evaluating the `with'. */
-        checkVarDefs2(done, defs, with);
-    }
+void ExprLet::bindVars(const StaticEnv & env)
+{
+    StaticEnv newEnv(false, &env);
     
-    else if (ATgetType(e) == AT_APPL) {
-        int arity = ATgetArity(ATgetAFun(e));
-        for (int i = 0; i < arity; ++i)
-            checkVarDefs2(done, defs, ATgetArgument(e, i));
-    }
-
-    else if (ATgetType(e) == AT_LIST)
-        for (ATermIterator i((ATermList) e); i; ++i)
-            checkVarDefs2(done, defs, *i);
+    unsigned int displ = 0;
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        newEnv.vars[i->first] = i->second.displ = displ++;
+    
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        if (i->second.inherited) i->second.var.bind(env);
+        else i->second.e->bindVars(newEnv);
+    
+    body->bindVars(newEnv);
 }
 
-
-void checkVarDefs(const ATermMap & defs, Expr e)
+void ExprWith::bindVars(const StaticEnv & env)
 {
-    set<Expr> done;
-    checkVarDefs2(done, defs, e);
-}
-
-
-struct Canonicalise : TermFun
-{
-    ATerm operator () (ATerm e)
-    {
-        /* Remove position info. */
-        ATerm path;
-        int line, column;
-        if (matchPos(e, path, line, column))
-            return makeNoPos();
-
-        /* Sort attribute sets. */
-        ATermList _;
-        if (matchAttrs(e, _)) {
-            ATermMap attrs;
-            queryAllAttrs(e, attrs);
-            StringSet names;
-            for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i)
-                names.insert(aterm2String(i->key));
-
-            ATermList attrs2 = ATempty;
-            for (StringSet::reverse_iterator i = names.rbegin(); i != names.rend(); ++i)
-                attrs2 = ATinsert(attrs2,
-                    makeBind(toATerm(*i), attrs.get(toATerm(*i)), makeNoPos()));
-
-            return makeAttrs(attrs2);
+    /* Does this `with' have an enclosing `with'?  If so, record its
+       level so that `lookupVar' can look up variables in the previous
+       `with' if this one doesn't contain the desired attribute. */
+    const StaticEnv * curEnv;
+    unsigned int level;
+    prevWith = 0;
+    for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++)
+        if (curEnv->isWith) {
+            prevWith = level;
+            break;
         }
-        
-        return e;
-    }
-};
+    
+    attrs->bindVars(env);    
+    StaticEnv newEnv(true, &env);
+    body->bindVars(newEnv);
+}
 
-
-Expr canonicaliseExpr(Expr e)
+void ExprIf::bindVars(const StaticEnv & env)
 {
-    Canonicalise canonicalise;
-    return bottomupRewrite(canonicalise, e);
+    cond->bindVars(env);
+    then->bindVars(env);
+    else_->bindVars(env);
 }
 
-
-Expr makeBool(bool b)
+void ExprAssert::bindVars(const StaticEnv & env)
 {
-    return b ? eTrue : eFalse;
+    cond->bindVars(env);
+    body->bindVars(env);
 }
 
-
-bool matchStr(Expr e, string & s, PathSet & context)
+void ExprOpNot::bindVars(const StaticEnv & env)
 {
-    ATermList l;
-    ATerm s_;
-
-    if (!matchStr(e, s_, l)) return false;
-
-    s = aterm2String(s_);
-
-    for (ATermIterator i(l); i; ++i)
-        context.insert(aterm2String(*i));
-
-    return true;
+    e->bindVars(env);
 }
 
-
-Expr makeStr(const string & s, const PathSet & context)
+void ExprConcatStrings::bindVars(const StaticEnv & env)
 {
-    return makeStr(toATerm(s), toATermList(context));
+    foreach (vector<Expr *>::iterator, i, *es)
+        (*i)->bindVars(env);
 }
 
 
-string showType(Expr e)
-{
-    ATerm t1, t2;
-    ATermList l1;
-    ATermBlob b1;
-    int i1;
-    Pattern p1;
-    if (matchStr(e, t1, l1)) return "a string";
-    if (matchPath(e, t1)) return "a path";
-    if (matchNull(e)) return "null";
-    if (matchInt(e, i1)) return "an integer";
-    if (matchBool(e, t1)) return "a boolean";
-    if (matchFunction(e, p1, t1, t2)) return "a function";
-    if (matchAttrs(e, l1)) return "an attribute set";
-    if (matchList(e, l1)) return "a list";
-    if (matchPrimOp(e, i1, b1, l1)) return "a partially applied built-in function";
-    return "an unknown type";
-}
-
-
-string showValue(Expr e)
-{
-    PathSet context;
-    string s;
-    ATerm s2;
-    int i;
-    if (matchStr(e, s, context)) {
-        string u;
-        for (string::iterator i = s.begin(); i != s.end(); ++i)
-            if (*i == '\"' || *i == '\\') u += "\\" + *i;
-            else if (*i == '\n') u += "\\n";
-            else if (*i == '\r') u += "\\r";
-            else if (*i == '\t') u += "\\t";
-            else u += *i;
-        return "\"" + u + "\"";
-    }
-    if (matchPath(e, s2)) return aterm2String(s2);
-    if (matchNull(e)) return "null";
-    if (matchInt(e, i)) return (format("%1%") % i).str();
-    if (e == eTrue) return "true";
-    if (e == eFalse) return "false";
-    /* !!! incomplete */
-    return "<unknown>";
-}
-
- 
 }

src/libexpr/nixexpr.hh

@@ -1,10 +1,9 @@
 #ifndef __NIXEXPR_H
 #define __NIXEXPR_H
 
-#include <map>
+#include "symbol-table.hh"
 
-#include "aterm-map.hh"
-#include "types.hh"
+#include <map>
 
 
 namespace nix {
@@ -16,107 +15,264 @@ MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
 MakeError(TypeError, EvalError)
+MakeError(ImportError, EvalError) // error building an imported derivation
 
 
-/* Nix expressions are represented as ATerms.  The maximal sharing
-   property of the ATerm library allows us to implement caching of
-   normals forms efficiently. */
-typedef ATerm Expr;
-typedef ATerm DefaultValue;
-typedef ATerm Pos;
-typedef ATerm Pattern;
-typedef ATerm ATermBool;
+/* Position objects. */
 
-
-/* A STL vector of ATerms.  Should be used with great care since it's
-   stored on the heap, and the elements are therefore not roots to the
-   ATerm garbage collector. */
-typedef vector<ATerm> ATermVector;
-
-
-/* A substitution is a linked list of ATermMaps that map names to
-   identifiers.  We use a list of ATermMaps rather than a single to
-   make it easy to grow or shrink a substitution when entering a
-   scope. */
-struct Substitution
+struct Pos
 {
-    ATermMap * map;
-    const Substitution * prev;
+    string file;
+    unsigned int line, column;
+    Pos() : line(0), column(0) { };
+    Pos(const string & file, unsigned int line, unsigned int column)
+        : file(file), line(line), column(column) { };
+};
 
-    Substitution(const Substitution * prev, ATermMap * map)
-    {
-        this->prev = prev;
-        this->map = map;
-    }
+extern Pos noPos;
+
+std::ostream & operator << (std::ostream & str, const Pos & pos);
+
+
+struct Env;
+struct Value;
+struct EvalState;
+struct StaticEnv;
+
+
+/* Abstract syntax of Nix expressions. */
+
+struct Expr
+{
+    virtual void show(std::ostream & str);
+    virtual void bindVars(const StaticEnv & env);
+    virtual void eval(EvalState & state, Env & env, Value & v);
+};
+
+std::ostream & operator << (std::ostream & str, Expr & e);
+
+#define COMMON_METHODS \
+    void show(std::ostream & str); \
+    void eval(EvalState & state, Env & env, Value & v); \
+    void bindVars(const StaticEnv & env);
+
+struct ExprInt : Expr
+{
+    int n;
+    ExprInt(int n) : n(n) { };
+    COMMON_METHODS
+};
+
+struct ExprString : Expr
+{
+    Symbol s;
+    ExprString(const Symbol & s) : s(s) { };
+    COMMON_METHODS
+};
+
+/* Temporary class used during parsing of indented strings. */
+struct ExprIndStr : Expr
+{
+    string s;
+    ExprIndStr(const string & s) : s(s) { };
+};
+
+struct ExprPath : Expr
+{
+    string s;
+    ExprPath(const string & s) : s(s) { };
+    COMMON_METHODS
+};
+
+struct VarRef
+{
+    Symbol name;
+
+    /* Whether the variable comes from an environment (e.g. a rec, let
+       or function argument) or from a "with". */
+    bool fromWith;
     
-    Expr lookup(Expr name) const
-    {
-        Expr x;
-        for (const Substitution * s(this); s; s = s->prev)
-            if ((x = s->map->get(name))) return x;
-        return 0;
-    }
+    /* In the former case, the value is obtained by going `level'
+       levels up from the current environment and getting the
+       `displ'th value in that environment.  In the latter case, the
+       value is obtained by getting the attribute named `name' from
+       the attribute set stored in the environment that is `level'
+       levels up from the current one.*/
+    unsigned int level;
+    unsigned int displ;
+
+    VarRef() { };
+    VarRef(const Symbol & name) : name(name) { };
+    void bind(const StaticEnv & env);
 };
 
-
-/* Show a position. */
-string showPos(ATerm pos);
-
-/* Generic bottomup traversal over ATerms.  The traversal first
-   recursively descends into subterms, and then applies the given term
-   function to the resulting term. */
-struct TermFun
+struct ExprVar : Expr
 {
-    virtual ~TermFun() { }
-    virtual ATerm operator () (ATerm e) = 0;
+    VarRef info;
+    ExprVar(const Symbol & name) : info(name) { };
+    COMMON_METHODS
+};
+
+struct ExprSelect : Expr
+{
+    Expr * e;
+    Symbol name;
+    ExprSelect(Expr * e, const Symbol & name) : e(e), name(name) { };
+    COMMON_METHODS
+};
+
+struct ExprOpHasAttr : Expr
+{
+    Expr * e;
+    Symbol name;
+    ExprOpHasAttr(Expr * e, const Symbol & name) : e(e), name(name) { };
+    COMMON_METHODS
+};
+
+struct ExprAttrs : Expr
+{
+    bool recursive;
+    struct AttrDef {
+        bool inherited;
+        Expr * e; // if not inherited
+        VarRef var; // if inherited
+        Pos pos;
+        unsigned int displ; // displacement
+        AttrDef(Expr * e, const Pos & pos) : inherited(false), e(e), pos(pos) { };
+        AttrDef(const Symbol & name, const Pos & pos) : inherited(true), var(name), pos(pos) { };
+        AttrDef() { };
+    };
+    typedef std::map<Symbol, AttrDef> AttrDefs;
+    AttrDefs attrs;
+    ExprAttrs() : recursive(false) { };
+    COMMON_METHODS
+};
+
+struct ExprList : Expr
+{
+    std::vector<Expr *> elems;
+    ExprList() { };
+    COMMON_METHODS
+};
+
+struct Formal
+{
+    Symbol name;
+    Expr * def;
+    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
+};
+
+struct Formals
+{
+    typedef std::list<Formal> Formals_;
+    Formals_ formals;
+    std::set<Symbol> argNames; // used during parsing
+    bool ellipsis;
+};
+
+struct ExprLambda : Expr
+{
+    Pos pos;
+    Symbol arg;
+    bool matchAttrs;
+    Formals * formals;
+    Expr * body;
+    ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
+        : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
+    {
+        if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
+            throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+                % arg % pos);
+    };
+    COMMON_METHODS
+};
+
+struct ExprLet : Expr
+{
+    ExprAttrs * attrs;
+    Expr * body;
+    ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { };
+    COMMON_METHODS
+};
+
+struct ExprWith : Expr
+{
+    Pos pos;
+    Expr * attrs, * body;
+    unsigned int prevWith;
+    ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
+    COMMON_METHODS
+};
+
+struct ExprIf : Expr
+{
+    Expr * cond, * then, * else_;
+    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
+    COMMON_METHODS
+};
+
+struct ExprAssert : Expr
+{
+    Pos pos;
+    Expr * cond, * body;
+    ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
+    COMMON_METHODS
+};
+
+struct ExprOpNot : Expr
+{
+    Expr * e;
+    ExprOpNot(Expr * e) : e(e) { };
+    COMMON_METHODS
+};
+
+#define MakeBinOp(name, s) \
+    struct Expr##name : Expr \
+    { \
+        Expr * e1, * e2; \
+        Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
+        void show(std::ostream & str) \
+        { \
+            str << *e1 << " " s " " << *e2; \
+        } \
+        void bindVars(const StaticEnv & env) \
+        { \
+            e1->bindVars(env); e2->bindVars(env); \
+        } \
+        void eval(EvalState & state, Env & env, Value & v); \
+    };
+
+MakeBinOp(App, "")
+MakeBinOp(OpEq, "==")
+MakeBinOp(OpNEq, "!=")
+MakeBinOp(OpAnd, "&&")
+MakeBinOp(OpOr, "||")
+MakeBinOp(OpImpl, "->")
+MakeBinOp(OpUpdate, "//")
+MakeBinOp(OpConcatLists, "++")
+
+struct ExprConcatStrings : Expr
+{
+    vector<Expr *> * es;
+    ExprConcatStrings(vector<Expr *> * es) : es(es) { };
+    COMMON_METHODS
 };
-ATerm bottomupRewrite(TermFun & f, ATerm e);
 
 
-/* Query all attributes in an attribute set expression.  The
-   expression must be in normal form. */
-void queryAllAttrs(Expr e, ATermMap & attrs, bool withPos = false);
-
-/* Query a specific attribute from an attribute set expression.  The
-   expression must be in normal form. */
-Expr queryAttr(Expr e, const string & name);
-Expr queryAttr(Expr e, const string & name, ATerm & pos);
-
-/* Create an attribute set expression from an Attrs value. */
-Expr makeAttrs(const ATermMap & attrs);
+/* Static environments are used to map variable names onto (level,
+   displacement) pairs used to obtain the value of the variable at
+   runtime. */
+struct StaticEnv
+{
+    bool isWith;
+    const StaticEnv * up;
+    typedef std::map<Symbol, unsigned int> Vars;
+    Vars vars;
+    StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { };
+};
 
 
-/* Perform a set of substitutions on an expression. */
-Expr substitute(const Substitution & subs, Expr e);
 
-
-/* Check whether all variables are defined in the given expression.
-   Throw an exception if this isn't the case. */
-void checkVarDefs(const ATermMap & def, Expr e);
-
-
-/* Canonicalise a Nix expression by sorting attributes and removing
-   location information. */
-Expr canonicaliseExpr(Expr e);
-
-
-/* Create an expression representing a boolean. */
-Expr makeBool(bool b);
-
-
-/* Manipulation of Str() nodes.  Note: matchStr() does not clear
-   context!  */
-bool matchStr(Expr e, string & s, PathSet & context);
-
-Expr makeStr(const string & s, const PathSet & context = PathSet());
-
-
-/* Showing types, values. */
-string showType(Expr e);
-
-string showValue(Expr e);
-
- 
 }
 
 

src/libexpr/parser.hh

@@ -8,12 +8,11 @@ namespace nix {
 
 
 /* Parse a Nix expression from the specified file.  If `path' refers
-   to a directory, the "/default.nix" is appended. */
-Expr parseExprFromFile(EvalState & state, Path path);
+   to a directory, then "/default.nix" is appended. */
+Expr * parseExprFromFile(EvalState & state, Path path);
 
 /* Parse a Nix expression from the specified string. */
-Expr parseExprFromString(EvalState & state, const string & s,
-    const Path & basePath);
+Expr * parseExprFromString(EvalState & state, const string & s, const Path & basePath);
 
 
 }

src/libexpr/parser.y

@@ -7,171 +7,126 @@
 %parse-param { yyscan_t scanner }
 %parse-param { ParseData * data }
 %lex-param { yyscan_t scanner }
+%lex-param { ParseData * data }
 
-
-%{
-/* Newer versions of Bison copy the declarations below to
-   parser-tab.hh, which sucks bigtime since lexer.l doesn't want that
-   stuff.  So allow it to be excluded. */
-#ifndef BISON_HEADER_HACK
-#define BISON_HEADER_HACK
+%code requires {
+    
+#ifndef BISON_HEADER
+#define BISON_HEADER
     
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "aterm.hh"
 #include "util.hh"
     
+#include "nixexpr.hh"
+
+namespace nix {
+
+    struct ParseData 
+    {
+        SymbolTable & symbols;
+        Expr * result;
+        Path basePath;
+        Path path;
+        string error;
+        Symbol sLetBody;
+        ParseData(SymbolTable & symbols)
+            : symbols(symbols)
+            , sLetBody(symbols.create("<let-body>"))
+            { };
+    };
+    
+}
+
+#define YY_DECL int yylex \
+    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data)
+
+#endif
+
+}
+
+%{
+
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
 #define YYSTYPE YYSTYPE // workaround a bug in Bison 2.4
 
-#include "nixexpr.hh"
-#include "nixexpr-ast.hh"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
+YY_DECL;
 
 using namespace nix;
 
 
 namespace nix {
-
     
-struct ParseData 
-{
-    Expr result;
-    Path basePath;
-    Path path;
-    string error;
-};
- 
 
-static string showAttrPath(ATermList attrPath)
+static string showAttrPath(const vector<Symbol> & attrPath)
 {
     string s;
-    for (ATermIterator i(attrPath); i; ++i) {
+    foreach (vector<Symbol>::const_iterator, i, attrPath) {
         if (!s.empty()) s += '.';
-        s += aterm2String(*i);
+        s += *i;
     }
     return s;
 }
- 
 
-struct Tree
+
+static void dupAttr(const vector<Symbol> & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    Expr leaf; ATerm pos; bool recursive;
-    typedef std::map<ATerm, Tree> Children;
-    Children children;
-    Tree() { leaf = 0; recursive = true; }
-};
-
-
-static ATermList buildAttrs(const Tree & t, ATermList & nonrec)
-{
-    ATermList res = ATempty;
-    for (Tree::Children::const_reverse_iterator i = t.children.rbegin();
-         i != t.children.rend(); ++i)
-        if (!i->second.recursive)
-            nonrec = ATinsert(nonrec, makeBind(i->first, i->second.leaf, i->second.pos));
-        else
-            res = ATinsert(res, i->second.leaf
-                ? makeBind(i->first, i->second.leaf, i->second.pos)
-                : makeBind(i->first, makeAttrs(buildAttrs(i->second, nonrec)), makeNoPos()));
-    return res;
+    throw ParseError(format("attribute `%1%' at %2% already defined at %3%")
+        % showAttrPath(attrPath) % pos % prevPos);
 }
  
 
-static Expr fixAttrs(bool recursive, ATermList as)
+static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    Tree attrs;
+    vector<Symbol> attrPath; attrPath.push_back(attr);
+    throw ParseError(format("attribute `%1%' at %2% already defined at %3%")
+        % showAttrPath(attrPath) % pos % prevPos);
+}
+ 
 
-    for (ATermIterator i(as); i; ++i) {
-        ATermList names, attrPath; Expr src, e; ATerm name, pos;
-
-        if (matchInherit(*i, src, names, pos)) {
-            bool fromScope = matchScope(src);
-            for (ATermIterator j(names); j; ++j) {
-                if (attrs.children.find(*j) != attrs.children.end()) 
-                    throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
-                        % showAttrPath(ATmakeList1(*j)) % showPos(pos));
-                Tree & t(attrs.children[*j]);
-                t.leaf = fromScope ? makeVar(*j) : makeSelect(src, *j);
-                t.pos = pos;
-                if (recursive && fromScope) t.recursive = false;
+static void addAttr(ExprAttrs * attrs, const vector<Symbol> & attrPath,
+    Expr * e, const Pos & pos)
+{
+    unsigned int n = 0;
+    foreach (vector<Symbol>::const_iterator, i, attrPath) {
+        n++;
+        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(*i);
+        if (j != attrs->attrs.end()) {
+            if (!j->second.inherited) {
+                ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
+                if (!attrs2 || n == attrPath.size()) dupAttr(attrPath, pos, j->second.pos);
+                attrs = attrs2;
+            } else
+                dupAttr(attrPath, pos, j->second.pos);
+        } else {
+            if (n == attrPath.size())
+                attrs->attrs[*i] = ExprAttrs::AttrDef(e, pos);
+            else {
+                ExprAttrs * nested = new ExprAttrs;
+                attrs->attrs[*i] = ExprAttrs::AttrDef(nested, pos);
+                attrs = nested;
             }
         }
-
-        else if (matchBindAttrPath(*i, attrPath, e, pos)) {
-
-            Tree * t(&attrs);
-            
-            for (ATermIterator j(attrPath); j; ) {
-                name = *j; ++j;
-                if (t->leaf) throw ParseError(format("attribute set containing `%1%' at %2% already defined at %3%")
-                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
-                t = &(t->children[name]);
-            }
-
-            if (t->leaf)
-                throw ParseError(format("duplicate definition of attribute `%1%' at %2% and %3%")
-                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
-            if (!t->children.empty())
-                throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
-                    % showAttrPath(attrPath) % showPos(pos));
-
-            t->leaf = e; t->pos = pos;
-        }
-
-        else abort(); /* can't happen */
     }
-
-    ATermList nonrec = ATempty;
-    ATermList rec = buildAttrs(attrs, nonrec);
-        
-    return recursive ? makeRec(rec, nonrec) : makeAttrs(rec);
 }
 
 
-static void checkPatternVars(ATerm pos, ATermMap & map, Pattern pat)
+static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
-    ATerm name;
-    ATermList formals;
-    Pattern pat1, pat2;
-    ATermBool ellipsis;
-    if (matchVarPat(pat, name)) {
-        if (map.get(name))
-            throw ParseError(format("duplicate formal function argument `%1%' at %2%")
-                % aterm2String(name) % showPos(pos));
-        map.set(name, name);
-    }
-    else if (matchAttrsPat(pat, formals, ellipsis)) { 
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1;
-            if (!matchFormal(*i, name, d1)) abort();
-            if (map.get(name))
-                throw ParseError(format("duplicate formal function argument `%1%' at %2%")
-                    % aterm2String(name) % showPos(pos));
-            map.set(name, name);
-        }
-    }
-    else if (matchAtPat(pat, pat1, pat2)) {
-        checkPatternVars(pos, map, pat1);
-        checkPatternVars(pos, map, pat2);
-    }
-    else abort();
+    if (formals->argNames.find(formal.name) != formals->argNames.end())
+        throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+            % formal.name % pos);
+    formals->formals.push_front(formal);
+    formals->argNames.insert(formal.name);
 }
 
 
-static void checkPatternVars(ATerm pos, Pattern pat)
+static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
 {
-    ATermMap map;
-    checkPatternVars(pos, map, pat);
-}
-
-
-static Expr stripIndentation(ATermList es)
-{
-    if (es == ATempty) return makeStr("");
+    if (es.empty()) return new ExprString(symbols.create(""));
     
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
@@ -179,9 +134,9 @@ static Expr stripIndentation(ATermList es)
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
     unsigned int minIndent = 1000000;
     unsigned int curIndent = 0;
-    ATerm e;
-    for (ATermIterator i(es); i; ++i) {
-        if (!matchIndStr(*i, e)) {
+    foreach (vector<Expr *>::iterator, i, es) {
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+        if (!e) {
             /* Anti-quotations end the current start-of-line whitespace. */
             if (atStartOfLine) {
                 atStartOfLine = false;
@@ -189,12 +144,11 @@ static Expr stripIndentation(ATermList es)
             }
             continue;
         }
-        string s = aterm2String(e);
-        for (unsigned int j = 0; j < s.size(); ++j) {
+        for (unsigned int j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
-                if (s[j] == ' ')
+                if (e->s[j] == ' ')
                     curIndent++;
-                else if (s[j] == '\n') {
+                else if (e->s[j] == '\n') {
                     /* Empty line, doesn't influence minimum
                        indentation. */
                     curIndent = 0;
@@ -202,7 +156,7 @@ static Expr stripIndentation(ATermList es)
                     atStartOfLine = false;
                     if (curIndent < minIndent) minIndent = curIndent;
                 }
-            } else if (s[j] == '\n') {
+            } else if (e->s[j] == '\n') {
                 atStartOfLine = true;
                 curIndent = 0;
             }
@@ -210,37 +164,37 @@ static Expr stripIndentation(ATermList es)
     }
 
     /* Strip spaces from each line. */
-    ATermList es2 = ATempty;
+    vector<Expr *> * es2 = new vector<Expr *>;
     atStartOfLine = true;
     unsigned int curDropped = 0;
-    unsigned int n = ATgetLength(es);
-    for (ATermIterator i(es); i; ++i, --n) {
-        if (!matchIndStr(*i, e)) {
+    unsigned int n = es.size();
+    for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+        if (!e) {
             atStartOfLine = false;
             curDropped = 0;
-            es2 = ATinsert(es2, *i);
+            es2->push_back(*i);
             continue;
         }
         
-        string s = aterm2String(e);
         string s2;
-        for (unsigned int j = 0; j < s.size(); ++j) {
+        for (unsigned int j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
-                if (s[j] == ' ') {
+                if (e->s[j] == ' ') {
                     if (curDropped++ >= minIndent)
-                        s2 += s[j];
+                        s2 += e->s[j];
                 }
-                else if (s[j] == '\n') {
+                else if (e->s[j] == '\n') {
                     curDropped = 0;
-                    s2 += s[j];
+                    s2 += e->s[j];
                 } else {
                     atStartOfLine = false;
                     curDropped = 0;
-                    s2 += s[j];
+                    s2 += e->s[j];
                 }
             } else {
-                s2 += s[j];
-                if (s[j] == '\n') atStartOfLine = true;
+                s2 += e->s[j];
+                if (e->s[j] == '\n') atStartOfLine = true;
             }
         }
 
@@ -251,11 +205,11 @@ static Expr stripIndentation(ATermList es)
             if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos)
                 s2 = string(s2, 0, p + 1);
         }
-            
-        es2 = ATinsert(es2, makeStr(s2));
+
+        es2->push_back(new ExprString(symbols.create(s2)));
     }
 
-    return makeConcatStrings(ATreverse(es2));
+    return new ExprConcatStrings(es2);
 }
 
 
@@ -263,13 +217,12 @@ void backToString(yyscan_t scanner);
 void backToIndString(yyscan_t scanner);
 
 
-static Pos makeCurPos(YYLTYPE * loc, ParseData * data)
+static Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return makePos(toATerm(data->path),
-        loc->first_line, loc->first_column);
+    return Pos(data->path, loc.first_line, loc.first_column);
 }
 
-#define CUR_POS makeCurPos(yylocp, data)
+#define CUR_POS makeCurPos(*yylocp, data)
 
 
 }
@@ -277,50 +230,40 @@ static Pos makeCurPos(YYLTYPE * loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = (format("%1%, at `%2%':%3%:%4%")
-        % error % data->path % loc->first_line % loc->first_column).str();
+    data->error = (format("%1%, at %2%")
+        % error % makeCurPos(*loc, data)).str();
 }
 
 
-/* Make sure that the parse stack is scanned by the ATerm garbage
-   collector. */
-static void * mallocAndProtect(size_t size)
-{
-    void * p = malloc(size);
-    if (p) ATprotectMemory(p, size);
-    return p;
-}
-
-static void freeAndUnprotect(void * p)
-{
-    ATunprotectMemory(p);
-    free(p);
-}
-
-#define YYMALLOC mallocAndProtect
-#define YYFREE freeAndUnprotect
-
-
-#endif
-
-
 %}
 
 %union {
-  ATerm t;
-  ATermList ts;
-  struct {
-    ATermList formals;
-    bool ellipsis;
-  } formals;
+  nix::Expr * e;
+  nix::ExprList * list;
+  nix::ExprAttrs * attrs;
+  nix::Formals * formals;
+  nix::Formal * formal;
+  int n;
+  char * id; // !!! -> Symbol
+  char * path;
+  char * uri;
+  std::vector<nix::Symbol> * ids;
+  std::vector<nix::Expr *> * string_parts;
 }
 
-%type <t> start expr expr_function expr_if expr_op
-%type <t> expr_app expr_select expr_simple bind inheritsrc formal
-%type <t> pattern pattern2
-%type <ts> binds ids attrpath expr_list string_parts ind_string_parts
+%type <e> start expr expr_function expr_if expr_op
+%type <e> expr_app expr_select expr_simple
+%type <list> expr_list
+%type <attrs> binds
 %type <formals> formals
-%token <t> ID INT STR IND_STR PATH URI
+%type <formal> formal
+%type <ids> ids attrpath
+%type <string_parts> string_parts ind_string_parts
+%token <id> ID ATTRPATH
+%token <e> STR IND_STR
+%token <n> INT
+%token <path> PATH
+%token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
@@ -344,163 +287,170 @@ start: expr { data->result = $1; };
 expr: expr_function;
 
 expr_function
-  : pattern ':' expr_function
-    { checkPatternVars(CUR_POS, $1); $$ = makeFunction($1, $3, CUR_POS); }
+  : ID ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
+  | '{' formals '}' ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
+  | '{' formals '}' '@' ID ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
+  | ID '@' '{' formals '}' ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
   | ASSERT expr ';' expr_function
-    { $$ = makeAssert($2, $4, CUR_POS); }
+    { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
-    { $$ = makeWith($2, $4, CUR_POS); }
+    { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
-    { $$ = makeSelect(fixAttrs(true, ATinsert($2, makeBindAttrPath(ATmakeList1(toATerm("<let-body>")), $4, CUR_POS))), toATerm("<let-body>")); }
+    { $$ = new ExprLet($2, $4); }
   | expr_if
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr
-    { $$ = makeIf($2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
   | expr_op
   ;
 
 expr_op
-  : '!' expr_op %prec NEG { $$ = makeOpNot($2); }
-  | expr_op EQ expr_op { $$ = makeOpEq($1, $3); }
-  | expr_op NEQ expr_op { $$ = makeOpNEq($1, $3); }
-  | expr_op AND expr_op { $$ = makeOpAnd($1, $3); }
-  | expr_op OR expr_op { $$ = makeOpOr($1, $3); }
-  | expr_op IMPL expr_op { $$ = makeOpImpl($1, $3); }
-  | expr_op UPDATE expr_op { $$ = makeOpUpdate($1, $3); }
-  | expr_op '~' expr_op { $$ = makeSubPath($1, $3); }
-  | expr_op '?' ID { $$ = makeOpHasAttr($1, $3); }
-  | expr_op '+' expr_op { $$ = makeOpPlus($1, $3); }
-  | expr_op CONCAT expr_op { $$ = makeOpConcat($1, $3); }
+  : '!' expr_op %prec NEG { $$ = new ExprOpNot($2); }
+  | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
+  | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
+  | expr_op AND expr_op { $$ = new ExprOpAnd($1, $3); }
+  | expr_op OR expr_op { $$ = new ExprOpOr($1, $3); }
+  | expr_op IMPL expr_op { $$ = new ExprOpImpl($1, $3); }
+  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate($1, $3); }
+  | expr_op '?' ID { $$ = new ExprOpHasAttr($1, data->symbols.create($3)); }
+  | expr_op '+' expr_op
+    { vector<Expr *> * l = new vector<Expr *>;
+      l->push_back($1);
+      l->push_back($3);
+      $$ = new ExprConcatStrings(l);
+    }
+  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists($1, $3); }
   | expr_app
   ;
 
 expr_app
   : expr_app expr_select
-    { $$ = makeCall($1, $2); }
+    { $$ = new ExprApp($1, $2); }
   | expr_select { $$ = $1; }
   ;
 
 expr_select
   : expr_select '.' ID
-    { $$ = makeSelect($1, $3); }
+    { $$ = new ExprSelect($1, data->symbols.create($3)); }
   | expr_simple { $$ = $1; }
   ;
 
 expr_simple
-  : ID { $$ = makeVar($1); }
-  | INT { $$ = makeInt(ATgetInt((ATermInt) $1)); }
+  : ID { $$ = new ExprVar(data->symbols.create($1)); }
+  | INT { $$ = new ExprInt($1); }
   | '"' string_parts '"' {
       /* For efficiency, and to simplify parse trees a bit. */
-      if ($2 == ATempty) $$ = makeStr(toATerm(""), ATempty);
-      else if (ATgetNext($2) == ATempty) $$ = ATgetFirst($2);
-      else $$ = makeConcatStrings(ATreverse($2));
+      if ($2->empty()) $$ = new ExprString(data->symbols.create(""));
+      else if ($2->size() == 1) $$ = $2->front();
+      else $$ = new ExprConcatStrings($2);
   }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = stripIndentation(ATreverse($2));
+      $$ = stripIndentation(data->symbols, *$2);
   }
-  | PATH { $$ = makePath(toATerm(absPath(aterm2String($1), data->basePath))); }
-  | URI { $$ = makeStr($1, ATempty); }
+  | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
+  | URI { $$ = new ExprString(data->symbols.create($1)); }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
   | LET '{' binds '}'
-    { $$ = makeSelect(fixAttrs(true, $3), toATerm("body")); }
+    { $3->recursive = true; $$ = new ExprSelect($3, data->symbols.create("body")); }
   | REC '{' binds '}'
-    { $$ = fixAttrs(true, $3); }
+    { $3->recursive = true; $$ = $3; }
   | '{' binds '}'
-    { $$ = fixAttrs(false, $2); }
-  | '[' expr_list ']' { $$ = makeList(ATreverse($2)); }
+    { $$ = $2; }
+  | '[' expr_list ']' { $$ = $2; }
   ;
 
 string_parts
-  : string_parts STR { $$ = ATinsert($1, $2); }
-  | string_parts DOLLAR_CURLY expr '}' { backToString(scanner); $$ = ATinsert($1, $3); }
-  | { $$ = ATempty; }
+  : string_parts STR { $$ = $1; $1->push_back($2); }
+  | string_parts DOLLAR_CURLY expr '}' { backToString(scanner); $$ = $1; $1->push_back($3); }
+  | { $$ = new vector<Expr *>; }
   ;
 
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = ATinsert($1, $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { backToIndString(scanner); $$ = ATinsert($1, $3); }
-  | { $$ = ATempty; }
-  ;
-
-pattern
-  : pattern2 '@' pattern { $$ = makeAtPat($1, $3); }
-  | pattern2
-  ;
-
-pattern2
-  : ID { $$ = makeVarPat($1); }
-  | '{' formals '}' { $$ = makeAttrsPat($2.formals, $2.ellipsis ? eTrue : eFalse); }
+  : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { backToIndString(scanner); $$ = $1; $1->push_back($3); }
+  | { $$ = new vector<Expr *>; }
   ;
 
 binds
-  : binds bind { $$ = ATinsert($1, $2); }
-  | { $$ = ATempty; }
+  : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); }
+  | binds INHERIT ids ';'
+    { $$ = $1;
+      foreach (vector<Symbol>::iterator, i, *$3) {
+          if ($$->attrs.find(*i) != $$->attrs.end())
+              dupAttr(*i, makeCurPos(@3, data), $$->attrs[*i].pos);
+          Pos pos = makeCurPos(@3, data);
+          $$->attrs[*i] = ExprAttrs::AttrDef(*i, pos);
+      }
+    }
+  | binds INHERIT '(' expr ')' ids ';'
+    { $$ = $1;
+      /* !!! Should ensure sharing of the expression in $4. */
+      foreach (vector<Symbol>::iterator, i, *$6) {
+          if ($$->attrs.find(*i) != $$->attrs.end())
+              dupAttr(*i, makeCurPos(@6, data), $$->attrs[*i].pos);
+          $$->attrs[*i] = ExprAttrs::AttrDef(new ExprSelect($4, *i), makeCurPos(@6, data));
+      }}
+
+  | { $$ = new ExprAttrs; }
   ;
 
-bind
-  : attrpath '=' expr ';'
-    { $$ = makeBindAttrPath(ATreverse($1), $3, CUR_POS); }
-  | INHERIT inheritsrc ids ';'
-    { $$ = makeInherit($2, $3, CUR_POS); }
+ids
+  : ids ID { $$ = $1; $1->push_back(data->symbols.create($2)); /* !!! dangerous */ }
+  | { $$ = new vector<Symbol>; }
   ;
 
-inheritsrc
-  : '(' expr ')' { $$ = $2; }
-  | { $$ = makeScope(); }
-  ;
-
-ids: ids ID { $$ = ATinsert($1, $2); } | { $$ = ATempty; };
-
 attrpath
-  : attrpath '.' ID { $$ = ATinsert($1, $3); }
-  | ID { $$ = ATmakeList1($1); }
+  : attrpath '.' ID { $$ = $1; $1->push_back(data->symbols.create($3)); }
+  | ID { $$ = new vector<Symbol>; $$->push_back(data->symbols.create($1)); }
   ;
 
 expr_list
-  : expr_list expr_select { $$ = ATinsert($1, $2); }
-  | { $$ = ATempty; }
+  : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ }
+  | { $$ = new ExprList; }
   ;
 
 formals
-  : formal ',' formals /* !!! right recursive */
-    { $$.formals = ATinsert($3.formals, $1); $$.ellipsis = $3.ellipsis; }
+  : formal ',' formals
+    { $$ = $3; addFormal(CUR_POS, $$, *$1); }
   | formal
-    { $$.formals = ATinsert(ATempty, $1); $$.ellipsis = false; }
+    { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; }
   |
-    { $$.formals = ATempty; $$.ellipsis = false; }
+    { $$ = new Formals; $$->ellipsis = false; }
   | ELLIPSIS
-    { $$.formals = ATempty; $$.ellipsis = true; }
+    { $$ = new Formals; $$->ellipsis = true; }
   ;
 
 formal
-  : ID { $$ = makeFormal($1, makeNoDefaultValue()); }
-  | ID '?' expr { $$ = makeFormal($1, makeDefaultValue($3)); }
+  : ID { $$ = new Formal(data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
   ;
   
 %%
 
 
-#include "eval.hh"  
-
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
 
+#include <eval.hh>
+
 
 namespace nix {
       
 
-static Expr parse(EvalState & state,
-    const char * text, const Path & path,
-    const Path & basePath)
+static Expr * parse(EvalState & state, const char * text,
+    const Path & path, const Path & basePath)
 {
     yyscan_t scanner;
-    ParseData data;
+    ParseData data(state.symbols);
     data.basePath = basePath;
     data.path = path;
 
@@ -512,7 +462,7 @@ static Expr parse(EvalState & state,
     if (res) throw ParseError(data.error);
 
     try {
-        checkVarDefs(state.primOps, data.result);
+        data.result->bindVars(state.staticBaseEnv);
     } catch (Error & e) {
         throw ParseError(format("%1%, in `%2%'") % e.msg() % path);
     }
@@ -521,16 +471,10 @@ static Expr parse(EvalState & state,
 }
 
 
-Expr parseExprFromFile(EvalState & state, Path path)
+Expr * parseExprFromFile(EvalState & state, Path path)
 {
     assert(path[0] == '/');
 
-#if 0
-    /* Perhaps this is already an imploded parse tree? */
-    Expr e = ATreadFromNamedFile(path.c_str());
-    if (e) return e;
-#endif
-
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
     struct stat st;
@@ -542,8 +486,6 @@ Expr parseExprFromFile(EvalState & state, Path path)
     }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (stat(path.c_str(), &st))
-        throw SysError(format("getting status of `%1%'") % path);
     if (S_ISDIR(st.st_mode))
         path = canonPath(path + "/default.nix");
 
@@ -552,7 +494,7 @@ Expr parseExprFromFile(EvalState & state, Path path)
 }
 
 
-Expr parseExprFromString(EvalState & state,
+Expr * parseExprFromString(EvalState & state,
     const string & s, const Path & basePath)
 {
     return parse(state, s.c_str(), "(string)", basePath);

src/libexpr/symbol-table.hh

@@ -0,0 +1,92 @@
+#ifndef __SYMBOL_TABLE_H
+#define __SYMBOL_TABLE_H
+
+#include "config.h"
+
+#include <map>
+
+#if HAVE_TR1_UNORDERED_SET
+#include <tr1/unordered_set>
+#endif
+
+#include "types.hh"
+
+namespace nix {
+
+/* Symbol table used by the parser and evaluator to represent and look
+   up identifiers and attribute sets efficiently.
+   SymbolTable::create() converts a string into a symbol.  Symbols
+   have the property that they can be compared efficiently (using a
+   pointer equality test), because the symbol table stores only one
+   copy of each string. */
+
+class Symbol
+{
+private:
+    const string * s; // pointer into SymbolTable
+    Symbol(const string * s) : s(s) { };
+    friend class SymbolTable;
+
+public:
+    Symbol() : s(0) { };
+    
+    bool operator == (const Symbol & s2) const
+    {
+        return s == s2.s;
+    }
+    
+    bool operator != (const Symbol & s2) const
+    {
+        return s != s2.s;
+    }
+    
+    bool operator < (const Symbol & s2) const
+    {
+        return s < s2.s;
+    }
+
+    operator const string & () const
+    {
+        return *s;
+    }
+
+    bool empty() const
+    {
+        return s->empty();
+    }
+
+    friend std::ostream & operator << (std::ostream & str, const Symbol & sym);
+};
+
+inline std::ostream & operator << (std::ostream & str, const Symbol & sym)
+{
+    str << *sym.s;
+    return str;
+}
+
+class SymbolTable
+{
+private:
+#if HAVE_TR1_UNORDERED_SET 
+    typedef std::tr1::unordered_set<string> Symbols;
+#else
+    typedef std::set<string> Symbols;
+#endif
+    Symbols symbols;
+
+public:
+    Symbol create(const string & s)
+    {
+        std::pair<Symbols::iterator, bool> res = symbols.insert(s);
+        return Symbol(&*res.first);
+    }
+
+    unsigned int size() const
+    {
+        return symbols.size();
+    }
+};
+
+}
+
+#endif /* !__SYMBOL_TABLE_H */

src/libexpr/value-to-xml.cc

@@ -0,0 +1,162 @@
+#include "value-to-xml.hh"
+#include "xml-writer.hh"
+#include "util.hh"
+
+#include <cstdlib>
+
+
+namespace nix {
+
+    
+static XMLAttrs singletonAttrs(const string & name, const string & value)
+{
+    XMLAttrs attrs;
+    attrs[name] = value;
+    return attrs;
+}
+
+
+static void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen);
+
+
+static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
+{
+    xmlAttrs["path"] = pos.file;
+    xmlAttrs["line"] = (format("%1%") % pos.line).str();
+    xmlAttrs["column"] = (format("%1%") % pos.column).str();
+}
+
+
+static void showAttrs(EvalState & state, bool strict, bool location,
+    Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
+{
+    StringSet names;
+    
+    foreach (Bindings::iterator, i, attrs)
+        names.insert(i->name);
+    
+    foreach (StringSet::iterator, i, names) {
+        Attr & a(*attrs.find(state.symbols.create(*i)));
+        
+        XMLAttrs xmlAttrs;
+        xmlAttrs["name"] = *i;
+        if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
+        
+        XMLOpenElement _(doc, "attr", xmlAttrs);
+        printValueAsXML(state, strict, location,
+            *a.value, doc, context, drvsSeen);
+    }
+}
+
+
+static void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
+{
+    checkInterrupt();
+
+    if (strict) state.forceValue(v);
+        
+    switch (v.type) {
+
+        case tInt:
+            doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
+            break;
+
+        case tBool:
+            doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
+            break;
+
+        case tString:
+            /* !!! show the context? */
+            copyContext(v, context);
+            doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
+            break;
+
+        case tPath:
+            doc.writeEmptyElement("path", singletonAttrs("value", v.path));
+            break;
+
+        case tNull:
+            doc.writeEmptyElement("null");
+            break;
+
+        case tAttrs:
+            if (state.isDerivation(v)) {
+                XMLAttrs xmlAttrs;
+            
+                Bindings::iterator a = v.attrs->find(state.symbols.create("derivation"));
+
+                Path drvPath;
+                a = v.attrs->find(state.sDrvPath);
+                if (a != v.attrs->end()) {
+                    if (strict) state.forceValue(*a->value);
+                    if (a->value->type == tString)
+                        xmlAttrs["drvPath"] = drvPath = a->value->string.s;
+                }
+        
+                a = v.attrs->find(state.sOutPath);
+                if (a != v.attrs->end()) {
+                    if (strict) state.forceValue(*a->value);
+                    if (a->value->type == tString)
+                        xmlAttrs["outPath"] = a->value->string.s;
+                }
+
+                XMLOpenElement _(doc, "derivation", xmlAttrs);
+
+                if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) {
+                    drvsSeen.insert(drvPath);
+                    showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
+                } else
+                    doc.writeEmptyElement("repeated");
+            }
+
+            else {
+                XMLOpenElement _(doc, "attrs");
+                showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
+            }
+            
+            break;
+
+        case tList: {
+            XMLOpenElement _(doc, "list");
+            for (unsigned int n = 0; n < v.list.length; ++n)
+                printValueAsXML(state, strict, location, *v.list.elems[n], doc, context, drvsSeen);
+            break;
+        }
+
+        case tLambda: {
+            XMLAttrs xmlAttrs;
+            if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
+            XMLOpenElement _(doc, "function", xmlAttrs);
+            
+            if (v.lambda.fun->matchAttrs) {
+                XMLAttrs attrs;
+                if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
+                if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
+                XMLOpenElement _(doc, "attrspat", attrs);
+                foreach (Formals::Formals_::iterator, i, v.lambda.fun->formals->formals)
+                    doc.writeEmptyElement("attr", singletonAttrs("name", i->name));
+            } else
+                doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg));
+            
+            break;
+        }
+
+        default:
+            doc.writeEmptyElement("unevaluated");
+    }
+}
+
+
+void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, std::ostream & out, PathSet & context)
+{
+    XMLWriter doc(true, out);
+    XMLOpenElement root(doc, "expr");
+    PathSet drvsSeen;    
+    printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
+}
+
+ 
+}

src/libexpr/value-to-xml.hh

@@ -0,0 +1,17 @@
+#ifndef __VALUE_TO_XML_H
+#define __VALUE_TO_XML_H
+
+#include "nixexpr.hh"
+#include "eval.hh"
+
+#include <string>
+#include <map>
+
+namespace nix {
+
+void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, std::ostream & out, PathSet & context);
+    
+}
+
+#endif /* !__VALUE_TO_XML_H */

src/libmain/Makefile.am

@@ -2,7 +2,7 @@ pkglib_LTLIBRARIES = libmain.la
 
 libmain_la_SOURCES = shared.cc
 
-libmain_la_LIBADD = ../libstore/libstore.la
+libmain_la_LIBADD = ../libstore/libstore.la @boehmgc_lib@
 
 pkginclude_HEADERS = shared.hh
 
@@ -15,5 +15,5 @@ AM_CXXFLAGS = \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
  -DNIX_VERSION=\"$(VERSION)\" \
- -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil \
+ -I$(srcdir)/.. -I$(srcdir)/../libutil \
  -I$(srcdir)/../libstore

src/libmain/shared.cc

@@ -13,7 +13,9 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-#include <aterm2.h>
+#if HAVE_BOEHMGC
+#include <gc/gc.h>
+#endif
 
 
 namespace nix {
@@ -87,21 +89,6 @@ static void setLogType(string lt)
 }
 
 
-unsigned long long getIntArg(const string & opt,
-    Strings::iterator & i, const Strings::iterator & end)
-{
-    ++i;
-    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
-    long long n;
-    if (!string2Int(*i, n) || n < 0)
-        throw UsageError(format("`%1%' requires a non-negative integer") % opt);
-    return n;
-}
-
-
-void initDerivationsHelpers();
-
-
 static void closeStore()
 {
     try {
@@ -152,26 +139,33 @@ static void initAndRun(int argc, char * * argv)
     /* Get some settings from the configuration file. */
     thisSystem = querySetting("system", SYSTEM);
     maxBuildJobs = queryIntSetting("build-max-jobs", 1);
+    buildCores = queryIntSetting("build-cores", 1);
     maxSilentTime = queryIntSetting("build-max-silent-time", 0);
 
     /* Catch SIGINT. */
-    struct sigaction act, oact;
+    struct sigaction act;
     act.sa_handler = sigintHandler;
     sigfillset(&act.sa_mask);
     act.sa_flags = 0;
-    if (sigaction(SIGINT, &act, &oact))
+    if (sigaction(SIGINT, &act, 0))
         throw SysError("installing handler for SIGINT");
-    if (sigaction(SIGTERM, &act, &oact))
+    if (sigaction(SIGTERM, &act, 0))
         throw SysError("installing handler for SIGTERM");
-    if (sigaction(SIGHUP, &act, &oact))
+    if (sigaction(SIGHUP, &act, 0))
         throw SysError("installing handler for SIGHUP");
 
     /* Ignore SIGPIPE. */
     act.sa_handler = SIG_IGN;
     act.sa_flags = 0;
-    if (sigaction(SIGPIPE, &act, &oact))
+    if (sigaction(SIGPIPE, &act, 0))
         throw SysError("ignoring SIGPIPE");
 
+    /* Reset SIGCHLD to its default. */
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    if (sigaction(SIGCHLD, &act, 0))
+        throw SysError("resetting SIGCHLD");
+
     /* There is no privacy in the Nix system ;-)  At least not for
        now.  In particular, store objects should be readable by
        everybody.  This prevents nasty surprises when using a shared
@@ -182,9 +176,6 @@ static void initAndRun(int argc, char * * argv)
     string lt = getEnv("NIX_LOG_TYPE");
     if (lt != "") setLogType(lt);
 
-    /* ATerm stuff.  !!! find a better place to put this */
-    initDerivationsHelpers();
-    
     /* Put the arguments in a vector. */
     Strings args, remaining;
     while (argc--) args.push_back(*argv++);
@@ -195,7 +186,7 @@ static void initAndRun(int argc, char * * argv)
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
         if (string(arg, 0, 4) == "-at-") ;
-        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-') {
+        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && !isdigit(arg[1])) {
             for (unsigned int j = 1; j < arg.length(); j++)
                 if (isalpha(arg[j]))
                     remaining.push_back((string) "-" + arg[j]);
@@ -239,11 +230,13 @@ static void initAndRun(int argc, char * * argv)
         else if (arg == "--fallback")
             tryFallback = true;
         else if (arg == "--max-jobs" || arg == "-j")
-            maxBuildJobs = getIntArg(arg, i, args.end());
+            maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end());
+        else if (arg == "--cores")
+            buildCores = getIntArg<unsigned int>(arg, i, args.end());
         else if (arg == "--readonly-mode")
             readOnlyMode = true;
         else if (arg == "--max-silent-time")
-            maxSilentTime = getIntArg(arg, i, args.end());
+            maxSilentTime = getIntArg<unsigned int>(arg, i, args.end());
         else if (arg == "--no-build-hook")
             useBuildHook = false;
         else if (arg == "--show-trace")
@@ -325,6 +318,14 @@ static void setuidInit()
 }
 
 
+/* Called when the Boehm GC runs out of memory. */
+static void * oomHandler(size_t requested)
+{
+    /* Convert this to a proper C++ exception. */
+    throw std::bad_alloc();
+}
+
+
 }
 
 
@@ -339,10 +340,6 @@ int main(int argc, char * * argv)
     if (argc == 0) abort();
     setuidInit();
     
-    /* ATerm setup. */
-    ATerm bottomOfStack;
-    ATinit(argc, argv, &bottomOfStack);
-
     /* Turn on buffering for cerr. */
 #if HAVE_PUBSETBUF
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
@@ -350,6 +347,26 @@ int main(int argc, char * * argv)
 
     std::ios::sync_with_stdio(false);
 
+#if HAVE_BOEHMGC
+    /* Initialise the Boehm garbage collector.  This isn't necessary
+       on most platforms, but for portability we do it anyway. */
+    GC_INIT();
+
+    GC_oom_fn = oomHandler;
+
+    /* Set the initial heap size to something fairly big (384 MiB) so
+       that in most cases we don't need to garbage collect at all.
+       (Collection has a fairly significant overhead, some.)  The heap
+       size can be overriden through libgc's GC_INITIAL_HEAP_SIZE
+       environment variable.  We should probably also provide a
+       nix.conf setting for this.  Note that GC_expand_hp() causes a
+       lot of virtual, but not physical (resident) memory to be
+       allocated.  This might be a problem on systems that don't
+       overcommit. */
+    if (!getenv("GC_INITIAL_HEAP_SIZE"))
+        GC_expand_hp(384 * 1024 * 1024);
+#endif
+
     try {
         try {
             initAndRun(argc, argv);

src/libmain/shared.hh

@@ -22,22 +22,30 @@ extern std::string programId;
 
 namespace nix {
 
+MakeError(UsageError, nix::Error);
+
 /* Ugh.  No better place to put this. */
 Path makeRootName(const Path & gcRoot, int & counter);
 void printGCWarning();
 
 void printMissing(const PathSet & paths);
 
-unsigned long long getIntArg(const string & opt,
-    Strings::iterator & i, const Strings::iterator & end);
+template<class N> N getIntArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end)
+{
+    ++i;
+    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
+    N n;
+    if (!string2Int(*i, n))
+        throw UsageError(format("`%1%' requires an integer argument") % opt);
+    return n;
+}
 
 /* Whether we're running setuid. */
 extern bool setuidMode;
 
 extern volatile ::sig_atomic_t blockInt;
 
-MakeError(UsageError, nix::Error);
-
 struct RemoveTempRoots 
 {
     ~RemoveTempRoots();    

src/libstore/Makefile.am

@@ -2,22 +2,15 @@ pkglib_LTLIBRARIES = libstore.la
 
 libstore_la_SOURCES = \
   store-api.cc local-store.cc remote-store.cc derivations.cc build.cc misc.cc \
-  globals.cc db.cc references.cc pathlocks.cc gc.cc upgrade-schema.cc \
+  globals.cc references.cc pathlocks.cc gc.cc \
   optimise-store.cc
 
 pkginclude_HEADERS = \
   store-api.hh local-store.hh remote-store.hh derivations.hh misc.hh \
-  globals.hh db.hh references.hh pathlocks.hh \
+  globals.hh references.hh pathlocks.hh \
   worker-protocol.hh
 
-libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la
-
-BUILT_SOURCES = derivations-ast.cc derivations-ast.hh
-
-EXTRA_DIST = derivations-ast.def derivations-ast.cc
+libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@
 
 AM_CXXFLAGS = -Wall \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} -I$(srcdir)/../libutil
-
-derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
-	$(perl) $(srcdir)/../aterm-helper.pl derivations-ast.hh derivations-ast.cc < $(srcdir)/derivations-ast.def
+ -I$(srcdir)/.. -I$(srcdir)/../libutil

src/libstore/build.cc

@@ -1,9 +1,12 @@
+#include "config.h"
+
 #include "references.hh"
 #include "pathlocks.hh"
 #include "misc.hh"
 #include "globals.hh"
 #include "local-store.hh"
 #include "util.hh"
+#include "archive.hh"
 
 #include <map>
 #include <iostream>
@@ -15,19 +18,20 @@
 
 #include <time.h>
 #include <sys/time.h>
+#include <sys/wait.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
+#include <stdio.h>
+#include <cstring>
 
 #include <pwd.h>
 #include <grp.h>
 
 
 /* Includes required for chroot support. */
-#include "config.h"
-
 #if HAVE_SYS_PARAM_H
 #include <sys/param.h>
 #endif
@@ -159,6 +163,7 @@ struct Child
 {
     WeakGoalPtr goal;
     set<int> fds;
+    bool monitorForSilence;
     bool inBuildSlot;
     time_t lastOutput; /* time we last got output on stdout/stderr */
 };
@@ -231,7 +236,7 @@ public:
     /* Registers a running child process.  `inBuildSlot' means that
        the process counts towards the jobs limit. */
     void childStarted(GoalPtr goal, pid_t pid,
-        const set<int> & fds, bool inBuildSlot);
+        const set<int> & fds, bool inBuildSlot, bool monitorForSilence);
 
     /* Unregisters a running child process.  `wakeSleepers' should be
        false if there is no sense in waking up goals that are sleeping
@@ -263,7 +268,7 @@ public:
 
 
 MakeError(SubstError, Error)
-MakeError(BuildError, Error) /* denoted a permanent build failure */
+MakeError(BuildError, Error) /* denotes a permanent build failure */
 
 
 //////////////////////////////////////////////////////////////////////
@@ -466,6 +471,8 @@ void UserLock::acquire()
         if (!pw)
             throw Error(format("the user `%1%' in the group `%2%' does not exist")
                 % *i % buildUsersGroup);
+
+        createDirs(nixStateDir + "/userpool");
         
         fnUserLock = (format("%1%/userpool/%2%") % nixStateDir % pw->pw_uid).str();
 
@@ -791,6 +798,9 @@ void DerivationGoal::init()
 {
     trace("init");
 
+    if (readOnlyMode)
+        throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
+
     /* The first thing to do is to make sure that the derivation
        exists.  If it doesn't, it may be created through a
        substitute. */
@@ -1254,7 +1264,7 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
     pid.setKillSignal(SIGTERM);
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(),
-        pid, singleton<set<int> >(logPipe.readSide), false);
+        pid, singleton<set<int> >(logPipe.readSide), false, false);
 
     toHook.readSide.close();
 
@@ -1310,18 +1320,18 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
         string s;
         foreach (PathSet::iterator, i, allInputs) s += *i + "\n";
         
-        writeStringToFile(inputListFN, s);
+        writeFile(inputListFN, s);
 
         /* The `outputs' file lists all outputs that have to be copied
            from the remote system. */
         s = "";
         foreach (DerivationOutputs::iterator, i, drv.outputs)
             s += i->second.path + "\n";
-        writeStringToFile(outputListFN, s);
+        writeFile(outputListFN, s);
 
         /* The `references' file has exactly the format accepted by
            `nix-store --register-validity'. */
-        writeStringToFile(referencesFN,
+        writeFile(referencesFN,
             makeValidityRegistration(allInputs, true, false));
 
         /* Tell the hook to proceed. */
@@ -1402,6 +1412,9 @@ void DerivationGoal::startBuilder()
        in the store or in the build directory). */
     env["NIX_STORE"] = nixStore;
 
+    /* The maximum number of cores to utilize for parallel building. */
+    env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str();
+
     /* Add all bindings specified in the derivation. */
     foreach (StringPairs::iterator, i, drv.env)
         env[i->first] = i->second;
@@ -1485,7 +1498,7 @@ void DerivationGoal::startBuilder()
         }
 
         /* Write closure info to `fileName'. */
-        writeStringToFile(tmpDir + "/" + fileName,
+        writeFile(tmpDir + "/" + fileName,
             makeValidityRegistration(paths, false, false));
     }
 
@@ -1557,16 +1570,23 @@ void DerivationGoal::startBuilder()
         createDirs(chrootTmpDir);
         chmod(chrootTmpDir, 01777);
 
-        /* Create a /etc/passwd with entries for the build user and
-           the nobody account.  The latter is kind of a hack to
-           support Samba-in-QEMU. */
+        /* Create a /etc/passwd with entries for the build user and the
+           nobody account.  The latter is kind of a hack to support
+           Samba-in-QEMU.  */
         createDirs(chrootRootDir + "/etc");
 
-        writeStringToFile(chrootRootDir + "/etc/passwd",
+        writeFile(chrootRootDir + "/etc/passwd",
             (format(
-                "nixbld:x:%1%:65534:Nix build user:/:/noshell\n"
+                "nixbld:x:%1%:%2%:Nix build user:/:/noshell\n"
                 "nobody:x:65534:65534:Nobody:/:/noshell\n")
-                % (buildUser.enabled() ? buildUser.getUID() : getuid())).str());
+                % (buildUser.enabled() ? buildUser.getUID() : getuid())
+	        % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
+
+	/* Declare the build user's group so that programs get a consistent
+	   view of the system (e.g., "id -gn").  */
+	writeFile(chrootRootDir + "/etc/group",
+		  (format("nixbld:!:%1%:\n")
+		   % (buildUser.enabled() ? buildUser.getGID() : getgid())).str());
 
         /* Bind-mount a user-configurable set of directories from the
            host file system.  The `/dev/pts' directory must be mounted
@@ -1600,8 +1620,18 @@ void DerivationGoal::startBuilder()
                 dirsInChroot.insert(*i);
             else {
                 Path p = chrootRootDir + *i;
-                if (link(i->c_str(), p.c_str()) == -1)
-                    throw SysError(format("linking `%1%' to `%2%'") % p % *i);
+                if (link(i->c_str(), p.c_str()) == -1) {
+                    /* Hard-linking fails if we exceed the maximum
+                       link count on a file (e.g. 32000 of ext3),
+                       which is quite possible after a `nix-store
+                       --optimise'.  Make a copy instead. */
+                    if (errno != EMLINK)
+                        throw SysError(format("linking `%1%' to `%2%'") % p % *i);
+                    StringSink sink;
+                    dumpPath(*i, sink);
+                    StringSource source(sink.s);
+                    restorePath(p, source);
+                }
             }
         }
         
@@ -1749,7 +1779,7 @@ void DerivationGoal::startBuilder()
     pid.setSeparatePG(true);
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(), pid,
-        singleton<set<int> >(logPipe.readSide), true);
+        singleton<set<int> >(logPipe.readSide), true, true);
 
     if (printBuildTrace) {
         printMsg(lvlError, format("@ build-started %1% %2% %3% %4%")
@@ -2107,6 +2137,9 @@ void SubstitutionGoal::init()
         return;
     }
 
+    if (readOnlyMode)
+        throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
+
     subs = substituters;
     
     tryNext();
@@ -2253,7 +2286,7 @@ void SubstitutionGoal::tryToRun()
     pid.setKillSignal(SIGTERM);
     logPipe.writeSide.close();
     worker.childStarted(shared_from_this(),
-        pid, singleton<set<int> >(logPipe.readSide), true);
+        pid, singleton<set<int> >(logPipe.readSide), true, true);
 
     state = &SubstitutionGoal::finished;
 
@@ -2453,13 +2486,15 @@ unsigned Worker::getNrLocalBuilds()
 
 
 void Worker::childStarted(GoalPtr goal,
-    pid_t pid, const set<int> & fds, bool inBuildSlot)
+    pid_t pid, const set<int> & fds, bool inBuildSlot,
+    bool monitorForSilence)
 {
     Child child;
     child.goal = goal;
     child.fds = fds;
     child.lastOutput = time(0);
     child.inBuildSlot = inBuildSlot;
+    child.monitorForSilence = monitorForSilence;
     children[pid] = child;
     if (inBuildSlot) nrLocalBuilds++;
 }
@@ -2580,12 +2615,16 @@ void Worker::waitForInput()
     if (maxSilentTime != 0) {
         time_t oldest = 0;
         foreach (Children::iterator, i, children) {
-            oldest = oldest == 0 || i->second.lastOutput < oldest
-                ? i->second.lastOutput : oldest;
+            if (i->second.monitorForSilence) {
+                oldest = oldest == 0 || i->second.lastOutput < oldest
+                    ? i->second.lastOutput : oldest;
+            }
+        }
+        if (oldest) {
+            useTimeout = true;
+            timeout.tv_sec = std::max((time_t) 0, oldest + maxSilentTime - before);
+            printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
         }
-        useTimeout = true;
-        timeout.tv_sec = std::max((time_t) 0, oldest + maxSilentTime - before);
-        printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec);
     }
 
     /* If we are polling goals that are waiting for a lock, then wake
@@ -2600,6 +2639,7 @@ void Worker::waitForInput()
         timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before);
     } else lastWokenUp = 0;
 
+    using namespace std;
     /* Use select() to wait for the input side of any logger pipe to
        become `available'.  Note that `available' (i.e., non-blocking)
        includes EOF. */
@@ -2660,6 +2700,7 @@ void Worker::waitForInput()
         }
 
         if (maxSilentTime != 0 &&
+            j->second.monitorForSilence &&
             after - j->second.lastOutput >= (time_t) maxSilentTime)
         {
             printMsg(lvlError,

src/libstore/db.cc

@@ -1,474 +0,0 @@
-#include "config.h"
-
-#ifdef OLD_DB_COMPAT
-
-#include "db.hh"
-#include "util.hh"
-#include "pathlocks.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-
-#include <memory>
-
-#include <db_cxx.h>
-
-
-namespace nix {
-
-
-/* Wrapper class to ensure proper destruction. */
-class DestroyDbc 
-{
-    Dbc * dbc;
-public:
-    DestroyDbc(Dbc * _dbc) : dbc(_dbc) { }
-    ~DestroyDbc() { dbc->close(); /* close() frees dbc */ }
-};
-
-
-class DestroyDbEnv
-{
-    DbEnv * dbenv;
-public:
-    DestroyDbEnv(DbEnv * _dbenv) : dbenv(_dbenv) { }
-    ~DestroyDbEnv() {
-        if (dbenv) {
-            if (dbenv->get_DB_ENV()) dbenv->close(0);
-            delete dbenv;
-        }
-    }
-    void release() { dbenv = 0; };
-};
-
-
-static void rethrow(DbException & e)
-{
-    throw Error(e.what());
-}
-
-
-Transaction::Transaction()
-    : txn(0)
-{
-}
-
-
-Transaction::Transaction(Database & db)
-    : txn(0)
-{
-    begin(db);
-}
-
-
-Transaction::~Transaction()
-{
-    if (txn) abort();
-}
-
-
-void Transaction::begin(Database & db)
-{
-    assert(txn == 0);
-    db.requireEnv();
-    try {
-        db.env->txn_begin(0, &txn, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::commit()
-{
-    if (!txn) throw Error("commit called on null transaction");
-    debug(format("committing transaction %1%") % (void *) txn);
-    DbTxn * txn2 = txn;
-    txn = 0;
-    try {
-        txn2->commit(0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::abort()
-{
-    if (!txn) throw Error("abort called on null transaction");
-    debug(format("aborting transaction %1%") % (void *) txn);
-    DbTxn * txn2 = txn;
-    txn = 0;
-    try {
-        txn2->abort();
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::moveTo(Transaction & t)
-{
-    if (t.txn) throw Error("target txn already exists");
-    t.txn = txn;
-    txn = 0;
-}
-
-
-void Database::requireEnv()
-{
-    checkInterrupt();
-    if (!env) throw Error("database environment is not open "
-        "(maybe you don't have sufficient permission?)");
-}
-
-
-Db * Database::getDb(TableId table)
-{
-    if (table == 0)
-        throw Error("database table is not open "
-            "(maybe you don't have sufficient permission?)");
-    std::map<TableId, Db *>::iterator i = tables.find(table);
-    if (i == tables.end())
-        throw Error("unknown table id");
-    return i->second;
-}
-
-
-Database::Database()
-    : env(0)
-    , nextId(1)
-{
-}
-
-
-Database::~Database()
-{
-    close();
-}
-
-
-void openEnv(DbEnv * & env, const string & path, u_int32_t flags)
-{
-    try {
-        createDirs(path);
-    } catch (SysError & e) {
-        if (e.errNo == EPERM || e.errNo == EACCES)
-            throw DbNoPermission(format("cannot create the Nix database in `%1%'") % path);
-        else
-            throw;
-    }
-        
-    try {
-        env->open(path.c_str(),
-            DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN |
-            DB_CREATE | flags,
-            0666);
-    } catch (DbException & e) {
-        printMsg(lvlError, format("environment open failed: %1%") % e.what());
-        throw;
-    }
-}
-
-
-static int my_fsync(int fd)
-{
-    return 0;
-}
-
-
-static void errorPrinter(const DbEnv * env, const char * errpfx, const char * msg)
-{
-    printMsg(lvlError, format("Berkeley DB error: %1%") % msg);
-}
-
-
-static void messagePrinter(const DbEnv * env, const char * msg)
-{
-    printMsg(lvlError, format("Berkeley DB message: %1%") % msg);
-}
-
-
-void Database::open2(const string & path, bool removeOldEnv)
-{
-    if (env) throw Error(format("environment already open"));
-
-    debug(format("opening database environment"));
-
-
-    /* Create the database environment object. */
-    DbEnv * env = new DbEnv(0);
-    DestroyDbEnv deleteEnv(env);
-
-    env->set_errcall(errorPrinter);
-    env->set_msgcall(messagePrinter);
-    if (getEnv("NIX_DEBUG_DB_REGISTER") == "1")
-        env->set_verbose(DB_VERB_REGISTER, 1);
-    env->set_verbose(DB_VERB_RECOVERY, 1);
-    
-    /* Smaller log files. */
-    env->set_lg_bsize(32 * 1024); /* default */
-    env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */
-    
-    /* Write the log, but don't sync.  This protects transactions
-       against application crashes, but if the system crashes, some
-       transactions may be undone.  An acceptable risk, I think. */
-    env->set_flags(DB_TXN_WRITE_NOSYNC | DB_LOG_AUTOREMOVE, 1);
-    
-    /* Increase the locking limits.  If you ever get `Dbc::get: Cannot
-       allocate memory' or similar, especially while running
-       `nix-store --verify', just increase the following number, then
-       run db_recover on the database to remove the existing DB
-       environment (since changes only take effect on new
-       environments). */
-    env->set_lk_max_locks(10000);
-    env->set_lk_max_lockers(10000);
-    env->set_lk_max_objects(10000);
-    env->set_lk_detect(DB_LOCK_DEFAULT);
-    
-    /* Dangerous, probably, but from the docs it *seems* that BDB
-       shouldn't sync when DB_TXN_WRITE_NOSYNC is used, but it still
-       fsync()s sometimes. */
-    db_env_set_func_fsync(my_fsync);
-
-    
-    if (removeOldEnv) {
-        printMsg(lvlError, "removing old Berkeley DB database environment...");
-        env->remove(path.c_str(), DB_FORCE);
-        return;
-    }
-
-    openEnv(env, path, DB_REGISTER | DB_RECOVER);
-
-    deleteEnv.release();
-    this->env = env;
-}
-
-
-void Database::open(const string & path)
-{
-    try {
-
-        open2(path, false);
-        
-    } catch (DbException e) {
-        
-        if (e.get_errno() == DB_VERSION_MISMATCH) {
-            /* Remove the environment while we are holding the global
-               lock.  If things go wrong there, we bail out.
-               !!! argh, we abolished the global lock :-( */
-            open2(path, true);
-
-            /* Try again. */
-            open2(path, false);
-
-            /* Force a checkpoint, as per the BDB docs. */
-            env->txn_checkpoint(DB_FORCE, 0, 0);
-
-            printMsg(lvlError, "database succesfully upgraded to new version");
-        }
-
-#if 0        
-        else if (e.get_errno() == DB_RUNRECOVERY) {
-            /* If recovery is needed, do it. */
-            printMsg(lvlError, "running recovery...");
-            open2(path, false, true);
-        }
-#endif        
-        
-        else
-            rethrow(e);
-    }
-}
-
-
-void Database::close()
-{
-    if (!env) return;
-
-    /* Close the database environment. */
-    debug(format("closing database environment"));
-
-    try {
-
-        for (std::map<TableId, Db *>::iterator i = tables.begin();
-             i != tables.end(); )
-        {
-            std::map<TableId, Db *>::iterator j = i;
-            ++j;
-            closeTable(i->first);
-            i = j;
-        }
-
-        /* Do a checkpoint every 128 kilobytes, or every 5 minutes. */
-        env->txn_checkpoint(128, 5, 0);
-        
-        env->close(0);
-
-    } catch (DbException e) { rethrow(e); }
-
-    delete env;
-
-    env = 0;
-}
-
-
-TableId Database::openTable(const string & tableName, bool sorted)
-{
-    requireEnv();
-    TableId table = nextId++;
-
-    try {
-
-        Db * db = new Db(env, 0);
-
-        try {
-            db->open(0, tableName.c_str(), 0, 
-                sorted ? DB_BTREE : DB_HASH,
-                DB_CREATE | DB_AUTO_COMMIT, 0666);
-        } catch (...) {
-            delete db;
-            throw;
-        }
-
-        tables[table] = db;
-
-    } catch (DbException e) { rethrow(e); }
-
-    return table;
-}
-
-
-void Database::closeTable(TableId table)
-{
-    try {
-        Db * db = getDb(table);
-        db->close(DB_NOSYNC);
-        delete db;
-        tables.erase(table);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::deleteTable(const string & table)
-{
-    try {
-        env->dbremove(0, table.c_str(), 0, DB_AUTO_COMMIT);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-bool Database::queryString(const Transaction & txn, TableId table, 
-    const string & key, string & data)
-{
-    checkInterrupt();
-
-    try {
-        Db * db = getDb(table);
-
-        Dbt kt((void *) key.c_str(), key.length());
-        Dbt dt;
-
-        int err = db->get(txn.txn, &kt, &dt, 0);
-        if (err) return false;
-
-        if (!dt.get_data())
-            data = "";
-        else
-            data = string((char *) dt.get_data(), dt.get_size());
-    
-    } catch (DbException e) { rethrow(e); }
-
-    return true;
-}
-
-
-bool Database::queryStrings(const Transaction & txn, TableId table, 
-    const string & key, Strings & data)
-{
-    string d;
-    if (!queryString(txn, table, key, d))
-        return false;
-    data = unpackStrings(d);
-    return true;
-}
-
-
-void Database::setString(const Transaction & txn, TableId table,
-    const string & key, const string & data)
-{
-    checkInterrupt();
-    try {
-        Db * db = getDb(table);
-        Dbt kt((void *) key.c_str(), key.length());
-        Dbt dt((void *) data.c_str(), data.length());
-        db->put(txn.txn, &kt, &dt, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::setStrings(const Transaction & txn, TableId table,
-    const string & key, const Strings & data, bool deleteEmpty)
-{
-    if (deleteEmpty && data.size() == 0)
-        delPair(txn, table, key);
-    else
-        setString(txn, table, key, packStrings(data));
-}
-
-
-void Database::delPair(const Transaction & txn, TableId table,
-    const string & key)
-{
-    checkInterrupt();
-    try {
-        Db * db = getDb(table);
-        Dbt kt((void *) key.c_str(), key.length());
-        db->del(txn.txn, &kt, 0);
-        /* Non-existence of a pair with the given key is not an
-           error. */
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::enumTable(const Transaction & txn, TableId table,
-    Strings & keys, const string & keyPrefix)
-{
-    try {
-        Db * db = getDb(table);
-
-        Dbc * dbc;
-        db->cursor(txn.txn, &dbc, 0);
-        DestroyDbc destroyDbc(dbc);
-
-        Dbt kt, dt;
-        u_int32_t flags = DB_NEXT;
-
-        if (!keyPrefix.empty()) {
-            flags = DB_SET_RANGE;
-            kt = Dbt((void *) keyPrefix.c_str(), keyPrefix.size());
-        }
-
-        while (dbc->get(&kt, &dt, flags) != DB_NOTFOUND) {
-            checkInterrupt();
-            string data((char *) kt.get_data(), kt.get_size());
-            if (!keyPrefix.empty() &&
-                string(data, 0, keyPrefix.size()) != keyPrefix)
-                break;
-            keys.push_back(data);
-            flags = DB_NEXT;
-        }
-
-    } catch (DbException e) { rethrow(e); }
-}
-
- 
-void Database::clearTable(const Transaction & txn, TableId table)
-{
-    try {
-        Db * db = getDb(table);
-        u_int32_t count;
-        db->truncate(txn.txn, &count, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-}
-
-#endif

src/libstore/db.hh

@@ -1,107 +0,0 @@
-#ifndef __DB_H
-#define __DB_H
-
-#include "types.hh"
-
-#include <map>
-
-
-/* Defined externally. */
-class DbTxn;
-class DbEnv;
-class Db;
-
-
-namespace nix {
-
-
-class Database;
-
-
-class Transaction
-{
-    friend class Database;
-
-private:
-    DbTxn * txn;
-    
-public:
-    Transaction();
-    Transaction(Database & _db);
-    ~Transaction();
-
-    void begin(Database & db);
-    void abort();
-    void commit();
-
-    void moveTo(Transaction & t);
-};
-
-
-#define noTxn Transaction()
-
-
-typedef unsigned int TableId; /* table handles */
-
-
-class Database
-{
-    friend class Transaction;
-
-private:
-    DbEnv * env;
-
-    TableId nextId;
-    std::map<TableId, Db *> tables;
-
-    void requireEnv();
-
-    Db * getDb(TableId table);
-
-    void open2(const string & path, bool removeOldEnv);
-    
-public:
-    Database();
-    ~Database();
-    
-    void open(const string & path);
-    void close();
-
-    TableId openTable(const string & table, bool sorted = false);
-    void closeTable(TableId table);
-    void deleteTable(const string & table);
-
-    bool queryString(const Transaction & txn, TableId table, 
-        const string & key, string & data);
-
-    bool queryStrings(const Transaction & txn, TableId table, 
-        const string & key, Strings & data);
-
-    void setString(const Transaction & txn, TableId table,
-        const string & key, const string & data);
-
-    void setStrings(const Transaction & txn, TableId table,
-        const string & key, const Strings & data,
-        bool deleteEmpty = true);
-
-    void delPair(const Transaction & txn, TableId table,
-        const string & key);
-
-    void enumTable(const Transaction & txn, TableId table,
-        Strings & keys, const string & keyPrefix = "");
-
-    void clearTable(const Transaction & txn, TableId table);
-};
-
-
-class DbNoPermission : public Error
-{
-public:
-    DbNoPermission(const format & f) : Error(f) { };
-};
-
- 
-}
-
-
-#endif /* !__DB_H */

src/libstore/derivations-ast.def

@@ -1,10 +0,0 @@
-init initDerivationsHelpers
-
-Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
-
-| string string | ATerm | EnvBinding |
-| string ATermList | ATerm | DerivationInput |
-| string string string string | ATerm | DerivationOutput |
-
-Closure | ATermList ATermList | ATerm | OldClosure |
-| string ATermList | ATerm | OldClosureElem |

src/libstore/derivations.cc

@@ -1,22 +1,12 @@
 #include "derivations.hh"
 #include "store-api.hh"
-#include "aterm.hh"
 #include "globals.hh"
 #include "util.hh"
 
-#include "derivations-ast.hh"
-#include "derivations-ast.cc"
-
 
 namespace nix {
 
 
-Hash hashTerm(ATerm t)
-{
-    return hashString(htSHA256, atPrint(t));
-}
-
-
 Path writeDerivation(const Derivation & drv, const string & name)
 {
     PathSet references;
@@ -27,137 +17,151 @@ Path writeDerivation(const Derivation & drv, const string & name)
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
     string suffix = name + drvExtension;
-    string contents = atPrint(unparseDerivation(drv));
+    string contents = unparseDerivation(drv);
     return readOnlyMode
         ? computeStorePathForText(suffix, contents, references)
         : store->addTextToStore(suffix, contents, references);
 }
 
 
-static void checkPath(const string & s)
+static Path parsePath(std::istream & str)
 {
+    string s = parseString(str);
     if (s.size() == 0 || s[0] != '/')
         throw Error(format("bad path `%1%' in derivation") % s);
+    return s;
 }
     
 
-static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
+static StringSet parseStrings(std::istream & str, bool arePaths)
 {
-    for (ATermIterator i(paths); i; ++i) {
-        if (ATgetType(*i) != AT_APPL)
-            throw badTerm("not a path", *i);
-        string s = aterm2String(*i);
-        if (arePaths) checkPath(s);
-        out.insert(s);
-    }
+    StringSet res;
+    while (!endOfList(str))
+        res.insert(arePaths ? parsePath(str) : parseString(str));
+    return res;
 }
+    
 
-
-/* Shut up warnings. */
-void throwBadDrv(ATerm t) __attribute__ ((noreturn));
-
-void throwBadDrv(ATerm t) 
-{
-    throw badTerm("not a valid derivation", t);
-}
-
-
-Derivation parseDerivation(ATerm t)
+Derivation parseDerivation(const string & s)
 {
     Derivation drv;
-    ATermList outs, inDrvs, inSrcs, args, bnds;
-    ATerm builder, platform;
+    std::istringstream str(s);
+    expect(str, "Derive([");
 
-    if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds))
-        throwBadDrv(t);
-
-    for (ATermIterator i(outs); i; ++i) {
-        ATerm id, path, hashAlgo, hash;
-        if (!matchDerivationOutput(*i, id, path, hashAlgo, hash))
-            throwBadDrv(t);
+    /* Parse the list of outputs. */
+    while (!endOfList(str)) {
         DerivationOutput out;
-        out.path = aterm2String(path);
-        checkPath(out.path);
-        out.hashAlgo = aterm2String(hashAlgo);
-        out.hash = aterm2String(hash);
-        drv.outputs[aterm2String(id)] = out;
+        expect(str, "("); string id = parseString(str);
+        expect(str, ","); out.path = parsePath(str);
+        expect(str, ","); out.hashAlgo = parseString(str);
+        expect(str, ","); out.hash = parseString(str);
+        expect(str, ")");
+        drv.outputs[id] = out;
     }
 
-    for (ATermIterator i(inDrvs); i; ++i) {
-        ATerm drvPath;
-        ATermList ids;
-        if (!matchDerivationInput(*i, drvPath, ids))
-            throwBadDrv(t);
-        Path drvPath2 = aterm2String(drvPath);
-        checkPath(drvPath2);
-        StringSet ids2;
-        parseStrings(ids, ids2, false);
-        drv.inputDrvs[drvPath2] = ids2;
+    /* Parse the list of input derivations. */
+    expect(str, ",[");
+    while (!endOfList(str)) {
+        expect(str, "(");
+        Path drvPath = parsePath(str);
+        expect(str, ",[");
+        drv.inputDrvs[drvPath] = parseStrings(str, false);
+        expect(str, ")");
+    }
+
+    expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
+    expect(str, ","); drv.platform = parseString(str);
+    expect(str, ","); drv.builder = parseString(str);
+
+    /* Parse the builder arguments. */
+    expect(str, ",[");
+    while (!endOfList(str))
+        drv.args.push_back(parseString(str));
+
+    /* Parse the environment variables. */
+    expect(str, ",[");
+    while (!endOfList(str)) {
+        expect(str, "("); string name = parseString(str);
+        expect(str, ","); string value = parseString(str);
+        expect(str, ")");
+        drv.env[name] = value;
     }
     
-    parseStrings(inSrcs, drv.inputSrcs, true);
-
-    drv.builder = aterm2String(builder);
-    drv.platform = aterm2String(platform);
-    
-    for (ATermIterator i(args); i; ++i) {
-        if (ATgetType(*i) != AT_APPL)
-            throw badTerm("string expected", *i);
-        drv.args.push_back(aterm2String(*i));
-    }
-
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm s1, s2;
-        if (!matchEnvBinding(*i, s1, s2))
-            throw badTerm("tuple of strings expected", *i);
-        drv.env[aterm2String(s1)] = aterm2String(s2);
-    }
-
+    expect(str, ")");
     return drv;
 }
 
 
-ATerm unparseDerivation(const Derivation & drv)
+static void printString(string & res, const string & s)
 {
-    ATermList outputs = ATempty;
-    for (DerivationOutputs::const_reverse_iterator i = drv.outputs.rbegin();
-         i != drv.outputs.rend(); ++i)
-        outputs = ATinsert(outputs,
-            makeDerivationOutput(
-                toATerm(i->first),
-                toATerm(i->second.path),
-                toATerm(i->second.hashAlgo),
-                toATerm(i->second.hash)));
+    res += '"';
+    for (const char * i = s.c_str(); *i; i++)
+        if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
+        else if (*i == '\n') res += "\\n";
+        else if (*i == '\r') res += "\\r";
+        else if (*i == '\t') res += "\\t";
+        else res += *i;
+    res += '"';
+}
 
-    ATermList inDrvs = ATempty;
-    for (DerivationInputs::const_reverse_iterator i = drv.inputDrvs.rbegin();
-         i != drv.inputDrvs.rend(); ++i)
-        inDrvs = ATinsert(inDrvs,
-            makeDerivationInput(
-                toATerm(i->first),
-                toATermList(i->second)));
+
+template<class ForwardIterator>
+static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
+{
+    res += '[';
+    bool first = true;
+    for ( ; i != j; ++i) {
+        if (first) first = false; else res += ',';
+        printString(res, *i);
+    }
+    res += ']';
+}
+
+
+string unparseDerivation(const Derivation & drv)
+{
+    string s;
+    s.reserve(65536);
+    s += "Derive([";
+
+    bool first = true;
+    foreach (DerivationOutputs::const_iterator, i, drv.outputs) {
+        if (first) first = false; else s += ',';
+        s += '('; printString(s, i->first);
+        s += ','; printString(s, i->second.path);
+        s += ','; printString(s, i->second.hashAlgo);
+        s += ','; printString(s, i->second.hash);
+        s += ')';
+    }
+
+    s += "],[";
+    first = true;
+    foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) {
+        if (first) first = false; else s += ',';
+        s += '('; printString(s, i->first);
+        s += ','; printStrings(s, i->second.begin(), i->second.end());
+        s += ')';
+    }
+
+    s += "],";
+    printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end());
     
-    ATermList args = ATempty;
-    for (Strings::const_reverse_iterator i = drv.args.rbegin();
-         i != drv.args.rend(); ++i)
-        args = ATinsert(args, toATerm(*i));
+    s += ','; printString(s, drv.platform);
+    s += ','; printString(s, drv.builder);
+    s += ','; printStrings(s, drv.args.begin(), drv.args.end());
 
-    ATermList env = ATempty;
-    for (StringPairs::const_reverse_iterator i = drv.env.rbegin();
-         i != drv.env.rend(); ++i)
-        env = ATinsert(env,
-            makeEnvBinding(
-                toATerm(i->first),
-                toATerm(i->second)));
-
-    return makeDerive(
-        outputs,
-        inDrvs,
-        toATermList(drv.inputSrcs),
-        toATerm(drv.platform),
-        toATerm(drv.builder),
-        args,
-        env);
+    s += ",[";
+    first = true;
+    foreach (StringPairs::const_iterator, i, drv.env) {
+        if (first) first = false; else s += ',';
+        s += '('; printString(s, i->first);
+        s += ','; printString(s, i->second);
+        s += ')';
+    }
+    
+    s += "])";
+    
+    return s;
 }
 
 

src/libstore/derivations.hh

@@ -1,8 +1,6 @@
 #ifndef __DERIVATIONS_H
 #define __DERIVATIONS_H
 
-typedef struct _ATerm * ATerm;
-
 #include "hash.hh"
 
 #include <map>
@@ -53,17 +51,14 @@ struct Derivation
 };
 
 
-/* Hash an aterm. */
-Hash hashTerm(ATerm t);
-
 /* Write a derivation to the Nix store, and return its path. */
 Path writeDerivation(const Derivation & drv, const string & name);
 
 /* Parse a derivation. */
-Derivation parseDerivation(ATerm t);
+Derivation parseDerivation(const string & s);
 
-/* Parse a derivation. */
-ATerm unparseDerivation(const Derivation & drv);
+/* Print a derivation. */
+string unparseDerivation(const Derivation & drv);
 
 /* Check whether a file name ends with the extensions for
    derivations. */

src/libstore/gc.cc

@@ -7,6 +7,7 @@
 
 #include <functional>
 #include <queue>
+#include <algorithm>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -14,11 +15,6 @@
 #include <fcntl.h>
 #include <unistd.h>
 
-#ifdef __CYGWIN__
-#include <windows.h>
-#include <sys/cygwin.h>
-#endif
-
 
 namespace nix {
 
@@ -178,15 +174,6 @@ void LocalStore::addTempRoot(const Path & path)
 
             fdGCLock.close();
       
-	    /* Note that on Cygwin a lot of the following complexity
-	       is unnecessary, since we cannot delete open lock
-	       files.  If we have the lock file open, then it's valid;
-	       if we can delete it, then it wasn't in use any more. 
-
-	       Also note that on Cygwin we cannot "upgrade" a lock
-	       from a read lock to a write lock. */
-
-#ifndef __CYGWIN__
             debug(format("acquiring read lock on `%1%'") % fnTempRoots);
             lockFile(fdTempRoots, ltRead, true);
 
@@ -200,10 +187,6 @@ void LocalStore::addTempRoot(const Path & path)
             /* The garbage collector deleted this file before we could
                get a lock.  (It won't delete the file after we get a
                lock.)  Try again. */
-
-#else
-            break;
-#endif
         }
 
     }
@@ -216,14 +199,9 @@ void LocalStore::addTempRoot(const Path & path)
     string s = path + '\0';
     writeFull(fdTempRoots, (const unsigned char *) s.c_str(), s.size());
 
-#ifndef __CYGWIN__
     /* Downgrade to a read lock. */
     debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
     lockFile(fdTempRoots, ltRead, true);
-#else
-    debug(format("releasing write lock on `%1%'") % fnTempRoots);
-    lockFile(fdTempRoots, ltNone, true);
-#endif
 }
 
 
@@ -251,19 +229,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
         Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
 
         debug(format("reading temporary root file `%1%'") % path);
-
-#ifdef __CYGWIN__
-	/* On Cygwin we just try to delete the lock file. */
-	char win32Path[MAX_PATH];
-	cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
-	if (DeleteFile(win32Path)) {
-            printMsg(lvlError, format("removed stale temporary roots file `%1%'")
-                % path);
-            continue;
-        } else
-            debug(format("delete of `%1%' failed: %2%") % path % GetLastError());
-#endif
-
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
         if (*fd == -1) {
             /* It's okay if the file has disappeared. */
@@ -275,7 +240,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
         //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
         //if (*fd == -1) continue;
 
-#ifndef __CYGWIN__
         /* Try to acquire a write lock without blocking.  This can
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
@@ -286,7 +250,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
             writeFull(*fd, (const unsigned char *) "d", 1);
             continue;
         }
-#endif
 
         /* Acquire a read lock.  This will prevent the owning process
            from upgrading to a write lock, therefore it will block in
@@ -401,7 +364,7 @@ static void addAdditionalRoots(PathSet & roots)
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
             if (roots.find(path) == roots.end() && store->isValidPath(path)) {
-                debug(format("found additional root `%1%'") % path);
+                debug(format("got additional root `%1%'") % path);
                 roots.insert(path);
             }
         }
@@ -439,147 +402,218 @@ Paths topoSortPaths(const PathSet & paths)
 }
 
 
-static time_t lastFileAccessTime(const Path & path)
-{
-    checkInterrupt();
-    
-    struct stat st;
-    if (lstat(path.c_str(), &st) == -1)
-        throw SysError(format("statting `%1%'") % path);
-
-    if (S_ISDIR(st.st_mode)) {
-        time_t last = 0;
-	Strings names = readDirectory(path);
-	foreach (Strings::iterator, i, names) {
-            time_t t = lastFileAccessTime(path + "/" + *i);
-            if (t > last) last = t;
-        }
-        return last;
-    }
-
-    else if (S_ISLNK(st.st_mode)) return 0;
-
-    else return st.st_atime;
-}
-
-
 struct GCLimitReached { };
 
 
-void LocalStore::gcPath(const GCOptions & options, GCResults & results, 
-    const Path & path)
+struct LocalStore::GCState
 {
-    results.paths.insert(path);
+    GCOptions options;
+    GCResults & results;
+    PathSet roots;
+    PathSet tempRoots;
+    PathSet deleted;
+    PathSet live;
+    PathSet busy;
+    bool gcKeepOutputs;
+    bool gcKeepDerivations;
 
-    if (!pathExists(path)) return;
-                
-    /* Okay, it's safe to delete. */
-    unsigned long long bytesFreed, blocksFreed;
-    deleteFromStore(path, bytesFreed, blocksFreed);
-    results.bytesFreed += bytesFreed;
-    results.blocksFreed += blocksFreed;
+    bool drvsIndexed;
+    typedef std::multimap<string, Path> DrvsByName;
+    DrvsByName drvsByName; // derivation paths hashed by name attribute
 
-    if (options.maxFreed && results.bytesFreed > options.maxFreed) {
-        printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % options.maxFreed);
-        throw GCLimitReached();
-    }
-
-    if (options.maxLinks) {
-        struct stat st;
-        if (stat(nixStore.c_str(), &st) == -1)
-            throw SysError(format("statting `%1%'") % nixStore);
-        if (st.st_nlink < options.maxLinks) {
-            printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % options.maxLinks);
-            throw GCLimitReached();
-        }
-    }
-}
-
-
-void LocalStore::gcPathRecursive(const GCOptions & options,
-    GCResults & results, PathSet & done, const Path & path)
-{
-    if (done.find(path) != done.end()) return;
-    done.insert(path);
-
-    startNest(nest, lvlDebug, format("looking at `%1%'") % path);
-        
-    /* Delete all the referrers first.  They must be garbage too,
-       since if they were live, then the current path would also be
-       live.  Note that deleteFromStore() below still makes sure that
-       the referrer set has become empty, just in case.  (However that
-       doesn't guard against deleting top-level paths that are only
-       reachable from GC roots.) */
-    PathSet referrers;
-    if (isValidPath(path))
-        queryReferrers(path, referrers);
-    foreach (PathSet::iterator, i, referrers)
-        if (*i != path) gcPathRecursive(options, results, done, *i);
-
-    printMsg(lvlInfo, format("deleting `%1%'") % path);
-            
-    gcPath(options, results, path);
-}
-
-
-struct CachingAtimeComparator : public std::binary_function<Path, Path, bool> 
-{
-    std::map<Path, time_t> cache;
-
-    time_t lookup(const Path & p)
+    GCState(GCResults & results_) : results(results_), drvsIndexed(false)
     {
-        std::map<Path, time_t>::iterator i = cache.find(p);
-        if (i != cache.end()) return i->second;
-        debug(format("computing atime of `%1%'") % p);
-        cache[p] = lastFileAccessTime(p);
-        assert(cache.find(p) != cache.end());
-        return cache[p];
-    }
-        
-    bool operator () (const Path & p1, const Path & p2)
-    {
-        return lookup(p2) < lookup(p1);
     }
 };
 
 
-static string showTime(const string & format, time_t t)
+static bool doDelete(GCOptions::GCAction action)
 {
-    char s[128];
-    strftime(s, sizeof s, format.c_str(), localtime(&t));
-    return string(s);
+    return action == GCOptions::gcDeleteDead
+        || action == GCOptions::gcDeleteSpecific;
 }
 
 
-static bool isLive(const Path & path, const PathSet & livePaths,
-    const PathSet & tempRoots, const PathSet & tempRootsClosed)
+bool LocalStore::isActiveTempFile(const GCState & state,
+    const Path & path, const string & suffix)
 {
-    if (livePaths.find(path) != livePaths.end() ||
-        tempRootsClosed.find(path) != tempRootsClosed.end()) return true;
+    return hasSuffix(path, suffix)
+        && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
+}
 
-    /* A lock file belonging to a path that we're building right
-       now isn't garbage. */
-    if (hasSuffix(path, ".lock") && tempRoots.find(string(path, 0, path.size() - 5)) != tempRoots.end())
-        return true;
 
-    /* Don't delete .chroot directories for derivations that are
-       currently being built. */
-    if (hasSuffix(path, ".chroot") && tempRoots.find(string(path, 0, path.size() - 7)) != tempRoots.end())
-        return true;
+/* Return all the derivations in the Nix store that have `path' as an
+   output.  This function assumes that derivations have the same name
+   as their outputs. */
+PathSet LocalStore::findDerivers(GCState & state, const Path & path)
+{
+    PathSet derivers;
 
+    Path deriver = queryDeriver(path);
+    if (deriver != "") derivers.insert(deriver);
+
+    if (!state.drvsIndexed) {
+        Paths entries = readDirectory(nixStore);
+        foreach (Paths::iterator, i, entries)
+            if (isDerivation(*i))
+                state.drvsByName.insert(std::pair<string, Path>(
+                        getNameOfStorePath(*i), nixStore + "/" + *i));
+        state.drvsIndexed = true;
+    }
+    
+    string name = getNameOfStorePath(path);
+
+    // Urgh, I should have used Haskell...
+    std::pair<GCState::DrvsByName::iterator, GCState::DrvsByName::iterator> range =
+        state.drvsByName.equal_range(name);
+
+    for (GCState::DrvsByName::iterator i = range.first; i != range.second; ++i)
+        if (isValidPath(i->second)) {
+            Derivation drv = derivationFromPath(i->second);
+            foreach (DerivationOutputs::iterator, j, drv.outputs)
+                if (j->second.path == path) derivers.insert(i->second);
+        }
+
+    return derivers;
+}
+
+    
+bool LocalStore::tryToDelete(GCState & state, const Path & path)
+{
+    if (!pathExists(path)) return true;
+    if (state.deleted.find(path) != state.deleted.end()) return true;
+    if (state.live.find(path) != state.live.end()) return false;
+
+    startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path);
+
+    if (state.roots.find(path) != state.roots.end()) {
+        printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path);
+        goto isLive;
+    }
+
+    if (isValidPath(path)) {
+
+        /* Recursively try to delete the referrers of this path.  If
+           any referrer can't be deleted, then this path can't be
+           deleted either. */
+        PathSet referrers;
+        queryReferrers(path, referrers);
+        foreach (PathSet::iterator, i, referrers)
+            if (*i != path && !tryToDelete(state, *i)) {
+                printMsg(lvlDebug, format("cannot delete `%1%' because it has live referrers") % path);
+                goto isLive;
+            }
+
+        /* If gc-keep-derivations is set and this is a derivation,
+           then don't delete the derivation if any of the outputs are
+           live. */
+        if (state.gcKeepDerivations && isDerivation(path)) {
+            Derivation drv = derivationFromPath(path);
+            foreach (DerivationOutputs::iterator, i, drv.outputs)
+                if (!tryToDelete(state, i->second.path)) {
+                    printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output is alive") % path);
+                    goto isLive;
+                }
+        }
+
+        /* If gc-keep-derivations and gc-keep-outputs are both set,
+           it's possible that the path has already been deleted (due
+           to the recursion below), so bail out. */
+        if (!pathExists(path)) return true;
+
+        /* If gc-keep-outputs is set, then don't delete this path if
+           its deriver is not garbage.  !!! Nix does not reliably
+           store derivers, so we have to look at all derivations to
+           determine which of them derive `path'.  Since this makes
+           the garbage collector very slow to start on large Nix
+           stores, here we just look for all derivations that have the
+           same name as `path' (where the name is the part of the
+           filename after the hash, i.e. the `name' attribute of the
+           derivation).  This is somewhat hacky: currently, the
+           deriver of a path always has the same name as the output,
+           but this might change in the future. */
+        if (state.gcKeepOutputs) {
+            PathSet derivers = findDerivers(state, path);
+            foreach (PathSet::iterator, deriver, derivers) {
+                /* Break an infinite recursion if gc-keep-derivations
+                   and gc-keep-outputs are both set by tentatively
+                   assuming that this path is garbage.  This is a safe
+                   assumption because at this point, the only thing
+                   that can prevent it from being garbage is the
+                   deriver.  Since tryToDelete() works "upwards"
+                   through the dependency graph, it won't encouter
+                   this path except in the call to tryToDelete() in
+                   the gc-keep-derivation branch. */
+                state.deleted.insert(path);
+                if (!tryToDelete(state, *deriver)) {
+                    state.deleted.erase(path);
+                    printMsg(lvlDebug, format("cannot delete `%1%' because its deriver `%2%' is alive") % path % *deriver);
+                    goto isLive;
+                }
+            }
+        }
+    }
+
+    else {
+
+        /* A lock file belonging to a path that we're building right
+           now isn't garbage. */
+        if (isActiveTempFile(state, path, ".lock")) return false;
+
+        /* Don't delete .chroot directories for derivations that are
+           currently being built. */
+        if (isActiveTempFile(state, path, ".chroot")) return false;
+
+    }
+
+    /* The path is garbage, so delete it. */
+    if (doDelete(state.options.action)) {
+        printMsg(lvlInfo, format("deleting `%1%'") % path);
+
+        unsigned long long bytesFreed, blocksFreed;
+        deleteFromStore(path, bytesFreed, blocksFreed);
+        state.results.bytesFreed += bytesFreed;
+        state.results.blocksFreed += blocksFreed;
+
+        if (state.options.maxFreed && state.results.bytesFreed > state.options.maxFreed) {
+            printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % state.options.maxFreed);
+            throw GCLimitReached();
+        }
+
+        if (state.options.maxLinks) {
+            struct stat st;
+            if (stat(nixStore.c_str(), &st) == -1)
+                throw SysError(format("statting `%1%'") % nixStore);
+            if (st.st_nlink < state.options.maxLinks) {
+                printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % state.options.maxLinks);
+                throw GCLimitReached();
+            }
+        }
+        
+    } else
+        printMsg(lvlTalkative, format("would delete `%1%'") % path);
+    
+    state.deleted.insert(path);
+    if (state.options.action != GCOptions::gcReturnLive)
+        state.results.paths.insert(path);
+    return true;
+
+ isLive:
+    state.live.insert(path);
+    if (state.options.action == GCOptions::gcReturnLive)
+        state.results.paths.insert(path);
     return false;
 }
 
 
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    bool gcKeepOutputs =
-        queryBoolSetting("gc-keep-outputs", false);
-    bool gcKeepDerivations =
-        queryBoolSetting("gc-keep-derivations", true);
-    int gcKeepOutputsThreshold = 
-        queryIntSetting ("gc-keep-outputs-threshold", defaultGcLevel);
-
+    GCState state(results);
+    state.options = options;    
+    
+    state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false);
+    state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true);
+    
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
        b) Processes from creating new temporary root files. */
@@ -590,198 +624,60 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printMsg(lvlError, format("finding garbage collector roots..."));
     Roots rootMap = options.ignoreLiveness ? Roots() : nix::findRoots(true);
 
-    PathSet roots;
-    foreach (Roots::iterator, i, rootMap) roots.insert(i->second);
+    foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second);
 
     /* Add additional roots returned by the program specified by the
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
     if (!options.ignoreLiveness)
-        addAdditionalRoots(roots);
-
-    if (options.action == GCOptions::gcReturnRoots) {
-        results.paths = roots;
-        return;
-    }
-
-    /* Determine the live paths which is just the closure of the
-       roots under the `references' relation. */
-    printMsg(lvlError, format("computing live paths..."));
-    PathSet livePaths;
-    foreach (PathSet::const_iterator, i, roots)
-        computeFSClosure(canonPath(*i), livePaths);
-
-    if (gcKeepDerivations) {
-        foreach (PathSet::iterator, i, livePaths) {
-            /* Note that the deriver need not be valid (e.g., if we
-               previously ran the collector with `gcKeepDerivations'
-               turned off). */
-            Path deriver = queryDeriver(*i);
-            if (deriver != "" && isValidPath(deriver))
-                computeFSClosure(deriver, livePaths);
-        }
-    }
-
-    if (gcKeepOutputs) {
-        /* Hmz, identical to storePathRequisites in nix-store. */
-        foreach (PathSet::iterator, i, livePaths)
-            if (isDerivation(*i)) {
-                Derivation drv = derivationFromPath(*i);
-
-		string gcLevelStr = drv.env["__gcLevel"];
-		int gcLevel;
-		if (!string2Int(gcLevelStr, gcLevel))
-		    gcLevel = defaultGcLevel;
-		
-		if (gcLevel >= gcKeepOutputsThreshold)    
-		    foreach (DerivationOutputs::iterator, j, drv.outputs)
-			if (isValidPath(j->second.path))
-			    computeFSClosure(j->second.path, livePaths);
-            }
-    }
-
-    if (options.action == GCOptions::gcReturnLive) {
-        results.paths = livePaths;
-        return;
-    }
+        addAdditionalRoots(state.roots);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
-    PathSet tempRoots;
     FDs fds;
-    readTempRoots(tempRoots, fds);
-
-    /* Close the temporary roots.  Note that we *cannot* do this in
-       readTempRoots(), because there we may not have all locks yet,
-       meaning that an invalid path can become valid (and thus add to
-       the references graph) after we have added it to the closure
-       (and computeFSClosure() assumes that the presence of a path
-       means that it has already been closed). */
-    PathSet tempRootsClosed;
-    foreach (PathSet::iterator, i, tempRoots)
-        if (isValidPath(*i))
-            computeFSClosure(*i, tempRootsClosed);
-        else
-            tempRootsClosed.insert(*i);
+    readTempRoots(state.tempRoots, fds);
+    state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
        increase, since we hold locks on everything.  So everything
-       that is not currently in in `livePaths' or `tempRootsClosed'
-       can be deleted. */
-    
-    /* Read the Nix store directory to find all currently existing
-       paths and filter out all live paths. */
-    printMsg(lvlError, format("reading the Nix store..."));
-    PathSet storePaths;
-    
-    if (options.action != GCOptions::gcDeleteSpecific) {
-        Paths entries = readDirectory(nixStore);
-        foreach (Paths::iterator, i, entries) {
-            Path path = canonPath(nixStore + "/" + *i);
-            if (!isLive(path, livePaths, tempRoots, tempRootsClosed)) storePaths.insert(path);
-        }
-    }
+       that is not reachable from `roots'. */
+
+    /* Now either delete all garbage paths, or just the specified
+       paths (for gcDeleteSpecific). */
+
+    if (options.action == GCOptions::gcDeleteSpecific) {
 
-    else {
         foreach (PathSet::iterator, i, options.pathsToDelete) {
             assertStorePath(*i);
-            storePaths.insert(*i);
-            if (isLive(*i, livePaths, tempRoots, tempRootsClosed))
+            if (!tryToDelete(state, *i))
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
         }
-    }
-
-    if (options.action == GCOptions::gcReturnDead) {
-        results.paths.insert(storePaths.begin(), storePaths.end());
-        return;
-    }
-
-    /* Delete all dead store paths (or until one of the stop
-       conditions is reached). */
-
-    PathSet done;
-    try {
-
-        if (!options.useAtime) {
-            /* Delete the paths, respecting the partial ordering
-               determined by the references graph. */
-            printMsg(lvlError, format("deleting garbage..."));
-            foreach (PathSet::iterator, i, storePaths)
-                gcPathRecursive(options, results, done, *i);
-        }
-
-        else {
-
-            /* Delete in order of ascending last access time, still
-               maintaining the partial ordering of the reference
-               graph.  Note that we can't use a topological sort for
-               this because that takes time O(V+E), and in this case
-               E=O(V^2) (i.e. the graph is dense because of the edges
-               due to the atime ordering).  So instead we put all
-               deletable paths in a priority queue (ordered by atime),
-               and after deleting a path, add additional paths that
-               have become deletable to the priority queue. */
-
-            CachingAtimeComparator atimeComp;
-
-            /* Create a priority queue that orders paths by ascending
-               atime.  This is why C++ needs type inferencing... */
-            std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> > prioQueue =
-                std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> >(binary_function_ref_adapter<CachingAtimeComparator>(&atimeComp));
-
-           /* Initially put the paths that are invalid or have no
-              referrers into the priority queue. */
-            printMsg(lvlError, format("finding deletable paths..."));
-            foreach (PathSet::iterator, i, storePaths) {
-                checkInterrupt();
-                /* We can safely delete a path if it's invalid or
-                   it has no referrers.  Note that all the invalid
-                   paths will be deleted in the first round. */
-                if (isValidPath(*i)) {
-                    if (queryReferrersNoSelf(*i).empty()) prioQueue.push(*i);
-                } else prioQueue.push(*i);
-            }
-
-            debug(format("%1% initially deletable paths") % prioQueue.size());
-
-            /* Now delete everything in the order of the priority
-               queue until nothing is left. */
-            printMsg(lvlError, format("deleting garbage..."));
-            while (!prioQueue.empty()) {
-                checkInterrupt();
-                Path path = prioQueue.top(); prioQueue.pop();
-
-                if (options.maxAtime != (time_t) -1 &&
-                    atimeComp.lookup(path) > options.maxAtime)
-                    continue;
-                
-                printMsg(lvlInfo, format("deleting `%1%' (last accessed %2%)") % path % showTime("%F %H:%M:%S", atimeComp.lookup(path)));
-
-                PathSet references;
-                if (isValidPath(path)) references = queryReferencesNoSelf(path);
-
-                gcPath(options, results, path);
-
-                /* For each reference of the current path, see if the
-                   reference has now become deletable (i.e. is in the
-                   set of dead paths and has no referrers left).  If
-                   so add it to the priority queue. */
-                foreach (PathSet::iterator, i, references) {
-                    if (storePaths.find(*i) != storePaths.end() &&
-                        queryReferrersNoSelf(*i).empty())
-                    {
-                        debug(format("path `%1%' has become deletable") % *i);
-                        prioQueue.push(*i);
-                    }
-                }
-            }
-            
-        }
         
-    } catch (GCLimitReached & e) {
-    }
+    } else {
+        
+        printMsg(lvlError, format("reading the Nix store..."));
+        Paths entries = readDirectory(nixStore);
+
+        /* Randomise the order in which we delete entries to make the
+           collector less biased towards deleting paths that come
+           alphabetically first (e.g. /nix/store/000...).  This
+           matters when using --max-freed etc. */
+        vector<Path> entries_(entries.begin(), entries.end());
+        random_shuffle(entries_.begin(), entries_.end());
+
+        if (doDelete(state.options.action))
+            printMsg(lvlError, format("deleting garbage..."));
+        else
+            printMsg(lvlError, format("determining live/dead paths..."));
+    
+        try {
+            foreach (vector<Path>::iterator, i, entries_)
+                tryToDelete(state, canonPath(nixStore + "/" + *i));
+        } catch (GCLimitReached & e) {
+        }
+    }        
 }
 
 

src/libstore/globals.cc

@@ -22,6 +22,7 @@ bool keepGoing = false;
 bool tryFallback = false;
 Verbosity buildVerbosity = lvlInfo;
 unsigned int maxBuildJobs = 1;
+unsigned int buildCores = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
 time_t maxSilentTime = 0;

src/libstore/globals.hh

@@ -55,6 +55,11 @@ extern Verbosity buildVerbosity;
 /* Maximum number of parallel build jobs.  0 means unlimited. */
 extern unsigned int maxBuildJobs;
 
+/* Number of CPU cores to utilize in parallel within a build, i.e. by passing
+   this number to Make via '-j'. 0 means that the number of actual CPU cores on
+   the local host ought to be auto-detected. */
+extern unsigned int buildCores;
+
 /* Read-only mode.  Don't copy stuff to the store, don't change the
    database. */
 extern bool readOnlyMode;

src/libstore/local-store.cc

@@ -3,8 +3,6 @@
 #include "globals.hh"
 #include "archive.hh"
 #include "pathlocks.hh"
-#include "aterm.hh"
-#include "derivations-ast.hh"
 #include "worker-protocol.hh"
     
 #include <iostream>
@@ -16,6 +14,7 @@
 #include <utime.h>
 #include <fcntl.h>
 #include <errno.h>
+#include <stdio.h>
 
 
 namespace nix {
@@ -47,8 +46,21 @@ LocalStore::LocalStore()
     
     if (readOnlyMode) return;
 
+    /* Create missing state directories if they don't already exist. */
     createDirs(nixStore);
-    
+    createDirs(nixDBPath + "/info");
+    createDirs(nixDBPath + "/referrer");
+    createDirs(nixDBPath + "/failed");
+    Path profilesDir = nixStateDir + "/profiles";
+    createDirs(nixStateDir + "/profiles");
+    createDirs(nixStateDir + "/temproots");
+    Path gcRootsDir = nixStateDir + "/gcroots";
+    if (!pathExists(gcRootsDir)) {
+        createDirs(gcRootsDir);
+        if (symlink(profilesDir.c_str(), (gcRootsDir + "/profiles").c_str()) == -1)
+            throw SysError(format("creating symlink to `%1%'") % profilesDir);
+    }
+  
     checkStoreNotSymlink();
 
     try {
@@ -64,11 +76,7 @@ LocalStore::LocalStore()
         printMsg(lvlError, "waiting for the big Nix store lock...");
         lockFile(globalLock, ltRead, true);
     }
-
-    createDirs(nixDBPath + "/info");
-    createDirs(nixDBPath + "/referrer");
-    createDirs(nixDBPath + "/failed");
-
+    
     int curSchema = getSchema();
     if (curSchema > nixSchemaVersion)
         throw Error(format("current Nix store schema is version %1%, but I only support %2%")
@@ -79,6 +87,8 @@ LocalStore::LocalStore()
     }
     if (curSchema == 1) throw Error("your Nix store is no longer supported");
     if (curSchema < nixSchemaVersion) upgradeStore12();
+
+    doFsync = queryBoolSetting("fsync-metadata", false);
 }
 
 
@@ -212,7 +222,7 @@ static Path tmpFileForAtomicUpdate(const Path & path)
 }
 
 
-static void appendReferrer(const Path & from, const Path & to, bool lock)
+void LocalStore::appendReferrer(const Path & from, const Path & to, bool lock)
 {
     Path referrersFile = referrersFileFor(from);
     
@@ -227,6 +237,8 @@ static void appendReferrer(const Path & from, const Path & to, bool lock)
     
     string s = " " + to;
     writeFull(fd, (const unsigned char *) s.c_str(), s.size());
+
+    if (doFsync) fsync(fd);
 }
 
 
@@ -257,6 +269,8 @@ void LocalStore::rewriteReferrers(const Path & path, bool purge, PathSet referre
     
     writeFull(fd, (const unsigned char *) s.c_str(), s.size());
 
+    if (doFsync) fsync(fd);
+    
     fd.close(); /* for Windows; can't rename open file */
 
     if (rename(tmpFile.c_str(), referrersFile.c_str()) == -1)
@@ -337,7 +351,7 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi
 
     /* Atomically rewrite the info file. */
     Path tmpFile = tmpFileForAtomicUpdate(infoFile);
-    writeFile(tmpFile, s);
+    writeFile(tmpFile, s, doFsync);
     if (rename(tmpFile.c_str(), infoFile.c_str()) == -1)
         throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile);
 
@@ -380,6 +394,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
 
     assertStorePath(path);
 
+    if (!isValidPath(path))
+        throw Error(format("path `%1%' is not valid") % path);
+
     std::map<Path, ValidPathInfo>::iterator lookup = pathInfoCache.find(path);
     if (lookup != pathInfoCache.end()) return lookup->second;
     
@@ -394,7 +411,11 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
 
     foreach (Strings::iterator, i, lines) {
         string::size_type p = i->find(':');
-        if (p == string::npos) continue; /* bad line */
+        if (p == string::npos) {
+            if (!ignoreErrors)
+                throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i);
+            continue; /* bad line */
+        }
         string name(*i, 0, p);
         string value(*i, p + 2);
         if (name == "References") {
@@ -425,7 +446,22 @@ bool LocalStore::isValidPath(const Path & path)
     /* Files in the info directory starting with a `.' are temporary
        files. */
     if (baseNameOf(path).at(0) == '.') return false;
-    return pathExists(infoFileFor(path));
+
+    /* A path is valid if its info file exists and has a non-zero
+       size.  (The non-zero size restriction is to be robust to
+       certain kinds of filesystem corruption, particularly with
+       ext4.) */
+    Path infoFile = infoFileFor(path);
+
+    struct stat st;
+    if (lstat(infoFile.c_str(), &st)) {
+        if (errno == ENOENT) return false;
+        throw SysError(format("getting status of `%1%'") % infoFile);
+    }
+
+    if (st.st_size == 0) return false;
+    
+    return true;
 }
 
 
@@ -509,6 +545,13 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
 
     case 0: /* child */
         try {
+            /* Hack to let "make check" succeed on Darwin.  The
+               libtool wrapper script sets DYLD_LIBRARY_PATH to our
+               libutil (among others), but Perl also depends on a
+               library named libutil.  As a result, substituters
+               written in Perl (i.e. all of them) fail. */
+            unsetenv("DYLD_LIBRARY_PATH");
+            
             fromPipe.readSide.close();
             toPipe.writeSide.close();
             if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
@@ -698,7 +741,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
                 StringSource source(dump);
                 restorePath(dstPath, source);
             } else
-                writeStringToFile(dstPath, dump);
+                writeFile(dstPath, dump);
 
             canonicalisePathMetaData(dstPath);
 
@@ -753,7 +796,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
 
             if (pathExists(dstPath)) deletePathWrapped(dstPath);
 
-            writeStringToFile(dstPath, s);
+            writeFile(dstPath, s);
 
             canonicalisePathMetaData(dstPath);
             
@@ -830,9 +873,11 @@ void LocalStore::exportPath(const Path & path, bool sign,
         writeInt(1, hashAndWriteSink);
         
         Path tmpDir = createTempDir();
+        PathLocks tmpDirLock(singleton<PathSet, Path>(tmpDir));
+        tmpDirLock.setDeletion(true);
         AutoDelete delTmp(tmpDir);
         Path hashFile = tmpDir + "/hash";
-        writeStringToFile(hashFile, printHash(hash));
+        writeFile(hashFile, printHash(hash));
 
         Path secretKey = nixConfDir + "/signing-key.sec";
         checkSecrecy(secretKey);
@@ -879,6 +924,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
        store path follows the archive data proper), and besides, we
        don't know yet whether the signature is valid. */
     Path tmpDir = createTempDir(nixStore);
+    PathLocks tmpDirLock(singleton<PathSet, Path>(tmpDir));
+    tmpDirLock.setDeletion(true);
     AutoDelete delTmp(tmpDir); /* !!! could be GC'ed! */
     Path unpacked = tmpDir + "/unpacked";
 
@@ -908,7 +955,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
 
         if (requireSignature) {
             Path sigFile = tmpDir + "/sig";
-            writeStringToFile(sigFile, signature);
+            writeFile(sigFile, signature);
 
             Strings args;
             args.push_back("rsautl");
@@ -1011,8 +1058,18 @@ void LocalStore::verifyStore(bool checkContents)
         } else if (!pathExists(*i)) {
             printMsg(lvlError, format("path `%1%' disappeared") % *i);
             invalidatePath(*i);
-        } else
-            validPaths.insert(*i);
+        } else {
+            Path infoFile = infoFileFor(*i);
+            struct stat st;
+            if (lstat(infoFile.c_str(), &st))
+                throw SysError(format("getting status of `%1%'") % infoFile);
+            if (st.st_size == 0) {
+                printMsg(lvlError, format("removing corrupt info file `%1%'") % infoFile);
+                if (unlink(infoFile.c_str()) == -1)
+                    throw SysError(format("unlinking `%1%'") % infoFile);
+            }
+            else validPaths.insert(*i);
+        }
     }
 
 
@@ -1116,4 +1173,16 @@ void LocalStore::verifyStore(bool checkContents)
 }
 
 
+/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12).  The
+   old schema uses Berkeley DB, the new one stores store path
+   meta-information in files. */
+void LocalStore::upgradeStore12()
+{
+    throw Error(
+        "Your Nix store has a database in Berkeley DB format,\n"
+        "which is no longer supported. To convert to the new format,\n"
+        "please upgrade Nix to version 0.12 first.");
+}
+
+
 }

src/libstore/local-store.hh

@@ -158,12 +158,17 @@ private:
     /* Store paths for which the referrers file must be purged. */
     PathSet delayedUpdates;
 
+    /* Whether to do an fsync() after writing Nix metadata. */
+    bool doFsync;
+
     int getSchema();
 
     void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false);
 
     ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false);
 
+    void appendReferrer(const Path & from, const Path & to, bool lock);
+    
     void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
 
     void flushDelayedUpdates();
@@ -174,20 +179,20 @@ private:
     
     void upgradeStore12();
 
-    void gcPath(const GCOptions & options, GCResults & results,
-        const Path & path);
-
-    void gcPathRecursive(const GCOptions & options,
-        GCResults & results, PathSet & done, const Path & path);
+    struct GCState;
 
+    bool tryToDelete(GCState & state, const Path & path);
+    
+    PathSet findDerivers(GCState & state, const Path & path);
+    
+    bool isActiveTempFile(const GCState & state,
+        const Path & path, const string & suffix);
+        
     void startSubstituter(const Path & substituter,
         RunningSubstituter & runningSubstituter);
 };
 
 
-/* Copy a path recursively. */
-void copyPath(const Path & src, const Path & dst);
-
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
    - the last modification date on each file is set to 1 (i.e.,

src/libstore/misc.cc

@@ -2,8 +2,6 @@
 #include "store-api.hh"
 #include "local-store.hh"
 
-#include <aterm2.h>
-
 
 namespace nix {
 
@@ -12,14 +10,12 @@ Derivation derivationFromPath(const Path & drvPath)
 {
     assertStorePath(drvPath);
     store->ensurePath(drvPath);
-    ATerm t = ATreadFromNamedFile(drvPath.c_str());
-    if (!t) throw Error(format("cannot read aterm from `%1%'") % drvPath);
-    return parseDerivation(t);
+    return parseDerivation(readFile(drvPath));
 }
 
 
 void computeFSClosure(const Path & storePath,
-    PathSet & paths, bool flipDirection)
+    PathSet & paths, bool flipDirection, bool includeOutputs)
 {
     if (paths.find(storePath) != paths.end()) return;
     paths.insert(storePath);
@@ -30,8 +26,15 @@ void computeFSClosure(const Path & storePath,
     else
         store->queryReferences(storePath, references);
 
+    if (includeOutputs && isDerivation(storePath)) {
+        Derivation drv = derivationFromPath(storePath);
+        foreach (DerivationOutputs::iterator, i, drv.outputs)
+            if (store->isValidPath(i->second.path))
+                computeFSClosure(i->second.path, paths, flipDirection, true);
+    }
+
     foreach (PathSet::iterator, i, references)
-        computeFSClosure(*i, paths, flipDirection);
+        computeFSClosure(*i, paths, flipDirection, includeOutputs);
 }
 
 

src/libstore/misc.hh

@@ -19,7 +19,8 @@ Derivation derivationFromPath(const Path & drvPath);
    `referrers' relation instead of the `references' relation is
    returned. */
 void computeFSClosure(const Path & storePath,
-    PathSet & paths, bool flipDirection = false);
+    PathSet & paths, bool flipDirection = false,
+    bool includeOutputs = false);
 
 /* Return the path corresponding to the output identifier `id' in the
    given derivation. */

src/libstore/optimise-store.cc

@@ -5,6 +5,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <errno.h>
+#include <stdio.h>
 
 
 namespace nix {
@@ -118,9 +119,23 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath,
             }
 
             /* Atomically replace the old file with the new hard link. */
-            if (rename(tempLink.c_str(), path.c_str()) == -1)
+            if (rename(tempLink.c_str(), path.c_str()) == -1) {
+                if (errno == EMLINK) {
+                    /* Some filesystems generate too many links on the
+                       rename, rather than on the original link.
+                       (Probably it temporarily increases the st_nlink
+                       field before decreasing it again.) */
+                    printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first);
+                    hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
+
+                    /* Unlink the temp link. */
+                    if (unlink(tempLink.c_str()) == -1)
+                        printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
+                    return;
+                }
                 throw SysError(format("cannot rename `%1%' to `%2%'")
                     % tempLink % path);
+            }
         } else
             printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
         

src/libstore/pathlocks.cc

@@ -8,11 +8,6 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
-#ifdef __CYGWIN__
-#include <windows.h>
-#include <sys/cygwin.h>
-#endif
-
 
 namespace nix {
 
@@ -21,72 +16,24 @@ int openLockFile(const Path & path, bool create)
 {
     AutoCloseFD fd;
 
-#ifdef __CYGWIN__
-    /* On Cygwin we have to open the lock file without "DELETE"
-       sharing mode; otherwise Windows will allow open lock files to
-       be deleted (which is almost but not quite what Unix does). */
-    char win32Path[MAX_PATH + 1];
-    cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
-
-    SECURITY_ATTRIBUTES sa; /* required, otherwise inexplicably bad shit happens */
-    sa.nLength = sizeof sa;
-    sa.lpSecurityDescriptor = 0;
-    sa.bInheritHandle = TRUE;
-    HANDLE h = CreateFile(win32Path, GENERIC_READ | GENERIC_WRITE,
-        FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, 
-        (create ? OPEN_ALWAYS : OPEN_EXISTING), 
-        FILE_ATTRIBUTE_NORMAL, 0);
-    if (h == INVALID_HANDLE_VALUE) {
-        if (create || GetLastError() != ERROR_FILE_NOT_FOUND)
-            throw Error(format("opening lock file `%1%'") % path);
-        fd = -1;
-    }
-    else
-        fd = cygwin_attach_handle_to_fd((char *) path.c_str(), -1, h, 1, O_RDWR);
-#else        
     fd = open(path.c_str(), O_RDWR | (create ? O_CREAT : 0), 0666);
     if (fd == -1 && (create || errno != ENOENT))
         throw SysError(format("opening lock file `%1%'") % path);
-#endif
 
     return fd.borrow();
 }
 
 
-void deleteLockFilePreClose(const Path & path, int fd)
+void deleteLockFile(const Path & path, int fd)
 {
-#ifndef __CYGWIN__
     /* Get rid of the lock file.  Have to be careful not to introduce
-       races. */
-    /* On Unix, write a (meaningless) token to the file to indicate to
+       races.  Write a (meaningless) token to the file to indicate to
        other processes waiting on this lock that the lock is stale
        (deleted). */
     unlink(path.c_str());
     writeFull(fd, (const unsigned char *) "d", 1);
     /* Note that the result of unlink() is ignored; removing the lock
        file is an optimisation, not a necessity. */
-#endif
-}
-
-
-void deleteLockFilePostClose(const Path & path)
-{
-#ifdef __CYGWIN__
-    /* On Windows, just try to delete the lock file.  This will fail
-       if anybody still has the file open.  We cannot use unlink()
-       here, because Cygwin emulates Unix semantics of allowing an
-       open file to be deleted (but fakes it - the file isn't actually
-       deleted until later, so a file with the same name cannot be
-       created in the meantime). */
-    char win32Path[MAX_PATH + 1];
-    cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
-    if (DeleteFile(win32Path))
-        debug(format("delete of `%1%' succeeded") % path.c_str());
-    else
-        /* Not an error: probably means that the lock is still opened
-           by someone else. */
-        debug(format("delete of `%1%' failed: %2%") % path.c_str() % GetLastError());
-#endif
 }
 
 
@@ -220,15 +167,13 @@ PathLocks::~PathLocks()
 void PathLocks::unlock()
 {
     foreach (list<FDPair>::iterator, i, fds) {
-        if (deletePaths) deleteLockFilePreClose(i->second, i->first);
+        if (deletePaths) deleteLockFile(i->second, i->first);
 
         lockedPaths.erase(i->second);
         if (close(i->first) == -1)
             printMsg(lvlError,
                 format("error (ignored): cannot close lock file on `%1%'") % i->second);
 
-	if (deletePaths) deleteLockFilePostClose(i->second);
-
         debug(format("lock released on `%1%'") % i->second);
     }
 

src/libstore/pathlocks.hh

@@ -12,10 +12,8 @@ namespace nix {
    because it doesn't exist.  Any other error throws an exception. */
 int openLockFile(const Path & path, bool create);
 
-/* Delete an open lock file.  Both must be called to be fully portable
-   between Unix and Windows. */
-void deleteLockFilePreClose(const Path & path, int fd);
-void deleteLockFilePostClose(const Path & path);
+/* Delete an open lock file. */
+void deleteLockFile(const Path & path, int fd);
 
 enum LockType { ltRead, ltWrite, ltNone };
 

src/libstore/remote-store.cc

@@ -13,6 +13,7 @@
 
 #include <iostream>
 #include <unistd.h>
+#include <cstring>
 
 
 namespace nix {
@@ -158,6 +159,7 @@ void RemoteStore::connectToDaemon()
     addr.sun_family = AF_UNIX;
     if (socketPathRel.size() >= sizeof(addr.sun_path))
         throw Error(format("socket path `%1%' is too long") % socketPathRel);
+    using namespace std;
     strcpy(addr.sun_path, socketPathRel.c_str());
     
     if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1)
@@ -196,6 +198,9 @@ void RemoteStore::setOptions()
         writeInt(logType, to);
         writeInt(printBuildTrace, to);
     }
+    if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) {
+        writeInt(buildCores, to);
+    }
     processStderr();
 }
 
@@ -426,8 +431,9 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     writeLongLong(options.maxFreed, to);
     writeInt(options.maxLinks, to);
     if (GET_PROTOCOL_MINOR(daemonVersion) >= 5) {
-        writeInt(options.useAtime, to);
-        writeInt(options.maxAtime, to);
+        /* removed options */
+        writeInt(0, to);
+        writeInt(0, to);
     }
     
     processStderr();

src/libstore/store-api.cc

@@ -1,6 +1,7 @@
 #include "store-api.hh"
 #include "globals.hh"
 #include "util.hh"
+#include "derivations.hh"
 
 #include <limits.h>
 
@@ -14,8 +15,6 @@ GCOptions::GCOptions()
     ignoreLiveness = false;
     maxFreed = 0;
     maxLinks = 0;
-    useAtime = false;
-    maxAtime = (time_t) -1;
 }
 
 
@@ -54,6 +53,18 @@ Path toStorePath(const Path & path)
 }
 
 
+string getNameOfStorePath(const Path & path)
+{
+    Path::size_type slash = path.rfind('/');
+    string p = slash == Path::npos ? path : string(path, slash + 1);
+    Path::size_type dash = p.find('-');
+    assert(dash != Path::npos);
+    string p2 = string(p, dash + 1);
+    if (isDerivation(p2)) p2 = string(p2, 0, p2.size() - 4);
+    return p2;
+}
+
+
 Path followLinksToStore(const Path & _path)
 {
     Path path = absPath(_path);

src/libstore/store-api.hh

@@ -1,14 +1,14 @@
 #ifndef __STOREAPI_H
 #define __STOREAPI_H
 
+#include "hash.hh"
+#include "serialise.hh"
+
 #include <string>
 #include <map>
 
 #include <boost/shared_ptr.hpp>
 
-#include "hash.hh"
-#include "serialise.hh"
-
 
 namespace nix {
 
@@ -16,16 +16,10 @@ namespace nix {
 typedef std::map<Path, Path> Roots;
 
 
-
-
 struct GCOptions
 {
     /* Garbage collector operation:
 
-       - `gcReturnRoots': find and return the set of roots for the
-         garbage collector.  These are the store paths symlinked to in
-         the `gcroots' directory.
-
        - `gcReturnLive': return the set of paths reachable from
          (i.e. in the closure of) the roots.
 
@@ -38,7 +32,6 @@ struct GCOptions
           `pathsToDelete', insofar as they are not reachable.
     */
     typedef enum {
-        gcReturnRoots,
         gcReturnLive,
         gcReturnDead,
         gcDeleteDead,
@@ -64,22 +57,6 @@ struct GCOptions
        has dropped below `maxLinks'. */
     unsigned int maxLinks;
 
-    /* Delete paths in order of ascending last access time.  I.e.,
-       prefer deleting unrecently used paths.  Useful in conjunction
-       with `maxFreed' and `maxLinks' (or manual interruption).  The
-       access time of a path is defined as the highest atime of any
-       non-directory, non-symlink file under that path.  Directories
-       and symlinks are ignored because their atimes are frequently
-       mass-updated, e.g. by `locate'.  Note that optimiseStore()
-       somewhat reduces the usefulness of this option: it hard-links
-       regular files and symlink together, giving them a "shared"
-       atime. */
-    bool useAtime;
-
-    /* Do not delete paths newer than `maxAtime'.  -1 means no age
-       limit. */
-    time_t maxAtime;
-
     GCOptions();
 };
 
@@ -264,6 +241,12 @@ void checkStoreName(const string & name);
 Path toStorePath(const Path & path);
 
 
+/* Get the "name" part of a store path, that is, the part after the
+   hash and the dash, and with any ".drv" suffix removed
+   (e.g. /nix/store/<hash>-foo-1.2.3.drv => foo-1.2.3). */
+string getNameOfStorePath(const Path & path);
+
+
 /* Follow symlinks until we end up with a path in the Nix store. */
 Path followLinksToStore(const Path & path);
 

src/libstore/upgrade-schema.cc

@@ -1,108 +0,0 @@
-#include "db.hh"
-#include "hash.hh"
-#include "util.hh"
-#include "local-store.hh"
-#include "globals.hh"
-#include "pathlocks.hh"
-#include "config.h"
-
-#include <iostream>
-
-
-namespace nix {
-
-
-Hash parseHashField(const Path & path, const string & s);
-
-
-/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12).  The
-   old schema uses Berkeley DB, the new one stores store path
-   meta-information in files. */
-void LocalStore::upgradeStore12()
-{
-#if OLD_DB_COMPAT
-    
-#ifdef __CYGWIN__
-    /* Cygwin can't upgrade a read lock to a write lock... */
-    lockFile(globalLock, ltNone, true);
-#endif
-
-    if (!lockFile(globalLock, ltWrite, false)) {
-        printMsg(lvlError, "waiting for exclusive access to the Nix store...");
-        lockFile(globalLock, ltWrite, true);
-    }
-
-    printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
-
-    if (getSchema() >= nixSchemaVersion) return; /* somebody else beat us to it */
-
-    /* Open the old Nix database and tables. */
-    Database nixDB;
-    nixDB.open(nixDBPath);
-    
-    /* dbValidPaths :: Path -> ()
-
-       The existence of a key $p$ indicates that path $p$ is valid
-       (that is, produced by a succesful build). */
-    TableId dbValidPaths = nixDB.openTable("validpaths");
-
-    /* dbReferences :: Path -> [Path]
-
-       This table lists the outgoing file system references for each
-       output path that has been built by a Nix derivation.  These are
-       found by scanning the path for the hash components of input
-       paths. */
-    TableId dbReferences = nixDB.openTable("references");
-
-    /* dbReferrers :: Path -> Path
-
-       This table is just the reverse mapping of dbReferences.  This
-       table can have duplicate keys, each corresponding value
-       denoting a single referrer. */
-    // Not needed for conversion: it's just the inverse of
-    // references.
-    // TableId dbReferrers = nixDB.openTable("referrers");
-
-    /* dbDerivers :: Path -> [Path]
-
-       This table lists the derivation used to build a path.  There
-       can only be multiple such paths for fixed-output derivations
-       (i.e., derivations specifying an expected hash). */
-    TableId dbDerivers = nixDB.openTable("derivers");
-
-    Paths paths;
-    nixDB.enumTable(noTxn, dbValidPaths, paths);
-    
-    foreach (Paths::iterator, i, paths) {
-        ValidPathInfo info;
-        info.path = *i;
-        
-        Paths references;
-        nixDB.queryStrings(noTxn, dbReferences, *i, references);
-        info.references.insert(references.begin(), references.end());
-        
-        string s;
-        nixDB.queryString(noTxn, dbValidPaths, *i, s);
-        info.hash = parseHashField(*i, s);
-        
-        nixDB.queryString(noTxn, dbDerivers, *i, info.deriver);
-        
-        registerValidPath(info, true);
-        std::cerr << ".";
-    }
-
-    std::cerr << std::endl;
-
-    writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
-
-    lockFile(globalLock, ltRead, true);
-
-#else
-    throw Error(
-        "Your Nix store has a database in Berkeley DB format. To convert\n"
-        "to the new format, please compile Nix with Berkeley DB support.");
-#endif
-}
-
-
-}

src/libstore/worker-protocol.hh

@@ -8,7 +8,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION 0x105
+#define PROTOCOL_VERSION 0x106
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 

src/libutil/Makefile.am

@@ -1,16 +1,16 @@
 pkglib_LTLIBRARIES = libutil.la
 
 libutil_la_SOURCES = util.cc hash.cc serialise.cc \
-  archive.cc aterm.cc aterm-map.cc xml-writer.cc
+  archive.cc xml-writer.cc
 
 libutil_la_LIBADD = ../boost/format/libformat.la
 
 pkginclude_HEADERS = util.hh hash.hh serialise.hh \
-  archive.hh aterm.hh aterm-map.hh xml-writer.hh types.hh
+  archive.hh xml-writer.hh types.hh
 
 if !HAVE_OPENSSL
 libutil_la_SOURCES += \
  md5.c md5.h sha1.c sha1.h sha256.c sha256.h md32_common.h
 endif
 
-AM_CXXFLAGS = -Wall -I$(srcdir)/.. ${aterm_include}
+AM_CXXFLAGS = -Wall -I$(srcdir)/..

src/libutil/archive.cc

@@ -1,3 +1,5 @@
+#include "config.h"
+
 #include <cerrno>
 #include <algorithm>
 #include <vector>
@@ -12,8 +14,6 @@
 #include "archive.hh"
 #include "util.hh"
 
-#include "config.h"
-
 
 namespace nix {
 
@@ -302,7 +302,12 @@ struct RestoreSink : ParseSink
 #if HAVE_POSIX_FALLOCATE
         if (len) {
             errno = posix_fallocate(fd, 0, len);
-            if (errno) throw SysError(format("preallocating file of %1% bytes") % len);
+            /* Note that EINVAL may indicate that the underlying
+               filesystem doesn't support preallocation (e.g. on
+               OpenSolaris).  Since preallocation is just an
+               optimisation, ignore it. */
+            if (errno && errno != EINVAL)
+                throw SysError(format("preallocating file of %1% bytes") % len);
         }
 #endif
     }

src/libutil/aterm-map.cc

@@ -1,332 +0,0 @@
-#include "aterm-map.hh"
-
-#include <iostream>
-
-#include <assert.h>
-#include <stdlib.h>
-
-#include <aterm2.h>
-
-
-namespace nix {
-
-
-static const unsigned int maxLoadFactor = /* 1 / */ 3;
-static unsigned int nrResizes = 0;
-static unsigned int sizeTotalAlloc = 0;
-static unsigned int sizeCurAlloc = 0;
-static unsigned int sizeMaxAlloc = 0;
-
-
-ATermMap::ATermMap(unsigned int expectedCount)
-{
-    init(expectedCount);
-}
-
-
-ATermMap::ATermMap(const ATermMap & map)
-{
-    init(map.maxCount);
-    copy(map.hashTable, map.capacity);
-}
-
-
-ATermMap & ATermMap::operator = (const ATermMap & map)
-{
-    if (this == &map) return *this;
-    free();
-    init(map.maxCount);
-    copy(map.hashTable, map.capacity);
-    return *this;
-}
-
-
-ATermMap::~ATermMap()
-{
-    free();
-}
-
-
-void ATermMap::init(unsigned int expectedCount)
-{
-    assert(sizeof(ATerm) * 2 == sizeof(KeyValue));
-    capacity = 0;
-    count = 0;
-    maxCount = 0;
-    hashTable = 0;
-    resizeTable(expectedCount);
-}
-
-
-void ATermMap::free()
-{
-    if (hashTable) {
-        ATunprotectArray((ATerm *) hashTable);
-        ::free(hashTable);
-        sizeCurAlloc -= sizeof(KeyValue) * capacity;
-        hashTable = 0;
-    }
-}
-
-
-static unsigned int roundToPowerOf2(unsigned int x)
-{
-    x--;
-    x |= x >> 1; x |= x >> 2; x |= x >> 4; x |= x >> 8; x |= x >> 16;
-    x++;
-    return x;
-}
-
-
-void ATermMap::resizeTable(unsigned int expectedCount)
-{
-    if (expectedCount == 0) expectedCount = 1;
-//     cout << maxCount << " -> " << expectedCount << endl;
-//     cout << maxCount << " " << size << endl;
-//     cout << (double) size / maxCount << endl;
-
-    unsigned int oldCapacity = capacity;
-    KeyValue * oldHashTable = hashTable;
-
-    maxCount = expectedCount;
-    capacity = roundToPowerOf2(maxCount * maxLoadFactor);
-    hashTable = (KeyValue *) calloc(sizeof(KeyValue), capacity);
-    sizeTotalAlloc += sizeof(KeyValue) * capacity;
-    sizeCurAlloc += sizeof(KeyValue) * capacity;
-    if (sizeCurAlloc > sizeMaxAlloc) sizeMaxAlloc = sizeCurAlloc;
-    ATprotectArray((ATerm *) hashTable, capacity * 2);
-    
-//     cout << capacity << endl;
-
-    /* Re-hash the elements in the old table. */
-    if (oldCapacity != 0) {
-        count = 0;
-        copy(oldHashTable, oldCapacity);
-        ATunprotectArray((ATerm *) oldHashTable);
-        ::free(oldHashTable);
-        sizeCurAlloc -= sizeof(KeyValue) * oldCapacity;
-        nrResizes++;
-    }
-}
-
-
-void ATermMap::copy(KeyValue * elements, unsigned int capacity)
-{
-    for (unsigned int i = 0; i < capacity; ++i)
-        if (elements[i].value) /* i.e., non-empty, non-deleted element */
-            set(elements[i].key, elements[i].value);
-}
-
-
-/* !!! use a bigger shift for 64-bit platforms? */
-static const unsigned int shift = 16;
-static const unsigned long knuth = (unsigned long) (0.6180339887 * (1 << shift));
-
-
-unsigned long ATermMap::hash1(ATerm key) const
-{
-    /* Don't care about the least significant bits of the ATerm
-       pointer since they're always 0. */
-    unsigned long key2 = ((unsigned long) key) >> 2;
-
-    /* Approximately equal to:
-    double d = key2 * 0.6180339887;
-    unsigned int h = (int) (capacity * (d - floor(d)));
-    */
- 
-    unsigned long h = (capacity * ((key2 * knuth) & ((1 << shift) - 1))) >> shift;
-
-    return h;
-}
-
-
-unsigned long ATermMap::hash2(ATerm key) const
-{
-    unsigned long key2 = ((unsigned long) key) >> 2;
-    /* Note: the result must be relatively prime to `capacity' (which
-       is a power of 2), so we make sure that the result is always
-       odd. */
-    unsigned long h = ((key2 * 134217689) & (capacity - 1)) | 1;
-    return h;
-}
-
-
-static unsigned int nrItemsSet = 0;
-static unsigned int nrSetProbes = 0;
-
-
-void ATermMap::set(ATerm key, ATerm value)
-{
-    if (count == maxCount) resizeTable(capacity * 2 / maxLoadFactor);
-    
-    nrItemsSet++;
-    for (unsigned int i = 0, h = hash1(key); i < capacity;
-         ++i, h = (h + hash2(key)) & (capacity - 1))
-    {
-        // assert(h < capacity);
-        nrSetProbes++;
-        /* Note: to see whether a slot is free, we check
-           hashTable[h].value, not hashTable[h].key, since we use
-           value == 0 to mark deleted slots. */
-        if (hashTable[h].value == 0 || hashTable[h].key == key) {
-            if (hashTable[h].value == 0) count++;
-            hashTable[h].key = key;
-            hashTable[h].value = value;
-            return;
-        }
-    }
-        
-    abort();
-}
-
-
-static unsigned int nrItemsGet = 0;
-static unsigned int nrGetProbes = 0;
-
-
-ATerm ATermMap::get(ATerm key) const
-{
-    nrItemsGet++;
-    for (unsigned int i = 0, h = hash1(key); i < capacity;
-         ++i, h = (h + hash2(key)) & (capacity - 1))
-    {
-        nrGetProbes++;
-        if (hashTable[h].key == 0) return 0;
-        if (hashTable[h].key == key) return hashTable[h].value;
-    }
-    return 0;
-}
-
-
-void ATermMap::remove(ATerm key)
-{
-    for (unsigned int i = 0, h = hash1(key); i < capacity;
-         ++i, h = (h + hash2(key)) & (capacity - 1))
-    {
-        if (hashTable[h].key == 0) return;
-        if (hashTable[h].key == key) {
-            if (hashTable[h].value != 0) {
-                hashTable[h].value = 0;
-                count--;
-            }
-            return;
-        }
-    }
-}
-
-
-unsigned int ATermMap::size()
-{
-    return count; /* STL nomenclature */
-}
-
-
-void printATermMapStats()
-{
-    using std::cerr;
-    using std::endl;
-    
-    cerr << "RESIZES: " << nrResizes << " "
-         << sizeTotalAlloc << " "
-         << sizeCurAlloc << " "
-         << sizeMaxAlloc << endl;
-        
-    cerr << "SET: "
-         << nrItemsSet << " "
-         << nrSetProbes << " "
-         << (double) nrSetProbes / nrItemsSet << endl;
-
-    cerr << "GET: "
-         << nrItemsGet << " "
-         << nrGetProbes << " "
-         << (double) nrGetProbes / nrItemsGet << endl;
-}
-
-
-#if 0
-int main(int argc, char * * argv)
-{
-    ATerm bottomOfStack;
-    ATinit(argc, argv, &bottomOfStack);
-
-    /* Make test terms. */
-    int nrTestTerms = 100000;
-    ATerm testTerms[nrTestTerms];
-
-    for (int i = 0; i < nrTestTerms; ++i) {
-        char name[10];
-        sprintf(name, "%d", (int) random() % 37);
-
-        int arity = i == 0 ? 0 : (random() % 37);
-        ATerm kids[arity];
-        for (int j = 0; j < arity; ++j)
-            kids[j] = testTerms[random() % i];
-        
-        testTerms[i] = (ATerm) ATmakeApplArray(ATmakeAFun(name, arity, ATfalse), kids);
-//         ATwriteToSharedTextFile(testTerms[i], stdout);
-//         printf("\n");
-    }
-
-
-    cout << "testing...\n";
-
-    
-    #define someTerm() (testTerms[(int) random() % nrTestTerms])
-
-
-    for (int test = 0; test < 100000; ++test) {
-        //cerr << test << endl;
-        unsigned int n = 300;
-        ATermMap map(300);
-        ATerm keys[n], values[n];
-        for (unsigned int i = 0; i < n; ++i) {
-            keys[i] = someTerm();
-            values[i] = someTerm();
-            map.set(keys[i], values[i]);
-            //cerr << "INSERT: " << keys[i] << " " << values[i] << endl;
-        }
-
-        unsigned int size = map.size();
-        assert(size <= n);
-        values[n - 1] = 0;
-        map.remove(keys[n - 1]);
-        assert(map.size() == size - 1);
-
-        unsigned int checksum;
-        unsigned int count = 0;
-        for (ATermMap::const_iterator i = map.begin(); i != map.end(); ++i, ++count) {
-            assert(i->key);
-            assert(i->value);
-            checksum += (unsigned int) (*i).key;
-            checksum += (unsigned int) (*i).value;
-            // cout << (*i).key << " " << (*i).value << endl;
-        }
-        assert(count == size - 1);
-
-        for (unsigned int i = 0; i < n; ++i) {
-            for (unsigned int j = i + 1; j < n; ++j)
-                if (keys[i] == keys[j]) goto x;
-            if (map.get(keys[i]) != values[i]) {
-                cerr << "MISMATCH: " << keys[i] << " " << values[i] << " " << map.get(keys[i]) << " " << i << endl;
-                abort();
-            }
-            if (values[i] != 0) {
-                checksum -= (unsigned int) keys[i];
-                checksum -= (unsigned int) values[i];
-            }
-        x:  ;
-        }
-
-        assert(checksum == 0);
-        
-        for (unsigned int i = 0; i < 100; ++i)
-            map.get(someTerm());
-        
-    }
-
-    printATermMapStats();
-}
-#endif
-
- 
-}