* Sync with the trunk.

when they are defined

.gitignore

@@ -0,0 +1,262 @@
+# START "git svn show-ignore"
+
+# /
+/Makefile
+/Makefile.in
+/aclocal.m4
+/autom4te.cache
+/config.*
+/configure
+/nix.spec
+/stamp-h1
+/svn-revision
+/NEWS
+/libtool
+
+# /config/
+/config/config.guess
+/config/config.sub
+/config/depcomp
+/config/install-sh
+/config/missing
+/config/mkinstalldirs
+/config/ltmain.sh
+
+# /corepkgs/
+/corepkgs/Makefile
+/corepkgs/Makefile.in
+
+# /corepkgs/buildenv/
+/corepkgs/buildenv/Makefile.in
+/corepkgs/buildenv/Makefile
+/corepkgs/buildenv/builder.pl
+
+# /corepkgs/channels/
+/corepkgs/channels/Makefile.in
+/corepkgs/channels/Makefile
+/corepkgs/channels/unpack.sh
+
+# /corepkgs/nar/
+/corepkgs/nar/Makefile
+/corepkgs/nar/Makefile.in
+/corepkgs/nar/nar.sh
+/corepkgs/nar/unnar.sh
+
+# /doc/
+/doc/Makefile
+/doc/Makefile.in
+
+# /doc/manual/
+/doc/manual/Makefile
+/doc/manual/Makefile.in
+/doc/manual/manual.html
+/doc/manual/manual.is-valid
+/doc/manual/*.1
+/doc/manual/*.8
+/doc/manual/images
+/doc/manual/version.txt
+/doc/manual/NEWS.html
+/doc/manual/NEWS.txt
+
+# /externals/
+/externals/Makefile
+/externals/Makefile.in
+/externals/aterm-*
+/externals/have-aterm
+/externals/build-aterm
+/externals/inst-aterm
+/externals/bzip2-*
+/externals/have-bzip2
+/externals/build-bzip2
+/externals/inst-bzip2
+
+# /make/examples/aterm/
+/make/examples/aterm/result*
+
+# /make/examples/aterm/aterm/
+/make/examples/aterm/aterm/*
+
+# /make/examples/aterm/test/
+/make/examples/aterm/test/*
+
+# /misc/
+/misc/Makefile.in
+/misc/Makefile
+
+# /misc/emacs/
+/misc/emacs/Makefile.in
+/misc/emacs/Makefile
+
+# /scripts/
+/scripts/Makefile
+/scripts/Makefile.in
+/scripts/nix-profile.sh
+/scripts/nix-pull
+/scripts/nix-push
+/scripts/nix-switch
+/scripts/nix-collect-garbage
+/scripts/nix-prefetch-url
+/scripts/nix-install-package
+/scripts/nix-channel
+/scripts/nix-build
+/scripts/nix-copy-closure
+/scripts/readmanifest.pm
+/scripts/readconfig.pm
+/scripts/download-using-manifests.pl
+/scripts/copy-from-other-stores.pl
+/scripts/generate-patches.pl
+/scripts/find-runtime-roots.pl
+/scripts/build-remote.pl
+
+# /src/
+/src/Makefile
+/src/Makefile.in
+
+# /src/bin2c/
+/src/bin2c/Makefile.in
+/src/bin2c/Makefile
+/src/bin2c/bin2c
+/src/bin2c/.deps
+/src/bin2c/.libs
+
+# /src/boost/
+/src/boost/Makefile
+/src/boost/Makefile.in
+
+# /src/boost/format/
+/src/boost/format/Makefile
+/src/boost/format/Makefile.in
+/src/boost/format/.deps
+/src/boost/format/libformat.a
+/src/boost/format/.libs
+
+# /src/bsdiff-4.3/
+/src/bsdiff-4.3/Makefile
+/src/bsdiff-4.3/Makefile.in
+/src/bsdiff-4.3/bsdiff
+/src/bsdiff-4.3/bspatch
+/src/bsdiff-4.3/.deps
+/src/bsdiff-4.3/.libs
+
+# /src/libexpr/
+/src/libexpr/Makefile
+/src/libexpr/Makefile.in
+/src/libexpr/.deps
+/src/libexpr/libexpr.a
+/src/libexpr/lexer-tab.cc
+/src/libexpr/lexer-tab.hh
+/src/libexpr/parser-tab.cc
+/src/libexpr/parser-tab.hh
+/src/libexpr/parser-tab.output
+/src/libexpr/nixexpr-ast.hh
+/src/libexpr/nixexpr-ast.cc
+/src/libexpr/.libs
+/src/libexpr/nix.tbl
+
+# /src/libmain/
+/src/libmain/Makefile
+/src/libmain/Makefile.in
+/src/libmain/.deps
+/src/libmain/libmain.a
+/src/libmain/.libs
+
+# /src/libstore/
+/src/libstore/Makefile
+/src/libstore/Makefile.in
+/src/libstore/.deps
+/src/libstore/libstore.a
+/src/libstore/derivations-ast.cc
+/src/libstore/derivations-ast.hh
+/src/libstore/.libs
+
+# /src/libutil/
+/src/libutil/Makefile
+/src/libutil/Makefile.in
+/src/libutil/.deps
+/src/libutil/libutil.a
+/src/libutil/.libs
+
+# /src/nix-env/
+/src/nix-env/Makefile.in
+/src/nix-env/Makefile
+/src/nix-env/.deps
+/src/nix-env/nix-env
+/src/nix-env/help.txt.hh
+/src/nix-env/.libs
+
+# /src/nix-hash/
+/src/nix-hash/Makefile
+/src/nix-hash/Makefile.in
+/src/nix-hash/.deps
+/src/nix-hash/.libs
+/src/nix-hash/nix-hash
+/src/nix-hash/help.txt.hh
+
+# /src/nix-instantiate/
+/src/nix-instantiate/Makefile.in
+/src/nix-instantiate/Makefile
+/src/nix-instantiate/.deps
+/src/nix-instantiate/nix-instantiate
+/src/nix-instantiate/help.txt.hh
+/src/nix-instantiate/.libs
+
+# /src/nix-log2xml/
+/src/nix-log2xml/Makefile.in
+/src/nix-log2xml/Makefile
+/src/nix-log2xml/.deps
+/src/nix-log2xml/nix-log2xml
+/src/nix-log2xml/test*.*
+/src/nix-log2xml/.libs
+/src/nix-log2xml/*.log
+/src/nix-log2xml/*.xml
+/src/nix-log2xml/*.html
+
+# /src/nix-setuid-helper/
+/src/nix-setuid-helper/Makefile.in
+/src/nix-setuid-helper/Makefile
+/src/nix-setuid-helper/.deps
+/src/nix-setuid-helper/nix-setuid-helper
+/src/nix-setuid-helper/help.txt.hh
+/src/nix-setuid-helper/.libs
+
+# /src/nix-store/
+/src/nix-store/Makefile
+/src/nix-store/Makefile.in
+/src/nix-store/.deps
+/src/nix-store/help.txt.hh
+/src/nix-store/nix-store
+/src/nix-store/.libs
+
+# /src/nix-worker/
+/src/nix-worker/Makefile.in
+/src/nix-worker/Makefile
+/src/nix-worker/.deps
+/src/nix-worker/nix-worker
+/src/nix-worker/help.txt.hh
+/src/nix-worker/.libs
+
+# /tests/
+/tests/Makefile
+/tests/Makefile.in
+/tests/test-tmp
+/tests/config.nix
+/tests/common.sh
+/tests/dummy
+
+# /tests/lang/
+/tests/lang/*.out
+/tests/lang/*.out.xml
+/tests/lang/*.ast
+
+# END "git svn show-ignore"
+
+*.lo
+*.la
+*.o
+*~
+
+# GNU Global
+GPATH
+GRTAGS
+GSYMS
+GTAGS

Makefile.am

@@ -2,6 +2,8 @@ SUBDIRS = externals src scripts corepkgs doc misc tests
 EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
   nix.conf.example NEWS version
 
+pkginclude_HEADERS = config.h
+
 include ./substitute.mk
 
 nix.spec: nix.spec.in
@@ -29,11 +31,9 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
-	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
-	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
-	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
+	-$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
 

aterm-gc.supp

@@ -115,3 +115,35 @@
    fun:*
    fun:AT_collect
 }
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:*
+   fun:mark_phase
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Cond
+   fun:*
+   fun:*
+   fun:mark_phase
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:*
+   fun:mark_phase_young
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Cond
+   fun:*
+   fun:*
+   fun:mark_phase_young
+}

bootstrap.sh

@@ -1,4 +1,5 @@
 #! /bin/sh -e
+rm -f aclocal.m4
 mkdir -p config
 libtoolize --copy
 aclocal

configure.ac

@@ -10,8 +10,8 @@ AC_CANONICAL_HOST
 
 # Construct a Nix system name (like "i686-linux").
 AC_MSG_CHECKING([for the canonical Nix system name])
-cpu_name=$(uname -p | tr 'A-Z ' 'a-z_')
-machine_name=$(uname -m | tr 'A-Z ' 'a-z_')
+cpu_name=$(uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
+machine_name=$(uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
 
 case $machine_name in
     i*86)
@@ -30,7 +30,7 @@ case $machine_name in
         ;;
 esac
 
-sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
+sys_name=$(uname -s | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
 
 case $sys_name in
     cygwin*)
@@ -50,46 +50,52 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
-# Whether to produce a statically linked binary.  On Cygwin, this is
-# the default: dynamically linking against the ATerm DLL does work,
-# except that it requires the ATerm "lib" directory to be in $PATH, as
-# Windows doesn't have anything like an RPATH embedded in executable.
-# Since this is kind of annoying, we use static libraries for now.
-    
-AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
-  [produce statically linked binaries]),
-  static_nix=$enableval, static_nix=no)
-
-if test "$sys_name" = cygwin; then
-   static_nix=yes
-fi
-  
-if test "$static_nix" = yes; then
+# Windows-specific stuff.  On Cygwin, dynamically linking against the
+# ATerm DLL works, except that it requires the ATerm "lib" directory
+# to be in $PATH, as Windows doesn't have anything like an RPATH
+# embedded in executable.  Since this is kind of annoying, we use
+# static libraries for now.
+if test "$sys_name" = "cygwin"; then
     AC_DISABLE_SHARED
     AC_ENABLE_STATIC
 fi
 
-  
-# Windows-specific stuff.
-if test "$sys_name" = "cygwin"; then
-    # We cannot delete open files.
-    AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.])
+
+# Solaris-specific stuff.
+if test "$sys_name" = "sunos"; then
+    # Solaris requires -lsocket -lnsl for network functions
+    LIBS="-lsocket -lnsl $LIBS"
 fi
 
 
 AC_PROG_CC
 AC_PROG_CXX
 
+# To build programs to be run in the build machine
+if test "$CC_FOR_BUILD" = ""; then
+    if test "$cross_compiling" = "yes"; then
+        AC_CHECK_PROGS(CC_FOR_BUILD, gcc cc)
+    else
+        CC_FOR_BUILD="$CC"
+    fi
+fi
+AC_SUBST([CC_FOR_BUILD])
 
 # We are going to use libtool.
 AC_DISABLE_STATIC
 AC_ENABLE_SHARED
 AC_PROG_LIBTOOL
 
+if test "$enable_shared" = yes; then
+   SUB_CONFIGURE_FLAGS="--enable-shared --disable-static"
+else
+   SUB_CONFIGURE_FLAGS="--enable-static --disable-shared"
+fi
+AC_SUBST(SUB_CONFIGURE_FLAGS)
+
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
-CFLAGS="-D_FILE_OFFSET_BITS=64 $CFLAGS"
-CXXFLAGS="-D_FILE_OFFSET_BITS=64 $CXXFLAGS"
+AC_SYS_LARGEFILE
 
 
 # Check for pubsetbuf.
@@ -122,11 +128,22 @@ AC_CHECK_HEADERS([locale], [], [], [])
 AC_LANG_POP(C++)
 
 
+# Check for <err.h>.
+AC_CHECK_HEADER([err.h], [], [bsddiff_compat_include="-Icompat-include"])
+AC_SUBST([bsddiff_compat_include])
+
+
 # Check whether we have the personality() syscall, which allows us to
 # do i686-linux builds on x86_64-linux machines.
 AC_CHECK_HEADERS([sys/personality.h])
 
 
+# Check for tr1/unordered_set.
+AC_LANG_PUSH(C++)
+AC_CHECK_HEADERS([tr1/unordered_set], [], [], [])
+AC_LANG_POP(C++)
+
+
 AC_DEFUN([NEED_PROG],
 [
 AC_PATH_PROG($1, $2)
@@ -140,11 +157,11 @@ NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
-AC_PATH_PROG(jing, jing, false) # needed because xmllint --relaxng seems broken
 AC_PATH_PROG(w3m, w3m, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
 NEED_PROG(perl, perl)
+NEED_PROG(sed, sed)
 NEED_PROG(tar, tar)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(dblatex, dblatex)
@@ -189,48 +206,6 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
   storedir=$withval, storedir='/nix/store')
 AC_SUBST(storedir)
 
-AC_ARG_ENABLE(old-db-compat, AC_HELP_STRING([--disable-old-db-compat],
-  [disable support for converting from old Berkeley DB-based Nix stores]),
-  old_db_compat=$enableval, old_db_compat=yes)
-AM_CONDITIONAL(OLD_DB_COMPAT, test "$old_db_compat" = "yes")
-
-AC_ARG_WITH(bdb, AC_HELP_STRING([--with-bdb=PATH],
-  [prefix of Berkeley DB (for Nix <= 0.11 compatibility)]),
-  bdb=$withval, bdb=)
-AM_CONDITIONAL(HAVE_BDB, test -n "$bdb")
-if test -z "$bdb"; then
-  bdb_lib='-L${top_builddir}/externals/inst-bdb/lib -ldb_cxx'
-  bdb_include='-I${top_builddir}/externals/inst-bdb/include'
-else
-  bdb_lib="-L$bdb/lib -ldb_cxx"
-  bdb_include="-I$bdb/include"
-fi
-if test "$old_db_compat" = "no"; then
-  bdb_lib=
-  bdb_include=
-else
-  AC_DEFINE(OLD_DB_COMPAT, 1, [Whether to support converting from old Berkeley DB-based Nix stores.])
-fi
-AC_SUBST(bdb_lib)
-AC_SUBST(bdb_include)
-
-AC_ARG_WITH(aterm, AC_HELP_STRING([--with-aterm=PATH],
-  [prefix of CWI ATerm library]),
-  aterm=$withval, aterm=)
-AM_CONDITIONAL(HAVE_ATERM, test -n "$aterm")
-if test -z "$aterm"; then
-  aterm_lib='-L${top_builddir}/externals/inst-aterm/lib -lATerm'
-  aterm_include='-I${top_builddir}/externals/inst-aterm/include'
-  aterm_bin='${top_builddir}/externals/inst-aterm/bin'
-else
-  aterm_lib="-L$aterm/lib -lATerm"
-  aterm_include="-I$aterm/include"
-  aterm_bin="$aterm/bin"
-fi
-AC_SUBST(aterm_lib)
-AC_SUBST(aterm_include)
-AC_SUBST(aterm_bin)
-
 AC_ARG_WITH(openssl, AC_HELP_STRING([--with-openssl=PATH],
   [prefix of the OpenSSL library]),
   openssl=$withval, openssl=)
@@ -246,6 +221,8 @@ AC_ARG_WITH(bzip2, AC_HELP_STRING([--with-bzip2=PATH],
   [prefix of bzip2]),
   bzip2=$withval, bzip2=)
 AM_CONDITIONAL(HAVE_BZIP2, test -n "$bzip2")
+ATERM_VERSION=2.5
+AC_SUBST(ATERM_VERSION)
 if test -z "$bzip2"; then
   # Headers and libraries will be used from the temporary installation
   # in externals/inst-bzip2.
@@ -260,14 +237,42 @@ else
   bzip2_include="-I$bzip2/include"
   bzip2_bin="$bzip2/bin"
   bzip2_bin_test="$bzip2/bin"
-fi   
+fi
 AC_SUBST(bzip2_lib)
 AC_SUBST(bzip2_include)
 AC_SUBST(bzip2_bin)
 AC_SUBST(bzip2_bin_test)
 
+AC_ARG_WITH(sqlite, AC_HELP_STRING([--with-sqlite=PATH],
+  [prefix of SQLite]),
+  sqlite=$withval, sqlite=)
+AM_CONDITIONAL(HAVE_SQLITE, test -n "$sqlite")
+SQLITE_VERSION=3070500
+AC_SUBST(SQLITE_VERSION)
+if test -z "$sqlite"; then
+  sqlite_lib='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)/libsqlite3.la'
+  sqlite_include='-I${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)'
+  sqlite_bin='${top_builddir}/externals/sqlite-autoconf-$(SQLITE_VERSION)'
+else
+  sqlite_lib="-L$sqlite/lib -lsqlite3"
+  sqlite_include="-I$sqlite/include"
+  sqlite_bin="$sqlite/bin"
+fi
+AC_SUBST(sqlite_lib)
+AC_SUBST(sqlite_include)
+AC_SUBST(sqlite_bin)
 
-AC_CHECK_LIB(pthread, pthread_mutex_init)
+# Whether to use the Boehm garbage collector.
+AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
+  [enable garbage collection in the Nix expression evaluator (requires Boehm GC)]),
+  gc=$enableval, gc=)
+if test -n "$gc"; then
+  PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
+  boehmgc_lib="-L$boehmgc/lib -lgc"
+  CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
+  AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
+fi
+AC_SUBST(boehmgc_lib)
 
 
 AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state],
@@ -281,25 +286,16 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
 
 
 # Nice to have, but not essential.
-AC_CHECK_FUNCS([strsignal])
-AC_CHECK_FUNCS([posix_fallocate])
+AC_CHECK_FUNCS([strsignal posix_fallocate nanosleep])
 
 
-# This is needed if ATerm, Berkeley DB or bzip2 are static libraries,
+# This is needed if ATerm or bzip2 are static libraries,
 # and the Nix libraries are dynamic.
 if test "$(uname)" = "Darwin"; then
     LDFLAGS="-all_load $LDFLAGS"
 fi
 
 
-if test "$static_nix" = yes; then
-    # `-all-static' has to be added at the end of configure, because
-    # the C compiler doesn't know about -all-static (it's filtered out
-    # by libtool, but configure doesn't use libtool).
-    LDFLAGS="-all-static $LDFLAGS"
-fi
-
-
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    externals/Makefile

corepkgs/buildenv/builder.pl.in

@@ -29,10 +29,18 @@ sub createLinks {
         $baseName =~ s/^.*\///g; # strip directory
         my $dstFile = "$dstDir/$baseName";
 
+        # The files below are special-cased so that they don't show up
+        # in user profiles, either because they are useless, or
+        # because they would cause pointless collisions (e.g., each
+        # Python package brings its own
+        # `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
         # Urgh, hacky...
-	if ($srcFile =~ /\/propagated-build-inputs$/ ||
+        if ($srcFile =~ /\/propagated-build-inputs$/ ||
             $srcFile =~ /\/nix-support$/ ||
             $srcFile =~ /\/perllocal.pod$/ ||
+            $srcFile =~ /\/easy-install.pth$/ ||
+            $srcFile =~ /\/site.py$/ ||
+            $srcFile =~ /\/site.pyc$/ ||
             $srcFile =~ /\/info\/dir$/ ||
             $srcFile =~ /\/log$/)
         {
@@ -160,4 +168,4 @@ while (scalar(keys %postponed) > 0) {
 print STDERR "created $symlinks symlinks in user environment\n";
 
 
-symlink($ENV{"manifest"}, "$out/manifest") or die "cannot create manifest";
+symlink($ENV{"manifest"}, "$out/manifest.nix") or die "cannot create manifest";

corepkgs/buildenv/default.nix

@@ -11,4 +11,8 @@ derivation {
   paths = derivations;
   active = map (x: if x ? meta && x.meta ? active then x.meta.active else "true") derivations;
   priority = map (x: if x ? meta && x.meta ? priority then x.meta.priority else "5") derivations;
+
+  # Building user environments remotely just causes huge amounts of
+  # network traffic, so don't do that.
+  preferLocalBuild = true;
 }

corepkgs/channels/unpack.sh.in

@@ -1,5 +1,8 @@
 #! @shell@ -e
 
+# Cygwin compatibility hack: bunzip2 expects cygwin.dll in $PATH.
+export PATH=@coreutils@
+
 @coreutils@/mkdir $out
 @coreutils@/mkdir $out/tmp
 cd $out/tmp

corepkgs/nar/nar.sh.in

@@ -7,8 +7,6 @@ dst=$out/tmp.nar.bz2
 
 @bzip2@ < tmp > $dst
 
-@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
-
 @bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
 
 @coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2

doc/manual/Makefile.am

@@ -6,7 +6,12 @@ XSLTPROC = $(xsltproc) $(xmlflags) \
  --param xref.with.number.and.title 1 \
  --param toc.section.depth 3 \
  --param admon.style \'\' \
- --param callout.graphics.extension \'.gif\'
+ --param callout.graphics.extension \'.gif\' \
+ --param contrib.inline.enabled 0
+
+dblatex_opts = \
+ -P doc.collab.show=0 \
+ -P latex.output.revhistory=0
 
 # Note: we use GIF for now, since the PNGs shipped with Docbook aren't
 # transparent.
@@ -29,13 +34,9 @@ MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  conf-file.xml release-notes.xml \
  style.css images
 
+# Note: RelaxNG validation requires xmllint >= 2.7.4.
 manual.is-valid: $(MANUAL_SRCS) version.txt
-#	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --relaxng $(docbookrng)/docbook.rng -
-	if test "$(jing)" != "false"; then \
-		$(XMLLINT) --xinclude $< | $(jing) $(docbookrng)/docbook.rng /dev/fd/0; \
-	else \
-		echo "Not validating."; \
-	fi
+	$(XMLLINT) --noout --nonet --xinclude --noxincludenode --relaxng $(docbookrng)/docbook.rng $<
 	touch $@
 
 version.txt:
@@ -50,7 +51,7 @@ manual.html: $(MANUAL_SRCS) manual.is-valid images
 
 manual.pdf: $(MANUAL_SRCS) manual.is-valid images
 	if test "$(dblatex)" != ""; then \
-		$(dblatex) manual.xml; \
+		$(dblatex) $(dblatex_opts) manual.xml; \
 	else \
 		echo "Please install dblatex and rerun configure."; \
 		exit 1; \

doc/manual/builtins.xml

@@ -324,6 +324,16 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.intersectAttrs</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return an attribute set consisting of the
+    attributes in the set <replaceable>e2</replaceable> that also
+    exist in the set <replaceable>e1</replaceable>.</para></listitem>
+
+  </varlistentry>
+        
+    
   <varlistentry><term><function>builtins.isAttrs</function>
   <replaceable>e</replaceable></term>
 

doc/manual/conf-file.xml

@@ -97,6 +97,25 @@ env-keep-derivations = false
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-build-cores"><term><literal>build-cores</literal></term>
+
+    <listitem><para>Sets the value of the
+    <envar>NIX_BUILD_CORES</envar> environment variable in the
+    invocation of builders.  Builders can use this variable at their
+    discretion to control the maximum amount of parallelism.  For
+    instance, in Nixpkgs, if the derivation attribute
+    <varname>enableParallelBuilding</varname> is set to
+    <literal>true</literal>, the builder passes the
+    <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+    It can be overriden using the <option
+    linkend='opt-cores'>--cores</option> command line switch and
+    defaults to <literal>1</literal>.  The value <literal>0</literal>
+    means that the builder should use all available CPU cores in the
+    system.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id="conf-build-max-silent-time"><term><literal>build-max-silent-time</literal></term>
 
     <listitem>
@@ -233,7 +252,17 @@ build-use-chroot = /dev /proc /bin</programlisting>
     <filename>configure</filename> at build time.</para></listitem>
 
   </varlistentry>
-  
+
+
+  <varlistentry><term><literal>fsync-metadata</literal></term>
+
+    <listitem><para>If set to <literal>true</literal>, changes to the
+    Nix store metadata (in <filename>/nix/var/nix/db</filename>) are
+    synchronously flushed to disk.  This improves robustness in case
+    of system crashes, but reduces performance.  The default is
+    <literal>true</literal>.</para></listitem>
+
+  </varlistentry>
     
 </variablelist>
 

doc/manual/env-common.xml

@@ -271,6 +271,17 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
 </varlistentry>
 
+
+<varlistentry><term><envar>GC_INITIAL_HEAP_SIZE</envar></term>
+
+  <listitem><para>If Nix has been configured to use the Boehm garbage
+  collector, this variable sets the initial size of the heap in bytes.
+  It defaults to 384 MiB.  Setting it to a low value reduces memory
+  consumption, but will increase runtime due to the overhead of
+  garbage collection.</para></listitem>
+
+</varlistentry>
+
     
 </variablelist>
 

doc/manual/installation.xml

@@ -96,23 +96,21 @@ ubiquitous 2.5.4a won't.  Note that these are only required if you
 modify the parser or when you are building from the Subversion
 repository.</para>
 
-<para>Nix uses CWI's ATerm library and the bzip2 compressor (including
-the bzip2 library).  These are included in the Nix source
-distribution.  If you build from the Subversion repository, you must
-download them yourself and place them in the
-<filename>externals/</filename> directory.  See
+<para>Nix uses the bzip2 compressor (including the bzip2 library).  It
+is included in the Nix source distribution.  If you build from the
+Subversion repository, you must download it yourself and place it in
+the <filename>externals/</filename> directory.  See
 <filename>externals/Makefile.am</filename> for the precise URLs of
-these packages.  Alternatively, if you already have them installed,
-you can use <command>configure</command>'s
-<option>--with-aterm</option> and <option>--with-bzip2</option>
+this packages.  Alternatively, if you already have it installed, you
+can use <command>configure</command>'s <option>--with-bzip2</option>
 options to point to their respective locations.</para>
 
-<para>If you want to be able to upgrade Nix stores from before version
-0.12pre12020, you need Sleepycat's Berkeley DB version version 4.5.
-(Other versions may not have compatible database formats.).  Berkeley
-DB 4.5 is included in the Nix source distribution.  If you do not need
-this ability, you can build Nix with the
-<option>--disable-old-db-compat</option> configure option.</para>
+<para>Nix can optionally use the <link
+xlink:href="http://www.hpl.hp.com/personal/Hans_Boehm/gc/">Boehm
+garbage collector</link> to reduce the evaluator’s memory consumption.
+To enable it, install <literal>pkgconfig</literal> and the Boehm
+garbage collector, and pass the flag <option>--enable-gc</option> to
+<command>configure</command>.</para>
 
 </section>
 
@@ -133,23 +131,32 @@ $ make install</screen>
 preceded by the command:
 
 <screen>
-$ ./bootstrap</screen>
+$ ./bootstrap.sh</screen>
 
 </para>
 
 <para>The installation path can be specified by passing the
 <option>--prefix=<replaceable>prefix</replaceable></option> to
 <command>configure</command>.  The default installation directory is
-<filename>/nix</filename>.  You can change this to any location you
-like.  You must have write permission to the
+<filename>/usr/local</filename>.  You can change this to any location
+you like.  You must have write permission to the
 <replaceable>prefix</replaceable> path.</para>
 
-<warning><para>It is best <emphasis>not</emphasis> to change the
-installation prefix from its default, since doing so makes it
-impossible to use pre-built binaries from the standard Nixpkgs
-channels.</para></warning>
+<para>Nix keeps its <emphasis>store</emphasis> (the place where
+packages are stored) in <filename>/nix/store</filename> by default.
+This can be changed using
+<option>--with-store-dir=<replaceable>path</replaceable></option>.</para>
 
-<para>If you want to rebuilt the documentation, pass the full path to
+<warning><para>It is best <emphasis>not</emphasis> to change the Nix
+store from its default, since doing so makes it impossible to use
+pre-built binaries from the standard Nixpkgs channels — that is, all
+packages will need to be built from source.</para></warning>
+
+<para>Nix keeps state (such as its database and log files) in
+<filename>/nix/var</filename> by default.  This can be changed using
+<option>--localstatedir=<replaceable>path</replaceable></option>.</para>
+
+<para>If you want to rebuild the documentation, pass the full path to
 the DocBook RELAX NG schemas and to the DocBook XSL stylesheets using
 the
 <option>--with-docbook-rng=<replaceable>path</replaceable></option>
@@ -160,27 +167,26 @@ options.</para>
 </section>
 
 
-<section><title>Installing from RPMs</title>
+<section><title>Installing a binary distribution</title>
 
-<para>RPM packages of Nix can be downloaded from <link
-xlink:href="http://nixos.org/" />.  These RPMs should work for most
-fairly recent releases of SuSE and Red Hat Linux.  They have been
-known to work work on SuSE Linux 8.1 and 9.0, and Red Hat 9.0.  In
-fact, it should work on any RPM-based Linux distribution based on
-<literal>glibc</literal> 2.3 or later.</para>
-
-<para>Once downloaded, the RPMs can be installed or upgraded using
-<command>rpm -U</command>.  For example,
+<para>RPM and Deb packages of Nix for a number of different versions
+of Fedora, openSUSE, Debian and Ubuntu can be downloaded from <link
+xlink:href="http://nixos.org/" />.  Once downloaded, the RPMs can be
+installed or upgraded using <command>rpm -U</command>.  For example,
 
 <screen>
-$ rpm -U nix-0.5pre664-1.i386.rpm</screen>
+$ rpm -U nix-0.13pre18104-1.i386.rpm</screen>
+
+Likewise, for a Deb package:
+
+<screen>
+$ dpkg -i nix_0.13pre18104-1_amd64.deb</screen>
 
 </para>
 
-<para>The RPMs install into the directory <filename>/nix</filename>.
-Nix can be uninstalled using <command>rpm -e nix</command>.  After
-this it will be necessary to manually remove the Nix store and other
-auxiliary data:
+<para>Nix can be uninstalled using <command>rpm -e nix</command> or
+<command>dpkg -r nix</command>.  After this you should manually remove
+the Nix store and other auxiliary data, if desired:
 
 <screen>
 $ rm -rf /nix/store
@@ -191,6 +197,7 @@ $ rm -rf /nix/var</screen>
 </section>
 
 
+<!-- TODO: should be updated
 <section><title>Upgrading Nix through Nix</title>
 
 <para>You can install the latest stable version of Nix through Nix
@@ -203,6 +210,7 @@ installation</link> by clicking on the package links at <link
 xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para>
 
 </section>
+-->
 
 
 <section><title>Security</title>

doc/manual/introduction.xml

@@ -113,7 +113,7 @@ $ nix-env --rollback
 
 <simplesect><title>Garbage collection</title>
 
-<para>When you install a package like this…
+<para>When you uninstall a package like this…
 
 <screen>
 $ nix-env --uninstall firefox
@@ -320,7 +320,7 @@ overview of NixOS is given in the HotOS XI paper <citetitle
 xlink:href="http://www.st.ewi.tudelft.nl/~dolstra/pubs/hotos-final.pdf">Purely
 Functional System Configuration Management</citetitle>.  The Nix
 homepage has <link
-xlink:href="http://nix.cs.uu.nl/docs/papers.html">an up-to-date list
+xlink:href="http://nixos.org/docs/papers.html">an up-to-date list
 of Nix-related papers</link>.</para>
 
 <para>Nix is the subject of Eelco Dolstra’s PhD thesis <citetitle

doc/manual/manual.xml

@@ -17,6 +17,7 @@
         <orgname>Delft University of Technology</orgname>
         <orgdiv>Department of Software Technology</orgdiv>
       </affiliation>
+      <contrib>Author</contrib>
     </author>
 
     <copyright>
@@ -25,10 +26,12 @@
       <year>2006</year>
       <year>2007</year>
       <year>2008</year>
+      <year>2009</year>
+      <year>2010</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>November 2008</date>
+    <date>August 2010</date>
     
   </info>
 
@@ -49,64 +52,25 @@
     
     <section>
       <title>Main commands</title>
-      <section xml:id="sec-nix-env">
-        <title>nix-env</title>
-        <xi:include href="nix-env.xml" />
-      </section>
-      <section xml:id="sec-nix-instantiate">
-        <title>nix-instantiate</title>
-        <xi:include href="nix-instantiate.xml" />
-      </section>
-      <section xml:id="sec-nix-store">
-        <title>nix-store</title>
-        <xi:include href="nix-store.xml" />
-      </section>
+      <xi:include href="nix-env.xml" />
+      <xi:include href="nix-instantiate.xml" />
+      <xi:include href="nix-store.xml" />
     </section>
     
     <section>
       <title>Utilities</title>
-      <section xml:id="sec-nix-build">
-        <title>nix-build</title>
-        <xi:include href="nix-build.xml" />
-      </section>
-      <section xml:id="sec-nix-channel">
-        <title>nix-channel</title>
-        <xi:include href="nix-channel.xml" />
-      </section>
-      <section xml:id="sec-nix-collect-garbage">
-        <title>nix-collect-garbage</title>
-        <xi:include href="nix-collect-garbage.xml" />
-      </section>
-      <section xml:id="sec-nix-copy-closure">
-        <title>nix-copy-closure</title>
-        <xi:include href="nix-copy-closure.xml" />
-      </section>
-      <section xml:id="sec-nix-hash">
-        <title>nix-hash</title>
-        <xi:include href="nix-hash.xml" />
-      </section>
-      <section xml:id="sec-nix-install-package">
-        <title>nix-install-package</title>
-        <xi:include href="nix-install-package.xml" />
-      </section>
-      <section xml:id="sec-nix-prefetch-url">
-        <title>nix-prefetch-url</title>
-        <xi:include href="nix-prefetch-url.xml" />
-      </section>
-      <section xml:id="sec-nix-pull">
-        <title>nix-pull</title>
-        <xi:include href="nix-pull.xml" />
-      </section>
-      <section xml:id="sec-nix-push">
-        <title>nix-push</title>
-        <xi:include href="nix-push.xml" />
-      </section>
-      <section xml:id="sec-nix-worker">
-        <title>nix-worker</title>
-        <xi:include href="nix-worker.xml" />
-      </section>
+      <xi:include href="nix-build.xml" />
+      <xi:include href="nix-channel.xml" />
+      <xi:include href="nix-collect-garbage.xml" />
+      <xi:include href="nix-copy-closure.xml" />
+      <xi:include href="nix-hash.xml" />
+      <xi:include href="nix-install-package.xml" />
+      <xi:include href="nix-prefetch-url.xml" />
+      <xi:include href="nix-pull.xml" />
+      <xi:include href="nix-push.xml" />
+      <xi:include href="nix-worker.xml" />
     </section>
-    
+
   </appendix>
 
   <xi:include href="troubleshooting.xml" />

doc/manual/nix-build.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-build">
 
 <refmeta>
   <refentrytitle>nix-build</refentrytitle>

doc/manual/nix-channel.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-channel">
   
 <refmeta>
   <refentrytitle>nix-channel</refentrytitle>

doc/manual/nix-collect-garbage.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-collect-garbage">
   
 <refmeta>
   <refentrytitle>nix-collect-garbage</refentrytitle>

doc/manual/nix-copy-closure.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-copy-closure">
 
 <refmeta>
   <refentrytitle>nix-copy-closure</refentrytitle>

doc/manual/nix-env.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-env">
   
 <refmeta>
   <refentrytitle>nix-env</refentrytitle>

doc/manual/nix-hash.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-hash">
   
 <refmeta>
   <refentrytitle>nix-hash</refentrytitle>

doc/manual/nix-install-package.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-install-package">
   
 <refmeta>
   <refentrytitle>nix-install-package</refentrytitle>

doc/manual/nix-instantiate.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-instantiate">
   
 <refmeta>
   <refentrytitle>nix-instantiate</refentrytitle>

doc/manual/nix-lang-ref.xml

@@ -178,100 +178,5 @@
     </productionset>
     
   </sect1>
-  
-
-
-  <sect1>
-    <title>Semantics</title>
-
-
-    
-    <sect2>
-      <title>Built-in functions</title>
-
-      <para>
-        The Nix language provides the following built-in function
-        (<quote>primops</quote>):
-      </para>
-
-      <variablelist>
-
-        <varlistentry>
-          <term><function>import</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield a path value.  The Nix expression
-              stored at this path in the file system is then read,
-              parsed, and evaluated.  Returns the result of the
-              evaluation of the Nix expression just read.
-            </para>
-
-            <para>
-              Example: <literal>import ./foo.nix</literal> evaluates
-              the expression stored in <filename>foo.nix</filename>
-              (in the directory containing the expression in which the
-              <function>import</function> occurs).
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>derivation</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield an attribute set.  [...]
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>baseNameOf</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>,
-              which must yield a string value, and returns a string
-              representing its <emphasis>base name</emphasis>.  This
-              is the substring following the last path separator
-              (<literal>/</literal>).
-            </para>
-
-            <para>
-              Example: <literal>baseNameOf "/foo/bar"</literal>
-              returns <literal>"bar"</literal>, and
-              <literal>baseNameOf "/foo/bar/"</literal> returns
-              <literal>""</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-
-        <varlistentry>
-          <term><function>toString</function>
-          <replaceable>e</replaceable></term>
-          <listitem>
-            <para>
-              Evaluates the expression <replaceable>e</replaceable>
-              and coerces it into a string, if possible.  Only
-              strings, paths, and URIs can be so coerced.
-            </para>
-
-            <para>
-              Example: <literal>toString
-              http://www.cs.uu.nl/</literal> returns
-              <literal>"http://www.cs.uu.nl/"</literal>.
-            </para>
-          </listitem>
-        </varlistentry>
-            
-      </variablelist>
-
-    </sect2>
-
-  </sect1>
-  
 
 </appendix>

doc/manual/nix-prefetch-url.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-prefetch-url">
   
 <refmeta>
   <refentrytitle>nix-prefetch-url</refentrytitle>

doc/manual/nix-pull.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-pull">
 
 <refmeta>
   <refentrytitle>nix-pull</refentrytitle>

doc/manual/nix-push.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-push">
 
 <refmeta>
   <refentrytitle>nix-push</refentrytitle>
@@ -119,7 +120,7 @@ dependencies used in the build, such as compilers).</para>
 dependencies, we can do:
       
 <screen>
-$ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
+$ nix-push <replaceable>urls</replaceable> $(nix-store -r $(nix-instantiate foo.nix))</screen>
     
 </para>
 

doc/manual/nix-store.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-store">
 
 <refmeta>
   <refentrytitle>nix-store</refentrytitle>
@@ -212,8 +213,6 @@ linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
   <arg><option>--max-links</option> <replaceable>nrlinks</replaceable></arg>
-  <arg><option>--max-atime</option> <replaceable>atime</replaceable></arg>
-  <arg><option>--use-atime</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -291,42 +290,6 @@ options control what gets deleted and in what order:
     
   </varlistentry>
 
-  <varlistentry><term><option>--max-atime</option> <replaceable>atime</replaceable></term>
-  
-    <listitem><para>Only delete a store path if its last-accessed time
-    is less than <replaceable>atime</replaceable>.  This allows you to
-    garbage-collect only packages that haven’t been used recently.
-    The time is expressed as the number of seconds in the Unix epoch,
-    i.e., since 1970-01-01 00:00:00 UTC.  An easy way to convert to
-    this format is <literal>date +%s -d "<replaceable>date
-    specification</replaceable>"</literal>.</para>
-
-    <para>For directories, the last-accessed time is the highest
-    last-accessed time of any regular file in the directory (or in any
-    of its subdirectories).  That is, the <literal>atime</literal>
-    field maintained by the filesystem is ignored for directories.
-    This is because operations such as rebuilding the
-    <command>locate</command> database tend to update the
-    <literal>atime</literal> values of all directories, so they’re not
-    a useful indicator of whether a package was recently used.</para>
-
-    <para>Note that <command>nix-store --optimise</command> reads all
-    regular files in the Nix store, and so causes all last-accessed
-    times to be set to the present time.  This makes
-    <option>--max-atime</option> ineffective (for a while at
-    least).</para></listitem>
-    
-  </varlistentry>
-
-  <varlistentry><term><option>--use-atime</option></term>
-
-    <listitem><para>Delete store paths in order of ascending
-    last-accessed time.  This is useful in conjunction with the other
-    options to delete only the least recently used
-    packages.</para></listitem>
-
-  </varlistentry>
-  
 </variablelist>
 
 </para>
@@ -357,13 +320,6 @@ deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
 
 </para>
 
-<para>To delete unreachable paths not accessed in the last two months:
-
-<screen>
-$ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
-
-</para>
-
 <para>To delete at least 100 MiBs of unreachable paths:
 
 <screen>
@@ -386,7 +342,7 @@ $ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
 
 <cmdsynopsis>
   <command>nix-store</command>
-  <arg choice='plain'><option>--gc</option></arg>
+  <arg choice='plain'><option>--delete</option></arg>
   <arg><option>--ignore-liveness</option></arg>
   <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
 </cmdsynopsis>
@@ -448,6 +404,8 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4'
     <arg choice='plain'><option>--tree</option></arg>
     <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg>
     <arg choice='plain'><option>--hash</option></arg>
+    <arg choice='plain'><option>--size</option></arg>
+    <arg choice='plain'><option>--roots</option></arg>
   </group>
   <arg><option>--use-output</option></arg>
   <arg><option>-u</option></arg>
@@ -630,9 +588,29 @@ query is applied to the target of the symlink.</para>
   <varlistentry><term><option>--hash</option></term>
   
     <listitem><para>Prints the SHA-256 hash of the contents of the
-    store path <replaceable>paths</replaceable>.  Since the hash is
-    stored in the Nix database, this is a fast
-    operation.</para></listitem>
+    store paths <replaceable>paths</replaceable> (that is, the hash of
+    the output of <command>nix-store --dump</command> on the given
+    paths).  Since the hash is stored in the Nix database, this is a
+    fast operation.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--size</option></term>
+  
+    <listitem><para>Prints the size in bytes of the contents of the
+    store paths <replaceable>paths</replaceable> — to be precise, the
+    size of the output of <command>nix-store --dump</command> on the
+    given paths.  Note that the actual disk space required by the
+    store paths may be higher, especially on filesystems with large
+    cluster sizes.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--roots</option></term>
+  
+    <listitem><para>Prints the garbage collector roots that point,
+    directly or indirectly, at the store paths
+    <replaceable>paths</replaceable>.</para></listitem>
 
   </varlistentry>
 
@@ -713,6 +691,18 @@ $ gv graph.ps</screen>
 
 </para>
 
+<para>Show every garbage collector root that points to a store path
+that depends on <command>svn</command>:
+
+<screen>
+$ nix-store -q --roots $(which svn)
+/nix/var/nix/profiles/default-81-link
+/nix/var/nix/profiles/default-82-link
+/nix/var/nix/profiles/per-user/eelco/profile-97-link
+</screen>
+
+</para>
+
 </refsection>
 
 

doc/manual/nix-worker.xml

@@ -1,6 +1,7 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
           xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
+          xmlns:xi="http://www.w3.org/2001/XInclude"
+          xml:id="sec-nix-worker">
 
 <refmeta>
   <refentrytitle>nix-worker</refentrytitle>

doc/manual/opt-common-syn.xml

@@ -13,6 +13,10 @@
   </group>
   <replaceable>number</replaceable>
 </arg>
+<arg>
+  <arg><option>--cores</option></arg>
+  <replaceable>number</replaceable>
+</arg>
 <arg>
   <arg><option>--max-silent-time</option></arg>
   <replaceable>number</replaceable>

doc/manual/opt-common.xml

@@ -98,7 +98,25 @@
   linkend='conf-build-max-jobs'><literal>build-max-jobs</literal></link>
   configuration setting, which itself defaults to
   <literal>1</literal>.  A higher value is useful on SMP systems or to
-  exploit I/O latency.  </para></listitem>
+  exploit I/O latency.</para></listitem>
+  
+</varlistentry>
+
+
+<varlistentry xml:id="opt-cores"><term><option>--cores</option></term>
+
+  <listitem><para>Sets the value of the <envar>NIX_BUILD_CORES</envar>
+  environment variable in the invocation of builders.  Builders can
+  use this variable at their discretion to control the maximum amount
+  of parallelism.  For instance, in Nixpkgs, if the derivation
+  attribute <varname>enableParallelBuilding</varname> is set to
+  <literal>true</literal>, the builder passes the
+  <option>-j<replaceable>N</replaceable></option> flag to GNU Make.
+  It defaults to the value of the <link
+  linkend='conf-build-cores'><literal>build-cores</literal></link>
+  configuration setting, if set, or <literal>1</literal> otherwise.
+  The value <literal>0</literal> means that the builder should use all
+  available CPU cores in the system.</para></listitem>
   
 </varlistentry>
 

doc/manual/package-management.xml

@@ -38,7 +38,7 @@ to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
 expressions based on it, or completely new ones.)  You can download
 the latest version from <link
-xlink:href='http://nixos.org/releases/full-index-nixpkgs.html' />.</para>
+xlink:href='http://nixos.org/nixpkgs/download.html' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available packages in the release:
@@ -331,7 +331,7 @@ default profile, respectively.  If the profile doesn’t exist, it will
 be created automatically.  You should be careful about storing a
 profile in another location than the <filename>profiles</filename>
 directory, since otherwise it might not be used as a root of the
-garbage collector (see section <xref linkend='sec-garbage-collection'
+garbage collector (see <xref linkend='sec-garbage-collection'
 />).</para>
 
 <para>All <command>nix-env</command> operations work on the profile
@@ -496,7 +496,7 @@ available in the subscribed channels.</para>
 
 <para>Often, when you want to install a specific package (e.g., from
 the <link
-xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/">Nix
+xlink:href="http://nixos.org/nixpkgs/">Nix
 Packages collection</link>), subscribing to a channel is a bit
 cumbersome.  And channels don’t help you at all if you want to install
 an older version of a package than the one provided by the current
@@ -507,19 +507,16 @@ click on it, and it will be installed with all the necessary
 dependencies.</para>
 
 <para>For instance, you can go to <link
-xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/" /> —
-or to any older release of Nix Packages — and click on any link for
-the individual packages for your platform (say, <link
-xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.10pre6622/pkgs/subversion-1.4.0-i686-linux.nixpkg'><literal>subversion-1.4.0</literal>
-for <literal>i686-linux</literal></link>).  The first time you do
-this, your browser will ask what to do with
-<literal>application/nix-package</literal> files.  You should open
-them with <filename>/nix/bin/nix-install-package</filename>.  This
-will open a window that asks you to confirm that you want to install
-the package.  When you answer <literal>Y</literal>, the package and
-all its dependencies will be installed.  This is a binary deployment
-mechanism — you get packages pre-compiled for the selected platform
-type.</para>
+xlink:href="http://hydra.nixos.org/jobset/nixpkgs/trunk/channel/latest"
+/> and click on any link for the individual packages for your
+platform.  The first time you do this, your browser will ask what to
+do with <literal>application/nix-package</literal> files.  You should
+open them with <filename>/nix/bin/nix-install-package</filename>.
+This will open a window that asks you to confirm that you want to
+install the package.  When you answer <literal>Y</literal>, the
+package and all its dependencies will be installed.  This is a binary
+deployment mechanism — you get packages pre-compiled for the selected
+platform type.</para>
 
 <para>You can also install <literal>application/nix-package</literal>
 files from the command line directly.  See <xref

doc/manual/quick-start.xml

@@ -1,5 +1,6 @@
 <chapter xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink">
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xml:id="chap-quick-start">
 
 <title>Quick Start</title>
 
@@ -10,9 +11,9 @@ to the following chapters.</para>
 
 <orderedlist>
 
-<listitem><para>Download a source tarball or RPM from <link
-xlink:href='http://nixos.org/'/>.  Build source
-distributions using the regular sequence:
+<listitem><para>Download a source tarball, RPM or Deb from <link
+xlink:href='http://nixos.org/'/>.  Build source distributions using
+the regular sequence:
         
 <screen>
 $ tar xvfj nix-<replaceable>version</replaceable>.tar.bz2
@@ -20,13 +21,21 @@ $ ./configure
 $ make
 $ make install <lineannotation>(as root)</lineannotation></screen>
 
-This will install Nix in <filename>/nix</filename>.  You shouldn't
-change the prefix if at all possible since that will make it
-impossible to use pre-built binaries from the Nixpkgs channel and
-other channels.  Alternatively, you could grab an RPM if you're on an
-RPM-based system.  You should also add
-<filename>/nix/etc/profile.d/nix.sh</filename> to your
-<filename>~/.bashrc</filename> (or some other login
+This will install the Nix binaries in <filename>/usr/local</filename>
+and keep the Nix store and other state in <filename>/nix</filename>.
+You can change the former by specifying
+<option>--prefix=<replaceable>path</replaceable></option>.  The
+location of the store can be changed using
+<option>--with-store-dir=<replaceable>path</replaceable></option>.
+However, you shouldn't change the store location, if at all possible,
+since that will make it impossible to use pre-built binaries from the
+Nixpkgs channel and other channels.  The location of the state can be
+changed using
+<option>--localstatedir=<replaceable>path</replaceable>.</option></para></listitem>
+
+<listitem><para>You should add
+<filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename>
+to your <filename>~/.bashrc</filename> (or some other login
 file).</para></listitem>
 
 <listitem><para>Subscribe to the Nix Packages channel.
@@ -51,7 +60,7 @@ available remotely.</para></listitem>
 in the channel:
 
 <screen>
-$ nix-env -qa ’*’ <lineannotation>(mind the quotes!)</lineannotation>
+$ nix-env -qa \*
 docbook-xml-4.2
 firefox-1.0pre-PR-0.10.1
 hello-2.1.1
@@ -99,7 +108,7 @@ numbers).</para></listitem>
 
 <listitem><para>You can also install specific packages directly from
 your web browser.  For instance, you can go to <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/" />
+xlink:href="http://hydra.nixos.org/jobset/nixpkgs/trunk/channel/latest" />
 and click on any link for the individual packages for your platform.
 Associate <literal>application/nix-package</literal> with the program
 <filename>/nix/bin/nix-install-package</filename>.  A window should

doc/manual/release-notes.xml

@@ -8,7 +8,137 @@
 
 <!--==================================================================-->
 
-<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (April NN,
+<section xml:id="ssec-relnotes-1.0"><title>Release 1.0 (TBA)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix can now optionally use the Boehm garbage collector.
+    This significantly reduces the Nix evaluator’s memory footprint,
+    especially when evaluating large NixOS system configurations.  It
+    can be enabled using the <option>--enable-gc</option> configure
+    option.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.16"><title>Release 0.16 (August 17, 2010)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The Nix expression evaluator is now much faster in most
+    cases: typically, <link
+    xlink:href="http://www.mail-archive.com/nix-dev@cs.uu.nl/msg04113.html">3
+    to 8 times compared to the old implementation</link>.  It also
+    uses less memory.  It no longer depends on the ATerm
+    library.</para>
+  </listitem>
+
+  <listitem>
+    <para>
+      Support for configurable parallelism inside builders.  Build
+      scripts have always had the ability to perform multiple build
+      actions in parallel (for instance, by running <command>make -j
+      2</command>), but this was not desirable because the number of
+      actions to be performed in parallel was not configurable.  Nix
+      now has an option <option>--cores
+      <replaceable>N</replaceable></option> as well as a configuration
+      setting <varname>build-cores =
+      <replaceable>N</replaceable></varname> that causes the
+      environment variable <envar>NIX_BUILD_CORES</envar> to be set to
+      <replaceable>N</replaceable> when the builder is invoked.  The
+      builder can use this at its discretion to perform a parallel
+      build, e.g., by calling <command>make -j
+      <replaceable>N</replaceable></command>.  In Nixpkgs, this can be
+      enabled on a per-package basis by setting the derivation
+      attribute <varname>enableParallelBuilding</varname> to
+      <literal>true</literal>.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para><command>nix-store -q</command> now supports XML output
+    through the <option>--xml</option> flag.</para>
+  </listitem>
+
+  <listitem>
+    <para>Several bug fixes.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.15"><title>Release 0.15 (March 17, 2010)</title>
+
+<para>This is a bug-fix release.  Among other things, it fixes
+building on Mac OS X (Snow Leopard), and improves the contents of
+<filename>/etc/passwd</filename> and <filename>/etc/group</filename>
+in <literal>chroot</literal> builds.</para>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.14"><title>Release 0.14 (February 4, 2010)</title>
+
+<para>This release has the following improvements:</para>
+
+<itemizedlist>
+
+  <listitem>
+    <para>The garbage collector now starts deleting garbage much
+    faster than before.  It no longer determines liveness of all paths
+    in the store, but does so on demand.</para>
+  </listitem>
+
+  <listitem>
+    <para>Added a new operation, <command>nix-store --query
+    --roots</command>, that shows the garbage collector roots that
+    directly or indirectly point to the given store paths.</para>
+  </listitem>
+
+  <listitem>
+    <para>Removed support for converting Berkeley DB-based Nix
+    databases to the new schema.</para>
+  </listitem>
+
+  <listitem>
+    <para>Removed the <option>--use-atime</option> and
+    <option>--max-atime</option> garbage collector options.  They were
+    not very useful in practice.</para>
+  </listitem>
+
+  <listitem>
+    <para>On Windows, Nix now requires Cygwin 1.7.x.</para>
+  </listitem>
+
+  <listitem>
+    <para>A few bug fixes.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (November 5,
 2009)</title>
 
 <para>This is primarily a bug fix release.  It has some new
@@ -85,7 +215,25 @@ features:</para>
     add a string to stack traces — useful for debugging),
     <varname>builtins.isBool</varname>,
     <varname>builtins.isString</varname>,
-    <varname>builtins.isInt</varname>.</para>
+    <varname>builtins.isInt</varname>,
+    <varname>builtins.intersectAttrs</varname>.</para>
+  </listitem>
+
+  <listitem>
+    <para>OpenSolaris support (Sander van der Burg).</para>
+  </listitem>
+
+  <listitem>
+    <para>Stack traces are no longer displayed unless the
+    <option>--show-trace</option> option is used.</para>
+  </listitem>
+
+  <listitem>
+    <para>The scoping rules for <literal>inherit
+    (<replaceable>e</replaceable>) ...</literal> in recursive
+    attribute sets have changed.  The expression
+    <replaceable>e</replaceable> can now refer to the attributes
+    defined in the containing set.</para>
   </listitem>
 
 </itemizedlist>

doc/manual/writing-nix-expressions.xml

@@ -1106,7 +1106,7 @@ used in the Nix expression for Subversion.</para>
     incompatibility might occur.</para>
   </callout>
 
-  <callout arearefs='ex-subversion-nix-co-2'>
+  <callout arearefs='ex-subversion-nix-co-3'>
     <para>This assertion says that in order for Subversion to have SSL
     support (so that it can access <literal>https</literal> URLs), an
     OpenSSL library must be passed.  Additionally, it says that

externals/Makefile.am

@@ -1,78 +1,3 @@
-# Berkeley DB
-
-DB = db-4.5.20
-
-if OLD_DB_COMPAT
-
-$(DB).tar.gz:
-	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.5.20 from"
-	@echo "  http://download-east.oracle.com/berkeley-db/db-4.5.20.tar.gz"
-	@echo "and place it in the externals/ directory."
-	false
-
-$(DB): $(DB).tar.gz
-	gunzip < $(srcdir)/$(DB).tar.gz | tar xvf -
-	(cd $(DB) && $(patch) -p1) < $(srcdir)/bdb-cygwin.patch
-
-have-db:
-	$(MAKE) $(DB)
-	touch have-db
-
-if HAVE_BDB
-build-db:
-else
-build-db: have-db
-	(pfx=`pwd` && \
-	cd $(DB)/build_unix && \
-	CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" \
-	 ../dist/configure --prefix=$$pfx/inst-bdb \
-	 --enable-cxx --disable-shared --disable-cryptography \
-         --disable-replication --disable-verify && \
-	$(MAKE) && \
-	$(MAKE) install_include install_lib)
-	touch build-db
-endif
-
-else
-
-build-db:
-
-endif
-
-
-# CWI ATerm
-
-ATERM = aterm-2.4.2-fixes-r2
-
-$(ATERM).tar.bz2:
-	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.4.2-fixes-r2 from"
-	@echo "  http://nixos.org/tarballs/aterm-2.4.2-fixes-r2.tar.bz2"
-	@echo "and place it in the externals/ directory."
-	false
-
-$(ATERM): $(ATERM).tar.bz2
-	bunzip2 < $(srcdir)/$(ATERM).tar.bz2 | tar xvf -
-
-have-aterm:
-	$(MAKE) $(ATERM)
-	touch have-aterm
-
-if HAVE_ATERM
-build-aterm:
-else
-build-aterm: have-aterm
-	(pfx=`pwd` && \
-	cd $(ATERM) && \
-	CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm \
-	  --disable-shared --enable-static && \
-	$(MAKE) && \
-	$(MAKE) install)
-	touch build-aterm
-endif
-
-
 # bzip2
 
 BZIP2 = bzip2-1.0.5
@@ -87,31 +12,56 @@ $(BZIP2).tar.gz:
 $(BZIP2): $(BZIP2).tar.gz
 	gunzip < $(srcdir)/$(BZIP2).tar.gz | tar xvf -
 
-have-bzip2:
-	$(MAKE) $(BZIP2)
-	touch have-bzip2
-
 if HAVE_BZIP2
 build-bzip2:
 else
-build-bzip2: have-bzip2
-	(pfx=`pwd` && \
-	cd $(BZIP2) && \
-	$(MAKE) && \
-	$(MAKE) install PREFIX=$$pfx/inst-bzip2)
+build-bzip2: $(BZIP2)
+	(cd $(BZIP2) && \
+	$(MAKE) CC="$(CC)" && \
+	$(MAKE) install PREFIX=$(abs_builddir)/inst-bzip2)
 	touch build-bzip2
 
-install:
+install-exec-local:: build-bzip2
 	mkdir -p $(DESTDIR)${bzip2_bin}
 	$(INSTALL_PROGRAM) $(bzip2_bin_test)/bzip2 $(bzip2_bin_test)/bunzip2 $(DESTDIR)${bzip2_bin}
 endif
 
 
-all: build-db build-aterm build-bzip2
+# SQLite
 
-EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.bz2 $(BZIP2).tar.gz \
-  bdb-cygwin.patch
+SQLITE = sqlite-autoconf-$(SQLITE_VERSION)
+SQLITE_TAR = sqlite-autoconf-$(SQLITE_VERSION).tar.gz
 
-ext-clean:
-	$(RM) -f have-db build-db have-aterm build-aterm have-bzip2 build-bzip2
-	$(RM) -rf $(DB) $(ATERM) $(BZIP2)
+$(SQLITE_TAR):
+	@echo "Nix requires the SQLite library to build."
+	@echo "Please download version $(SQLITE_VERSION) from"
+	@echo "  http://www.sqlite.org/$(SQLITE_TAR)"
+	@echo "and place it in the externals/ directory."
+	false
+
+$(SQLITE): $(SQLITE_TAR)
+	gzip -d < $(srcdir)/$(SQLITE_TAR) | tar xvf -
+
+if HAVE_SQLITE
+build-sqlite:
+else
+build-sqlite: $(SQLITE)
+	(cd $(SQLITE) && \
+	CC="$(CC)" CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \
+	$(MAKE) )
+	touch build-sqlite
+
+install-exec-local:: build-sqlite
+	cd $(SQLITE) && $(MAKE) install
+	rm -rf "$(DESTDIR)/$(pkglibdir)/dummy"
+endif
+
+
+all: build-bzip2 build-sqlite
+
+EXTRA_DIST = $(BZIP2).tar.gz $(SQLITE_TAR)
+
+clean:
+	$(RM) -f build-bzip2 build-sqlite
+	$(RM) -rf $(BZIP2) $(SQLITE)
+	$(RM) -rf inst-bzip2

externals/bdb-cygwin.patch

@@ -1,22 +0,0 @@
-diff -rc db-4.5.20-orig/os/os_flock.c db-4.5.20/os/os_flock.c
-*** db-4.5.20-orig/os/os_flock.c	2006-10-13 12:36:12.000000000 +0200
---- db-4.5.20/os/os_flock.c	2006-10-13 12:40:11.000000000 +0200
-***************
-*** 30,35 ****
---- 30,44 ----
-  
-  	DB_ASSERT(dbenv, F_ISSET(fhp, DB_FH_OPENED) && fhp->fd != -1);
-  
-+ #ifdef __CYGWIN__
-+ 	/*
-+ 	 * Windows file locking interferes with read/write operations, so we
-+ 	 * map the ranges to an area past the end of the file.
-+ 	 */
-+ 	DB_ASSERT(dbenv, offset < (off_t) 1 << 62);
-+ 	offset += (off_t) 1 << 62;
-+ #endif
-+ 
-  	fl.l_start = offset;
-  	fl.l_len = 1;
-  	fl.l_type = acquire ? F_WRLCK : F_UNLCK;
-Only in db-4.5.20/os: os_flock.c~

nix.conf.example

@@ -59,6 +59,17 @@
 #build-max-jobs = 1
 
 
+### Option `build-cores'
+#
+# This option defines the number of CPU cores to utilize in parallel
+# within a build job, i.e. by passing an appropriate `-jN' flag to GNU
+# Make. The default is 1, meaning that parallel building within jobs
+# is disabled.  Passing the special value `0' causes Nix to try and
+# auto-detect the number of available cores on the local host.  This
+# setting can be overridden using the `--cores' command line switch.
+#build-cores = 1
+
+
 ### Option `build-max-silent-time'
 #
 # This option defines the maximum number of seconds that a builder can

release.nix

@@ -1,4 +1,7 @@
-{ nixpkgs ? ../nixpkgs }:
+{ nixpkgs ? ../nixpkgs
+, nix ? { outPath = ./.; rev = 1234; }
+, officialRelease ? false
+}:
 
 let
 
@@ -6,10 +9,6 @@ let
 
 
     tarball =
-      { nix ? {outPath = ./.; rev = 1234;}
-      , officialRelease ? false
-      }:
-
       with import nixpkgs {};
 
       releaseTools.sourceTarball {
@@ -18,7 +17,10 @@ let
         src = nix;
         inherit officialRelease;
 
-        buildInputs = [curl bison flex2533 perl libxml2 libxslt w3m bzip2 jing_tools tetex dblatex];
+        buildInputs =
+          [ curl bison24 flex2535 perl libxml2 libxslt w3m bzip2
+            tetex dblatex nukeReferences pkgconfig
+          ];
 
         configureFlags = ''
           --with-docbook-rng=${docbook5}/xml/rng/docbook
@@ -26,27 +28,16 @@ let
           --with-xml-flags=--nonet
         '';
 
-        # Include the BDB, ATerm and Bzip2 tarballs in the distribution.
+        # Include the Bzip2 tarball in the distribution.
         preConfigure = ''
-          stripHash ${db45.src}
-          # Remove unnecessary stuff from the Berkeley DB tarball.
-          ( mkdir bdb-temp
-            cd bdb-temp
-            tar xfz ${db45.src}
-            cd *
-            rm -rf docs test tcl perl libdb_java java rpc_server build_vxworks \
-              examples_java examples_c examples_cxx dist/tags
-            mkdir test
-            touch test/include.tcl
-            cd ..
-            tar cvfz ../externals/$strippedName *
-          )
-
-          stripHash ${aterm242fixes.src}
-          cp -pv ${aterm242fixes.src} externals/$strippedName
-
           stripHash ${bzip2.src}
           cp -pv ${bzip2.src} externals/$strippedName
+
+          stripHash ${sqlite.src}
+          cp -pv ${sqlite.src} externals/$strippedName
+
+          # TeX needs a writable font cache.
+          export VARTEXFONTS=$TMPDIR/texfonts
         '';
 
         preDist = ''
@@ -54,6 +45,14 @@ let
           
           make -C doc/manual manual.pdf prefix=$out
           cp doc/manual/manual.pdf $out/manual.pdf
+
+          # The PDF containes filenames of included graphics (see
+          # http://www.tug.org/pipermail/pdftex/2007-August/007290.html).
+          # This causes a retained dependency on dblatex, which Hydra
+          # doesn't like (the output of the tarball job is distributed
+          # to Windows and Macs, so there should be no Linux binaries
+          # in the closure).
+          nuke-refs $out/manual.pdf
           
           echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
           echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products
@@ -63,68 +62,43 @@ let
 
 
     build =
-      { tarball ? jobs.tarball {}
-      , system ? "i686-linux"
-      }:
+      { system ? "i686-linux" }:
 
-      with import nixpkgs {inherit system;};
+      with import nixpkgs { inherit system; };
 
       releaseTools.nixBuild {
         name = "nix";
         src = tarball;
 
-        buildInputs = [curl perl bzip2 openssl];
+        buildInputs = [ curl perl bzip2 openssl pkgconfig boehmgc ];
 
         configureFlags = ''
           --disable-init-state
-          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --with-bzip2=${bzip2} --with-sqlite=${sqlite}
+          --enable-gc
         '';
       };
 
-      
-    static =
-      { tarball ? jobs.tarball {}
-      , system ? "i686-linux"
-      }:
 
-      with import nixpkgs {inherit system;};
-
-      releaseTools.binaryTarball {
-        name = "nix-static-tarball";
-        src = tarball;
-
-        buildInputs = [curl perl bzip2];
-
-        configureFlags = ''
-          --disable-init-state
-          --disable-old-db-compat --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
-          --enable-static-nix
-        '';
-      };
-
-      
     coverage =
-      { tarball ? jobs.tarball {}
-      }:
-
-      with import nixpkgs {};
+      with import nixpkgs { system = "x86_64-linux"; };
 
       releaseTools.coverageAnalysis {
         name = "nix-build";
         src = tarball;
 
-        buildInputs = [
-          curl perl bzip2 openssl
-          # These are for "make check" only:
-          graphviz libxml2 libxslt
-        ];
+        buildInputs =
+          [ curl perl bzip2 openssl
+            # These are for "make check" only:
+            graphviz libxml2 libxslt
+          ];
 
         configureFlags = ''
           --disable-init-state --disable-shared
-          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --with-bzip2=${bzip2} --with-sqlite=${sqlite}
         '';
 
-        lcovFilter = ["*/boost/*" "*-tab.*"];
+        lcovFilter = [ "*/boost/*" "*-tab.*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some
@@ -133,22 +107,32 @@ let
       };
 
       
-    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 20;
-    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 50;
-    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 50;
-    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 40;
-    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 40;
+    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 10;
+    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 20;
+    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 20;
+    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 30;
+    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 30;
+    rpm_fedora11i386 = makeRPM_i686 (diskImages: diskImages.fedora11i386) 40;
+    rpm_fedora11x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora11x86_64) 40;
+    rpm_fedora12i386 = makeRPM_i686 (diskImages: diskImages.fedora12i386) 50;
+    rpm_fedora12x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora12x86_64) 50;
     rpm_opensuse103i386 = makeRPM_i686 (diskImages: diskImages.opensuse103i386) 40;
+    rpm_opensuse110i386 = makeRPM_i686 (diskImages: diskImages.opensuse110i386) 50;
+    rpm_opensuse110x86_64 = makeRPM_x86_64 (diskImages: diskImages.opensuse110x86_64) 50;
 
     
     deb_debian40i386 = makeDeb_i686 (diskImages: diskImages.debian40i386) 40;
     deb_debian40x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian40x86_64) 40;
-    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 30;
-    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 30;
-    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 50;
-    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 50;
-    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 40;
-    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 40;
+    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 50;
+    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 50;
+    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 20;
+    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 20;
+    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 30;
+    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 30;
+    deb_ubuntu904i386 = makeDeb_i686 (diskImages: diskImages.ubuntu904i386) 40;
+    deb_ubuntu904x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu904x86_64) 40;
+    deb_ubuntu910i386 = makeDeb_i686 (diskImages: diskImages.ubuntu910i386) 50;
+    deb_ubuntu910x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu910x86_64) 50;
 
 
   };
@@ -159,17 +143,15 @@ let
 
   makeRPM = 
     system: diskImageFun: prio:
-    { tarball ? jobs.tarball {}
-    }:
 
-    with import nixpkgs {inherit system;};
+    with import nixpkgs { inherit system; };
 
     releaseTools.rpmBuild rec {
       name = "nix-rpm-${diskImage.name}";
-      src = tarball;
+      src = jobs.tarball;
       diskImage = diskImageFun vmTools.diskImages;
       memSize = 1024;
-      meta = { schedulingPriority = toString prio; };
+      meta.schedulingPriority = prio;
     };
 
 
@@ -178,18 +160,17 @@ let
   
   makeDeb =
     system: diskImageFun: prio:
-    { tarball ? jobs.tarball {}
-    }:
 
-    with import nixpkgs {inherit system;};
+    with import nixpkgs { inherit system; };
 
     releaseTools.debBuild {
       name = "nix-deb";
-      src = tarball;
+      src = jobs.tarball;
       diskImage = diskImageFun vmTools.diskImages;
       memSize = 1024;
-      meta = { schedulingPriority = toString prio; };
+      meta.schedulingPriority = prio;
       configureFlags = "--sysconfdir=/etc";
+      debRequires = [ "curl" ];
     };
 
 

scripts/GeneratePatches.pm.in

@@ -0,0 +1,334 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use File::Temp qw(tempdir);
+
+
+# Some patch generations options.
+
+# Max size of NAR archives to generate patches for.
+my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
+$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize;
+
+# If patch is bigger than this fraction of full archive, reject.
+my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
+$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
+
+my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"};
+$timeLimit = 180 if !defined $timeLimit;
+
+my $hashAlgo = "sha256";
+
+
+sub findOutputPaths {
+    my $narFiles = shift;
+
+    my %outPaths;
+    
+    foreach my $p (keys %{$narFiles}) {
+
+        # Ignore derivations.
+        next if ($p =~ /\.drv$/);
+        
+        # Ignore builders (too much ambiguity -- they're all called
+        # `builder.sh').
+        next if ($p =~ /\.sh$/);
+        next if ($p =~ /\.patch$/);
+        
+        # Don't bother including tar files etc.
+        next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/);
+
+        $outPaths{$p} = 1;
+    }
+
+    return %outPaths;
+}
+
+
+sub getNameVersion {
+    my $p = shift;
+    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
+    my $name = $1;
+    my $version = $2;
+    return undef unless defined $name && defined $version;
+    $name =~ s/^-//;
+    $version =~ s/^-//;
+    return ($name, $version);
+}
+
+
+# A quick hack to get a measure of the `distance' between two
+# versions: it's just the position of the first character that differs
+# (or 999 if they are the same).
+sub versionDiff {
+    my $s = shift;
+    my $t = shift;
+    my $i;
+    return 999 if $s eq $t;
+    for ($i = 0; $i < length $s; $i++) {
+        return $i if $i >= length $t or
+            substr($s, $i, 1) ne substr($t, $i, 1);
+    }
+    return $i;
+}
+
+
+sub getNarBz2 {
+    my $narPath = shift;
+    my $narFiles = shift;
+    my $storePath = shift;
+    
+    my $narFileList = $$narFiles{$storePath};
+    die "missing path $storePath" unless defined $narFileList;
+
+    my $narFile = @{$narFileList}[0];
+    die unless defined $narFile;
+
+    $narFile->{url} =~ /\/([^\/]+)$/;
+    die unless defined $1;
+    return "$narPath/$1";
+}
+
+
+sub containsPatch {
+    my $patches = shift;
+    my $storePath = shift;
+    my $basePath = shift;
+    my $patchList = $$patches{$storePath};
+    return 0 if !defined $patchList;
+    my $found = 0;
+    foreach my $patch (@{$patchList}) {
+        # !!! baseHash might differ
+        return 1 if $patch->{basePath} eq $basePath;
+    }
+    return 0;
+}
+
+
+sub generatePatches {
+    my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_;
+
+    my %srcOutPaths = findOutputPaths $srcNarFiles;
+    my %dstOutPaths = findOutputPaths $dstNarFiles;
+
+    # For each output path in the destination, see if we need to / can
+    # create a patch.
+
+    print STDERR "creating patches...\n";
+
+    foreach my $p (keys %dstOutPaths) {
+
+        # If exactly the same path already exists in the source, skip it.
+        next if defined $srcOutPaths{$p};
+    
+        print "  $p\n";
+
+        # If not, then we should find the paths in the source that are
+        # `most' likely to be present on a system that wants to
+        # install this path.
+
+        (my $name, my $version) = getNameVersion $p;
+        next unless defined $name && defined $version;
+
+        my @closest = ();
+        my $closestVersion;
+        my $minDist = -1; # actually, larger means closer
+
+        # Find all source paths with the same name.
+
+        foreach my $q (keys %srcOutPaths) {
+            (my $name2, my $version2) = getNameVersion $q;
+            next unless defined $name2 && defined $version2;
+
+            if ($name eq $name2) {
+
+                my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system};
+                my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system};
+                if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) {
+                    print "    SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n";
+                    next;
+                }
+
+                # If the sizes differ too much, then skip.  This
+                # disambiguates between, e.g., a real component and a
+                # wrapper component (cf. Firefox in Nixpkgs).
+                my $srcSize = @{$$srcNarFiles{$q}}[0]->{size};
+                my $dstSize = @{$$dstNarFiles{$p}}[0]->{size};
+                my $ratio = $srcSize / $dstSize;
+                $ratio = 1 / $ratio if $ratio < 1;
+                # print "  SIZE $srcSize $dstSize $ratio $q\n";
+
+                if ($ratio >= 3) {
+                    print "    SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n";
+                    next;
+                }
+
+                # If there are multiple matching names, include the
+                # ones with the closest version numbers.
+                my $dist = versionDiff $version, $version2;
+                if ($dist > $minDist) {
+                    $minDist = $dist;
+                    @closest = ($q);
+                    $closestVersion = $version2;
+                } elsif ($dist == $minDist) {
+                    push @closest, $q;
+                }
+            }
+        }
+
+        if (scalar(@closest) == 0) {
+            print "    NO BASE: $p\n";
+            next;
+        }
+
+        foreach my $closest (@closest) {
+
+            # Generate a patch between $closest and $p.
+            print STDERR "  $p <- $closest\n";
+
+            # If the patch already exists, skip it.
+            if (containsPatch($srcPatches, $p, $closest) ||
+                containsPatch($dstPatches, $p, $closest))
+            {
+                print "    skipping, already exists\n";
+                next;
+            }
+
+            my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest;
+            my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p;
+
+            if (! -f $srcNarBz2) {
+                warn "patch source archive $srcNarBz2 is missing\n";
+                next;
+            }
+
+            system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
+                or die "cannot unpack $srcNarBz2";
+
+            if ((stat "$tmpDir/A")[7] >= $maxNarSize) {
+                print "    skipping, source is too large\n";
+                next;
+            }
+        
+            system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0
+                or die "cannot unpack $dstNarBz2";
+
+            if ((stat "$tmpDir/B")[7] >= $maxNarSize) {
+                print "    skipping, destination is too large\n";
+                next;
+            }
+        
+            my $time1 = time();
+            my $res = system("ulimit -t $timeLimit; @libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF");
+            my $time2 = time();
+            if ($res) {
+                warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n";
+                next;
+            }
+
+            my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die;
+            chomp $baseHash;
+
+            my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die;
+            chomp $narHash;
+
+            my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die;
+            chomp $narDiffHash;
+
+            my $narDiffSize = (stat "$tmpDir/DIFF")[7];
+            my $dstNarBz2Size = (stat $dstNarBz2)[7];
+
+            print "    size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n";
+        
+            if ($narDiffSize >= $dstNarBz2Size) {
+                print "    rejecting; patch bigger than full archive\n";
+                next;
+            }
+    
+            if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
+                print "    rejecting; patch too large relative to full archive\n";
+                next;
+            }
+    
+            my $finalName = "$narDiffHash.nar-bsdiff";
+
+            if (-e "$patchesPath/$finalName") {
+                print "    not copying, already exists\n";
+            }
+
+            else {
+                system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0
+                    or die "cannot copy diff";
+                rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName")
+                    or die "cannot rename $patchesPath/$finalName.tmp";
+            }
+        
+            # Add the patch to the manifest.
+            addPatch $dstPatches, $p,
+                { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
+                , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
+                , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
+                };
+        }
+    }
+}
+
+
+# Propagate useful patches from $srcPatches to $dstPatches.  A patch
+# is useful if it produces either paths in the $dstNarFiles or paths
+# that can be used as the base for other useful patches.
+sub propagatePatches {
+    my ($srcPatches, $dstNarFiles, $dstPatches) = @_;
+
+    print STDERR "propagating patches...\n";
+
+    my $changed;
+    do {
+        # !!! we repeat this to reach the transitive closure; inefficient
+        $changed = 0;
+
+        print STDERR "loop\n";
+
+        my %dstBasePaths;
+        foreach my $q (keys %{$dstPatches}) {
+            foreach my $patch (@{$$dstPatches{$q}}) {
+                $dstBasePaths{$patch->{basePath}} = 1;
+            }
+        }
+
+        foreach my $p (keys %{$srcPatches}) {
+            my $patchList = $$srcPatches{$p};
+
+            my $include = 0;
+
+            # Is path $p included in the destination?  If so, include
+            # patches that produce it.
+            $include = 1 if defined $$dstNarFiles{$p};
+
+            # Is path $p a path that serves as a base for paths in the
+            # destination?  If so, include patches that produce it.
+            # !!! check baseHash
+            $include = 1 if defined $dstBasePaths{$p};
+
+            if ($include) {
+                foreach my $patch (@{$patchList}) {
+                    $changed = 1 if addPatch $dstPatches, $p, $patch;
+                }
+            }
+        
+        }
+    
+    } while $changed;
+}
+
+
+# Add all new patches in $srcPatches to $dstPatches.
+sub copyPatches {
+    my ($srcPatches, $dstPatches) = @_;
+    foreach my $p (keys %{$srcPatches}) {
+        addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}};
+    }
+}
+
+
+return 1;

scripts/Makefile.am

@@ -1,22 +1,23 @@
 bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build \
-  nix-copy-closure 
+  nix-copy-closure nix-generate-patches
 
-noinst_SCRIPTS = nix-profile.sh generate-patches.pl \
+noinst_SCRIPTS = nix-profile.sh GeneratePatches.pm \
   find-runtime-roots.pl build-remote.pl nix-reduce-build \
   copy-from-other-stores.pl nix-http-export.cgi
 
-nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl
+nix-pull nix-push: NixManifest.pm NixConfig.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
+install-exec-local: NixManifest.pm GeneratePatches.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
-	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) NixManifest.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) NixConfig.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) SSH.pm $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_DATA) GeneratePatches.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
@@ -29,14 +30,16 @@ EXTRA_DIST = nix-collect-garbage.in \
   nix-pull.in nix-push.in nix-profile.sh.in \
   nix-prefetch-url.in nix-install-package.in \
   nix-channel.in \
-  readmanifest.pm.in \
-  readconfig.pm.in \
+  NixManifest.pm.in \
+  NixConfig.pm.in \
+  SSH.pm \
+  GeneratePatches.pm.in \
   nix-build.in \
   download-using-manifests.pl.in \
   copy-from-other-stores.pl.in \
-  generate-patches.pl.in \
   nix-copy-closure.in \
   find-runtime-roots.pl.in \
   build-remote.pl.in \
   nix-reduce-build.in \
-  nix-http-export.cgi.in 
+  nix-http-export.cgi.in \
+  nix-generate-patches.in

scripts/NixManifest.pm.in

@@ -33,18 +33,8 @@ sub readManifest {
 
     my $manifestVersion = 2;
 
-    my $storePath;
-    my $url;
-    my $hash;
-    my $size;
-    my $basePath;
-    my $baseHash;
-    my $patchType;
-    my $narHash;
-    my $references;
-    my $deriver;
-    my $hashAlgo;
-    my $copyFrom;
+    my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType);
+    my ($narHash, $narSize, $references, $deriver, $hashAlgo, $copyFrom, $system);
 
     while (<MANIFEST>) {
         chomp;
@@ -62,9 +52,11 @@ sub readManifest {
                 undef $hash;
                 undef $size;
                 undef $narHash;
+                undef $narSize;
                 undef $basePath;
                 undef $baseHash;
                 undef $patchType;
+                undef $system;
                 $references = "";
                 $deriver = "";
                 $hashAlgo = "md5";
@@ -89,8 +81,10 @@ sub readManifest {
                     if (!$found) {
                         push @{$narFileList},
                             { url => $url, hash => $hash, size => $size
-                            , narHash => $narHash, references => $references
+                            , narHash => $narHash, narSize => $narSize
+                            , references => $references
                             , deriver => $deriver, hashAlgo => $hashAlgo
+                            , system => $system
                             };
                     }
                 
@@ -100,8 +94,8 @@ sub readManifest {
                     addPatch $patches, $storePath,
                         { url => $url, hash => $hash, size => $size
                         , basePath => $basePath, baseHash => $baseHash
-                        , narHash => $narHash, patchType => $patchType
-                        , hashAlgo => $hashAlgo
+                        , narHash => $narHash, narSize => $narSize
+                        , patchType => $patchType, hashAlgo => $hashAlgo
                         };
                 }
 
@@ -132,9 +126,11 @@ sub readManifest {
             elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
             elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
             elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
+            elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; }
             elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
             elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
             elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
+            elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; }
 
             # Compatibility;
             elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
@@ -150,7 +146,7 @@ sub readManifest {
 
 
 sub writeManifest {
-    my ($manifest, $narFiles, $patches) = @_;
+    my ($manifest, $narFiles, $patches, $noCompress) = @_;
 
     open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
 
@@ -165,12 +161,14 @@ sub writeManifest {
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $narFile->{url}\n";
             print MANIFEST "  Hash: $narFile->{hash}\n" if defined $narFile->{hash};
-            print MANIFEST "  NarHash: $narFile->{narHash}\n";
             print MANIFEST "  Size: $narFile->{size}\n" if defined $narFile->{size};
+            print MANIFEST "  NarHash: $narFile->{narHash}\n";
+            print MANIFEST "  NarSize: $narFile->{narSize}\n" if $narFile->{narSize};
             print MANIFEST "  References: $narFile->{references}\n"
                 if defined $narFile->{references} && $narFile->{references} ne "";
             print MANIFEST "  Deriver: $narFile->{deriver}\n"
                 if defined $narFile->{deriver} && $narFile->{deriver} ne "";
+            print MANIFEST "  System: $narFile->{system}\n" if defined $narFile->{system};
             print MANIFEST "}\n";
         }
     }
@@ -182,8 +180,9 @@ sub writeManifest {
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $patch->{url}\n";
             print MANIFEST "  Hash: $patch->{hash}\n";
-            print MANIFEST "  NarHash: $patch->{narHash}\n";
             print MANIFEST "  Size: $patch->{size}\n";
+            print MANIFEST "  NarHash: $patch->{narHash}\n";
+            print MANIFEST "  NarSize: $patch->{narSize}\n" if $patch->{narSize};
             print MANIFEST "  BasePath: $patch->{basePath}\n";
             print MANIFEST "  BaseHash: $patch->{baseHash}\n";
             print MANIFEST "  Type: $patch->{patchType}\n";
@@ -199,11 +198,13 @@ sub writeManifest {
 
 
     # Create a bzipped manifest.
-    system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0
-        or die "cannot compress manifest";
+    unless (defined $noCompress) {
+	system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0
+	    or die "cannot compress manifest";
 
-    rename("$manifest.bz2.tmp", "$manifest.bz2")
-        or die "cannot rename $manifest.bz2.tmp: $!";
+	rename("$manifest.bz2.tmp", "$manifest.bz2")
+	    or die "cannot rename $manifest.bz2.tmp: $!";
+    }
 }
 
 

scripts/SSH.pm

@@ -0,0 +1,52 @@
+use strict;
+use File::Temp qw(tempdir);
+
+our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
+
+push @sshOpts, "-x";
+
+my $sshStarted = 0;
+my $sshHost;
+
+# Open a master SSH connection to `host', unless there already is a
+# running master connection (as determined by `-O check').
+sub openSSHConnection {
+    my ($host) = @_;
+    die if $sshStarted;
+    $sshHost = $host;
+    return 1 if system("ssh $sshHost @sshOpts -O check 2> /dev/null") == 0;
+
+    my $tmpDir = tempdir("nix-ssh.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+        or die "cannot create a temporary directory";
+    
+    push @sshOpts, "-S", "$tmpDir/control";
+
+    # Start the master.  We can't use the `-f' flag (fork into
+    # background after establishing the connection) because then the
+    # child continues to run if we are killed.  So instead make SSH
+    # print "started" when it has established the connection, and wait
+    # until we see that.
+    open SSHPIPE, "ssh $sshHost @sshOpts -M -N -o LocalCommand='echo started' -o PermitLocalCommand=yes |" or die;
+
+    while (<SSHPIPE>) {
+        chomp;
+        if ($_ eq "started") {
+            $sshStarted = 1;
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+# Tell the master SSH client to exit.
+sub closeSSHConnection {
+    if ($sshStarted) {
+        system("ssh $sshHost @sshOpts -O exit 2> /dev/null") == 0
+            or warn "unable to stop SSH master: $?";
+    }
+}
+
+END { my $saved = $?; closeSSHConnection; $? = $saved; }
+
+return 1;

scripts/build-remote.pl.in

@@ -1,9 +1,11 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
 
-use strict;
 use Fcntl ':flock';
 use English '-no_match_vars';
 use IO::Handle;
+use SSH qw/sshOpts openSSHConnection/;
+no warnings('once');
+
 
 # General operation:
 #
@@ -22,191 +24,246 @@ use IO::Handle;
 # The nice thing about this scheme is that if we die prematurely, the
 # locks are released automatically.
 
-my $loadIncreased = 0;
 
-my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV;
-$maxSilentTime = 0 unless defined $maxSilentTime;
+# Make sure that we don't get any SSH passphrase or host key popups -
+# if there is any problem it should fail, not do something
+# interactive.
+$ENV{"DISPLAY"} = "";
+$ENV{"SSH_ASKPASS"} = "";
+
 
 sub sendReply {
     my $reply = shift;
     print STDERR "# $reply\n";
 }
 
-sub decline {
-    sendReply "decline";
-    exit 0;
-}
+sub all { $_ || return 0 for @_; 1 }
+
+
+# Initialisation.
+my $loadIncreased = 0;
+
+my ($localSystem, $maxSilentTime, $printBuildTrace) = @ARGV;
+$maxSilentTime = 0 unless defined $maxSilentTime;
 
 my $currentLoad = $ENV{"NIX_CURRENT_LOAD"};
-decline unless defined $currentLoad;
-mkdir $currentLoad, 0777 or die unless -d $currentLoad;
-
 my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
-decline if !defined $conf || ! -e $conf;
-
-my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
 
 
-# Otherwise find a willing remote machine.
-my @machines;
-my %curJobs;
+sub openSlotLock {
+    my ($machine, $slot) = @_;
+    my $slotLockFn = "$currentLoad/" . (join '+', @{$machine->{systemTypes}}) . "-" . $machine->{hostName} . "-$slot";
+    my $slotLock = new IO::Handle;
+    open $slotLock, ">>$slotLockFn" or die;
+    return $slotLock;
+}
 
 
 # Read the list of machines.
-open CONF, "< $conf" or die;
-
-while (<CONF>) {
-    chomp;
-    s/\#.*$//g;
-    next if /^\s*$/;
-    /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die;
-    push @machines,
-        { hostName => $1
-        , systemType => $2
-        , sshKeys => $3
-        , maxJobs => $4
-        };
+my @machines;
+if (defined $conf && -e $conf) {
+    open CONF, "< $conf" or die;
+    while (<CONF>) {
+        chomp;
+        s/\#.*$//g;
+        next if /^\s*$/;
+        my @tokens = split /\s/, $_;
+        push @machines,
+            { hostName => $tokens[0]
+            , systemTypes => [ split(/,/, $tokens[1]) ]
+            , sshKeys => $tokens[2]
+            , maxJobs => int($tokens[3])
+            , speedFactor => 1.0 * (defined $tokens[4] ? int($tokens[4]) : 1)
+            , features => [ split(/,/, $tokens[5] || "") ]
+            , enabled => 1
+            };
+    }
+    close CONF;
 }
 
-close CONF;
 
 
-# Acquire the exclusive lock on $currentLoad/main-lock.
-my $mainLock = "$currentLoad/main-lock";
-open MAINLOCK, ">>$mainLock" or die;
-flock(MAINLOCK, LOCK_EX) or die;
+# Wait for the calling process to ask us whether we can build some derivation.
+my ($drvPath, $hostName, $slotLock);
 
+REQ: while (1) {
+    $_ = <STDIN> || exit 0;
+    my ($amWilling, $neededSystem);
+    ($amWilling, $neededSystem, $drvPath, $requiredFeatures) = split;
+    my @requiredFeatures = split /,/, $requiredFeatures;
 
-# Find a suitable system.
-my $rightType = 0;
-my $machine;
-my $slotLock;
-LOOP: foreach my $cur (@machines) {
-    if ($neededSystem eq $cur->{systemType}
-        || ($neededSystem eq "i686-linux" && $cur->{systemType} eq "x86_64-linux"))
-    {
-        $rightType = 1;
+    my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
 
-        # We have a machine of the right type.  Try to get a lock on
-        # one of the machine's lock files.
-        my $slot = 0;
-        while ($slot < $cur->{maxJobs}) {
-            my $slotLockFn = "$currentLoad/" . $cur->{systemType} . "-" . $cur->{hostName} . "-$slot";
-            $slotLock = new IO::Handle;
-            open $slotLock, ">>$slotLockFn" or die;
-            if (flock($slotLock, LOCK_EX | LOCK_NB)) {
-                utime undef, undef, $slotLock;
-                $machine = $cur;
-                last LOOP;
+    if (!defined $currentLoad) {
+        sendReply "decline";
+        next;
+    }
+    
+    # Acquire the exclusive lock on $currentLoad/main-lock.
+    mkdir $currentLoad, 0777 or die unless -d $currentLoad;
+    my $mainLock = "$currentLoad/main-lock";
+    open MAINLOCK, ">>$mainLock" or die;
+    flock(MAINLOCK, LOCK_EX) or die;
+    
+    
+    while (1) {
+        # Find all machine that can execute this build, i.e., that
+        # support builds for the given platform and features, and are
+        # not at their job limit.
+        my $rightType = 0;
+        my @available = ();
+        LOOP: foreach my $cur (@machines) {
+            if ($cur->{enabled}
+                && (grep { $neededSystem eq $_ } @{$cur->{systemTypes}})
+                && all(map { my $f = $_; 0 != grep { $f eq $_ } @{$cur->{features}} } @requiredFeatures))
+            {
+                $rightType = 1;
+
+                # We have a machine of the right type.  Determine the load on
+                # the machine.
+                my $slot = 0;
+                my $load = 0;
+                my $free;
+                while ($slot < $cur->{maxJobs}) {
+                    my $slotLock = openSlotLock($cur, $slot);
+                    if (flock($slotLock, LOCK_EX | LOCK_NB)) {
+                        $free = $slot unless defined $free;
+                        flock($slotLock, LOCK_UN) or die;
+                    } else {
+                        $load++;
+                    }
+                    close $slotLock;
+                    $slot++;
+                }
+                
+                push @available, { machine => $cur, load => $load, free => $free }
+                if $load < $cur->{maxJobs};
             }
-            close $slotLock;
-            $slot++;
         }
+
+        if (defined $ENV{NIX_DEBUG_HOOK}) {
+            print STDERR "load on " . $_->{machine}->{hostName} . " = " . $_->{load} . "\n"
+                foreach @available;
+        }
+
+
+        # Didn't find any available machine?  Then decline or postpone.
+        if (scalar @available == 0) {
+            # Postpone if we have a machine of the right type, except
+            # if the local system can and wants to do the build.
+            if ($rightType && !$canBuildLocally) {
+                sendReply "postpone";
+            } else {
+                sendReply "decline";                
+            }
+            close MAINLOCK;
+            next REQ;
+        }
+
+
+        # Prioritise the available machines as follows:
+        # - First by load divided by speed factor, rounded to the nearest
+        #   integer.  This causes fast machines to be preferred over slow
+        #   machines with similar loads.
+        # - Then by speed factor.
+        # - Finally by load.
+        sub lf { my $x = shift; return int($x->{load} / $x->{machine}->{speedFactor} + 0.4999); }
+        @available = sort
+            { lf($a) <=> lf($b)
+                  || $b->{machine}->{speedFactor} <=> $a->{machine}->{speedFactor}
+                  || $a->{load} <=> $b->{load}
+            } @available;
+
+
+        # Select the best available machine and lock a free slot.
+        my $selected = $available[0]; 
+        my $machine = $selected->{machine};
+        
+        $slotLock = openSlotLock($machine, $selected->{free});
+        flock($slotLock, LOCK_EX | LOCK_NB) or die;
+        utime undef, undef, $slotLock;
+
+        close MAINLOCK;
+
+
+        # Connect to the selected machine.
+        @sshOpts = ("-i", $machine->{sshKeys}, "-x");
+        $hostName = $machine->{hostName};
+        last REQ if openSSHConnection $hostName;
+    
+        warn "unable to open SSH connection to $hostName, trying other available machines...\n";
+        $machine->{enabled} = 0;
     }
 }
 
-close MAINLOCK;
 
-
-# Didn't find one?  Then decline or postpone.
-if (!defined $machine) {
-    # Postpone if we have a machine of the right type, except if the
-    # local system can and wants to do the build.
-    if ($rightType && !$canBuildLocally) {
-        sendReply "postpone";
-        exit 0;
-    } else {
-        decline;
-    }
-}
-
-# Yes we did, accept.
+# Tell Nix we've accepted the build.
 sendReply "accept";
-my $x = <STDIN>;
-chomp $x;
-
-if ($x ne "okay") {
-    exit 0;
-}
+my @inputs = split /\s/, readline(STDIN);
+my @outputs = split /\s/, readline(STDIN);
 
 
-# Do the actual job.
-my $hostName = $machine->{hostName};
 print STDERR "building `$drvPath' on `$hostName'\n";
+print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace;
 
-# Make sure that we don't get any SSH passphrase or host key popups -
-# if there is any problem it should fail, not do something
-# interactive.
-$ENV{"DISPLAY"} = "";
-$ENV{"SSH_PASSWORD_FILE="} = "";
-$ENV{"SSH_ASKPASS="} = "";
-
-my $sshOpts = "-i " . $machine->{sshKeys} . " -x";
-
-# Hack to support Cygwin: if we login without a password, we don't
-# have exactly the same rights as when we do.  This causes the
-# Microsoft C compiler to fail with certain flags:
-#
-#   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676
-#
-# So as a workaround, we pass a verbatim password.  ssh tries to makes
-# this very hard; the trick is to make it call SSH_ASKPASS to get the
-# password.  (It only calls this command when there is no controlling
-# terminal, but Nix ensures that is is the case.  When doing this
-# manually, use setsid(1).)
-if ($machine->{sshKeys} =~ /^password:/) {
-    my $passwordFile = $machine->{sshKeys};
-    $passwordFile =~ s/^password://;
-    $sshOpts = "ssh -x";
-    $ENV{"SSH_PASSWORD_FILE"} = $passwordFile;
-    $ENV{"SSH_ASKPASS"} = "/tmp/writepass";
-
-    open WRITEPASS, ">/tmp/writepass" or die;
-    print WRITEPASS "#! /bin/sh\ncat \"\$SSH_PASSWORD_FILE\"";
-    close WRITEPASS;
-    chmod 0755, "/tmp/writepass" or die;
-}
-
-my $inputs = `cat inputs`; die if ($? != 0);
-$inputs =~ s/\n/ /g;
-
-my $outputs = `cat outputs`; die if ($? != 0);
-$outputs =~ s/\n/ /g;
-
-print "copying inputs...\n";
 
 my $maybeSign = "";
 $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
 
-system("NIX_SSHOPTS=\"$sshOpts\" @bindir@/nix-copy-closure --gzip $hostName $maybeSign $drvPath $inputs") == 0
+
+# Register the derivation as a temporary GC root.  Note that $PPID is
+# the PID of the remote SSH process, which, due to the use of a
+# persistant SSH connection, should be the same across all remote
+# command invocations for this session.
+my $rootsDir = "@localstatedir@/nix/gcroots/tmp";
+system("ssh $hostName @sshOpts 'mkdir -m 1777 -p $rootsDir; ln -sfn $drvPath $rootsDir/\$PPID.drv'");
+
+sub removeRoots {
+    system("ssh $hostName @sshOpts 'rm -f $rootsDir/\$PPID.drv $rootsDir/\$PPID.out'");
+}
+
+
+# Copy the derivation and its dependencies to the build machine.
+system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0
     or die "cannot copy inputs to $hostName: $?";
 
-print "building...\n";
 
-my $buildFlags = "--max-silent-time $maxSilentTime";
+# Perform the build.
+my $buildFlags = "--max-silent-time $maxSilentTime --fallback --add-root $rootsDir/\$PPID.out --option verbosity 0";
 
-# `-tt' forces allocation of a pseudo-terminal.  This is required to
-# make the remote nix-store process receive a signal when the
-# connection dies.  Without it, the remote process might continue to
-# run indefinitely (that is, until it next tries to write to
-# stdout/stderr).
-if (system("ssh -tt $sshOpts $hostName 'nix-store --realise -K $buildFlags $drvPath > /dev/null'") != 0) {
-    # If we couldn't run ssh or there was an ssh problem (indicated by
-    # exit code 255), then we return exit code 1; otherwise we assume
-    # that the builder failed, which we indicated to Nix using exit
-    # code 100.  It's important to distinguish between the two because
-    # the first is a transient failure and the latter is permanent.
-    my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100;
-    print STDERR "build of `$drvPath' on `$hostName' failed with exit code $?\n";
+# We let the remote side kill its process group when the connection is
+# closed unexpectedly.  This is necessary to ensure that no processes
+# are left running on the remote system if the local Nix process is
+# killed.  (SSH itself doesn't kill child processes if the connection
+# is interrupted unless the `-tt' flag is used to force a pseudo-tty,
+# in which case every child receives SIGHUP; however, `-tt' doesn't
+# work on some platforms when connection sharing is used.)
+pipe STDIN, DUMMY; # make sure we have a readable STDIN
+if (system("ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) {
+    # Note that if we get exit code 100 from `nix-store -r', it
+    # denotes a permanent build failure (as opposed to an SSH problem
+    # or a temporary Nix problem).  We propagate this to the caller to
+    # allow it to distinguish between transient and permanent
+    # failures.
+    my $res = $? >> 8;
+    print STDERR "build of `$drvPath' on `$hostName' failed with exit code $res\n";
+    removeRoots;
     exit $res;
 }
 
-print "build of `$drvPath' on `$hostName' succeeded\n";
+#print "build of `$drvPath' on `$hostName' succeeded\n";
 
-foreach my $output (split '\n', $outputs) {
+
+# Copy the output from the build machine.
+foreach my $output (@outputs) {
     my $maybeSignRemote = "";
     $maybeSignRemote = "--sign" if $UID != 0;
     
-    system("ssh $sshOpts $hostName 'nix-store --export $maybeSignRemote $output | gzip' | gunzip | @bindir@/nix-store --import > /dev/null") == 0
+    system("ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" .
+           "| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0
 	or die "cannot copy $output from $hostName: $?";
 }
+
+
+# Get rid of the temporary GC roots.
+removeRoots;

scripts/copy-from-other-stores.pl.in

@@ -17,25 +17,19 @@ foreach my $dir (@remoteStoresAll) {
 }
 
 
+$ENV{"NIX_REMOTE"} = "";
+
+
 sub findStorePath {
     my $storePath = shift;
-    
-    my $storePathName = basename $storePath;
-    
     foreach my $store (@remoteStores) {
-        # Determine whether $storePath exists by looking for the
-        # existence of the info file, and if so, get store path info
-        # from that file.  This rather breaks abstraction: we should
-        # be using `nix-store' for that.  But right now there is no
-        # good way to tell nix-store to access a store mounted under a
-        # different location (there's $NIX_STORE, but that only works
-        # if the remote store is mounted under its "real" location).
-        my $infoFile = "$store/var/nix/db/info/$storePathName";
-        my $storePath2 = "$store/store/$storePathName";
-        if (-f $infoFile && -e $storePath2) {
-            return ($infoFile, $storePath2);
-        }
+        my $sourcePath = "$store/store/" . basename $storePath;
+        next unless -e $sourcePath || -l $sourcePath;
+        $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
+        return ($store, $sourcePath) if
+            system("@bindir@/nix-store --check-validity $storePath") == 0;
     }
+    return undef;
 }
 
 
@@ -46,37 +40,38 @@ if ($ARGV[0] eq "--query") {
 
         if ($cmd eq "have") {
             my $storePath = <STDIN>; chomp $storePath;
-            (my $infoFile) = findStorePath $storePath;
-            print STDOUT ($infoFile ? "1\n" : "0\n");
+            print STDOUT (defined findStorePath($storePath) ? "1\n" : "0\n");
         }
 
         elsif ($cmd eq "info") {
             my $storePath = <STDIN>; chomp $storePath;
-            (my $infoFile) = findStorePath $storePath;
-            if (!$infoFile) {
+            my ($store, $sourcePath) = findStorePath($storePath);
+            if (!defined $store) {
                 print "0\n";
                 next; # not an error
             }
             print "1\n";
 
-            my $deriver = "";
-            my @references = ();
+            $ENV{"NIX_DB_DIR"} = "$store/var/nix/db";
+            
+            my $deriver = `@bindir@/nix-store --query --deriver $storePath`;
+            die "cannot query deriver of `$storePath'" if $? != 0;
+            chomp $deriver;
+            $deriver = "" if $deriver eq "unknown-deriver";
 
-            open INFO, "<$infoFile" or die "cannot read info file $infoFile\n";
-            while (<INFO>) {
-                chomp;
-                /^([\w-]+): (.*)$/ or die "bad info file";
-                my $key = $1;
-                my $value = $2;
-                if ($key eq "Deriver") { $deriver = $value; }
-                elsif ($key eq "References") { @references = split ' ', $value; }
-            }
-            close INFO;
+            my @references = split "\n",
+                `@bindir@/nix-store --query --references $storePath`;
+            die "cannot query references of `$storePath'" if $? != 0;
+
+            my $narSize = `@bindir@/nix-store --query --size $storePath`;
+            die "cannot query size of `$storePath'" if $? != 0;
+            chomp $narSize;
 
             print "$deriver\n";
             print scalar @references, "\n";
             print "$_\n" foreach @references;
-            print "0\n"; # !!! showing size not supported (yet)
+            print "$narSize\n";
+            print "$narSize\n";
         }
 
         else { die "unknown command `$cmd'"; }
@@ -87,8 +82,8 @@ if ($ARGV[0] eq "--query") {
 elsif ($ARGV[0] eq "--substitute") {
     die unless scalar @ARGV == 2;
     my $storePath = $ARGV[1];
-    (my $infoFile, my $sourcePath) = findStorePath $storePath;
-    die unless $infoFile;
+    my ($store, $sourcePath) = findStorePath $storePath;
+    die unless $store;
     print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
     system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
         or die "cannot copy `$sourcePath' to `$storePath'";

scripts/download-using-manifests.pl.in

@@ -1,7 +1,7 @@
 #! @perl@ -w -I@libexecdir@/nix
 
 use strict;
-use readmanifest;
+use NixManifest;
 use POSIX qw(strftime);
 use File::Temp qw(tempdir);
 
@@ -12,6 +12,10 @@ STDOUT->autoflush(1);
 my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests");
 my $logFile = "@localstatedir@/log/nix/downloads";
 
+# For queries, skip expensive calls to nix-hash etc.  We're just
+# estimating the expected download size.
+my $fast = 1;
+
 
 # Load all manifests.
 my %narFiles;
@@ -31,6 +35,151 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") {
 }
 
 
+sub isValidPath {
+    my $p = shift;
+    if ($fast) {
+        return -e $p;
+    } else {
+        return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
+    }
+}
+
+
+sub parseHash {
+    my $hash = shift;
+    if ($hash =~ /^(.+):(.+)$/) {
+        return ($1, $2);
+    } else {
+        return ("md5", $hash);
+    }
+}
+
+
+# Compute the most efficient sequence of downloads to produce the
+# given path.
+sub computeSmallestDownload {
+    my $targetPath = shift;
+    
+    # Build a graph of all store paths that might contribute to the
+    # construction of $targetPath, and the special node "start".  The
+    # edges are either patch operations, or downloads of full NAR
+    # files.  The latter edges only occur between "start" and a store
+    # path.
+    my %graph;
+
+    $graph{"start"} = {d => 0, pred => undef, edges => []};
+
+    my @queue = ();
+    my $queueFront = 0;
+    my %done;
+
+    sub addNode {
+        my $graph = shift;
+        my $u = shift;
+        $$graph{$u} = {d => 999999999999, pred => undef, edges => []}
+            unless defined $$graph{$u};
+    }
+
+    sub addEdge {
+        my $graph = shift;
+        my $u = shift;
+        my $v = shift;
+        my $w = shift;
+        my $type = shift;
+        my $info = shift;
+        addNode $graph, $u;
+        push @{$$graph{$u}->{edges}},
+            {weight => $w, start => $u, end => $v, type => $type, info => $info};
+        my $n = scalar @{$$graph{$u}->{edges}};
+    }
+
+    push @queue, $targetPath;
+
+    while ($queueFront < scalar @queue) {
+        my $u = $queue[$queueFront++];
+        next if defined $done{$u};
+        $done{$u} = 1;
+
+        addNode \%graph, $u;
+
+        # If the path already exists, it has distance 0 from the
+        # "start" node.
+        if (isValidPath($u)) {
+            addEdge \%graph, "start", $u, 0, "present", undef;
+        }
+
+        else {
+
+            # Add patch edges.
+            my $patchList = $patches{$u};
+            foreach my $patch (@{$patchList}) {
+                if (isValidPath($patch->{basePath})) {
+                    # !!! this should be cached
+                    my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
+                    my $format = "--base32";
+                    $format = "" if $baseHashAlgo eq "md5";
+                    my $hash = $fast && $baseHashAlgo eq "sha256"
+                        ? `$binDir/nix-store -q --hash "$patch->{basePath}"`
+                        : `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
+                    chomp $hash;
+                    $hash =~ s/.*://;
+                    next if $hash ne $baseHash;
+                }
+                push @queue, $patch->{basePath};
+                addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
+            }
+
+            # Add NAR file edges to the start node.
+            my $narFileList = $narFiles{$u};
+            foreach my $narFile (@{$narFileList}) {
+                # !!! how to handle files whose size is not known in advance?
+                # For now, assume some arbitrary size (1 MB).
+                addEdge \%graph, "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
+            }
+        }
+    }
+
+
+    # Run Dijkstra's shortest path algorithm to determine the shortest
+    # sequence of download and/or patch actions that will produce
+    # $targetPath.
+
+    my @todo = keys %graph;
+
+    while (scalar @todo > 0) {
+
+        # Remove the closest element from the todo list.
+        # !!! inefficient, use a priority queue
+        @todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo;
+        my $u = pop @todo;
+
+        my $u_ = $graph{$u};
+
+        foreach my $edge (@{$u_->{edges}}) {
+            my $v_ = $graph{$edge->{end}};
+            if ($v_->{d} > $u_->{d} + $edge->{weight}) {
+                $v_->{d} = $u_->{d} + $edge->{weight};
+                # Store the edge; to edge->start is actually the
+                # predecessor.
+                $v_->{pred} = $edge; 
+            }
+        }
+    }
+
+
+    # Retrieve the shortest path from "start" to $targetPath.
+    my @path = ();
+    my $cur = $targetPath;
+    return () unless defined $graph{$targetPath}->{pred};
+    while ($cur ne "start") {
+        push @path, $graph{$cur}->{pred};
+        $cur = $graph{$cur}->{pred}->{start};
+    }
+
+    return @path;
+}
+
+
 # Parse the arguments.
 
 if ($ARGV[0] eq "--query") {
@@ -46,6 +195,7 @@ if ($ARGV[0] eq "--query") {
 
         elsif ($cmd eq "info") {
             my $storePath = <STDIN>; chomp $storePath;
+
             my $info;
             if (defined $narFiles{$storePath}) {
                 $info = @{$narFiles{$storePath}}[0];
@@ -57,13 +207,32 @@ if ($ARGV[0] eq "--query") {
                 print "0\n";
                 next; # not an error
             }
+
             print "1\n";
             print "$info->{deriver}\n";
             my @references = split " ", $info->{references};
             print scalar @references, "\n";
             print "$_\n" foreach @references;
-            my $size = $info->{size} || 0;
-            print "$size\n";
+
+            my @path = computeSmallestDownload $storePath;
+            
+            my $downloadSize = 0;
+            while (scalar @path > 0) {
+                my $edge = pop @path;
+                my $u = $edge->{start};
+                my $v = $edge->{end};
+                if ($edge->{type} eq "patch") {
+                    $downloadSize += $edge->{info}->{size} || 0;
+                }
+                elsif ($edge->{type} eq "narfile") {
+                    $downloadSize += $edge->{info}->{size} || 0;
+                }
+            }
+                
+            print "$downloadSize\n";
+            
+            my $narSize = $info->{narSize} || 0;
+            print "$narSize\n";
         }
         
         else { die "unknown command `$cmd'"; }
@@ -73,20 +242,19 @@ if ($ARGV[0] eq "--query") {
 }
 
 elsif ($ARGV[0] ne "--substitute") {
-    die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n";
+    die;
 }
 
 
 die unless scalar @ARGV == 2;
 my $targetPath = $ARGV[1];
+$fast = 0;
 
 
 # Create a temporary directory.
 my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-chdir $tmpDir or die "cannot change to `$tmpDir': $!";
-
 my $tmpNar = "$tmpDir/nar";
 my $tmpNar2 = "$tmpDir/nar2";
 
@@ -112,148 +280,9 @@ foreach my $localPath (@{$localPathList}) {
 }
 
 
-# Build a graph of all store paths that might contribute to the
-# construction of $targetPath, and the special node "start".  The
-# edges are either patch operations, or downloads of full NAR files.
-# The latter edges only occur between "start" and a store path.
-
-my %graph;
-
-$graph{"start"} = {d => 0, pred => undef, edges => []};
-
-my @queue = ();
-my $queueFront = 0;
-my %done;
-
-sub addToQueue {
-    my $v = shift;
-    return if defined $done{$v};
-    $done{$v} = 1;
-    push @queue, $v;
-}
-
-sub addNode {
-    my $u = shift;
-    $graph{$u} = {d => 999999999999, pred => undef, edges => []}
-        unless defined $graph{$u};
-}
-
-sub addEdge {
-    my $u = shift;
-    my $v = shift;
-    my $w = shift;
-    my $type = shift;
-    my $info = shift;
-    addNode $u;
-    push @{$graph{$u}->{edges}},
-        {weight => $w, start => $u, end => $v, type => $type, info => $info};
-    my $n = scalar @{$graph{$u}->{edges}};
-}
-
-addToQueue $targetPath;
-
-sub isValidPath {
-    my $p = shift;
-    return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
-}
-
-sub parseHash {
-    my $hash = shift;
-    if ($hash =~ /^(.+):(.+)$/) {
-        return ($1, $2);
-    } else {
-        return ("md5", $hash);
-    }
-}
-
-while ($queueFront < scalar @queue) {
-    my $u = $queue[$queueFront++];
-#    print "$u\n";
-
-    addNode $u;
-
-    # If the path already exists, it has distance 0 from the "start"
-    # node.
-    if (isValidPath($u)) {
-        addEdge "start", $u, 0, "present", undef;
-    }
-
-    else {
-
-        # Add patch edges.
-        my $patchList = $patches{$u};
-        foreach my $patch (@{$patchList}) {
-            if (isValidPath($patch->{basePath})) {
-                # !!! this should be cached
-                my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
-                my $format = "--base32";
-                $format = "" if $baseHashAlgo eq "md5";
-                my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
-                chomp $hash;
-                if ($hash ne $baseHash) {
-                    print LOGFILE "$$ rejecting $patch->{basePath}\n";
-                    next;
-                }
-            }
-            addToQueue $patch->{basePath};
-            addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
-        }
-
-        # Add NAR file edges to the start node.
-        my $narFileList = $narFiles{$u};
-        foreach my $narFile (@{$narFileList}) {
-            # !!! how to handle files whose size is not known in advance?
-            # For now, assume some arbitrary size (1 MB).
-            addEdge "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
-            if ($u eq $targetPath) {
-                my $size = $narFile->{size} || -1;
-                print LOGFILE "$$ full-download-would-be $size\n";
-            }
-        }
-
-    }
-}
-
-
-# Run Dijkstra's shortest path algorithm to determine the shortest
-# sequence of download and/or patch actions that will produce
-# $targetPath.
-
-sub byDistance { # sort by distance, reversed
-    return -($graph{$a}->{d} <=> $graph{$b}->{d});
-}
-
-my @todo = keys %graph;
-
-while (scalar @todo > 0) {
-
-    # Remove the closest element from the todo list.
-    @todo = sort byDistance @todo;
-    my $u = pop @todo;
-
-    my $u_ = $graph{$u};
-
-    foreach my $edge (@{$u_->{edges}}) {
-        my $v_ = $graph{$edge->{end}};
-        if ($v_->{d} > $u_->{d} + $edge->{weight}) {
-            $v_->{d} = $u_->{d} + $edge->{weight};
-            # Store the edge; to edge->start is actually the
-            # predecessor.
-            $v_->{pred} = $edge; 
-        }
-    }
-}
-
-
-# Retrieve the shortest path from "start" to $targetPath.
-my @path = ();
-my $cur = $targetPath;
-die "don't know how to produce $targetPath\n"
-    unless defined $graph{$targetPath}->{pred};
-while ($cur ne "start") {
-    push @path, $graph{$cur}->{pred};
-    $cur = $graph{$cur}->{pred}->{start};
-}
+# Compute the shortest path.
+my @path = computeSmallestDownload $targetPath;
+die "don't know how to produce $targetPath\n" if scalar @path == 0;
 
 
 # Traverse the shortest path, perform the actions described by the

scripts/generate-patches.pl.in

@@ -1,408 +0,0 @@
-#! @perl@ -w -I@libexecdir@/nix
-
-use strict;
-use File::Temp qw(tempdir);
-use readmanifest;
-
-
-# Some patch generations options.
-
-# Max size of NAR archives to generate patches for.
-my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
-$maxNarSize = 100 * 1024 * 1024 if !defined $maxNarSize;
-
-# If patch is bigger than this fraction of full archive, reject.
-my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
-$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
-
-
-die unless scalar @ARGV == 5;
-
-my $hashAlgo = "sha256";
-
-my $cacheDir = $ARGV[0];
-my $patchesDir = $ARGV[1];
-my $patchesURL = $ARGV[2];
-my $srcDir = $ARGV[3];
-my $dstDir = $ARGV[4];
-
-my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-print "TEMP = $tmpDir\n";
-
-#END { rmdir $tmpDir; }
-
-my %srcNarFiles;
-my %srcLocalPaths;
-my %srcPatches;
-
-my %dstNarFiles;
-my %dstLocalPaths;
-my %dstPatches;
-
-readManifest "$srcDir/MANIFEST",
-    \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
-
-readManifest "$dstDir/MANIFEST",
-    \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
-
-
-sub findOutputPaths {
-    my $narFiles = shift;
-
-    my %outPaths;
-    
-    foreach my $p (keys %{$narFiles}) {
-
-        # Ignore store expressions.
-        next if ($p =~ /\.store$/);
-        next if ($p =~ /\.drv$/);
-        
-        # Ignore builders (too much ambiguity -- they're all called
-        # `builder.sh').
-        next if ($p =~ /\.sh$/);
-        next if ($p =~ /\.patch$/);
-        
-        # Don't bother including tar files etc.
-        next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
-
-        $outPaths{$p} = 1;
-    }
-
-    return %outPaths;
-}
-
-print "finding src output paths...\n";
-my %srcOutPaths = findOutputPaths \%srcNarFiles;
-
-print "finding dst output paths...\n";
-my %dstOutPaths = findOutputPaths \%dstNarFiles;
-
-
-sub getNameVersion {
-    my $p = shift;
-    $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
-    my $name = $1;
-    my $version = $2;
-    $name =~ s/^-//;
-    $version =~ s/^-//;
-    return ($name, $version);
-}
-
-
-# A quick hack to get a measure of the `distance' between two
-# versions: it's just the position of the first character that differs
-# (or 999 if they are the same).
-sub versionDiff {
-    my $s = shift;
-    my $t = shift;
-    my $i;
-    return 999 if $s eq $t;
-    for ($i = 0; $i < length $s; $i++) {
-        return $i if $i >= length $t or
-            substr($s, $i, 1) ne substr($t, $i, 1);
-    }
-    return $i;
-}
-
-
-sub getNarBz2 {
-    my $narFiles = shift;
-    my $storePath = shift;
-    
-    my $narFileList = $$narFiles{$storePath};
-    die "missing store expression $storePath" unless defined $narFileList;
-
-    my $narFile = @{$narFileList}[0];
-    die unless defined $narFile;
-
-    $narFile->{url} =~ /\/([^\/]+)$/;
-    die unless defined $1;
-    return "$cacheDir/$1";
-}
-
-
-sub containsPatch {
-    my $patches = shift;
-    my $storePath = shift;
-    my $basePath = shift;
-    my $patchList = $$patches{$storePath};
-    return 0 if !defined $patchList;
-    my $found = 0;
-    foreach my $patch (@{$patchList}) {
-        # !!! baseHash might differ
-        return 1 if $patch->{basePath} eq $basePath;
-    }
-    return 0;
-}
-
-
-# Compute the "weighted" number of uses of a path in the build graph.
-sub computeUses {
-    my $narFiles = shift;
-    my $path = shift;
-
-    # Find the deriver of $path.
-    return 1 unless defined $$narFiles{$path};
-    my $deriver = @{$$narFiles{$path}}[0]->{deriver};
-    return 1 unless defined $deriver && $deriver ne "";
-
-#    print "  DERIVER $deriver\n";
-
-    # Optimisation: build the referrers graph from the references
-    # graph.
-    my %referrers;
-    foreach my $q (keys %{$narFiles}) {
-        my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
-        foreach my $r (@refs) {
-            $referrers{$r} = [] unless defined $referrers{$r};
-            push @{$referrers{$r}}, $q;
-        }
-    }
-
-    # Determine the shortest path from $deriver to all other reachable
-    # paths in the `referrers' graph.
-
-    my %dist;
-    $dist{$deriver} = 0;
-
-    my @queue = ($deriver);
-    my $pos = 0;
-    
-    while ($pos < scalar @queue) {
-        my $p = $queue[$pos];
-        $pos++;
-
-        foreach my $q (@{$referrers{$p}}) {
-            if (!defined $dist{$q}) {
-                $dist{$q} = $dist{$p} + 1;
-#                print "    $q $dist{$q}\n";
-                push @queue, $q;
-            }
-        }
-    }
-
-    my $wuse = 1.0;
-    foreach my $user (keys %dist) {
-        next if $user eq $deriver;
-#        print "    $user $dist{$user}\n";
-        $wuse += 1.0 / 2.0**$dist{$user};
-    }
-
-#    print "    XXX $path $wuse\n";
-    
-    return $wuse;
-}
-
-
-# For each output path in the destination, see if we need to / can
-# create a patch.
-
-print "creating patches...\n";
-
-foreach my $p (keys %dstOutPaths) {
-
-    # If exactly the same path already exists in the source, skip it.
-    next if defined $srcOutPaths{$p};
-    
-    print "  $p\n";
-
-    # If not, then we should find the paths in the source that are
-    # `most' likely to be present on a system that wants to install
-    # this path.
-
-    (my $name, my $version) = getNameVersion $p;
-
-    my @closest = ();
-    my $closestVersion;
-    my $minDist = -1; # actually, larger means closer
-
-    # Find all source paths with the same name.
-
-    foreach my $q (keys %srcOutPaths) {
-        (my $name2, my $version2) = getNameVersion $q;
-        if ($name eq $name2) {
-
-            # If the sizes differ too much, then skip.  This
-            # disambiguates between, e.g., a real component and a
-            # wrapper component (cf. Firefox in Nixpkgs).
-            my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
-            my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
-            my $ratio = $srcSize / $dstSize;
-            $ratio = 1 / $ratio if $ratio < 1;
-#            print "  SIZE $srcSize $dstSize $ratio $q\n";
-
-            if ($ratio >= 3) {
-                print "  SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
-                next;
-            }
-
-            # If the numbers of weighted uses differ too much, then
-            # skip.  This disambiguates between, e.g., the bootstrap
-            # GCC and the final GCC in Nixpkgs.
-            my $srcUses = computeUses \%srcNarFiles, $q;
-            my $dstUses = computeUses \%dstNarFiles, $p;
-            $ratio = $srcUses / $dstUses;
-            $ratio = 1 / $ratio if $ratio < 1;
-            print "  USE $srcUses $dstUses $ratio $q\n";
-            
-#            if ($ratio >= 2) {
-#                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
-#                next;
-#            }
-
-            # If there are multiple matching names, include the ones
-            # with the closest version numbers.
-            my $dist = versionDiff $version, $version2;
-            if ($dist > $minDist) {
-                $minDist = $dist;
-                @closest = ($q);
-                $closestVersion = $version2;
-            } elsif ($dist == $minDist) {
-                push @closest, $q;
-            }
-        }
-    }
-
-    if (scalar(@closest) == 0) {
-        print "  NO BASE: $p\n";
-        next;
-    }
-
-    foreach my $closest (@closest) {
-
-        # Generate a patch between $closest and $p.
-        print "  $p <- $closest\n";
-
-        # If the patch already exists, skip it.
-        if (containsPatch(\%srcPatches, $p, $closest) ||
-            containsPatch(\%dstPatches, $p, $closest))
-        {
-            print "    skipping, already exists\n";
-            next;
-        }
-
-#        next;
-        
-        my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
-        my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
-
-        system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
-            or die "cannot unpack $srcNarBz2";
-
-        if ((stat "$tmpDir/A")[7] >= $maxNarSize) {
-            print "    skipping, source is too large\n";
-            next;
-        }
-        
-        system("@bunzip2@ < $dstNarBz2 > $tmpDir/B") == 0
-            or die "cannot unpack $dstNarBz2";
-
-        if ((stat "$tmpDir/B")[7] >= $maxNarSize) {
-            print "    skipping, destination is too large\n";
-            next;
-        }
-        
-        system("@libexecdir@/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF") == 0
-            or die "cannot compute binary diff";
-
-        my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die;
-        chomp $baseHash;
-
-        my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die;
-        chomp $narHash;
-
-        my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die;
-        chomp $narDiffHash;
-
-        my $narDiffSize = (stat "$tmpDir/DIFF")[7];
-        my $dstNarBz2Size = (stat $dstNarBz2)[7];
-
-        print "    size $narDiffSize; full size $dstNarBz2Size\n";
-        
-        if ($narDiffSize >= $dstNarBz2Size) {
-            print "    rejecting; patch bigger than full archive\n";
-            next;
-        }
-    
-        if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
-            print "    rejecting; patch too large relative to full archive\n";
-            next;
-        }
-    
-        my $finalName =
-            "$narDiffHash.nar-bsdiff";
-
-        if (-e "$patchesDir/$finalName") {
-            print "    not copying, already exists\n";
-        }
-
-        else {
-
-            system("cp '$tmpDir/DIFF' '$patchesDir/$finalName.tmp'") == 0
-                or die "cannot copy diff";
-            
-            rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
-                or die "cannot rename $patchesDir/$finalName.tmp";
-            
-        }
-        
-        # Add the patch to the manifest.
-        addPatch \%dstPatches, $p,
-            { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
-            , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
-            , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
-            }, 0;
-    }
-}
-
-
-# Add in any potentially useful patches in the source (namely, those
-# patches that produce either paths in the destination or paths that
-# can be used as the base for other useful patches).
-
-print "propagating patches...\n";
-
-my $changed;
-do {
-    # !!! we repeat this to reach the transitive closure; inefficient
-    $changed = 0;
-
-    print "loop\n";
-
-    my %dstBasePaths;
-    foreach my $q (keys %dstPatches) {
-	foreach my $patch (@{$dstPatches{$q}}) {
-	    $dstBasePaths{$patch->{basePath}} = 1;
-	}
-    }
-
-    foreach my $p (keys %srcPatches) {
-        my $patchList = $srcPatches{$p};
-
-        my $include = 0;
-
-        # Is path $p included in the destination?  If so, include
-        # patches that produce it.
-        $include = 1 if defined $dstNarFiles{$p};
-
-        # Is path $p a path that serves as a base for paths in the
-        # destination?  If so, include patches that produce it.
-	# !!! check baseHash
-        $include = 1 if defined $dstBasePaths{$p};
-
-        if ($include) {
-            foreach my $patch (@{$patchList}) {
-                $changed = 1 if addPatch \%dstPatches, $p, $patch;
-            }
-        }
-        
-    }
-    
-} while $changed;
-
-
-# Rewrite the manifest of the destination (with the new patches).
-writeManifest "$dstDir/MANIFEST",
-    \%dstNarFiles, \%dstPatches;

scripts/nix-build.in

@@ -12,6 +12,7 @@ my $outLink;
 my $drvLink;
 
 my $dryRun = 0;
+my $verbose = 0;
 
 my @instArgs = ();
 my @buildArgs = ();
@@ -101,7 +102,7 @@ EOF
         $n += 2;
     }
 
-    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type") {
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @buildArgs, ($arg, $ARGV[$n]);
@@ -112,6 +113,21 @@ EOF
         $dryRun = 1;
     }
     
+    elsif ($arg eq "--show-trace") {
+        push @instArgs, $arg;
+    }
+    
+    elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") {
+        push @buildArgs, $arg;
+        push @instArgs, $arg;
+        $verbose = 1;
+    }
+    
+    elsif ($arg eq "--quiet") {
+        push @buildArgs, $arg;
+        push @instArgs, $arg;
+    }
+    
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -149,12 +165,12 @@ foreach my $expr (@exprs) {
 
     foreach my $drvPath (@drvPaths) {
         my $target = readlink $drvPath or die "cannot read symlink `$drvPath'";
-        print STDERR "store derivation is $target\n";
+        print STDERR "derivation is $target\n" if $verbose;
     }
 
     # Build.
     my @outPaths;
-    $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-rv",
+    $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-r",
         @buildArgs, @drvPaths;
     while (<OUTPATHS>) {chomp; push @outPaths, $_;}
     if (!close OUTPATHS) {

scripts/nix-channel.in

@@ -78,6 +78,9 @@ sub removeChannel {
 sub update {
     readChannels;
 
+    # Create the manifests directory if it doesn't exist.
+    mkdir "$stateDir/manifests", 0755 unless -e "$stateDir/manifests";
+
     # Do we have write permission to the manifests directory?  If not,
     # then just skip pulling the manifest and just download the Nix
     # expressions.  If the user is a non-privileged user in a
@@ -85,11 +88,6 @@ sub update {
     # source.
     if (-W "$stateDir/manifests") {
 
-        # Remove all the old manifests.
-        for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
-            unlink $manifest or die "cannot remove `$manifest': $!";
-        }
-
         # Pull cache manifests.
         foreach my $url (@channels) {
             #print "pulling cache manifest from `$url'\n";

scripts/nix-copy-closure.in

@@ -1,4 +1,6 @@
-#! @perl@ -w
+#! @perl@ -w -I@libexecdir@/nix
+
+use SSH;
 
 my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
@@ -14,12 +16,11 @@ EOF
 
 # Get the target host.
 my $sshHost;
-my @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
 
 my $sign = 0;
 
-my $compressor = "cat";
-my $decompressor = "cat";
+my $compressor = "";
+my $decompressor = "";
 
 my $toMode = 1;
 
@@ -34,8 +35,8 @@ while (@ARGV) {
         $sign = 1;
     }
     elsif ($arg eq "--gzip") {
-        $compressor = "gzip";
-        $decompressor = "gunzip";
+        $compressor = "| gzip";
+        $decompressor = "gunzip |";
     }
     elsif ($arg eq "--from") {
         $toMode = 0;
@@ -52,12 +53,15 @@ while (@ARGV) {
 }
 
 
+openSSHConnection $sshHost or die "$0: unable to start SSH\n";
+
+
 if ($toMode) { # Copy TO the remote machine.
 
     my @allStorePaths;
 
     # Get the closure of this path.
-    my $pid = open(READ, "$binDir/nix-store --query --requisites @storePaths|") or die;
+    my $pid = open(READ, "set -f; $binDir/nix-store --query --requisites @storePaths|") or die;
     
     while (<READ>) {
         chomp;
@@ -69,11 +73,10 @@ if ($toMode) { # Copy TO the remote machine.
 
 
     # Ask the remote host which paths are invalid.
-    open(READ, "ssh @sshOpts $sshHost nix-store --check-validity --print-invalid @allStorePaths|");
+    open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|");
     my @missing = ();
     while (<READ>) {
         chomp;
-        print STDERR "target machine needs $_\n";
         push @missing, $_;
     }
     close READ or die;
@@ -81,9 +84,11 @@ if ($toMode) { # Copy TO the remote machine.
 
     # Export the store paths and import them on the remote machine.
     if (scalar @missing > 0) {
+        print STDERR "copying these missing paths:\n";
+        print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("nix-store --export $extraOpts @missing | $compressor | ssh @sshOpts $sshHost '$decompressor | nix-store --import'") == 0
+        system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
             or die "copying store paths to remote machine `$sshHost' failed: $?";
     }
 
@@ -96,7 +101,7 @@ else { # Copy FROM the remote machine.
     # machine.  Paths are assumed to be store paths; there is no
     # resolution (following of symlinks).
     my $pid = open(READ,
-        "ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
+        "set -f; ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
     
     my @allStorePaths;
 
@@ -110,11 +115,10 @@ else { # Copy FROM the remote machine.
 
 
     # What paths are already valid locally?
-    open(READ, "@bindir@/nix-store --check-validity --print-invalid @allStorePaths|");
+    open(READ, "set -f; @bindir@/nix-store --check-validity --print-invalid @allStorePaths|");
     my @missing = ();
     while (<READ>) {
         chomp;
-        print STDERR "local machine needs $_\n";
         push @missing, $_;
     }
     close READ or die;
@@ -122,10 +126,12 @@ else { # Copy FROM the remote machine.
 
     # Export the store paths on the remote machine and import them on locally.
     if (scalar @missing > 0) {
+        print STDERR "copying these missing paths:\n";
+        print STDERR "  $_\n" foreach @missing;
         my $extraOpts = "";
         $extraOpts .= "--sign" if $sign == 1;
-        system("ssh @sshOpts $sshHost 'nix-store --export $extraOpts @missing | $compressor' | $decompressor | @bindir@/nix-store --import") == 0
-            or die "copying store paths to remote machine `$sshHost' failed: $?";
+        system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor @bindir@/nix-store --import") == 0
+            or die "copying store paths from remote machine `$sshHost' failed: $?";
     }
 
 }

scripts/nix-generate-patches.in

@@ -0,0 +1,42 @@
+#! @perl@ -w -I@libexecdir@/nix
+
+use strict;
+use File::Temp qw(tempdir);
+use NixManifest;
+use GeneratePatches;
+
+if (scalar @ARGV != 5) {
+    print STDERR <<EOF;
+Usage: nix-generate-patches NAR-DIR PATCH-DIR PATCH-URI OLD-MANIFEST NEW-MANIFEST
+
+This command generates binary patches between NAR files listed in
+OLD-MANIFEST and NEW-MANIFEST.  The patches are written to the
+directory PATCH-DIR, and the prefix PATCH-URI is used to generate URIs
+for the patches.  The patches are added to NEW-MANIFEST.  All NARs are
+required to exist in NAR-DIR.  Patches are generated between
+succeeding versions of packages with the same name.
+EOF
+    exit 1;
+}
+
+my $narPath = $ARGV[0];
+my $patchesPath = $ARGV[1];
+my $patchesURL = $ARGV[2];
+my $srcManifest = $ARGV[3];
+my $dstManifest = $ARGV[4];
+
+my (%srcNarFiles, %srcLocalPaths, %srcPatches);
+readManifest $srcManifest, \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
+
+my (%dstNarFiles, %dstLocalPaths, %dstPatches);
+readManifest $dstManifest, \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
+
+my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+    or die "cannot create a temporary directory";
+
+generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches,
+    $narPath, $patchesPath, $patchesURL, $tmpDir;
+
+propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches;
+
+writeManifest $dstManifest, \%dstNarFiles, \%dstPatches;

scripts/nix-prefetch-url.in

@@ -21,7 +21,10 @@ if test -z "$url"; then
     exit 1
 fi
 
-name=$(basename "$url")
+# Handle escaped characters in the URI.  `+', `=' and `?' are the only
+# characters that are valid in Nix store path names but have a special
+# meaning in URIs.
+name=$(basename "$url" | @sed@ -e 's/%2b/+/g' -e 's/%3d/=/g' -e 's/%3f/\?/g')
 if test -z "$name"; then echo "invalid url"; exit 1; fi
 
 

scripts/nix-pull.in

@@ -2,7 +2,7 @@
 
 use strict;
 use File::Temp qw(tempdir);
-use readmanifest;
+use NixManifest;
 
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
@@ -18,6 +18,12 @@ my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "$stateDir/manifests");
 umask 0022;
 
 
+# Create the manifests directory if it doesn't exist.
+if (! -e $manifestDir) {
+    mkdir $manifestDir, 0755 or die "cannot create directory `$manifestDir'";
+}
+
+
 # Process the URLs specified on the command line.
 my %narFiles;
 my %localPaths;
@@ -45,7 +51,7 @@ sub processURL {
 
     # First see if a bzipped manifest is available.
     if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) {
-        print "obtaining list of Nix archives at `$url.bz2'...\n";
+        print "fetching list of Nix archives at `$url.bz2'...\n";
         my $bzipped = downloadFile "$url.bz2";
 
         $manifest = "$tmpDir/MANIFEST";
@@ -86,11 +92,31 @@ sub processURL {
     my $hash = `$binDir/nix-hash --flat '$manifest'`
         or die "cannot hash `$manifest'";
     chomp $hash;
+
+    my $urlFile = "$manifestDir/$baseName-$hash.url";
+    open URL, ">$urlFile" or die "cannot create `$urlFile'";
+    print URL "$url";
+    close URL;
     
     my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest";
-    
-    system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
+
+    unlink $finalPath if -e $finalPath;
+        
+    symlink("$manifest", "$finalPath")
         or die "cannot link `$finalPath to `$manifest'";
+
+    # Delete all old manifests downloaded from this URL.
+    for my $urlFile2 (glob "$manifestDir/*.url") {
+        next if $urlFile eq $urlFile2;
+        open URL, "<$urlFile2" or die;
+        my $url2 = <URL>;
+        chomp $url2;
+        close URL;
+        next unless $url eq $url2;
+        my $base = $urlFile2; $base =~ s/.url$//;
+        unlink "${base}.url";
+        unlink "${base}.nixmanifest";
+    }
 }
 
 while (@ARGV) {

scripts/nix-push.in

@@ -2,7 +2,7 @@
 
 use strict;
 use File::Temp qw(tempdir);
-use readmanifest;
+use NixManifest;
 
 my $hashAlgo = "sha256";
 
@@ -172,12 +172,6 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
     $narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
     close HASH;
 
-    open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
-    my $narHash = <HASH>;
-    chomp $narHash;
-    $narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
-    close HASH;
-    
     my $narName = "$narbz2Hash.nar.bz2";
 
     my $narFile = "$narDir/$narName";
@@ -195,6 +189,14 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
     chomp $deriver;
     $deriver = "" if $deriver eq "unknown-deriver";
 
+    my $narHash = `$binDir/nix-store --query --hash '$storePath'`;
+    die "cannot query hash for `$storePath'" if $? != 0;
+    chomp $narHash;
+
+    my $narSize = `$binDir/nix-store --query --size '$storePath'`;
+    die "cannot query size for `$storePath'" if $? != 0;
+    chomp $narSize;
+
     my $url;
     if ($localCopy) {
         $url = "$targetArchivesUrl/$narName";
@@ -205,7 +207,8 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
         { url => $url
         , hash => "$hashAlgo:$narbz2Hash"
         , size => $narbz2Size
-        , narHash => "$hashAlgo:$narHash"
+        , narHash => "$narHash"
+        , narSize => $narSize
         , references => $references
         , deriver => $deriver
         }

src/Makefile.am

@@ -1,5 +1,3 @@
 SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
  libexpr nix-instantiate nix-env nix-worker nix-setuid-helper \
  nix-log2xml bsdiff-4.3
-
-EXTRA_DIST = aterm-helper.pl

src/aterm-helper.pl

@@ -1,178 +0,0 @@
-#! /usr/bin/perl -w
-
-# This program generates C/C++ code for efficiently manipulating
-# ATerms.  It generates functions to build and match ATerms according
-# to a set of constructor definitions defined in a file read from
-# standard input.  A constructor is defined by a line with the
-# following format:
-#
-#   SYM | ARGS | TYPE | FUN?
-#
-# where SYM is the name of the constructor, ARGS is a
-# whitespace-separated list of argument types, TYPE is the type of the
-# resulting ATerm (which should be `ATerm' or a type synonym for
-# `ATerm'), and the optional FUN is used to construct the names of the
-# build and match functions (it defaults to SYM; overriding it is
-# useful if there are overloaded constructors, e.g., with different
-# arities).  Note that SYM may be empty.
-#
-# A line of the form
-#
-#   VAR = EXPR
-#
-# causes a ATerm variable to be generated that is initialised to the
-# value EXPR.
-#
-# Finally, a line of the form
-#
-#   init NAME
-#
-# causes the initialisation function to be called `NAME'.  This
-# function must be called before any of the build/match functions or
-# the generated variables are used.
-
-die if scalar @ARGV != 2;
-
-my $syms = "";
-my $init = "";
-my $initFun = "init";
-
-open HEADER, ">$ARGV[0]";
-open IMPL, ">$ARGV[1]";
-
-print HEADER "#include <aterm2.h>\n";
-print HEADER "#ifdef __cplusplus\n";
-print HEADER "namespace nix {\n";
-print HEADER "#endif\n\n\n";
-print IMPL "namespace nix {\n";
-
-while (<STDIN>) {
-    s/\#.*//;
-    next if (/^\s*$/);
-    
-    if (/^\s*(\w*)\s*\|([^\|]*)\|\s*(\w+)\s*\|\s*(\w+)?/) {
-        my $const = $1;
-        my @types = split ' ', $2;
-        my $result = $3;
-        my $funname = $4;
-        $funname = $const unless defined $funname;
-
-        my $formals = "";
-        my $formals2 = "";
-        my $args = "";
-        my $unpack = "";
-        my $n = 1;
-        foreach my $type (@types) {
-            my $realType = $type;
-            $args .= ", ";
-            if ($type eq "string") {
-#                $args .= "(ATerm) ATmakeAppl0(ATmakeAFun((char *) e$n, 0, ATtrue))";
-#                $type = "const char *";
-                $type = "ATerm";
-                $args .= "e$n";
-                # !!! in the matcher, we should check that the
-                # argument is a string (i.e., a nullary application).
-            } elsif ($type eq "int") {
-                $args .= "(ATerm) ATmakeInt(e$n)";
-            } elsif ($type eq "ATermList" || $type eq "ATermBlob") {
-                $args .= "(ATerm) e$n";
-            } else {
-                $args .= "e$n";
-            }
-            $formals .= ", " if $formals ne "";
-            $formals .= "$type e$n";
-            $formals2 .= ", ";
-            $formals2 .= "$type & e$n";
-            my $m = $n - 1;
-            # !!! more checks here
-            if ($type eq "int") {
-                $unpack .= "    e$n = ATgetInt((ATermInt) ATgetArgument(e, $m));\n";
-            } elsif ($type eq "ATermList") {
-                $unpack .= "    e$n = (ATermList) ATgetArgument(e, $m);\n";
-            } elsif ($type eq "ATermBlob") {
-                $unpack .= "    e$n = (ATermBlob) ATgetArgument(e, $m);\n";
-            } elsif ($realType eq "string") {
-                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
-                $unpack .= "    if (ATgetType(e$n) != AT_APPL) return false;\n";
-            } else {
-                $unpack .= "    e$n = ATgetArgument(e, $m);\n";
-            }
-            $n++;
-        }
-
-        my $arity = scalar @types;
-
-        print HEADER "extern AFun sym$funname;\n\n";
-        
-        print IMPL "AFun sym$funname = 0;\n";
-        
-        if ($arity == 0) {
-            print HEADER "extern ATerm const$funname;\n\n";
-            print IMPL "ATerm const$funname = 0;\n";
-        }
-        
-        print HEADER "static inline $result make$funname($formals) __attribute__ ((pure, nothrow));\n";
-        print HEADER "static inline $result make$funname($formals) {\n";
-        if ($arity == 0) {
-            print HEADER "    return const$funname;\n";
-        }
-        elsif ($arity <= 6) {
-            print HEADER "    return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
-        } else {
-            $args =~ s/^,//;
-            print HEADER "    ATerm array[$arity] = {$args};\n";
-            print HEADER "    return (ATerm) ATmakeApplArray(sym$funname, array);\n";
-        }
-        print HEADER "}\n\n";
-
-        print HEADER "#ifdef __cplusplus\n";
-        print HEADER "static inline bool match$funname(ATerm e$formals2) {\n";
-        print HEADER "    if (ATgetType(e) != AT_APPL || (AFun) ATgetAFun(e) != sym$funname) return false;\n";
-        print HEADER "$unpack";
-        print HEADER "    return true;\n";
-        print HEADER "}\n";
-        print HEADER "#endif\n\n\n";
-
-        $init .= "    sym$funname = ATmakeAFun(\"$const\", $arity, ATfalse);\n";
-        $init .= "    ATprotectAFun(sym$funname);\n";
-        if ($arity == 0) {
-            $init .= "    const$funname = (ATerm) ATmakeAppl0(sym$funname);\n";
-            $init .= "    ATprotect(&const$funname);\n";
-        }
-    }
-
-    elsif (/^\s*(\w+)\s*=\s*(.*)$/) {
-        my $name = $1;
-        my $value = $2;
-        print HEADER "extern ATerm $name;\n";
-        print IMPL "ATerm $name = 0;\n";
-        $init .= "    $name = $value;\n";
-    }
-
-    elsif (/^\s*init\s+(\w+)\s*$/) {
-        $initFun = $1;
-    }
-
-    else {
-        die "bad line: `$_'";
-    }
-}
-
-print HEADER "void $initFun();\n\n";
-
-print HEADER "static inline const char * aterm2String(ATerm t) {\n";
-print HEADER "    return (const char *) ATgetName(ATgetAFun(t));\n";
-print HEADER "}\n\n";
-
-print IMPL "\n";
-print IMPL "void $initFun() {\n";
-print IMPL "$init";
-print IMPL "}\n";
-
-print HEADER "#ifdef __cplusplus\n";
-print HEADER "}\n";
-print HEADER "#endif\n\n\n";
-print IMPL "}\n";
-
-close HEADER;
-close IMPL;

src/bin2c/Makefile.am

@@ -1,3 +1,6 @@
 noinst_PROGRAMS = bin2c
 
 bin2c_SOURCES = bin2c.c
+
+bin2c$(EXEEXT): bin2c.c
+	$(CC_FOR_BUILD) $(CFLAGS_FOR_BUILD) -o bin2c bin2c.c

src/bin2c/bin2c.c

@@ -14,10 +14,10 @@ int main(int argc, char * * argv)
 {
     int c;
     if (argc != 2) abort();
-    print("static unsigned char %s[] = {", argv[1]);
+    print("static unsigned char %s[] = { ", argv[1]);
     while ((c = getchar()) != EOF) {
         print("0x%02x, ", (unsigned char) c);
     }
-    print("};\n");
+    print("0 };\n");
     return 0;
 }

src/bsdiff-4.3/Makefile.am

@@ -1,3 +1,5 @@
+EXTRA_DIST = compat-include
+
 libexec_PROGRAMS = bsdiff bspatch
 
 bsdiff_SOURCES = bsdiff.c
@@ -8,4 +10,4 @@ bspatch_SOURCES = bspatch.c
 
 bspatch_LDADD = ${bzip2_lib}
 
-AM_CFLAGS = -O3 ${bzip2_include}
+AM_CFLAGS = -O3 ${bzip2_include} ${bsddiff_compat_include}

src/bsdiff-4.3/bsdiff.c

@@ -277,6 +277,7 @@ int main(int argc,char *argv[])
 		for(scsc=scan+=len;scan<newsize;scan++) {
 			len=search(I,old,oldsize,new+scan,newsize-scan,
 					0,oldsize,&pos);
+			if (len > 64 * 1024) break;
 
 			for(;scsc<scan+len;scsc++)
 			if((scsc+lastoffset<oldsize) &&

src/bsdiff-4.3/compat-include/err.h

@@ -0,0 +1,12 @@
+/* Simulate BSD's <err.h> functionality. */
+
+#ifndef COMPAT_ERR_H_INCLUDED
+#define COMPAT_ERR_H_INCLUDED 1
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define err(rc,...)  do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
+#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
+
+#endif

src/libexpr/Makefile.am

@@ -2,27 +2,25 @@ pkglib_LTLIBRARIES = libexpr.la
 
 libexpr_la_SOURCES = \
  nixexpr.cc eval.cc primops.cc lexer-tab.cc parser-tab.cc \
- get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc \
- names.cc normal-forms.cc
+ get-drvs.cc attr-path.cc value-to-xml.cc common-opts.cc \
+ names.cc
 
 pkginclude_HEADERS = \
  nixexpr.hh eval.hh parser.hh lexer-tab.hh parser-tab.hh \
- get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh \
- names.hh nixexpr-ast.hh
+ get-drvs.hh attr-path.hh value-to-xml.hh common-opts.hh \
+ names.hh symbol-table.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
- ../boost/format/libformat.la
+ ../boost/format/libformat.la @boehmgc_lib@
 
-BUILT_SOURCES = nixexpr-ast.cc nixexpr-ast.hh \
+BUILT_SOURCES = \
  parser-tab.hh lexer-tab.hh parser-tab.cc lexer-tab.cc
 
-EXTRA_DIST = lexer.l parser.y nixexpr-ast.def nixexpr-ast.cc
+EXTRA_DIST = lexer.l parser.y
 
 AM_CXXFLAGS = \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} \
+ -I$(srcdir)/.. \
  -I$(srcdir)/../libutil -I$(srcdir)/../libstore
-AM_CFLAGS = \
- ${aterm_include}
 
 
 # Parser generation.
@@ -34,15 +32,6 @@ lexer-tab.cc lexer-tab.hh: lexer.l
 	$(flex) --outfile lexer-tab.cc --header-file=lexer-tab.hh $(srcdir)/lexer.l 
 
 
-# ATerm helper function generation.
-
-nixexpr-ast.cc nixexpr-ast.hh: ../aterm-helper.pl nixexpr-ast.def
-	$(perl) $(srcdir)/../aterm-helper.pl nixexpr-ast.hh nixexpr-ast.cc < $(srcdir)/nixexpr-ast.def
-
-
-CLEANFILES =
-
-
 # SDF stuff (not built by default).
 nix.tbl: nix.sdf
 	sdf2table -m Nix -s -i nix.sdf -o nix.tbl

src/libexpr/attr-path.cc

@@ -1,23 +1,13 @@
 #include "attr-path.hh"
-#include "nixexpr-ast.hh"
 #include "util.hh"
 
 
 namespace nix {
 
 
-bool isAttrs(EvalState & state, Expr e, ATermMap & attrs)
-{
-    e = evalExpr(state, e);
-    ATermList dummy;
-    if (!matchAttrs(e, dummy)) return false;
-    queryAllAttrs(e, attrs, false);
-    return true;
-}
-
-
-Expr findAlongAttrPath(EvalState & state, const string & attrPath,
-    const ATermMap & autoArgs, Expr e)
+// !!! Shouldn't we return a pointer to a Value?
+void findAlongAttrPath(EvalState & state, const string & attrPath,
+    Bindings & autoArgs, Expr * e, Value & v)
 {
     Strings tokens = tokenizeString(attrPath, ".");
 
@@ -25,8 +15,10 @@ Expr findAlongAttrPath(EvalState & state, const string & attrPath,
         Error(format("attribute selection path `%1%' does not match expression") % attrPath);
 
     string curPath;
+
+    state.mkThunk_(v, e);
     
-    for (Strings::iterator i = tokens.begin(); i != tokens.end(); ++i) {
+    foreach (Strings::iterator, i, tokens) {
 
         if (!curPath.empty()) curPath += ".";
         curPath += *i;
@@ -38,7 +30,10 @@ Expr findAlongAttrPath(EvalState & state, const string & attrPath,
         if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
-        e = evalExpr(state, autoCallFunction(evalExpr(state, e), autoArgs));
+        Value vTmp;
+        state.autoCallFunction(autoArgs, v, vTmp);
+        v = vTmp;
+        state.forceValue(v);
 
         /* It should evaluate to either an attribute set or an
            expression, according to what is specified in the
@@ -46,36 +41,31 @@ Expr findAlongAttrPath(EvalState & state, const string & attrPath,
 
         if (apType == apAttr) {
 
-            ATermMap attrs;
-
-            if (!isAttrs(state, e, attrs))
+            if (v.type != tAttrs)
                 throw TypeError(
                     format("the expression selected by the selection path `%1%' should be an attribute set but is %2%")
-                    % curPath % showType(e));
-                
-            e = attrs.get(toATerm(attr));
-            if (!e)
-                throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % curPath);
+                    % curPath % showType(v));
 
+            Bindings::iterator a = v.attrs->find(state.symbols.create(attr));
+            if (a == v.attrs->end())
+                throw Error(format("attribute `%1%' in selection path `%2%' not found") % attr % curPath);
+            v = *a->value;
         }
 
         else if (apType == apIndex) {
 
-            ATermList es;
-            if (!matchList(e, es))
+            if (v.type != tList)
                 throw TypeError(
                     format("the expression selected by the selection path `%1%' should be a list but is %2%")
-                    % curPath % showType(e));
+                    % curPath % showType(v));
 
-            e = ATelementAt(es, attrIndex);
-            if (!e)
-                throw Error(format("list index %1% in selection path `%2%' not found") % attrIndex % curPath);
-            
+            if (attrIndex >= v.list.length)
+                throw Error(format("list index %1% in selection path `%2%' is out of range") % attrIndex % curPath);
+
+            v = *v.list.elems[attrIndex];
         }
         
     }
-    
-    return e;
 }
 
  

src/libexpr/attr-path.hh

@@ -1,17 +1,17 @@
 #ifndef __ATTR_PATH_H
 #define __ATTR_PATH_H
 
+#include "eval.hh"
+
 #include <string>
 #include <map>
 
-#include "eval.hh"
-
 
 namespace nix {
 
     
-Expr findAlongAttrPath(EvalState & state, const string & attrPath,
-    const ATermMap & autoArgs, Expr e);
+void findAlongAttrPath(EvalState & state, const string & attrPath,
+    Bindings & autoArgs, Expr * e, Value & v);
 
     
 }

src/libexpr/common-opts.cc

@@ -9,7 +9,7 @@ namespace nix {
 
 bool parseOptionArg(const string & arg, Strings::iterator & i,
     const Strings::iterator & argsEnd, EvalState & state,
-    ATermMap & autoArgs)
+    Bindings & autoArgs)
 {
     if (arg != "--arg" && arg != "--argstr") return false;
 
@@ -19,12 +19,18 @@ bool parseOptionArg(const string & arg, Strings::iterator & i,
     string name = *i++;
     if (i == argsEnd) throw error;
     string value = *i++;
-    
-    Expr e = arg == "--arg"
-        ? parseExprFromString(state, value, absPath("."))
-        : makeStr(value);
-    autoArgs.set(toATerm(name), e);
-    
+
+    /* !!! check for duplicates! */
+    Value * v = state.allocValue();
+    autoArgs.push_back(Attr(state.symbols.create(name), v));
+
+    if (arg == "--arg")
+        state.mkThunk_(*v, parseExprFromString(state, value, absPath(".")));
+    else
+        mkString(*v, value);
+
+    autoArgs.sort(); // !!! inefficient
+
     return true;
 }
 

src/libexpr/common-opts.hh

@@ -9,7 +9,7 @@ namespace nix {
 /* Some common option parsing between nix-env and nix-instantiate. */
 bool parseOptionArg(const string & arg, Strings::iterator & i,
     const Strings::iterator & argsEnd, EvalState & state,
-    ATermMap & autoArgs);
+    Bindings & autoArgs);
     
 }
 

src/libexpr/eval.hh

@@ -1,19 +1,193 @@
 #ifndef __EVAL_H
 #define __EVAL_H
 
+#include "nixexpr.hh"
+#include "symbol-table.hh"
+#include "hash.hh"
+
 #include <map>
 
-#include "aterm.hh"
-#include "nixexpr.hh"
+#if HAVE_BOEHMGC
+#include <gc/gc_allocator.h>
+#endif
 
 
 namespace nix {
 
 
-class Hash;
-    
+class EvalState;
+struct Env;
+struct Value;
+struct Attr;
+
+
+/* Attribute sets are represented as a vector of attributes, sorted by
+   symbol (i.e. pointer to the attribute name in the symbol table). */
+#if HAVE_BOEHMGC
+typedef std::vector<Attr, gc_allocator<Attr> > BindingsBase;
+#else
+typedef std::vector<Attr> BindingsBase;
+#endif
+
+
+class Bindings : public BindingsBase
+{
+public:
+    iterator find(const Symbol & name);
+    void sort();
+};
+
+
+typedef enum {
+    tInt = 1,
+    tBool,
+    tString,
+    tPath,
+    tNull,
+    tAttrs,
+    tList,
+    tThunk,
+    tApp,
+    tLambda,
+    tBlackhole,
+    tPrimOp,
+    tPrimOpApp,
+} ValueType;
+
+
+typedef void (* PrimOpFun) (EvalState & state, Value * * args, Value & v);
+
+
+struct PrimOp
+{
+    PrimOpFun fun;
+    unsigned int arity;
+    Symbol name;
+    PrimOp(PrimOpFun fun, unsigned int arity, Symbol name)
+        : fun(fun), arity(arity), name(name) { }
+};
+
+
+struct Value
+{
+    ValueType type;
+    union 
+    {
+        int integer;
+        bool boolean;
+        
+        /* Strings in the evaluator carry a so-called `context' (the
+           ATermList) which is a list of strings representing store
+           paths.  This is to allow users to write things like
+
+             "--with-freetype2-library=" + freetype + "/lib"
+
+           where `freetype' is a derivation (or a source to be copied
+           to the store).  If we just concatenated the strings without
+           keeping track of the referenced store paths, then if the
+           string is used as a derivation attribute, the derivation
+           will not have the correct dependencies in its inputDrvs and
+           inputSrcs.
+
+           The semantics of the context is as follows: when a string
+           with context C is used as a derivation attribute, then the
+           derivations in C will be added to the inputDrvs of the
+           derivation, and the other store paths in C will be added to
+           the inputSrcs of the derivations.
+
+           For canonicity, the store paths should be in sorted order. */
+        struct {
+            const char * s;
+            const char * * context; // must be in sorted order
+        } string;
+        
+        const char * path;
+        Bindings * attrs;
+        struct {
+            unsigned int length;
+            Value * * elems;
+        } list;
+        struct {
+            Env * env;
+            Expr * expr;
+        } thunk;
+        struct {
+            Value * left, * right;
+        } app;
+        struct {
+            Env * env;
+            ExprLambda * fun;
+        } lambda;
+        PrimOp * primOp;
+        struct {
+            Value * left, * right;
+        } primOpApp;
+    };
+};
+
+
+struct Env
+{
+    Env * up;
+    unsigned int prevWith; // nr of levels up to next `with' environment
+    Value * values[0];
+};
+
+
+struct Attr
+{
+    Symbol name;
+    Value * value;
+    Pos * pos;
+    Attr(Symbol name, Value * value, Pos * pos = &noPos)
+        : name(name), value(value), pos(pos) { };
+    Attr() : pos(&noPos) { };
+    bool operator < (const Attr & a) const
+    {
+        return name < a.name;
+    }
+};
+
+
+/* After overwriting an app node, be sure to clear pointers in the
+   Value to ensure that the target isn't kept alive unnecessarily. */
+static inline void clearValue(Value & v)
+{
+    v.app.right = 0;
+}
+
+
+static inline void mkInt(Value & v, int n)
+{
+    clearValue(v);
+    v.type = tInt;
+    v.integer = n;
+}
+
+
+static inline void mkBool(Value & v, bool b)
+{
+    clearValue(v);
+    v.type = tBool;
+    v.boolean = b;
+}
+
+
+static inline void mkApp(Value & v, Value & left, Value & right)
+{
+    v.type = tApp;
+    v.app.left = &left;
+    v.app.right = &right;
+}
+
+
+void mkString(Value & v, const char * s);
+void mkString(Value & v, const string & s, const PathSet & context = PathSet());
+void mkPath(Value & v, const char * s);
+
+void copyContext(const Value & v, PathSet & context);
+
 
-typedef std::map<Path, PathSet> DrvRoots;
 typedef std::map<Path, Hash> DrvHashes;
 
 /* Cache for calls to addToStore(); maps source paths to the store
@@ -22,81 +196,160 @@ typedef std::map<Path, Path> SrcToStore;
 
 struct EvalState;
 
-/* Note: using a ATermVector is safe here, since when we call a primop
-   we also have an ATermList on the stack. */
-typedef Expr (* PrimOp) (EvalState &, const ATermVector & args);
+
+std::ostream & operator << (std::ostream & str, const Value & v);
 
 
-struct EvalState 
+class EvalState 
 {
-    ATermMap normalForms;
-    ATermMap sessionNormalForms;
-    ATermMap primOps;
-    DrvRoots drvRoots;
+public:
     DrvHashes drvHashes; /* normalised derivation hashes */
+
+    SymbolTable symbols;
+
+    const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName,
+        sSystem, sOverrides;
+
+private:
     SrcToStore srcToStore; 
 
-    unsigned int nrEvaluated;
-    unsigned int nrCached;
-    unsigned int nrDepthAfterReset;
-
     bool allowUnsafeEquality;
-    bool safeCache;
 
+    std::map<Path, Expr *> parseTrees;
+
+public:
+    
     EvalState();
     ~EvalState();
 
-    void addPrimOps();
+    /* Evaluate an expression read from the given file to normal
+       form. */
+    void evalFile(const Path & path, Value & v);
+
+    /* Evaluate an expression to normal form, storing the result in
+       value `v'. */
+    void eval(Expr * e, Value & v);
+    void eval(Env & env, Expr * e, Value & v);
+
+    /* Evaluation the expression, then verify that it has the expected
+       type. */
+    bool evalBool(Env & env, Expr * e);
+    void evalAttrs(Env & env, Expr * e, Value & v);
+
+    /* If `v' is a thunk, enter it and overwrite `v' with the result
+       of the evaluation of the thunk.  If `v' is a delayed function
+       application, call the function and overwrite `v' with the
+       result.  Otherwise, this is a no-op. */
+    void forceValue(Value & v);
+
+    /* Force a value, then recursively force list elements and
+       attributes. */
+    void strictForceValue(Value & v);
+
+    /* Force `v', and then verify that it has the expected type. */
+    int forceInt(Value & v);
+    bool forceBool(Value & v);
+    void forceAttrs(Value & v);
+    void forceList(Value & v);
+    void forceFunction(Value & v); // either lambda or primop
+    string forceString(Value & v);
+    string forceString(Value & v, PathSet & context);
+    string forceStringNoCtx(Value & v);
+
+    /* Return true iff the value `v' denotes a derivation (i.e. a
+       set with attribute `type = "derivation"'). */
+    bool isDerivation(Value & v);
+
+    /* String coercion.  Converts strings, paths and derivations to a
+       string.  If `coerceMore' is set, also converts nulls, integers,
+       booleans and lists to a string.  If `copyToStore' is set,
+       referenced paths are copied to the Nix store as a side effect.q */
+    string coerceToString(Value & v, PathSet & context,
+        bool coerceMore = false, bool copyToStore = true);
+
+    /* Path coercion.  Converts strings, paths and derivations to a
+       path.  The result is guaranteed to be a canonicalised, absolute
+       path.  Nothing is copied to the store. */
+    Path coerceToPath(Value & v, PathSet & context);
+
+private:
+
+    /* The base environment, containing the builtin functions and
+       values. */
+    Env & baseEnv;
+
+    unsigned int baseEnvDispl;
+
+public:
+    
+    /* The same, but used during parsing to resolve variables. */
+    StaticEnv staticBaseEnv; // !!! should be private
+
+private:
+    
+    void createBaseEnv();
+    
+    void addConstant(const string & name, Value & v);
+
     void addPrimOp(const string & name,
-        unsigned int arity, PrimOp primOp);
-    void loadNormalForms();
-    void saveNormalForms();
+        unsigned int arity, PrimOpFun primOp);
+
+    Value * lookupVar(Env * env, const VarRef & var);
+    
+    friend class ExprVar;
+    friend class ExprAttrs;
+    friend class ExprLet;
+
+public:
+    
+    /* Do a deep equality test between two values.  That is, list
+       elements and attributes are compared recursively. */
+    bool eqValues(Value & v1, Value & v2);
+
+    void callFunction(Value & fun, Value & arg, Value & v);
+
+    /* Automatically call a function for which each argument has a
+       default value or has a binding in the `args' map. */
+    void autoCallFunction(Bindings & args, Value & fun, Value & res);
+    
+    /* Allocation primitives. */
+    Value * allocValue();
+    Env & allocEnv(unsigned int size);
+
+    Value * allocAttr(Value & vAttrs, const Symbol & name);
+
+    void mkList(Value & v, unsigned int length);
+    void mkAttrs(Value & v, unsigned int expected);
+    void mkThunk_(Value & v, Expr * expr);
+
+    Value * maybeThunk(Env & env, Expr * expr);
+    
+    /* Print statistics. */
+    void printStats();
+
+private:
+    
+    unsigned long nrEnvs;
+    unsigned long nrValuesInEnvs;
+    unsigned long nrValues;
+    unsigned long nrListElems;
+    unsigned long nrEvaluated;
+    unsigned long nrAttrsets;
+    unsigned long nrOpUpdates;
+    unsigned long nrOpUpdateValuesCopied;
+    unsigned int recursionDepth;
+    unsigned int maxRecursionDepth;
+    char * deepestStack; /* for measuring stack usage */
+    
+    friend class RecursionCounter;
+    friend class ExprOpUpdate;
 };
 
 
-/* Evaluate an expression to normal form. */
-Expr evalExpr(EvalState & state, Expr e);
+/* Return a string representing the type of the value `v'. */
+string showType(const Value & v);
 
-/* Evaluate an expression read from the given file to normal form. */
-Expr evalFile(EvalState & state, const Path & path);
 
-/* Evaluate an expression, and recursively evaluate list elements and
-   attributes.  If `canonicalise' is true, we remove things like
-   position information and make sure that attribute sets are in
-   sorded order. */
-Expr strictEvalExpr(EvalState & state, Expr e);
-
-/* Specific results. */
-string evalString(EvalState & state, Expr e, PathSet & context);
-string evalStringNoCtx(EvalState & state, Expr e);
-int evalInt(EvalState & state, Expr e);
-bool evalBool(EvalState & state, Expr e);
-ATermList evalList(EvalState & state, Expr e);
-
-/* Flatten nested lists into a single list (or expand a singleton into
-   a list). */
-ATermList flattenList(EvalState & state, Expr e);
-
-/* String coercion.  Converts strings, paths and derivations to a
-   string.  If `coerceMore' is set, also converts nulls, integers,
-   booleans and lists to a string. */
-string coerceToString(EvalState & state, Expr e, PathSet & context,
-    bool coerceMore = false, bool copyToStore = true);
-
-/* Path coercion.  Converts strings, paths and derivations to a path.
-   The result is guaranteed to be an canonicalised, absolute path.
-   Nothing is copied to the store. */
-Path coerceToPath(EvalState & state, Expr e, PathSet & context);
-
-/* Automatically call a function for which each argument has a default
-   value or has a binding in the `args' map.  Note: result is a call,
-   not a normal form; it should be evaluated by calling evalExpr(). */
-Expr autoCallFunction(Expr e, const ATermMap & args);
-
-/* Print statistics. */
-void printEvalStats(EvalState & state);
-
- 
 }
 
 

src/libexpr/expr-to-xml.cc

@@ -1,156 +0,0 @@
-#include "expr-to-xml.hh"
-#include "xml-writer.hh"
-#include "nixexpr-ast.hh"
-#include "aterm.hh"
-#include "util.hh"
-
-#include <cstdlib>
-
-
-namespace nix {
-
-    
-static XMLAttrs singletonAttrs(const string & name, const string & value)
-{
-    XMLAttrs attrs;
-    attrs[name] = value;
-    return attrs;
-}
-
-
-/* set<Expr> is safe because all the expressions are also reachable
-   from the stack, therefore can't be garbage-collected. */
-typedef set<Expr> ExprSet;
-
-
-static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
-    ExprSet & drvsSeen);
-
-
-static void showAttrs(const ATermMap & attrs, XMLWriter & doc,
-    PathSet & context, ExprSet & drvsSeen)
-{
-    StringSet names;
-    for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i)
-        names.insert(aterm2String(i->key));
-    for (StringSet::iterator i = names.begin(); i != names.end(); ++i) {
-        XMLOpenElement _(doc, "attr", singletonAttrs("name", *i));
-        printTermAsXML(attrs.get(toATerm(*i)), doc, context, drvsSeen);
-    }
-}
-
-
-static void printPatternAsXML(Pattern pat, XMLWriter & doc)
-{
-    ATerm name;
-    ATermList formals;
-    Pattern pat1, pat2;
-    ATermBool ellipsis;
-    if (matchVarPat(pat, name))
-        doc.writeEmptyElement("varpat", singletonAttrs("name", aterm2String(name)));
-    else if (matchAttrsPat(pat, formals, ellipsis)) {
-        XMLOpenElement _(doc, "attrspat");
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ATerm dummy;
-            if (!matchFormal(*i, name, dummy)) abort();
-            doc.writeEmptyElement("attr", singletonAttrs("name", aterm2String(name)));
-        }
-        if (ellipsis == eTrue) doc.writeEmptyElement("ellipsis");
-    }
-    else if (matchAtPat(pat, pat1, pat2)) {
-        XMLOpenElement _(doc, "at");
-        printPatternAsXML(pat1, doc);
-        printPatternAsXML(pat2, doc);
-    }
-}
-
-
-static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
-    ExprSet & drvsSeen)
-{
-    XMLAttrs attrs;
-    string s;
-    ATerm s2;
-    int i;
-    ATermList as, es;
-    ATerm pat, body, pos;
-
-    checkInterrupt();
-
-    if (matchStr(e, s, context)) /* !!! show the context? */
-        doc.writeEmptyElement("string", singletonAttrs("value", s));
-
-    else if (matchPath(e, s2))
-        doc.writeEmptyElement("path", singletonAttrs("value", aterm2String(s2)));
-
-    else if (matchNull(e))
-        doc.writeEmptyElement("null");
-
-    else if (matchInt(e, i))
-        doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % i).str()));
-
-    else if (e == eTrue)
-        doc.writeEmptyElement("bool", singletonAttrs("value", "true"));
-
-    else if (e == eFalse)
-        doc.writeEmptyElement("bool", singletonAttrs("value", "false"));
-
-    else if (matchAttrs(e, as)) {
-        ATermMap attrs;
-        queryAllAttrs(e, attrs);
-
-        Expr a = attrs.get(toATerm("type"));
-        if (a && matchStr(a, s, context) && s == "derivation") {
-
-            XMLAttrs xmlAttrs;
-            Path outPath, drvPath;
-            
-            a = attrs.get(toATerm("drvPath"));
-            if (matchStr(a, drvPath, context))
-                xmlAttrs["drvPath"] = drvPath;
-        
-            a = attrs.get(toATerm("outPath"));
-            if (matchStr(a, outPath, context))
-                xmlAttrs["outPath"] = outPath;
-        
-            XMLOpenElement _(doc, "derivation", xmlAttrs);
-
-            if (drvsSeen.find(e) == drvsSeen.end()) {
-                drvsSeen.insert(e);
-                showAttrs(attrs, doc, context, drvsSeen);
-            } else
-                doc.writeEmptyElement("repeated");
-        }
-
-        else {
-            XMLOpenElement _(doc, "attrs");
-            showAttrs(attrs, doc, context, drvsSeen);
-        }
-    }
-
-    else if (matchList(e, es)) {
-        XMLOpenElement _(doc, "list");
-        for (ATermIterator i(es); i; ++i)
-            printTermAsXML(*i, doc, context, drvsSeen);
-    }
-
-    else if (matchFunction(e, pat, body, pos)) {
-        XMLOpenElement _(doc, "function");
-        printPatternAsXML(pat, doc);
-    }
-
-    else
-        doc.writeEmptyElement("unevaluated");
-}
-
-
-void printTermAsXML(Expr e, std::ostream & out, PathSet & context)
-{
-    XMLWriter doc(true, out);
-    XMLOpenElement root(doc, "expr");
-    ExprSet drvsSeen;    
-    printTermAsXML(e, doc, context, drvsSeen);
-}
-
- 
-}

src/libexpr/expr-to-xml.hh

@@ -1,16 +0,0 @@
-#ifndef __EXPR_TO_XML_H
-#define __EXPR_TO_XML_H
-
-#include <string>
-#include <map>
-
-#include "nixexpr.hh"
-#include "aterm.hh"
-
-namespace nix {
-
-void printTermAsXML(Expr e, std::ostream & out, PathSet & context);
-    
-}
-
-#endif /* !__EXPR_TO_XML_H */

src/libexpr/get-drvs.cc

@@ -1,5 +1,4 @@
 #include "get-drvs.hh"
-#include "nixexpr-ast.hh"
 #include "util.hh"
 
 
@@ -8,17 +7,10 @@ namespace nix {
 
 string DrvInfo::queryDrvPath(EvalState & state) const
 {
-    if (drvPath == "") {
-        Expr a = attrs->get(toATerm("drvPath"));
-
-        /* Backwards compatibility hack with user environments made by
-           Nix <= 0.10: these contain illegal Path("") expressions. */
-        ATerm t;
-        if (a && matchPath(evalExpr(state, a), t))
-            return aterm2String(t);
-        
+    if (drvPath == "" && attrs) {
+        Bindings::iterator i = attrs->find(state.sDrvPath);
         PathSet context;
-        (string &) drvPath = a ? coerceToPath(state, a, context) : "";
+        (string &) drvPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : "";
     }
     return drvPath;
 }
@@ -26,11 +18,10 @@ string DrvInfo::queryDrvPath(EvalState & state) const
 
 string DrvInfo::queryOutPath(EvalState & state) const
 {
-    if (outPath == "") {
-        Expr a = attrs->get(toATerm("outPath"));
-        if (!a) throw TypeError("output path missing");
+    if (outPath == "" && attrs) {
+        Bindings::iterator i = attrs->find(state.sOutPath);
         PathSet context;
-        (string &) outPath = coerceToPath(state, a, context);
+        (string &) outPath = i != attrs->end() ? state.coerceToPath(*i->value, context) : "";
     }
     return outPath;
 }
@@ -38,35 +29,30 @@ string DrvInfo::queryOutPath(EvalState & state) const
 
 MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
 {
-    MetaInfo meta;
+    if (metaInfoRead) return meta;
     
-    Expr a = attrs->get(toATerm("meta"));
-    if (!a) return meta; /* fine, empty meta information */
+    (bool &) metaInfoRead = true;
+    
+    Bindings::iterator a = attrs->find(state.sMeta);
+    if (a == attrs->end()) return meta; /* fine, empty meta information */
 
-    ATermMap attrs2;
-    queryAllAttrs(evalExpr(state, a), attrs2);
+    state.forceAttrs(*a->value);
 
-    for (ATermMap::const_iterator i = attrs2.begin(); i != attrs2.end(); ++i) {
-        Expr e = evalExpr(state, i->value);
-        string s;
-        PathSet context;
+    foreach (Bindings::iterator, i, *a->value->attrs) {
         MetaValue value;
-        int n;
-        ATermList es;
-        if (matchStr(e, s, context)) {
+        state.forceValue(*i->value);
+        if (i->value->type == tString) {
             value.type = MetaValue::tpString;
-            value.stringValue = s;
-            meta[aterm2String(i->key)] = value;
-        } else if (matchInt(e, n)) {
+            value.stringValue = i->value->string.s;
+        } else if (i->value->type == tInt) {
             value.type = MetaValue::tpInt;
-            value.intValue = n;
-            meta[aterm2String(i->key)] = value;
-        } else if (matchList(e, es)) {
+            value.intValue = i->value->integer;
+        } else if (i->value->type == tList) {
             value.type = MetaValue::tpStrings;
-            for (ATermIterator j(es); j; ++j)
-                value.stringValues.push_back(evalStringNoCtx(state, *j));
-            meta[aterm2String(i->key)] = value;
-        }
+            for (unsigned int j = 0; j < i->value->list.length; ++j)
+                value.stringValues.push_back(state.forceStringNoCtx(*i->value->list.elems[j]));
+        } else continue;
+        ((MetaInfo &) meta)[i->name] = value;
     }
 
     return meta;
@@ -82,73 +68,46 @@ MetaValue DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
 
 void DrvInfo::setMetaInfo(const MetaInfo & meta)
 {
-    ATermMap metaAttrs;
-    foreach (MetaInfo::const_iterator, i, meta) {
-        Expr e;
-        switch (i->second.type) {
-            case MetaValue::tpInt: e = makeInt(i->second.intValue); break;
-            case MetaValue::tpString: e = makeStr(i->second.stringValue); break;
-            case MetaValue::tpStrings: {
-                ATermList es = ATempty;
-                foreach (Strings::const_iterator, j, i->second.stringValues)
-                    es = ATinsert(es, makeStr(*j));
-                e = makeList(ATreverse(es));
-                break;
-            }
-            default: abort();
-        }
-        metaAttrs.set(toATerm(i->first), makeAttrRHS(e, makeNoPos()));
-    }
-    attrs->set(toATerm("meta"), makeAttrs(metaAttrs));
+    metaInfoRead = true;
+    this->meta = meta;
 }
 
 
-/* Cache for already evaluated derivations.  Usually putting ATerms in
-   a STL container is unsafe (they're not scanning for GC roots), but
-   here it doesn't matter; everything in this set is reachable from
-   the stack as well. */
-typedef set<Expr> Exprs;
+/* Cache for already considered attrsets. */
+typedef set<Bindings *> Done;
 
 
-/* Evaluate expression `e'.  If it evaluates to an attribute set of
-   type `derivation', then put information about it in `drvs' (unless
-   it's already in `doneExprs').  The result boolean indicates whether
-   it makes sense for the caller to recursively search for derivations
-   in `e'. */
-static bool getDerivation(EvalState & state, Expr e,
-    const string & attrPath, DrvInfos & drvs, Exprs & doneExprs)
+/* Evaluate value `v'.  If it evaluates to an attribute set of type
+   `derivation', then put information about it in `drvs' (unless it's
+   already in `doneExprs').  The result boolean indicates whether it
+   makes sense for the caller to recursively search for derivations in
+   `v'. */
+static bool getDerivation(EvalState & state, Value & v,
+    const string & attrPath, DrvInfos & drvs, Done & done)
 {
     try {
-        
-        ATermList es;
-        e = evalExpr(state, e);
-        if (!matchAttrs(e, es)) return true;
-
-        boost::shared_ptr<ATermMap> attrs(new ATermMap());
-        queryAllAttrs(e, *attrs, false);
-        
-        Expr a = attrs->get(toATerm("type"));
-        if (!a || evalStringNoCtx(state, a) != "derivation") return true;
+        state.forceValue(v);
+        if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., an attribute set like
            `rec { x = derivation {...}; y = x;}'. */
-        if (doneExprs.find(e) != doneExprs.end()) return false;
-        doneExprs.insert(e);
+        if (done.find(v.attrs) != done.end()) return false;
+        done.insert(v.attrs);
 
         DrvInfo drv;
     
-        a = attrs->get(toATerm("name"));
+        Bindings::iterator i = v.attrs->find(state.sName);
         /* !!! We really would like to have a decent back trace here. */
-        if (!a) throw TypeError("derivation name missing");
-        drv.name = evalStringNoCtx(state, a);
+        if (i == v.attrs->end()) throw TypeError("derivation name missing");
+        drv.name = state.forceStringNoCtx(*i->value);
 
-        a = attrs->get(toATerm("system"));
-        if (!a)
+        Bindings::iterator i2 = v.attrs->find(state.sSystem);
+        if (i2 == v.attrs->end())
             drv.system = "unknown";
         else
-            drv.system = evalStringNoCtx(state, a);
+            drv.system = state.forceStringNoCtx(*i2->value);
 
-        drv.attrs = attrs;
+        drv.attrs = v.attrs;
 
         drv.attrPath = attrPath;
 
@@ -161,11 +120,11 @@ static bool getDerivation(EvalState & state, Expr e,
 }
 
 
-bool getDerivation(EvalState & state, Expr e, DrvInfo & drv)
+bool getDerivation(EvalState & state, Value & v, DrvInfo & drv)
 {
-    Exprs doneExprs;
+    Done done;
     DrvInfos drvs;
-    getDerivation(state, e, "", drvs, doneExprs);
+    getDerivation(state, v, "", drvs, done);
     if (drvs.size() != 1) return false;
     drv = drvs.front();
     return true;
@@ -178,83 +137,73 @@ static string addToPath(const string & s1, const string & s2)
 }
 
 
-static void getDerivations(EvalState & state, Expr e,
-    const string & pathPrefix, const ATermMap & autoArgs,
-    DrvInfos & drvs, Exprs & doneExprs)
+static void getDerivations(EvalState & state, Value & vIn,
+    const string & pathPrefix, Bindings & autoArgs,
+    DrvInfos & drvs, Done & done)
 {
-    e = evalExpr(state, autoCallFunction(evalExpr(state, e), autoArgs));
-
+    Value v;
+    state.autoCallFunction(autoArgs, vIn, v);
+    
     /* Process the expression. */
-    ATermList es;
     DrvInfo drv;
 
-    if (!getDerivation(state, e, pathPrefix, drvs, doneExprs))
-        return;
+    if (!getDerivation(state, v, pathPrefix, drvs, done)) ;
 
-    if (matchAttrs(e, es)) {
-        ATermMap drvMap(ATgetLength(es));
-        queryAllAttrs(e, drvMap);
+    else if (v.type == tAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = drvMap.get(toATerm("_combineChannels"));
+        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        typedef std::map<string, Expr> AttrsSorted;
-        AttrsSorted attrsSorted;
-        foreach (ATermMap::const_iterator, i, drvMap)
-            attrsSorted[aterm2String(i->key)] = i->value;
+        typedef std::map<string, Symbol> SortedSymbols;
+        SortedSymbols attrs;
+        foreach (Bindings::iterator, i, *v.attrs)
+            attrs.insert(std::pair<string, Symbol>(i->name, i->name));
 
-        foreach (AttrsSorted::iterator, i, attrsSorted) {
+        foreach (SortedSymbols::iterator, i, attrs) {
             startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first);
             string pathPrefix2 = addToPath(pathPrefix, i->first);
+            Value & v2(*v.attrs->find(i->second)->value);
             if (combineChannels)
-                getDerivations(state, i->second, pathPrefix2, autoArgs, drvs, doneExprs);
-            else if (getDerivation(state, i->second, pathPrefix2, drvs, doneExprs)) {
+                getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done);
+            else if (getDerivation(state, v2, pathPrefix2, drvs, done)) {
                 /* If the value of this attribute is itself an
                    attribute set, should we recurse into it?  => Only
                    if it has a `recurseForDerivations = true'
                    attribute. */
-                ATermList es;
-                Expr e = evalExpr(state, i->second), e2;
-                if (matchAttrs(e, es)) {
-                    ATermMap attrs(ATgetLength(es));
-                    queryAllAttrs(e, attrs, false);
-                    if (((e2 = attrs.get(toATerm("recurseForDerivations")))
-                            && evalBool(state, e2)))
-                        getDerivations(state, e, pathPrefix2, autoArgs, drvs, doneExprs);
+                if (v2.type == tAttrs) {
+                    Bindings::iterator j = v2.attrs->find(state.symbols.create("recurseForDerivations"));
+                    if (j != v2.attrs->end() && state.forceBool(*j->value))
+                        getDerivations(state, v2, pathPrefix2, autoArgs, drvs, done);
                 }
             }
         }
-        
-        return;
     }
 
-    if (matchList(e, es)) {
-        int n = 0;
-        for (ATermIterator i(es); i; ++i, ++n) {
+    else if (v.type == tList) {
+        for (unsigned int n = 0; n < v.list.length; ++n) {
             startNest(nest, lvlDebug,
                 format("evaluating list element"));
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *i, pathPrefix2, drvs, doneExprs))
-                getDerivations(state, *i, pathPrefix2, autoArgs, drvs, doneExprs);
+            if (getDerivation(state, *v.list.elems[n], pathPrefix2, drvs, done))
+                getDerivations(state, *v.list.elems[n], pathPrefix2, autoArgs, drvs, done);
         }
-        return;
     }
 
-    throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
+    else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)");
 }
 
 
-void getDerivations(EvalState & state, Expr e, const string & pathPrefix,
-    const ATermMap & autoArgs, DrvInfos & drvs)
+void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+    Bindings & autoArgs, DrvInfos & drvs)
 {
-    Exprs doneExprs;
-    getDerivations(state, e, pathPrefix, autoArgs, drvs, doneExprs);
+    Done done;
+    getDerivations(state, v, pathPrefix, autoArgs, drvs, done);
 }
 
  

src/libexpr/get-drvs.hh

@@ -1,13 +1,13 @@
 #ifndef __GET_DRVS_H
 #define __GET_DRVS_H
 
+#include "eval.hh"
+
 #include <string>
 #include <map>
 
 #include <boost/shared_ptr.hpp>
 
-#include "eval.hh"
-
 
 namespace nix {
 
@@ -29,16 +29,19 @@ struct DrvInfo
 private:
     string drvPath;
     string outPath;
+
+    bool metaInfoRead;
+    MetaInfo meta;
     
 public:
     string name;
     string attrPath; /* path towards the derivation */
     string system;
 
-    /* !!! these should really be hidden, and setMetaInfo() should
-       make a copy since the ATermMap can be shared between multiple
-       DrvInfos. */
-    boost::shared_ptr<ATermMap> attrs;
+    /* !!! make this private */
+    Bindings * attrs;
+
+    DrvInfo() : metaInfoRead(false), attrs(0) { };
 
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
@@ -59,16 +62,19 @@ public:
 };
 
 
+#if HAVE_BOEHMGC
+typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
+#else
 typedef list<DrvInfo> DrvInfos;
+#endif
 
 
-/* Evaluate expression `e'.  If it evaluates to a derivation, store
-   information about the derivation in `drv' and return true.
-   Otherwise, return false. */
-bool getDerivation(EvalState & state, Expr e, DrvInfo & drv);
+/* If value `v' denotes a derivation, store information about the
+   derivation in `drv' and return true.  Otherwise, return false. */
+bool getDerivation(EvalState & state, Value & v, DrvInfo & drv);
 
-void getDerivations(EvalState & state, Expr e, const string & pathPrefix,
-    const ATermMap & autoArgs, DrvInfos & drvs);
+void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+    Bindings & autoArgs, DrvInfos & drvs);
 
  
 }

src/libexpr/lexer.l

@@ -8,9 +8,7 @@
 
 
 %{
-#include "aterm.hh"
 #include "nixexpr.hh"
-#include "nixexpr-ast.hh"
 #define BISON_HEADER_HACK
 #include "parser-tab.hh"
 
@@ -21,13 +19,16 @@ namespace nix {
     
 static void initLoc(YYLTYPE * loc)
 {
-    loc->first_line = 1;
-    loc->first_column = 1;
+    loc->first_line = loc->last_line = 1;
+    loc->first_column = loc->last_column = 1;
 }
 
     
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
+    loc->first_line = loc->last_line;
+    loc->first_column = loc->last_column;
+
     while (len--) {
        switch (*s++) {
        case '\r':
@@ -35,17 +36,17 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
                s++;
            /* fall through */
        case '\n': 
-           ++loc->first_line;
-           loc->first_column = 1;
+           ++loc->last_line;
+           loc->last_column = 1;
            break;
        default:
-           ++loc->first_column;
+           ++loc->last_column;
        }
     }
 }
 
 
-static Expr unescapeStr(const char * s)
+static Expr * unescapeStr(SymbolTable & symbols, const char * s)
 {
     string t;
     char c;
@@ -65,7 +66,7 @@ static Expr unescapeStr(const char * s)
         }
         else t += c;
     }
-    return makeStr(toATerm(t), ATempty);
+    return new ExprString(symbols.create(t));
 }
 
  
@@ -105,19 +106,20 @@ inherit     { return INHERIT; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
 
-{ID}        { yylval->t = toATerm(yytext); return ID; /* !!! alloc */ }
+{ID}        { yylval->id = strdup(yytext); return ID; }
 {INT}       { int n = atoi(yytext); /* !!! overflow */
-              yylval->t = ATmake("<int>", n);
+              yylval->n = n;
               return INT;
             }
 
 \"          { BEGIN(STRING); return '"'; }
 <STRING>([^\$\"\\]|\$[^\{\"]|\\.)+ {
-/* !!! Not quite right: we want a follow restriction on "$", it
-   shouldn't be followed by a "{".  Right now "$\"" will be consumed
-   as part of a string, rather than a "$" followed by the string
-   terminator.  Disallow "$\"" for now. */
-              yylval->t = unescapeStr(yytext); /* !!! alloc */ 
+              /* !!! Not quite right: we want a follow restriction on
+                 "$", it shouldn't be followed by a "{".  Right now
+                 "$\"" will be consumed as part of a string, rather
+                 than a "$" followed by the string terminator.
+                 Disallow "$\"" for now. */
+              yylval->e = unescapeStr(data->symbols, yytext);
               return STR;
             }
 <STRING>\$\{  { BEGIN(INITIAL); return DOLLAR_CURLY; }
@@ -126,31 +128,31 @@ inherit     { return INHERIT; }
 
 \'\'(\ *\n)?     { BEGIN(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
-                   yylval->t = makeIndStr(toATerm(yytext));
+                   yylval->e = new ExprIndStr(yytext);
                    return IND_STR;
                  }
 <IND_STRING>\'\'\$ {
-                   yylval->t = makeIndStr(toATerm("$"));
+                   yylval->e = new ExprIndStr("$");
                    return IND_STR;
                  }
 <IND_STRING>\'\'\' {
-                   yylval->t = makeIndStr(toATerm("''"));
+                   yylval->e = new ExprIndStr("''");
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\. {
-                   yylval->t = unescapeStr(yytext + 2);
+                   yylval->e = unescapeStr(data->symbols, yytext + 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { BEGIN(INITIAL); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { BEGIN(INITIAL); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
-                   yylval->t = makeIndStr(toATerm("'"));
+                   yylval->e = new ExprIndStr("'");
                    return IND_STR;
                  }
 <IND_STRING>.    return yytext[0]; /* just in case: shouldn't be reached */
 
-{PATH}      { yylval->t = toATerm(yytext); return PATH; /* !!! alloc */ }
-{URI}       { yylval->t = toATerm(yytext); return URI; /* !!! alloc */ }
+{PATH}      { yylval->path = strdup(yytext); return PATH; }
+{URI}       { yylval->uri = strdup(yytext); return URI; }
 
 [ \t\r\n]+    /* eat up whitespace */
 \#[^\r\n]*    /* single-line comments */

src/libexpr/nixexpr-ast.def

@@ -1,97 +0,0 @@
-init initNixExprHelpers
-
-Pos | string int int | Pos |
-NoPos | | Pos |
-
-Function | Pattern Expr Pos | Expr |
-Assert | Expr Expr Pos | Expr |
-With | Expr Expr Pos | Expr |
-If | Expr Expr Expr | Expr |
-OpNot | Expr | Expr |
-OpEq | Expr Expr | Expr |
-OpNEq | Expr Expr | Expr |
-OpAnd | Expr Expr | Expr |
-OpOr | Expr Expr | Expr |
-OpImpl | Expr Expr | Expr |
-OpUpdate | Expr Expr | Expr |
-SubPath | Expr Expr | Expr |
-OpHasAttr | Expr string | Expr |
-OpPlus | Expr Expr | Expr |
-OpConcat | Expr Expr | Expr |
-ConcatStrings | ATermList | Expr |
-Call | Expr Expr | Expr |
-Select | Expr string | Expr |
-Var | string | Expr |
-Int | int | Expr |
-
-# Strings in the evaluator carry a so-called `context' (the ATermList)
-# which is a list of strings representing store paths.  This is to
-# allow users to write things like
-#
-#   "--with-freetype2-library=" + freetype + "/lib"
-#
-# where `freetype' is a derivation (or a source to be copied to the
-# store).  If we just concatenated the strings without keeping track
-# of the referenced store paths, then if the string is used as a
-# derivation attribute, the derivation will not have the correct
-# dependencies in its inputDrvs and inputSrcs.
-#
-# The semantics of the context is as follows: when a string with
-# context C is used as a derivation attribute, then the derivations in
-# C will be added to the inputDrvs of the derivation, and the other
-# store paths in C will be added to the inputSrcs of the derivations.
-#
-# For canonicity, the store paths should be in sorted order.
-Str | string ATermList | Expr |
-Str | string | Expr | ObsoleteStr
-
-# Internal to the parser, doesn't occur in ASTs.
-IndStr | string | Expr |
-
-# A path is a reference to a file system object that is to be copied
-# to the Nix store when used as a derivation attribute.  When it is
-# concatenated to a string (i.e., `str + path'), it is also copied and
-# the resulting store path is concatenated to the string (with the
-# store path in the context).  If a string or path is concatenated to
-# a path (i.e., `path + str' or `path + path'), the result is a new
-# path (if the right-hand side is a string, the context must be
-# empty).
-Path | string | Expr |
-
-List | ATermList | Expr |
-BlackHole | | Expr |
-Undefined | | Expr |
-Removed | | Expr |
-PrimOp | int ATermBlob ATermList | Expr |
-Attrs | ATermList | Expr |
-Closed | Expr | Expr |
-Rec | ATermList ATermList | Expr |
-Bool | ATermBool | Expr |
-Null | | Expr |
-
-Bind | string Expr Pos | ATerm |
-BindAttrPath | ATermList Expr Pos | ATerm | # desugared during parsing
-Bind | string Expr | ATerm | ObsoleteBind
-Inherit | Expr ATermList Pos | ATerm |
-
-Scope | | Expr |
-
-VarPat | string | Pattern |
-AttrsPat | ATermList ATermBool | Pattern | # bool = `...'
-AtPat | Pattern Pattern | Pattern |
-
-Formal | string DefaultValue | ATerm |
-
-DefaultValue | Expr | DefaultValue |
-NoDefaultValue | | DefaultValue |
-
-True | | ATermBool |
-False | | ATermBool |
-
-PrimOpDef | int ATermBlob | ATerm |
-
-AttrRHS | Expr Pos | ATerm |
-
-eTrue = makeBool(makeTrue())
-eFalse = makeBool(makeFalse())
-sOverrides = toATerm("__overrides")

src/libexpr/nixexpr.cc

@@ -1,407 +1,314 @@
 #include "nixexpr.hh"
 #include "derivations.hh"
 #include "util.hh"
-#include "aterm.hh"
-
-#include "nixexpr-ast.hh"
-#include "nixexpr-ast.cc"
 
 #include <cstdlib>
 
 
 namespace nix {
-    
 
-string showPos(ATerm pos)
+
+/* Displaying abstract syntax trees. */
+
+std::ostream & operator << (std::ostream & str, Expr & e)
 {
-    ATerm path;
-    int line, column;
-    if (matchNoPos(pos)) return "undefined position";
-    if (!matchPos(pos, path, line, column))
-        throw badTerm("position expected", pos);
-    return (format("`%1%:%2%:%3%'") % aterm2String(path) % line % column).str();
+    e.show(str);
+    return str;
 }
-    
 
-ATerm bottomupRewrite(TermFun & f, ATerm e)
+void Expr::show(std::ostream & str)
 {
-    checkInterrupt();
+    abort();
+}
 
-    if (ATgetType(e) == AT_APPL) {
-        AFun fun = ATgetAFun(e);
-        int arity = ATgetArity(fun);
-        ATerm args[arity];
+void ExprInt::show(std::ostream & str)
+{
+    str << n;
+}
 
-        for (int i = 0; i < arity; ++i)
-            args[i] = bottomupRewrite(f, ATgetArgument(e, i));
+void ExprString::show(std::ostream & str)
+{
+    str << "\"" << s << "\""; // !!! escaping
+}
+
+void ExprPath::show(std::ostream & str)
+{
+    str << s;
+}
+
+void ExprVar::show(std::ostream & str)
+{
+    str << info.name;
+}
+
+void ExprSelect::show(std::ostream & str)
+{
+    str << "(" << *e << ")." << name;
+}
+
+void ExprOpHasAttr::show(std::ostream & str)
+{
+    str << "(" << *e << ") ? " << name;
+}
+
+void ExprAttrs::show(std::ostream & str)
+{
+    if (recursive) str << "rec ";
+    str << "{ ";
+    foreach (AttrDefs::iterator, i, attrs)
+        if (i->second.inherited)
+            str << "inherit " << i->first << " " << "; ";
+        else
+            str << i->first << " = " << *i->second.e << "; ";
+    str << "}";
+}
+
+void ExprList::show(std::ostream & str)
+{
+    str << "[ ";
+    foreach (vector<Expr *>::iterator, i, elems)
+        str << "(" << **i << ") ";
+    str << "]";
+}
+
+void ExprLambda::show(std::ostream & str)
+{
+    str << "(";
+    if (matchAttrs) {
+        str << "{ ";
+        bool first = true;
+        foreach (Formals::Formals_::iterator, i, formals->formals) {
+            if (first) first = false; else str << ", ";
+            str << i->name;
+            if (i->def) str << " ? " << *i->def;
+        }
+        str << " }";
+        if (!arg.empty()) str << " @ ";
+    }
+    if (!arg.empty()) str << arg;
+    str << ": " << *body << ")";
+}
+
+void ExprLet::show(std::ostream & str)
+{
+    str << "let ";
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        if (i->second.inherited)
+            str << "inherit " << i->first << "; ";
+        else
+            str << i->first << " = " << *i->second.e << "; ";
+    str << "in " << *body;
+}
+
+void ExprWith::show(std::ostream & str)
+{
+    str << "with " << *attrs << "; " << *body;
+}
+
+void ExprIf::show(std::ostream & str)
+{
+    str << "if " << *cond << " then " << *then << " else " << *else_;
+}
+
+void ExprAssert::show(std::ostream & str)
+{
+    str << "assert " << *cond << "; " << *body;
+}
+
+void ExprOpNot::show(std::ostream & str)
+{
+    str << "! " << *e;
+}
+
+void ExprConcatStrings::show(std::ostream & str)
+{
+    bool first = true;
+    foreach (vector<Expr *>::iterator, i, *es) {
+        if (first) first = false; else str << " + ";
+        str << **i;
+    }
+}
+
+
+std::ostream & operator << (std::ostream & str, const Pos & pos)
+{
+    if (!pos.line)
+        str << "undefined position";
+    else
+        str << (format("`%1%:%2%:%3%'") % pos.file % pos.line % pos.column).str();
+    return str;
+}
+
+
+Pos noPos;
+
+
+/* Computing levels/displacements for variables. */
+
+void Expr::bindVars(const StaticEnv & env)
+{
+    abort();
+}
+
+void ExprInt::bindVars(const StaticEnv & env)
+{
+}
+
+void ExprString::bindVars(const StaticEnv & env)
+{
+}
+
+void ExprPath::bindVars(const StaticEnv & env)
+{
+}
+
+void VarRef::bind(const StaticEnv & env)
+{
+    /* Check whether the variable appears in the environment.  If so,
+       set its level and displacement. */
+    const StaticEnv * curEnv;
+    unsigned int level;
+    int withLevel = -1;
+    for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) {
+        if (curEnv->isWith) {
+            if (withLevel == -1) withLevel = level;
+        } else {
+            StaticEnv::Vars::const_iterator i = curEnv->vars.find(name);
+            if (i != curEnv->vars.end()) {
+                fromWith = false;
+                this->level = level;
+                displ = i->second;
+                return;
+            }
+        }
+    }
+
+    /* Otherwise, the variable must be obtained from the nearest
+       enclosing `with'.  If there is no `with', then we can issue an
+       "undefined variable" error now. */
+    if (withLevel == -1) throw EvalError(format("undefined variable `%1%'") % name);
+
+    fromWith = true;
+    this->level = withLevel;
+}
+
+void ExprVar::bindVars(const StaticEnv & env)
+{
+    info.bind(env);
+}
+
+void ExprSelect::bindVars(const StaticEnv & env)
+{
+    e->bindVars(env);
+}
+
+void ExprOpHasAttr::bindVars(const StaticEnv & env)
+{
+    e->bindVars(env);
+}
+
+void ExprAttrs::bindVars(const StaticEnv & env)
+{
+    if (recursive) {
+        StaticEnv newEnv(false, &env);
+    
+        unsigned int displ = 0;
+        foreach (AttrDefs::iterator, i, attrs)
+            newEnv.vars[i->first] = i->second.displ = displ++;
         
-        e = (ATerm) ATmakeApplArray(fun, args);
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited) i->second.var.bind(env);
+            else i->second.e->bindVars(newEnv);
     }
 
-    else if (ATgetType(e) == AT_LIST) {
-        ATermList in = (ATermList) e;
-        ATermList out = ATempty;
-
-        for (ATermIterator i(in); i; ++i)
-            out = ATinsert(out, bottomupRewrite(f, *i));
-
-        e = (ATerm) ATreverse(out);
-    }
-
-    return f(e);
+    else
+        foreach (AttrDefs::iterator, i, attrs)
+            if (i->second.inherited) i->second.var.bind(env);
+            else i->second.e->bindVars(env);
 }
 
-
-void queryAllAttrs(Expr e, ATermMap & attrs, bool withPos)
+void ExprList::bindVars(const StaticEnv & env)
 {
-    ATermList bnds;
-    if (!matchAttrs(e, bnds))
-        throw TypeError(format("value is %1% while an attribute set was expected") % showType(e));
-
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm name;
-        Expr e;
-        ATerm pos;
-        if (!matchBind(*i, name, e, pos)) abort(); /* can't happen */
-        attrs.set(name, withPos ? makeAttrRHS(e, pos) : e);
-    }
+    foreach (vector<Expr *>::iterator, i, elems)
+        (*i)->bindVars(env);
 }
 
-
-Expr queryAttr(Expr e, const string & name)
+void ExprLambda::bindVars(const StaticEnv & env)
 {
-    ATerm dummy;
-    return queryAttr(e, name, dummy);
-}
-
-
-Expr queryAttr(Expr e, const string & name, ATerm & pos)
-{
-    ATermList bnds;
-    if (!matchAttrs(e, bnds))
-        throw TypeError(format("value is %1% while an attribute set was expected") % showType(e));
-
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm name2, pos2;
-        Expr e;
-        if (!matchBind(*i, name2, e, pos2))
-            abort(); /* can't happen */
-        if (aterm2String(name2) == name) {
-            pos = pos2;
-            return e;
-        }
-    }
-
-    return 0;
-}
-
-
-Expr makeAttrs(const ATermMap & attrs)
-{
-    ATermList bnds = ATempty;
-    for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i) {
-        Expr e;
-        ATerm pos;
-        if (!matchAttrRHS(i->value, e, pos))
-            abort(); /* can't happen */
-        bnds = ATinsert(bnds, makeBind(i->key, e, pos));
-    }
-    return makeAttrs(bnds);
-}
-
-
-static void varsBoundByPattern(ATermMap & map, Pattern pat)
-{
-    ATerm name;
-    ATermList formals;
-    Pattern pat1, pat2;
-    ATermBool ellipsis;
-    /* Use makeRemoved() so that it can be used directly in
-       substitute(). */
-    if (matchVarPat(pat, name))
-        map.set(name, makeRemoved());
-    else if (matchAttrsPat(pat, formals, ellipsis)) { 
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1;
-            if (!matchFormal(*i, name, d1)) abort();
-            map.set(name, makeRemoved());
-        }
-    }
-    else if (matchAtPat(pat, pat1, pat2)) {
-        varsBoundByPattern(map, pat1);
-        varsBoundByPattern(map, pat2);
-    }
-    else abort();
-}
-
-
-Expr substitute(const Substitution & subs, Expr e)
-{
-    checkInterrupt();
-
-    //if (subs.size() == 0) return e;
-
-    ATerm name, pos, e2;
-
-    /* As an optimisation, don't substitute in subterms known to be
-       closed. */
-    if (matchClosed(e, e2)) return e;
-
-    if (matchVar(e, name)) {
-        Expr sub = subs.lookup(name);
-        if (sub == makeRemoved()) sub = 0;
-        Expr wrapped;
-        /* Add a "closed" wrapper around terms that aren't already
-           closed.  The check is necessary to prevent repeated
-           wrapping, e.g., closed(closed(closed(...))), which kills
-           caching. */
-        return sub ? (matchClosed(sub, wrapped) ? sub : makeClosed(sub)) : e;
-    }
-
-    /* In case of a function, filter out all variables bound by this
-       function. */
-    Pattern pat;
-    ATerm body;
-    if (matchFunction(e, pat, body, pos)) {
-        ATermMap map(16);
-        varsBoundByPattern(map, pat);
-        Substitution subs2(&subs, &map);
-        return makeFunction(
-            (Pattern) substitute(subs2, (Expr) pat),
-            substitute(subs2, body), pos);
-    }
-
-    /* Idem for a mutually recursive attribute set. */
-    ATermList rbnds, nrbnds;
-    if (matchRec(e, rbnds, nrbnds)) {
-        ATermMap map(ATgetLength(rbnds) + ATgetLength(nrbnds));
-        for (ATermIterator i(rbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
-        for (ATermIterator i(nrbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
-        return makeRec(
-            (ATermList) substitute(Substitution(&subs, &map), (ATerm) rbnds),
-            (ATermList) substitute(subs, (ATerm) nrbnds));
-    }
-
-    if (ATgetType(e) == AT_APPL) {
-        AFun fun = ATgetAFun(e);
-        int arity = ATgetArity(fun);
-        ATerm args[arity];
-        bool changed = false;
-
-        for (int i = 0; i < arity; ++i) {
-            ATerm arg = ATgetArgument(e, i);
-            args[i] = substitute(subs, arg);
-            if (args[i] != arg) changed = true;
-        }
-        
-        return changed ? (ATerm) ATmakeApplArray(fun, args) : e;
-    }
-
-    if (ATgetType(e) == AT_LIST) {
-        unsigned int len = ATgetLength((ATermList) e);
-        ATerm es[len];
-        ATermIterator i((ATermList) e);
-        for (unsigned int j = 0; i; ++i, ++j)
-            es[j] = substitute(subs, *i);
-        ATermList out = ATempty;
-        for (unsigned int j = len; j; --j)
-            out = ATinsert(out, es[j - 1]);
-        return (ATerm) out;
-    }
-
-    return e;
-}
-
-
-/* We use memoisation to prevent exponential complexity on heavily
-   shared ATerms (remember, an ATerm is a graph, not a tree!).  Note
-   that using an STL set is fine here wrt to ATerm garbage collection
-   since all the ATerms in the set are already reachable from
-   somewhere else. */
-static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
-{
-    if (done.find(e) != done.end()) return;
-    done.insert(e);
+    StaticEnv newEnv(false, &env);
     
-    ATerm name, pos, value;
-    ATerm with, body;
-    ATermList rbnds, nrbnds;
-    Pattern pat;
-
-    /* Closed terms don't have free variables, so we don't have to
-       check by definition. */
-    if (matchClosed(e, value)) return;
+    unsigned int displ = 0;
     
-    else if (matchVar(e, name)) {
-        if (!defs.get(name))
-            throw EvalError(format("undefined variable `%1%'")
-                % aterm2String(name));
+    if (!arg.empty()) newEnv.vars[arg] = displ++;
+
+    if (matchAttrs) {
+        foreach (Formals::Formals_::iterator, i, formals->formals)
+            newEnv.vars[i->name] = displ++;
+
+        foreach (Formals::Formals_::iterator, i, formals->formals)
+            if (i->def) i->def->bindVars(newEnv);
     }
 
-    else if (matchFunction(e, pat, body, pos)) {
-        ATermMap defs2(defs);
-        varsBoundByPattern(defs2, pat);
-        set<Expr> done2;
-        checkVarDefs2(done2, defs2, pat);
-        checkVarDefs2(done2, defs2, body);
-    }
-        
-    else if (matchRec(e, rbnds, nrbnds)) {
-        checkVarDefs2(done, defs, (ATerm) nrbnds);
-        ATermMap defs2(defs);
-        for (ATermIterator i(rbnds); i; ++i) {
-            if (!matchBind(*i, name, value, pos)) abort(); /* can't happen */
-            defs2.set(name, (ATerm) ATempty);
-        }
-        for (ATermIterator i(nrbnds); i; ++i) {
-            if (!matchBind(*i, name, value, pos)) abort(); /* can't happen */
-            defs2.set(name, (ATerm) ATempty);
-        }
-        set<Expr> done2;
-        checkVarDefs2(done2, defs2, (ATerm) rbnds);
-    }
+    body->bindVars(newEnv);
+}
 
-    else if (matchWith(e, with, body, pos)) {
-        /* We can't check the body without evaluating the definitions
-           (which is an arbitrary expression), so we don't do that
-           here but only when actually evaluating the `with'. */
-        checkVarDefs2(done, defs, with);
-    }
+void ExprLet::bindVars(const StaticEnv & env)
+{
+    StaticEnv newEnv(false, &env);
     
-    else if (ATgetType(e) == AT_APPL) {
-        int arity = ATgetArity(ATgetAFun(e));
-        for (int i = 0; i < arity; ++i)
-            checkVarDefs2(done, defs, ATgetArgument(e, i));
-    }
-
-    else if (ATgetType(e) == AT_LIST)
-        for (ATermIterator i((ATermList) e); i; ++i)
-            checkVarDefs2(done, defs, *i);
+    unsigned int displ = 0;
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        newEnv.vars[i->first] = i->second.displ = displ++;
+    
+    foreach (ExprAttrs::AttrDefs::iterator, i, attrs->attrs)
+        if (i->second.inherited) i->second.var.bind(env);
+        else i->second.e->bindVars(newEnv);
+    
+    body->bindVars(newEnv);
 }
 
-
-void checkVarDefs(const ATermMap & defs, Expr e)
+void ExprWith::bindVars(const StaticEnv & env)
 {
-    set<Expr> done;
-    checkVarDefs2(done, defs, e);
-}
-
-
-struct Canonicalise : TermFun
-{
-    ATerm operator () (ATerm e)
-    {
-        /* Remove position info. */
-        ATerm path;
-        int line, column;
-        if (matchPos(e, path, line, column))
-            return makeNoPos();
-
-        /* Sort attribute sets. */
-        ATermList _;
-        if (matchAttrs(e, _)) {
-            ATermMap attrs;
-            queryAllAttrs(e, attrs);
-            StringSet names;
-            for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i)
-                names.insert(aterm2String(i->key));
-
-            ATermList attrs2 = ATempty;
-            for (StringSet::reverse_iterator i = names.rbegin(); i != names.rend(); ++i)
-                attrs2 = ATinsert(attrs2,
-                    makeBind(toATerm(*i), attrs.get(toATerm(*i)), makeNoPos()));
-
-            return makeAttrs(attrs2);
+    /* Does this `with' have an enclosing `with'?  If so, record its
+       level so that `lookupVar' can look up variables in the previous
+       `with' if this one doesn't contain the desired attribute. */
+    const StaticEnv * curEnv;
+    unsigned int level;
+    prevWith = 0;
+    for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++)
+        if (curEnv->isWith) {
+            prevWith = level;
+            break;
         }
-        
-        return e;
-    }
-};
+    
+    attrs->bindVars(env);    
+    StaticEnv newEnv(true, &env);
+    body->bindVars(newEnv);
+}
 
-
-Expr canonicaliseExpr(Expr e)
+void ExprIf::bindVars(const StaticEnv & env)
 {
-    Canonicalise canonicalise;
-    return bottomupRewrite(canonicalise, e);
+    cond->bindVars(env);
+    then->bindVars(env);
+    else_->bindVars(env);
 }
 
-
-Expr makeBool(bool b)
+void ExprAssert::bindVars(const StaticEnv & env)
 {
-    return b ? eTrue : eFalse;
+    cond->bindVars(env);
+    body->bindVars(env);
 }
 
-
-bool matchStr(Expr e, string & s, PathSet & context)
+void ExprOpNot::bindVars(const StaticEnv & env)
 {
-    ATermList l;
-    ATerm s_;
-
-    if (!matchStr(e, s_, l)) return false;
-
-    s = aterm2String(s_);
-
-    for (ATermIterator i(l); i; ++i)
-        context.insert(aterm2String(*i));
-
-    return true;
+    e->bindVars(env);
 }
 
-
-Expr makeStr(const string & s, const PathSet & context)
+void ExprConcatStrings::bindVars(const StaticEnv & env)
 {
-    return makeStr(toATerm(s), toATermList(context));
+    foreach (vector<Expr *>::iterator, i, *es)
+        (*i)->bindVars(env);
 }
 
 
-string showType(Expr e)
-{
-    ATerm t1, t2;
-    ATermList l1;
-    ATermBlob b1;
-    int i1;
-    Pattern p1;
-    if (matchStr(e, t1, l1)) return "a string";
-    if (matchPath(e, t1)) return "a path";
-    if (matchNull(e)) return "null";
-    if (matchInt(e, i1)) return "an integer";
-    if (matchBool(e, t1)) return "a boolean";
-    if (matchFunction(e, p1, t1, t2)) return "a function";
-    if (matchAttrs(e, l1)) return "an attribute set";
-    if (matchList(e, l1)) return "a list";
-    if (matchPrimOp(e, i1, b1, l1)) return "a partially applied built-in function";
-    return "an unknown type";
-}
-
-
-string showValue(Expr e)
-{
-    PathSet context;
-    string s;
-    ATerm s2;
-    int i;
-    if (matchStr(e, s, context)) {
-        string u;
-        for (string::iterator i = s.begin(); i != s.end(); ++i)
-            if (*i == '\"' || *i == '\\') u += "\\" + *i;
-            else if (*i == '\n') u += "\\n";
-            else if (*i == '\r') u += "\\r";
-            else if (*i == '\t') u += "\\t";
-            else u += *i;
-        return "\"" + u + "\"";
-    }
-    if (matchPath(e, s2)) return aterm2String(s2);
-    if (matchNull(e)) return "null";
-    if (matchInt(e, i)) return (format("%1%") % i).str();
-    if (e == eTrue) return "true";
-    if (e == eFalse) return "false";
-    /* !!! incomplete */
-    return "<unknown>";
-}
-
- 
 }

src/libexpr/nixexpr.hh

@@ -1,10 +1,9 @@
 #ifndef __NIXEXPR_H
 #define __NIXEXPR_H
 
-#include <map>
+#include "symbol-table.hh"
 
-#include "aterm-map.hh"
-#include "types.hh"
+#include <map>
 
 
 namespace nix {
@@ -16,107 +15,264 @@ MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
 MakeError(TypeError, EvalError)
+MakeError(ImportError, EvalError) // error building an imported derivation
 
 
-/* Nix expressions are represented as ATerms.  The maximal sharing
-   property of the ATerm library allows us to implement caching of
-   normals forms efficiently. */
-typedef ATerm Expr;
-typedef ATerm DefaultValue;
-typedef ATerm Pos;
-typedef ATerm Pattern;
-typedef ATerm ATermBool;
+/* Position objects. */
 
-
-/* A STL vector of ATerms.  Should be used with great care since it's
-   stored on the heap, and the elements are therefore not roots to the
-   ATerm garbage collector. */
-typedef vector<ATerm> ATermVector;
-
-
-/* A substitution is a linked list of ATermMaps that map names to
-   identifiers.  We use a list of ATermMaps rather than a single to
-   make it easy to grow or shrink a substitution when entering a
-   scope. */
-struct Substitution
+struct Pos
 {
-    ATermMap * map;
-    const Substitution * prev;
+    string file;
+    unsigned int line, column;
+    Pos() : line(0), column(0) { };
+    Pos(const string & file, unsigned int line, unsigned int column)
+        : file(file), line(line), column(column) { };
+};
 
-    Substitution(const Substitution * prev, ATermMap * map)
-    {
-        this->prev = prev;
-        this->map = map;
-    }
+extern Pos noPos;
+
+std::ostream & operator << (std::ostream & str, const Pos & pos);
+
+
+struct Env;
+struct Value;
+struct EvalState;
+struct StaticEnv;
+
+
+/* Abstract syntax of Nix expressions. */
+
+struct Expr
+{
+    virtual void show(std::ostream & str);
+    virtual void bindVars(const StaticEnv & env);
+    virtual void eval(EvalState & state, Env & env, Value & v);
+};
+
+std::ostream & operator << (std::ostream & str, Expr & e);
+
+#define COMMON_METHODS \
+    void show(std::ostream & str); \
+    void eval(EvalState & state, Env & env, Value & v); \
+    void bindVars(const StaticEnv & env);
+
+struct ExprInt : Expr
+{
+    int n;
+    ExprInt(int n) : n(n) { };
+    COMMON_METHODS
+};
+
+struct ExprString : Expr
+{
+    Symbol s;
+    ExprString(const Symbol & s) : s(s) { };
+    COMMON_METHODS
+};
+
+/* Temporary class used during parsing of indented strings. */
+struct ExprIndStr : Expr
+{
+    string s;
+    ExprIndStr(const string & s) : s(s) { };
+};
+
+struct ExprPath : Expr
+{
+    string s;
+    ExprPath(const string & s) : s(s) { };
+    COMMON_METHODS
+};
+
+struct VarRef
+{
+    Symbol name;
+
+    /* Whether the variable comes from an environment (e.g. a rec, let
+       or function argument) or from a "with". */
+    bool fromWith;
     
-    Expr lookup(Expr name) const
-    {
-        Expr x;
-        for (const Substitution * s(this); s; s = s->prev)
-            if ((x = s->map->get(name))) return x;
-        return 0;
-    }
+    /* In the former case, the value is obtained by going `level'
+       levels up from the current environment and getting the
+       `displ'th value in that environment.  In the latter case, the
+       value is obtained by getting the attribute named `name' from
+       the attribute set stored in the environment that is `level'
+       levels up from the current one.*/
+    unsigned int level;
+    unsigned int displ;
+
+    VarRef() { };
+    VarRef(const Symbol & name) : name(name) { };
+    void bind(const StaticEnv & env);
 };
 
-
-/* Show a position. */
-string showPos(ATerm pos);
-
-/* Generic bottomup traversal over ATerms.  The traversal first
-   recursively descends into subterms, and then applies the given term
-   function to the resulting term. */
-struct TermFun
+struct ExprVar : Expr
 {
-    virtual ~TermFun() { }
-    virtual ATerm operator () (ATerm e) = 0;
+    VarRef info;
+    ExprVar(const Symbol & name) : info(name) { };
+    COMMON_METHODS
+};
+
+struct ExprSelect : Expr
+{
+    Expr * e;
+    Symbol name;
+    ExprSelect(Expr * e, const Symbol & name) : e(e), name(name) { };
+    COMMON_METHODS
+};
+
+struct ExprOpHasAttr : Expr
+{
+    Expr * e;
+    Symbol name;
+    ExprOpHasAttr(Expr * e, const Symbol & name) : e(e), name(name) { };
+    COMMON_METHODS
+};
+
+struct ExprAttrs : Expr
+{
+    bool recursive;
+    struct AttrDef {
+        bool inherited;
+        Expr * e; // if not inherited
+        VarRef var; // if inherited
+        Pos pos;
+        unsigned int displ; // displacement
+        AttrDef(Expr * e, const Pos & pos) : inherited(false), e(e), pos(pos) { };
+        AttrDef(const Symbol & name, const Pos & pos) : inherited(true), var(name), pos(pos) { };
+        AttrDef() { };
+    };
+    typedef std::map<Symbol, AttrDef> AttrDefs;
+    AttrDefs attrs;
+    ExprAttrs() : recursive(false) { };
+    COMMON_METHODS
+};
+
+struct ExprList : Expr
+{
+    std::vector<Expr *> elems;
+    ExprList() { };
+    COMMON_METHODS
+};
+
+struct Formal
+{
+    Symbol name;
+    Expr * def;
+    Formal(const Symbol & name, Expr * def) : name(name), def(def) { };
+};
+
+struct Formals
+{
+    typedef std::list<Formal> Formals_;
+    Formals_ formals;
+    std::set<Symbol> argNames; // used during parsing
+    bool ellipsis;
+};
+
+struct ExprLambda : Expr
+{
+    Pos pos;
+    Symbol arg;
+    bool matchAttrs;
+    Formals * formals;
+    Expr * body;
+    ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
+        : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
+    {
+        if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
+            throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+                % arg % pos);
+    };
+    COMMON_METHODS
+};
+
+struct ExprLet : Expr
+{
+    ExprAttrs * attrs;
+    Expr * body;
+    ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { };
+    COMMON_METHODS
+};
+
+struct ExprWith : Expr
+{
+    Pos pos;
+    Expr * attrs, * body;
+    unsigned int prevWith;
+    ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { };
+    COMMON_METHODS
+};
+
+struct ExprIf : Expr
+{
+    Expr * cond, * then, * else_;
+    ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { };
+    COMMON_METHODS
+};
+
+struct ExprAssert : Expr
+{
+    Pos pos;
+    Expr * cond, * body;
+    ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { };
+    COMMON_METHODS
+};
+
+struct ExprOpNot : Expr
+{
+    Expr * e;
+    ExprOpNot(Expr * e) : e(e) { };
+    COMMON_METHODS
+};
+
+#define MakeBinOp(name, s) \
+    struct Expr##name : Expr \
+    { \
+        Expr * e1, * e2; \
+        Expr##name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \
+        void show(std::ostream & str) \
+        { \
+            str << *e1 << " " s " " << *e2; \
+        } \
+        void bindVars(const StaticEnv & env) \
+        { \
+            e1->bindVars(env); e2->bindVars(env); \
+        } \
+        void eval(EvalState & state, Env & env, Value & v); \
+    };
+
+MakeBinOp(App, "")
+MakeBinOp(OpEq, "==")
+MakeBinOp(OpNEq, "!=")
+MakeBinOp(OpAnd, "&&")
+MakeBinOp(OpOr, "||")
+MakeBinOp(OpImpl, "->")
+MakeBinOp(OpUpdate, "//")
+MakeBinOp(OpConcatLists, "++")
+
+struct ExprConcatStrings : Expr
+{
+    vector<Expr *> * es;
+    ExprConcatStrings(vector<Expr *> * es) : es(es) { };
+    COMMON_METHODS
 };
-ATerm bottomupRewrite(TermFun & f, ATerm e);
 
 
-/* Query all attributes in an attribute set expression.  The
-   expression must be in normal form. */
-void queryAllAttrs(Expr e, ATermMap & attrs, bool withPos = false);
-
-/* Query a specific attribute from an attribute set expression.  The
-   expression must be in normal form. */
-Expr queryAttr(Expr e, const string & name);
-Expr queryAttr(Expr e, const string & name, ATerm & pos);
-
-/* Create an attribute set expression from an Attrs value. */
-Expr makeAttrs(const ATermMap & attrs);
+/* Static environments are used to map variable names onto (level,
+   displacement) pairs used to obtain the value of the variable at
+   runtime. */
+struct StaticEnv
+{
+    bool isWith;
+    const StaticEnv * up;
+    typedef std::map<Symbol, unsigned int> Vars;
+    Vars vars;
+    StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { };
+};
 
 
-/* Perform a set of substitutions on an expression. */
-Expr substitute(const Substitution & subs, Expr e);
 
-
-/* Check whether all variables are defined in the given expression.
-   Throw an exception if this isn't the case. */
-void checkVarDefs(const ATermMap & def, Expr e);
-
-
-/* Canonicalise a Nix expression by sorting attributes and removing
-   location information. */
-Expr canonicaliseExpr(Expr e);
-
-
-/* Create an expression representing a boolean. */
-Expr makeBool(bool b);
-
-
-/* Manipulation of Str() nodes.  Note: matchStr() does not clear
-   context!  */
-bool matchStr(Expr e, string & s, PathSet & context);
-
-Expr makeStr(const string & s, const PathSet & context = PathSet());
-
-
-/* Showing types, values. */
-string showType(Expr e);
-
-string showValue(Expr e);
-
- 
 }
 
 

src/libexpr/normal-forms.cc

@@ -1,64 +0,0 @@
-#include "eval.hh"
-#include "util.hh"
-#include "globals.hh"
-#include "serialise.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-namespace nix {
-
-
-void EvalState::loadNormalForms()
-{
-    //    nixCacheFile = getEnv("NIX_CACHE_FILE", nixStateDir + "/reduce-cache");
-    nixCacheFile = getEnv("NIX_CACHE_FILE");
-    string loadFlag = getEnv("NIX_CACHE_FILE_LOAD", "1");
-
-    if(nixCacheFile == "" || loadFlag == "")
-        return;
-    if(!pathExists(nixCacheFile))
-        return;
-
-    printMsg(lvlInfo, format("Load cache: ..."));
-    try {
-        int fd = open(nixCacheFile.c_str(), O_RDONLY);
-        if (fd == -1)
-            throw SysError(format("opening file `%1%'") % nixCacheFile);
-        AutoCloseFD auto_fd = fd;
-
-        FdSource source = fd;
-        normalForms = readATermMap(source);
-    } catch (Error & e) {
-        e.addPrefix(format("Cannot load cached reduce operations from %1%:\n") % nixCacheFile);
-        throw;
-    }
-    printMsg(lvlInfo, format("Load cache: end"));
-}
-
-void EvalState::saveNormalForms()
-{
-    string saveFlag = getEnv("NIX_CACHE_FILE_SAVE", "");
-
-    if(nixCacheFile == "" || saveFlag == "")
-        return;
-
-    printMsg(lvlInfo, format("Save cache: ..."));
-    try {
-        int fd = open(nixCacheFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
-        if (fd == -1)
-            throw SysError(format("opening file `%1%'") % nixCacheFile);
-        AutoCloseFD auto_fd = fd;
-
-        FdSink sink = fd;
-        writeATermMap(normalForms, sink);
-    } catch (Error & e) {
-        e.addPrefix(format("Cannot save cached reduce operations to %1%:\n") % nixCacheFile);
-        throw;
-    }
-    printMsg(lvlInfo, format("Save cache: end"));
-}
-
-
-}

src/libexpr/parser.hh

@@ -8,12 +8,11 @@ namespace nix {
 
 
 /* Parse a Nix expression from the specified file.  If `path' refers
-   to a directory, the "/default.nix" is appended. */
-Expr parseExprFromFile(EvalState & state, Path path);
+   to a directory, then "/default.nix" is appended. */
+Expr * parseExprFromFile(EvalState & state, Path path);
 
 /* Parse a Nix expression from the specified string. */
-Expr parseExprFromString(EvalState & state, const string & s,
-    const Path & basePath);
+Expr * parseExprFromString(EvalState & state, const string & s, const Path & basePath);
 
 
 }

src/libexpr/parser.y

@@ -7,171 +7,126 @@
 %parse-param { yyscan_t scanner }
 %parse-param { ParseData * data }
 %lex-param { yyscan_t scanner }
+%lex-param { ParseData * data }
 
-
-%{
-/* Newer versions of Bison copy the declarations below to
-   parser-tab.hh, which sucks bigtime since lexer.l doesn't want that
-   stuff.  So allow it to be excluded. */
-#ifndef BISON_HEADER_HACK
-#define BISON_HEADER_HACK
+%code requires {
+    
+#ifndef BISON_HEADER
+#define BISON_HEADER
     
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "aterm.hh"
 #include "util.hh"
     
+#include "nixexpr.hh"
+
+namespace nix {
+
+    struct ParseData 
+    {
+        SymbolTable & symbols;
+        Expr * result;
+        Path basePath;
+        Path path;
+        string error;
+        Symbol sLetBody;
+        ParseData(SymbolTable & symbols)
+            : symbols(symbols)
+            , sLetBody(symbols.create("<let-body>"))
+            { };
+    };
+    
+}
+
+#define YY_DECL int yylex \
+    (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data)
+
+#endif
+
+}
+
+%{
+
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
 #define YYSTYPE YYSTYPE // workaround a bug in Bison 2.4
 
-#include "nixexpr.hh"
-#include "nixexpr-ast.hh"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
+YY_DECL;
 
 using namespace nix;
 
 
 namespace nix {
-
     
-struct ParseData 
-{
-    Expr result;
-    Path basePath;
-    Path path;
-    string error;
-};
- 
 
-static string showAttrPath(ATermList attrPath)
+static string showAttrPath(const vector<Symbol> & attrPath)
 {
     string s;
-    for (ATermIterator i(attrPath); i; ++i) {
+    foreach (vector<Symbol>::const_iterator, i, attrPath) {
         if (!s.empty()) s += '.';
-        s += aterm2String(*i);
+        s += *i;
     }
     return s;
 }
- 
 
-struct Tree
+
+static void dupAttr(const vector<Symbol> & attrPath, const Pos & pos, const Pos & prevPos)
 {
-    Expr leaf; ATerm pos; bool recursive;
-    typedef std::map<ATerm, Tree> Children;
-    Children children;
-    Tree() { leaf = 0; recursive = true; }
-};
-
-
-static ATermList buildAttrs(const Tree & t, ATermList & nonrec)
-{
-    ATermList res = ATempty;
-    for (Tree::Children::const_reverse_iterator i = t.children.rbegin();
-         i != t.children.rend(); ++i)
-        if (!i->second.recursive)
-            nonrec = ATinsert(nonrec, makeBind(i->first, i->second.leaf, i->second.pos));
-        else
-            res = ATinsert(res, i->second.leaf
-                ? makeBind(i->first, i->second.leaf, i->second.pos)
-                : makeBind(i->first, makeAttrs(buildAttrs(i->second, nonrec)), makeNoPos()));
-    return res;
+    throw ParseError(format("attribute `%1%' at %2% already defined at %3%")
+        % showAttrPath(attrPath) % pos % prevPos);
 }
  
 
-static Expr fixAttrs(bool recursive, ATermList as)
+static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
-    Tree attrs;
+    vector<Symbol> attrPath; attrPath.push_back(attr);
+    throw ParseError(format("attribute `%1%' at %2% already defined at %3%")
+        % showAttrPath(attrPath) % pos % prevPos);
+}
+ 
 
-    for (ATermIterator i(as); i; ++i) {
-        ATermList names, attrPath; Expr src, e; ATerm name, pos;
-
-        if (matchInherit(*i, src, names, pos)) {
-            bool fromScope = matchScope(src);
-            for (ATermIterator j(names); j; ++j) {
-                if (attrs.children.find(*j) != attrs.children.end()) 
-                    throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
-                        % showAttrPath(ATmakeList1(*j)) % showPos(pos));
-                Tree & t(attrs.children[*j]);
-                t.leaf = fromScope ? makeVar(*j) : makeSelect(src, *j);
-                t.pos = pos;
-                if (recursive && fromScope) t.recursive = false;
+static void addAttr(ExprAttrs * attrs, const vector<Symbol> & attrPath,
+    Expr * e, const Pos & pos)
+{
+    unsigned int n = 0;
+    foreach (vector<Symbol>::const_iterator, i, attrPath) {
+        n++;
+        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(*i);
+        if (j != attrs->attrs.end()) {
+            if (!j->second.inherited) {
+                ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
+                if (!attrs2 || n == attrPath.size()) dupAttr(attrPath, pos, j->second.pos);
+                attrs = attrs2;
+            } else
+                dupAttr(attrPath, pos, j->second.pos);
+        } else {
+            if (n == attrPath.size())
+                attrs->attrs[*i] = ExprAttrs::AttrDef(e, pos);
+            else {
+                ExprAttrs * nested = new ExprAttrs;
+                attrs->attrs[*i] = ExprAttrs::AttrDef(nested, pos);
+                attrs = nested;
             }
         }
-
-        else if (matchBindAttrPath(*i, attrPath, e, pos)) {
-
-            Tree * t(&attrs);
-            
-            for (ATermIterator j(attrPath); j; ) {
-                name = *j; ++j;
-                if (t->leaf) throw ParseError(format("attribute set containing `%1%' at %2% already defined at %3%")
-                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
-                t = &(t->children[name]);
-            }
-
-            if (t->leaf)
-                throw ParseError(format("duplicate definition of attribute `%1%' at %2% and %3%")
-                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
-            if (!t->children.empty())
-                throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
-                    % showAttrPath(attrPath) % showPos(pos));
-
-            t->leaf = e; t->pos = pos;
-        }
-
-        else abort(); /* can't happen */
     }
-
-    ATermList nonrec = ATempty;
-    ATermList rec = buildAttrs(attrs, nonrec);
-        
-    return recursive ? makeRec(rec, nonrec) : makeAttrs(rec);
 }
 
 
-static void checkPatternVars(ATerm pos, ATermMap & map, Pattern pat)
+static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
-    ATerm name;
-    ATermList formals;
-    Pattern pat1, pat2;
-    ATermBool ellipsis;
-    if (matchVarPat(pat, name)) {
-        if (map.get(name))
-            throw ParseError(format("duplicate formal function argument `%1%' at %2%")
-                % aterm2String(name) % showPos(pos));
-        map.set(name, name);
-    }
-    else if (matchAttrsPat(pat, formals, ellipsis)) { 
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1;
-            if (!matchFormal(*i, name, d1)) abort();
-            if (map.get(name))
-                throw ParseError(format("duplicate formal function argument `%1%' at %2%")
-                    % aterm2String(name) % showPos(pos));
-            map.set(name, name);
-        }
-    }
-    else if (matchAtPat(pat, pat1, pat2)) {
-        checkPatternVars(pos, map, pat1);
-        checkPatternVars(pos, map, pat2);
-    }
-    else abort();
+    if (formals->argNames.find(formal.name) != formals->argNames.end())
+        throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+            % formal.name % pos);
+    formals->formals.push_front(formal);
+    formals->argNames.insert(formal.name);
 }
 
 
-static void checkPatternVars(ATerm pos, Pattern pat)
+static Expr * stripIndentation(SymbolTable & symbols, vector<Expr *> & es)
 {
-    ATermMap map;
-    checkPatternVars(pos, map, pat);
-}
-
-
-static Expr stripIndentation(ATermList es)
-{
-    if (es == ATempty) return makeStr("");
+    if (es.empty()) return new ExprString(symbols.create(""));
     
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
@@ -179,9 +134,9 @@ static Expr stripIndentation(ATermList es)
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
     unsigned int minIndent = 1000000;
     unsigned int curIndent = 0;
-    ATerm e;
-    for (ATermIterator i(es); i; ++i) {
-        if (!matchIndStr(*i, e)) {
+    foreach (vector<Expr *>::iterator, i, es) {
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+        if (!e) {
             /* Anti-quotations end the current start-of-line whitespace. */
             if (atStartOfLine) {
                 atStartOfLine = false;
@@ -189,12 +144,11 @@ static Expr stripIndentation(ATermList es)
             }
             continue;
         }
-        string s = aterm2String(e);
-        for (unsigned int j = 0; j < s.size(); ++j) {
+        for (unsigned int j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
-                if (s[j] == ' ')
+                if (e->s[j] == ' ')
                     curIndent++;
-                else if (s[j] == '\n') {
+                else if (e->s[j] == '\n') {
                     /* Empty line, doesn't influence minimum
                        indentation. */
                     curIndent = 0;
@@ -202,7 +156,7 @@ static Expr stripIndentation(ATermList es)
                     atStartOfLine = false;
                     if (curIndent < minIndent) minIndent = curIndent;
                 }
-            } else if (s[j] == '\n') {
+            } else if (e->s[j] == '\n') {
                 atStartOfLine = true;
                 curIndent = 0;
             }
@@ -210,37 +164,37 @@ static Expr stripIndentation(ATermList es)
     }
 
     /* Strip spaces from each line. */
-    ATermList es2 = ATempty;
+    vector<Expr *> * es2 = new vector<Expr *>;
     atStartOfLine = true;
     unsigned int curDropped = 0;
-    unsigned int n = ATgetLength(es);
-    for (ATermIterator i(es); i; ++i, --n) {
-        if (!matchIndStr(*i, e)) {
+    unsigned int n = es.size();
+    for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+        if (!e) {
             atStartOfLine = false;
             curDropped = 0;
-            es2 = ATinsert(es2, *i);
+            es2->push_back(*i);
             continue;
         }
         
-        string s = aterm2String(e);
         string s2;
-        for (unsigned int j = 0; j < s.size(); ++j) {
+        for (unsigned int j = 0; j < e->s.size(); ++j) {
             if (atStartOfLine) {
-                if (s[j] == ' ') {
+                if (e->s[j] == ' ') {
                     if (curDropped++ >= minIndent)
-                        s2 += s[j];
+                        s2 += e->s[j];
                 }
-                else if (s[j] == '\n') {
+                else if (e->s[j] == '\n') {
                     curDropped = 0;
-                    s2 += s[j];
+                    s2 += e->s[j];
                 } else {
                     atStartOfLine = false;
                     curDropped = 0;
-                    s2 += s[j];
+                    s2 += e->s[j];
                 }
             } else {
-                s2 += s[j];
-                if (s[j] == '\n') atStartOfLine = true;
+                s2 += e->s[j];
+                if (e->s[j] == '\n') atStartOfLine = true;
             }
         }
 
@@ -251,11 +205,11 @@ static Expr stripIndentation(ATermList es)
             if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos)
                 s2 = string(s2, 0, p + 1);
         }
-            
-        es2 = ATinsert(es2, makeStr(s2));
+
+        es2->push_back(new ExprString(symbols.create(s2)));
     }
 
-    return makeConcatStrings(ATreverse(es2));
+    return new ExprConcatStrings(es2);
 }
 
 
@@ -263,13 +217,12 @@ void backToString(yyscan_t scanner);
 void backToIndString(yyscan_t scanner);
 
 
-static Pos makeCurPos(YYLTYPE * loc, ParseData * data)
+static Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return makePos(toATerm(data->path),
-        loc->first_line, loc->first_column);
+    return Pos(data->path, loc.first_line, loc.first_column);
 }
 
-#define CUR_POS makeCurPos(yylocp, data)
+#define CUR_POS makeCurPos(*yylocp, data)
 
 
 }
@@ -277,50 +230,40 @@ static Pos makeCurPos(YYLTYPE * loc, ParseData * data)
 
 void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
 {
-    data->error = (format("%1%, at `%2%':%3%:%4%")
-        % error % data->path % loc->first_line % loc->first_column).str();
+    data->error = (format("%1%, at %2%")
+        % error % makeCurPos(*loc, data)).str();
 }
 
 
-/* Make sure that the parse stack is scanned by the ATerm garbage
-   collector. */
-static void * mallocAndProtect(size_t size)
-{
-    void * p = malloc(size);
-    if (p) ATprotectMemory(p, size);
-    return p;
-}
-
-static void freeAndUnprotect(void * p)
-{
-    ATunprotectMemory(p);
-    free(p);
-}
-
-#define YYMALLOC mallocAndProtect
-#define YYFREE freeAndUnprotect
-
-
-#endif
-
-
 %}
 
 %union {
-  ATerm t;
-  ATermList ts;
-  struct {
-    ATermList formals;
-    bool ellipsis;
-  } formals;
+  nix::Expr * e;
+  nix::ExprList * list;
+  nix::ExprAttrs * attrs;
+  nix::Formals * formals;
+  nix::Formal * formal;
+  int n;
+  char * id; // !!! -> Symbol
+  char * path;
+  char * uri;
+  std::vector<nix::Symbol> * ids;
+  std::vector<nix::Expr *> * string_parts;
 }
 
-%type <t> start expr expr_function expr_if expr_op
-%type <t> expr_app expr_select expr_simple bind inheritsrc formal
-%type <t> pattern pattern2
-%type <ts> binds ids attrpath expr_list string_parts ind_string_parts
+%type <e> start expr expr_function expr_if expr_op
+%type <e> expr_app expr_select expr_simple
+%type <list> expr_list
+%type <attrs> binds
 %type <formals> formals
-%token <t> ID INT STR IND_STR PATH URI
+%type <formal> formal
+%type <ids> ids attrpath
+%type <string_parts> string_parts ind_string_parts
+%token <id> ID ATTRPATH
+%token <e> STR IND_STR
+%token <n> INT
+%token <path> PATH
+%token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
@@ -344,163 +287,170 @@ start: expr { data->result = $1; };
 expr: expr_function;
 
 expr_function
-  : pattern ':' expr_function
-    { checkPatternVars(CUR_POS, $1); $$ = makeFunction($1, $3, CUR_POS); }
+  : ID ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
+  | '{' formals '}' ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
+  | '{' formals '}' '@' ID ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
+  | ID '@' '{' formals '}' ':' expr_function
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
   | ASSERT expr ';' expr_function
-    { $$ = makeAssert($2, $4, CUR_POS); }
+    { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
-    { $$ = makeWith($2, $4, CUR_POS); }
+    { $$ = new ExprWith(CUR_POS, $2, $4); }
   | LET binds IN expr_function
-    { $$ = makeSelect(fixAttrs(true, ATinsert($2, makeBindAttrPath(ATmakeList1(toATerm("<let-body>")), $4, CUR_POS))), toATerm("<let-body>")); }
+    { $$ = new ExprLet($2, $4); }
   | expr_if
   ;
 
 expr_if
-  : IF expr THEN expr ELSE expr
-    { $$ = makeIf($2, $4, $6); }
+  : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); }
   | expr_op
   ;
 
 expr_op
-  : '!' expr_op %prec NEG { $$ = makeOpNot($2); }
-  | expr_op EQ expr_op { $$ = makeOpEq($1, $3); }
-  | expr_op NEQ expr_op { $$ = makeOpNEq($1, $3); }
-  | expr_op AND expr_op { $$ = makeOpAnd($1, $3); }
-  | expr_op OR expr_op { $$ = makeOpOr($1, $3); }
-  | expr_op IMPL expr_op { $$ = makeOpImpl($1, $3); }
-  | expr_op UPDATE expr_op { $$ = makeOpUpdate($1, $3); }
-  | expr_op '~' expr_op { $$ = makeSubPath($1, $3); }
-  | expr_op '?' ID { $$ = makeOpHasAttr($1, $3); }
-  | expr_op '+' expr_op { $$ = makeOpPlus($1, $3); }
-  | expr_op CONCAT expr_op { $$ = makeOpConcat($1, $3); }
+  : '!' expr_op %prec NEG { $$ = new ExprOpNot($2); }
+  | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
+  | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
+  | expr_op AND expr_op { $$ = new ExprOpAnd($1, $3); }
+  | expr_op OR expr_op { $$ = new ExprOpOr($1, $3); }
+  | expr_op IMPL expr_op { $$ = new ExprOpImpl($1, $3); }
+  | expr_op UPDATE expr_op { $$ = new ExprOpUpdate($1, $3); }
+  | expr_op '?' ID { $$ = new ExprOpHasAttr($1, data->symbols.create($3)); }
+  | expr_op '+' expr_op
+    { vector<Expr *> * l = new vector<Expr *>;
+      l->push_back($1);
+      l->push_back($3);
+      $$ = new ExprConcatStrings(l);
+    }
+  | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists($1, $3); }
   | expr_app
   ;
 
 expr_app
   : expr_app expr_select
-    { $$ = makeCall($1, $2); }
+    { $$ = new ExprApp($1, $2); }
   | expr_select { $$ = $1; }
   ;
 
 expr_select
   : expr_select '.' ID
-    { $$ = makeSelect($1, $3); }
+    { $$ = new ExprSelect($1, data->symbols.create($3)); }
   | expr_simple { $$ = $1; }
   ;
 
 expr_simple
-  : ID { $$ = makeVar($1); }
-  | INT { $$ = makeInt(ATgetInt((ATermInt) $1)); }
+  : ID { $$ = new ExprVar(data->symbols.create($1)); }
+  | INT { $$ = new ExprInt($1); }
   | '"' string_parts '"' {
       /* For efficiency, and to simplify parse trees a bit. */
-      if ($2 == ATempty) $$ = makeStr(toATerm(""), ATempty);
-      else if (ATgetNext($2) == ATempty) $$ = ATgetFirst($2);
-      else $$ = makeConcatStrings(ATreverse($2));
+      if ($2->empty()) $$ = new ExprString(data->symbols.create(""));
+      else if ($2->size() == 1) $$ = $2->front();
+      else $$ = new ExprConcatStrings($2);
   }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = stripIndentation(ATreverse($2));
+      $$ = stripIndentation(data->symbols, *$2);
   }
-  | PATH { $$ = makePath(toATerm(absPath(aterm2String($1), data->basePath))); }
-  | URI { $$ = makeStr($1, ATempty); }
+  | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
+  | URI { $$ = new ExprString(data->symbols.create($1)); }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
   | LET '{' binds '}'
-    { $$ = makeSelect(fixAttrs(true, $3), toATerm("body")); }
+    { $3->recursive = true; $$ = new ExprSelect($3, data->symbols.create("body")); }
   | REC '{' binds '}'
-    { $$ = fixAttrs(true, $3); }
+    { $3->recursive = true; $$ = $3; }
   | '{' binds '}'
-    { $$ = fixAttrs(false, $2); }
-  | '[' expr_list ']' { $$ = makeList(ATreverse($2)); }
+    { $$ = $2; }
+  | '[' expr_list ']' { $$ = $2; }
   ;
 
 string_parts
-  : string_parts STR { $$ = ATinsert($1, $2); }
-  | string_parts DOLLAR_CURLY expr '}' { backToString(scanner); $$ = ATinsert($1, $3); }
-  | { $$ = ATempty; }
+  : string_parts STR { $$ = $1; $1->push_back($2); }
+  | string_parts DOLLAR_CURLY expr '}' { backToString(scanner); $$ = $1; $1->push_back($3); }
+  | { $$ = new vector<Expr *>; }
   ;
 
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = ATinsert($1, $2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { backToIndString(scanner); $$ = ATinsert($1, $3); }
-  | { $$ = ATempty; }
-  ;
-
-pattern
-  : pattern2 '@' pattern { $$ = makeAtPat($1, $3); }
-  | pattern2
-  ;
-
-pattern2
-  : ID { $$ = makeVarPat($1); }
-  | '{' formals '}' { $$ = makeAttrsPat($2.formals, $2.ellipsis ? eTrue : eFalse); }
+  : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { backToIndString(scanner); $$ = $1; $1->push_back($3); }
+  | { $$ = new vector<Expr *>; }
   ;
 
 binds
-  : binds bind { $$ = ATinsert($1, $2); }
-  | { $$ = ATempty; }
+  : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); }
+  | binds INHERIT ids ';'
+    { $$ = $1;
+      foreach (vector<Symbol>::iterator, i, *$3) {
+          if ($$->attrs.find(*i) != $$->attrs.end())
+              dupAttr(*i, makeCurPos(@3, data), $$->attrs[*i].pos);
+          Pos pos = makeCurPos(@3, data);
+          $$->attrs[*i] = ExprAttrs::AttrDef(*i, pos);
+      }
+    }
+  | binds INHERIT '(' expr ')' ids ';'
+    { $$ = $1;
+      /* !!! Should ensure sharing of the expression in $4. */
+      foreach (vector<Symbol>::iterator, i, *$6) {
+          if ($$->attrs.find(*i) != $$->attrs.end())
+              dupAttr(*i, makeCurPos(@6, data), $$->attrs[*i].pos);
+          $$->attrs[*i] = ExprAttrs::AttrDef(new ExprSelect($4, *i), makeCurPos(@6, data));
+      }}
+
+  | { $$ = new ExprAttrs; }
   ;
 
-bind
-  : attrpath '=' expr ';'
-    { $$ = makeBindAttrPath(ATreverse($1), $3, CUR_POS); }
-  | INHERIT inheritsrc ids ';'
-    { $$ = makeInherit($2, $3, CUR_POS); }
+ids
+  : ids ID { $$ = $1; $1->push_back(data->symbols.create($2)); /* !!! dangerous */ }
+  | { $$ = new vector<Symbol>; }
   ;
 
-inheritsrc
-  : '(' expr ')' { $$ = $2; }
-  | { $$ = makeScope(); }
-  ;
-
-ids: ids ID { $$ = ATinsert($1, $2); } | { $$ = ATempty; };
-
 attrpath
-  : attrpath '.' ID { $$ = ATinsert($1, $3); }
-  | ID { $$ = ATmakeList1($1); }
+  : attrpath '.' ID { $$ = $1; $1->push_back(data->symbols.create($3)); }
+  | ID { $$ = new vector<Symbol>; $$->push_back(data->symbols.create($1)); }
   ;
 
 expr_list
-  : expr_list expr_select { $$ = ATinsert($1, $2); }
-  | { $$ = ATempty; }
+  : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ }
+  | { $$ = new ExprList; }
   ;
 
 formals
-  : formal ',' formals /* !!! right recursive */
-    { $$.formals = ATinsert($3.formals, $1); $$.ellipsis = $3.ellipsis; }
+  : formal ',' formals
+    { $$ = $3; addFormal(CUR_POS, $$, *$1); }
   | formal
-    { $$.formals = ATinsert(ATempty, $1); $$.ellipsis = false; }
+    { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; }
   |
-    { $$.formals = ATempty; $$.ellipsis = false; }
+    { $$ = new Formals; $$->ellipsis = false; }
   | ELLIPSIS
-    { $$.formals = ATempty; $$.ellipsis = true; }
+    { $$ = new Formals; $$->ellipsis = true; }
   ;
 
 formal
-  : ID { $$ = makeFormal($1, makeNoDefaultValue()); }
-  | ID '?' expr { $$ = makeFormal($1, makeDefaultValue($3)); }
+  : ID { $$ = new Formal(data->symbols.create($1), 0); }
+  | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); }
   ;
   
 %%
 
 
-#include "eval.hh"  
-
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
 
+#include <eval.hh>
+
 
 namespace nix {
       
 
-static Expr parse(EvalState & state,
-    const char * text, const Path & path,
-    const Path & basePath)
+static Expr * parse(EvalState & state, const char * text,
+    const Path & path, const Path & basePath)
 {
     yyscan_t scanner;
-    ParseData data;
+    ParseData data(state.symbols);
     data.basePath = basePath;
     data.path = path;
 
@@ -512,7 +462,7 @@ static Expr parse(EvalState & state,
     if (res) throw ParseError(data.error);
 
     try {
-        checkVarDefs(state.primOps, data.result);
+        data.result->bindVars(state.staticBaseEnv);
     } catch (Error & e) {
         throw ParseError(format("%1%, in `%2%'") % e.msg() % path);
     }
@@ -521,16 +471,10 @@ static Expr parse(EvalState & state,
 }
 
 
-Expr parseExprFromFile(EvalState & state, Path path)
+Expr * parseExprFromFile(EvalState & state, Path path)
 {
     assert(path[0] == '/');
 
-#if 0
-    /* Perhaps this is already an imploded parse tree? */
-    Expr e = ATreadFromNamedFile(path.c_str());
-    if (e) return e;
-#endif
-
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
     struct stat st;
@@ -542,8 +486,6 @@ Expr parseExprFromFile(EvalState & state, Path path)
     }
 
     /* If `path' refers to a directory, append `/default.nix'. */
-    if (stat(path.c_str(), &st))
-        throw SysError(format("getting status of `%1%'") % path);
     if (S_ISDIR(st.st_mode))
         path = canonPath(path + "/default.nix");
 
@@ -552,7 +494,7 @@ Expr parseExprFromFile(EvalState & state, Path path)
 }
 
 
-Expr parseExprFromString(EvalState & state,
+Expr * parseExprFromString(EvalState & state,
     const string & s, const Path & basePath)
 {
     return parse(state, s.c_str(), "(string)", basePath);

src/libexpr/symbol-table.hh

@@ -0,0 +1,92 @@
+#ifndef __SYMBOL_TABLE_H
+#define __SYMBOL_TABLE_H
+
+#include "config.h"
+
+#include <map>
+
+#if HAVE_TR1_UNORDERED_SET
+#include <tr1/unordered_set>
+#endif
+
+#include "types.hh"
+
+namespace nix {
+
+/* Symbol table used by the parser and evaluator to represent and look
+   up identifiers and attribute sets efficiently.
+   SymbolTable::create() converts a string into a symbol.  Symbols
+   have the property that they can be compared efficiently (using a
+   pointer equality test), because the symbol table stores only one
+   copy of each string. */
+
+class Symbol
+{
+private:
+    const string * s; // pointer into SymbolTable
+    Symbol(const string * s) : s(s) { };
+    friend class SymbolTable;
+
+public:
+    Symbol() : s(0) { };
+    
+    bool operator == (const Symbol & s2) const
+    {
+        return s == s2.s;
+    }
+    
+    bool operator != (const Symbol & s2) const
+    {
+        return s != s2.s;
+    }
+    
+    bool operator < (const Symbol & s2) const
+    {
+        return s < s2.s;
+    }
+
+    operator const string & () const
+    {
+        return *s;
+    }
+
+    bool empty() const
+    {
+        return s->empty();
+    }
+
+    friend std::ostream & operator << (std::ostream & str, const Symbol & sym);
+};
+
+inline std::ostream & operator << (std::ostream & str, const Symbol & sym)
+{
+    str << *sym.s;
+    return str;
+}
+
+class SymbolTable
+{
+private:
+#if HAVE_TR1_UNORDERED_SET 
+    typedef std::tr1::unordered_set<string> Symbols;
+#else
+    typedef std::set<string> Symbols;
+#endif
+    Symbols symbols;
+
+public:
+    Symbol create(const string & s)
+    {
+        std::pair<Symbols::iterator, bool> res = symbols.insert(s);
+        return Symbol(&*res.first);
+    }
+
+    unsigned int size() const
+    {
+        return symbols.size();
+    }
+};
+
+}
+
+#endif /* !__SYMBOL_TABLE_H */

src/libexpr/value-to-xml.cc

@@ -0,0 +1,162 @@
+#include "value-to-xml.hh"
+#include "xml-writer.hh"
+#include "util.hh"
+
+#include <cstdlib>
+
+
+namespace nix {
+
+    
+static XMLAttrs singletonAttrs(const string & name, const string & value)
+{
+    XMLAttrs attrs;
+    attrs[name] = value;
+    return attrs;
+}
+
+
+static void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen);
+
+
+static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
+{
+    xmlAttrs["path"] = pos.file;
+    xmlAttrs["line"] = (format("%1%") % pos.line).str();
+    xmlAttrs["column"] = (format("%1%") % pos.column).str();
+}
+
+
+static void showAttrs(EvalState & state, bool strict, bool location,
+    Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
+{
+    StringSet names;
+    
+    foreach (Bindings::iterator, i, attrs)
+        names.insert(i->name);
+    
+    foreach (StringSet::iterator, i, names) {
+        Attr & a(*attrs.find(state.symbols.create(*i)));
+        
+        XMLAttrs xmlAttrs;
+        xmlAttrs["name"] = *i;
+        if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
+        
+        XMLOpenElement _(doc, "attr", xmlAttrs);
+        printValueAsXML(state, strict, location,
+            *a.value, doc, context, drvsSeen);
+    }
+}
+
+
+static void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
+{
+    checkInterrupt();
+
+    if (strict) state.forceValue(v);
+        
+    switch (v.type) {
+
+        case tInt:
+            doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
+            break;
+
+        case tBool:
+            doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
+            break;
+
+        case tString:
+            /* !!! show the context? */
+            copyContext(v, context);
+            doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
+            break;
+
+        case tPath:
+            doc.writeEmptyElement("path", singletonAttrs("value", v.path));
+            break;
+
+        case tNull:
+            doc.writeEmptyElement("null");
+            break;
+
+        case tAttrs:
+            if (state.isDerivation(v)) {
+                XMLAttrs xmlAttrs;
+            
+                Bindings::iterator a = v.attrs->find(state.symbols.create("derivation"));
+
+                Path drvPath;
+                a = v.attrs->find(state.sDrvPath);
+                if (a != v.attrs->end()) {
+                    if (strict) state.forceValue(*a->value);
+                    if (a->value->type == tString)
+                        xmlAttrs["drvPath"] = drvPath = a->value->string.s;
+                }
+        
+                a = v.attrs->find(state.sOutPath);
+                if (a != v.attrs->end()) {
+                    if (strict) state.forceValue(*a->value);
+                    if (a->value->type == tString)
+                        xmlAttrs["outPath"] = a->value->string.s;
+                }
+
+                XMLOpenElement _(doc, "derivation", xmlAttrs);
+
+                if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) {
+                    drvsSeen.insert(drvPath);
+                    showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
+                } else
+                    doc.writeEmptyElement("repeated");
+            }
+
+            else {
+                XMLOpenElement _(doc, "attrs");
+                showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
+            }
+            
+            break;
+
+        case tList: {
+            XMLOpenElement _(doc, "list");
+            for (unsigned int n = 0; n < v.list.length; ++n)
+                printValueAsXML(state, strict, location, *v.list.elems[n], doc, context, drvsSeen);
+            break;
+        }
+
+        case tLambda: {
+            XMLAttrs xmlAttrs;
+            if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
+            XMLOpenElement _(doc, "function", xmlAttrs);
+            
+            if (v.lambda.fun->matchAttrs) {
+                XMLAttrs attrs;
+                if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
+                if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
+                XMLOpenElement _(doc, "attrspat", attrs);
+                foreach (Formals::Formals_::iterator, i, v.lambda.fun->formals->formals)
+                    doc.writeEmptyElement("attr", singletonAttrs("name", i->name));
+            } else
+                doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg));
+            
+            break;
+        }
+
+        default:
+            doc.writeEmptyElement("unevaluated");
+    }
+}
+
+
+void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, std::ostream & out, PathSet & context)
+{
+    XMLWriter doc(true, out);
+    XMLOpenElement root(doc, "expr");
+    PathSet drvsSeen;    
+    printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
+}
+
+ 
+}

src/libexpr/value-to-xml.hh

@@ -0,0 +1,17 @@
+#ifndef __VALUE_TO_XML_H
+#define __VALUE_TO_XML_H
+
+#include "nixexpr.hh"
+#include "eval.hh"
+
+#include <string>
+#include <map>
+
+namespace nix {
+
+void printValueAsXML(EvalState & state, bool strict, bool location,
+    Value & v, std::ostream & out, PathSet & context);
+    
+}
+
+#endif /* !__VALUE_TO_XML_H */

src/libmain/Makefile.am

@@ -2,7 +2,7 @@ pkglib_LTLIBRARIES = libmain.la
 
 libmain_la_SOURCES = shared.cc
 
-libmain_la_LIBADD = ../libstore/libstore.la
+libmain_la_LIBADD = ../libstore/libstore.la @boehmgc_lib@
 
 pkginclude_HEADERS = shared.hh
 
@@ -15,5 +15,5 @@ AM_CXXFLAGS = \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
  -DNIX_BIN_DIR=\"$(bindir)\" \
  -DNIX_VERSION=\"$(VERSION)\" \
- -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil \
+ -I$(srcdir)/.. -I$(srcdir)/../libutil \
  -I$(srcdir)/../libstore

src/libmain/shared.cc

@@ -13,7 +13,9 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-#include <aterm2.h>
+#if HAVE_BOEHMGC
+#include <gc/gc.h>
+#endif
 
 
 namespace nix {
@@ -52,25 +54,26 @@ void printGCWarning()
 
 void printMissing(const PathSet & paths)
 {
-    unsigned long long downloadSize;
+    unsigned long long downloadSize, narSize;
     PathSet willBuild, willSubstitute, unknown;
-    queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize);
+    queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
 
     if (!willBuild.empty()) {
-        printMsg(lvlInfo, format("the following derivations will be built:"));
+        printMsg(lvlInfo, format("these derivations will be built:"));
         foreach (PathSet::iterator, i, willBuild)
             printMsg(lvlInfo, format("  %1%") % *i);
     }
 
     if (!willSubstitute.empty()) {
-        printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") %
-            (downloadSize / (1024.0 * 1024.0)));
+        printMsg(lvlInfo, format("these paths will be downloaded/copied (%.2f MiB download, %.2f MiB unpacked):")
+            % (downloadSize / (1024.0 * 1024.0))
+            % (narSize / (1024.0 * 1024.0)));
         foreach (PathSet::iterator, i, willSubstitute)
             printMsg(lvlInfo, format("  %1%") % *i);
     }
 
     if (!unknown.empty()) {
-        printMsg(lvlInfo, format("don't know how to build the following paths%1%:")
+        printMsg(lvlInfo, format("don't know how to build these paths%1%:")
             % (readOnlyMode ? " (may be caused by read-only store access)" : ""));
         foreach (PathSet::iterator, i, unknown)
             printMsg(lvlInfo, format("  %1%") % *i);
@@ -87,21 +90,6 @@ static void setLogType(string lt)
 }
 
 
-unsigned long long getIntArg(const string & opt,
-    Strings::iterator & i, const Strings::iterator & end)
-{
-    ++i;
-    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
-    long long n;
-    if (!string2Int(*i, n) || n < 0)
-        throw UsageError(format("`%1%' requires a non-negative integer") % opt);
-    return n;
-}
-
-
-void initDerivationsHelpers();
-
-
 static void closeStore()
 {
     try {
@@ -152,26 +140,33 @@ static void initAndRun(int argc, char * * argv)
     /* Get some settings from the configuration file. */
     thisSystem = querySetting("system", SYSTEM);
     maxBuildJobs = queryIntSetting("build-max-jobs", 1);
+    buildCores = queryIntSetting("build-cores", 1);
     maxSilentTime = queryIntSetting("build-max-silent-time", 0);
 
     /* Catch SIGINT. */
-    struct sigaction act, oact;
+    struct sigaction act;
     act.sa_handler = sigintHandler;
     sigfillset(&act.sa_mask);
     act.sa_flags = 0;
-    if (sigaction(SIGINT, &act, &oact))
+    if (sigaction(SIGINT, &act, 0))
         throw SysError("installing handler for SIGINT");
-    if (sigaction(SIGTERM, &act, &oact))
+    if (sigaction(SIGTERM, &act, 0))
         throw SysError("installing handler for SIGTERM");
-    if (sigaction(SIGHUP, &act, &oact))
+    if (sigaction(SIGHUP, &act, 0))
         throw SysError("installing handler for SIGHUP");
 
     /* Ignore SIGPIPE. */
     act.sa_handler = SIG_IGN;
     act.sa_flags = 0;
-    if (sigaction(SIGPIPE, &act, &oact))
+    if (sigaction(SIGPIPE, &act, 0))
         throw SysError("ignoring SIGPIPE");
 
+    /* Reset SIGCHLD to its default. */
+    act.sa_handler = SIG_DFL;
+    act.sa_flags = 0;
+    if (sigaction(SIGCHLD, &act, 0))
+        throw SysError("resetting SIGCHLD");
+
     /* There is no privacy in the Nix system ;-)  At least not for
        now.  In particular, store objects should be readable by
        everybody.  This prevents nasty surprises when using a shared
@@ -182,9 +177,6 @@ static void initAndRun(int argc, char * * argv)
     string lt = getEnv("NIX_LOG_TYPE");
     if (lt != "") setLogType(lt);
 
-    /* ATerm stuff.  !!! find a better place to put this */
-    initDerivationsHelpers();
-    
     /* Put the arguments in a vector. */
     Strings args, remaining;
     while (argc--) args.push_back(*argv++);
@@ -195,7 +187,7 @@ static void initAndRun(int argc, char * * argv)
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
         if (string(arg, 0, 4) == "-at-") ;
-        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-') {
+        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && !isdigit(arg[1])) {
             for (unsigned int j = 1; j < arg.length(); j++)
                 if (isalpha(arg[j]))
                     remaining.push_back((string) "-" + arg[j]);
@@ -209,17 +201,16 @@ static void initAndRun(int argc, char * * argv)
     remaining.clear();
 
     /* Process default options. */
+    int verbosityDelta = 0;
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
         string arg = *i;
-        if (arg == "--verbose" || arg == "-v")
-            verbosity = (Verbosity) ((int) verbosity + 1);
+        if (arg == "--verbose" || arg == "-v") verbosityDelta++;
+        else if (arg == "--quiet") verbosityDelta--;
         else if (arg == "--log-type") {
             ++i;
             if (i == args.end()) throw UsageError("`--log-type' requires an argument");
             setLogType(*i);
         }
-        else if (arg == "--build-output" || arg == "-B")
-            ; /* !!! obsolete - remove eventually */
         else if (arg == "--no-build-output" || arg == "-Q")
             buildVerbosity = lvlVomit;
         else if (arg == "--print-build-trace")
@@ -239,11 +230,13 @@ static void initAndRun(int argc, char * * argv)
         else if (arg == "--fallback")
             tryFallback = true;
         else if (arg == "--max-jobs" || arg == "-j")
-            maxBuildJobs = getIntArg(arg, i, args.end());
+            maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end());
+        else if (arg == "--cores")
+            buildCores = getIntArg<unsigned int>(arg, i, args.end());
         else if (arg == "--readonly-mode")
             readOnlyMode = true;
         else if (arg == "--max-silent-time")
-            maxSilentTime = getIntArg(arg, i, args.end());
+            maxSilentTime = getIntArg<unsigned int>(arg, i, args.end());
         else if (arg == "--no-build-hook")
             useBuildHook = false;
         else if (arg == "--show-trace")
@@ -258,6 +251,9 @@ static void initAndRun(int argc, char * * argv)
         else remaining.push_back(arg);
     }
 
+    verbosityDelta += queryIntSetting("verbosity", lvlInfo);
+    verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta);
+    
     /* Automatically clean up the temporary roots file when we
        exit. */
     RemoveTempRoots removeTempRoots __attribute__((unused));
@@ -325,6 +321,14 @@ static void setuidInit()
 }
 
 
+/* Called when the Boehm GC runs out of memory. */
+static void * oomHandler(size_t requested)
+{
+    /* Convert this to a proper C++ exception. */
+    throw std::bad_alloc();
+}
+
+
 }
 
 
@@ -339,10 +343,6 @@ int main(int argc, char * * argv)
     if (argc == 0) abort();
     setuidInit();
     
-    /* ATerm setup. */
-    ATerm bottomOfStack;
-    ATinit(argc, argv, &bottomOfStack);
-
     /* Turn on buffering for cerr. */
 #if HAVE_PUBSETBUF
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
@@ -350,6 +350,26 @@ int main(int argc, char * * argv)
 
     std::ios::sync_with_stdio(false);
 
+#if HAVE_BOEHMGC
+    /* Initialise the Boehm garbage collector.  This isn't necessary
+       on most platforms, but for portability we do it anyway. */
+    GC_INIT();
+
+    GC_oom_fn = oomHandler;
+
+    /* Set the initial heap size to something fairly big (384 MiB) so
+       that in most cases we don't need to garbage collect at all.
+       (Collection has a fairly significant overhead, some.)  The heap
+       size can be overriden through libgc's GC_INITIAL_HEAP_SIZE
+       environment variable.  We should probably also provide a
+       nix.conf setting for this.  Note that GC_expand_hp() causes a
+       lot of virtual, but not physical (resident) memory to be
+       allocated.  This might be a problem on systems that don't
+       overcommit. */
+    if (!getenv("GC_INITIAL_HEAP_SIZE"))
+        GC_expand_hp(384 * 1024 * 1024);
+#endif
+
     try {
         try {
             initAndRun(argc, argv);
@@ -373,7 +393,7 @@ int main(int argc, char * * argv)
         printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg());
         if (e.prefix() != "" && !showTrace)
             printMsg(lvlError, "(use `--show-trace' to show detailed location information)");
-        return 1;
+        return e.status;
     } catch (std::exception & e) {
         printMsg(lvlError, format("error: %1%") % e.what());
         return 1;

src/libmain/shared.hh

@@ -1,7 +1,7 @@
 #ifndef __SHARED_H
 #define __SHARED_H
 
-#include "types.hh"
+#include "util.hh"
 
 #include <signal.h>
 
@@ -22,22 +22,30 @@ extern std::string programId;
 
 namespace nix {
 
+MakeError(UsageError, nix::Error);
+
 /* Ugh.  No better place to put this. */
 Path makeRootName(const Path & gcRoot, int & counter);
 void printGCWarning();
 
 void printMissing(const PathSet & paths);
 
-unsigned long long getIntArg(const string & opt,
-    Strings::iterator & i, const Strings::iterator & end);
+template<class N> N getIntArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end)
+{
+    ++i;
+    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
+    N n;
+    if (!string2Int(*i, n))
+        throw UsageError(format("`%1%' requires an integer argument") % opt);
+    return n;
+}
 
 /* Whether we're running setuid. */
 extern bool setuidMode;
 
 extern volatile ::sig_atomic_t blockInt;
 
-MakeError(UsageError, nix::Error);
-
 struct RemoveTempRoots 
 {
     ~RemoveTempRoots();    

src/libstore/Makefile.am

@@ -2,22 +2,22 @@ pkglib_LTLIBRARIES = libstore.la
 
 libstore_la_SOURCES = \
   store-api.cc local-store.cc remote-store.cc derivations.cc build.cc misc.cc \
-  globals.cc db.cc references.cc pathlocks.cc gc.cc upgrade-schema.cc \
+  globals.cc references.cc pathlocks.cc gc.cc \
   optimise-store.cc
 
 pkginclude_HEADERS = \
   store-api.hh local-store.hh remote-store.hh derivations.hh misc.hh \
-  globals.hh db.hh references.hh pathlocks.hh \
+  globals.hh references.hh pathlocks.hh \
   worker-protocol.hh
 
-libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la
+libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib}
 
-BUILT_SOURCES = derivations-ast.cc derivations-ast.hh
-
-EXTRA_DIST = derivations-ast.def derivations-ast.cc
+EXTRA_DIST = schema.sql
 
 AM_CXXFLAGS = -Wall \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} -I$(srcdir)/../libutil
+ ${sqlite_include} -I$(srcdir)/.. -I$(srcdir)/../libutil
 
-derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
-	$(perl) $(srcdir)/../aterm-helper.pl derivations-ast.hh derivations-ast.cc < $(srcdir)/derivations-ast.def
+local-store.lo: schema.sql.hh
+
+%.sql.hh: %.sql
+	../bin2c/bin2c schema < $< > $@ || (rm $@ && exit 1)

src/libstore/db.cc

@@ -1,474 +0,0 @@
-#include "config.h"
-
-#ifdef OLD_DB_COMPAT
-
-#include "db.hh"
-#include "util.hh"
-#include "pathlocks.hh"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-
-#include <memory>
-
-#include <db_cxx.h>
-
-
-namespace nix {
-
-
-/* Wrapper class to ensure proper destruction. */
-class DestroyDbc 
-{
-    Dbc * dbc;
-public:
-    DestroyDbc(Dbc * _dbc) : dbc(_dbc) { }
-    ~DestroyDbc() { dbc->close(); /* close() frees dbc */ }
-};
-
-
-class DestroyDbEnv
-{
-    DbEnv * dbenv;
-public:
-    DestroyDbEnv(DbEnv * _dbenv) : dbenv(_dbenv) { }
-    ~DestroyDbEnv() {
-        if (dbenv) {
-            if (dbenv->get_DB_ENV()) dbenv->close(0);
-            delete dbenv;
-        }
-    }
-    void release() { dbenv = 0; };
-};
-
-
-static void rethrow(DbException & e)
-{
-    throw Error(e.what());
-}
-
-
-Transaction::Transaction()
-    : txn(0)
-{
-}
-
-
-Transaction::Transaction(Database & db)
-    : txn(0)
-{
-    begin(db);
-}
-
-
-Transaction::~Transaction()
-{
-    if (txn) abort();
-}
-
-
-void Transaction::begin(Database & db)
-{
-    assert(txn == 0);
-    db.requireEnv();
-    try {
-        db.env->txn_begin(0, &txn, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::commit()
-{
-    if (!txn) throw Error("commit called on null transaction");
-    debug(format("committing transaction %1%") % (void *) txn);
-    DbTxn * txn2 = txn;
-    txn = 0;
-    try {
-        txn2->commit(0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::abort()
-{
-    if (!txn) throw Error("abort called on null transaction");
-    debug(format("aborting transaction %1%") % (void *) txn);
-    DbTxn * txn2 = txn;
-    txn = 0;
-    try {
-        txn2->abort();
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Transaction::moveTo(Transaction & t)
-{
-    if (t.txn) throw Error("target txn already exists");
-    t.txn = txn;
-    txn = 0;
-}
-
-
-void Database::requireEnv()
-{
-    checkInterrupt();
-    if (!env) throw Error("database environment is not open "
-        "(maybe you don't have sufficient permission?)");
-}
-
-
-Db * Database::getDb(TableId table)
-{
-    if (table == 0)
-        throw Error("database table is not open "
-            "(maybe you don't have sufficient permission?)");
-    std::map<TableId, Db *>::iterator i = tables.find(table);
-    if (i == tables.end())
-        throw Error("unknown table id");
-    return i->second;
-}
-
-
-Database::Database()
-    : env(0)
-    , nextId(1)
-{
-}
-
-
-Database::~Database()
-{
-    close();
-}
-
-
-void openEnv(DbEnv * & env, const string & path, u_int32_t flags)
-{
-    try {
-        createDirs(path);
-    } catch (SysError & e) {
-        if (e.errNo == EPERM || e.errNo == EACCES)
-            throw DbNoPermission(format("cannot create the Nix database in `%1%'") % path);
-        else
-            throw;
-    }
-        
-    try {
-        env->open(path.c_str(),
-            DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN |
-            DB_CREATE | flags,
-            0666);
-    } catch (DbException & e) {
-        printMsg(lvlError, format("environment open failed: %1%") % e.what());
-        throw;
-    }
-}
-
-
-static int my_fsync(int fd)
-{
-    return 0;
-}
-
-
-static void errorPrinter(const DbEnv * env, const char * errpfx, const char * msg)
-{
-    printMsg(lvlError, format("Berkeley DB error: %1%") % msg);
-}
-
-
-static void messagePrinter(const DbEnv * env, const char * msg)
-{
-    printMsg(lvlError, format("Berkeley DB message: %1%") % msg);
-}
-
-
-void Database::open2(const string & path, bool removeOldEnv)
-{
-    if (env) throw Error(format("environment already open"));
-
-    debug(format("opening database environment"));
-
-
-    /* Create the database environment object. */
-    DbEnv * env = new DbEnv(0);
-    DestroyDbEnv deleteEnv(env);
-
-    env->set_errcall(errorPrinter);
-    env->set_msgcall(messagePrinter);
-    if (getEnv("NIX_DEBUG_DB_REGISTER") == "1")
-        env->set_verbose(DB_VERB_REGISTER, 1);
-    env->set_verbose(DB_VERB_RECOVERY, 1);
-    
-    /* Smaller log files. */
-    env->set_lg_bsize(32 * 1024); /* default */
-    env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */
-    
-    /* Write the log, but don't sync.  This protects transactions
-       against application crashes, but if the system crashes, some
-       transactions may be undone.  An acceptable risk, I think. */
-    env->set_flags(DB_TXN_WRITE_NOSYNC | DB_LOG_AUTOREMOVE, 1);
-    
-    /* Increase the locking limits.  If you ever get `Dbc::get: Cannot
-       allocate memory' or similar, especially while running
-       `nix-store --verify', just increase the following number, then
-       run db_recover on the database to remove the existing DB
-       environment (since changes only take effect on new
-       environments). */
-    env->set_lk_max_locks(10000);
-    env->set_lk_max_lockers(10000);
-    env->set_lk_max_objects(10000);
-    env->set_lk_detect(DB_LOCK_DEFAULT);
-    
-    /* Dangerous, probably, but from the docs it *seems* that BDB
-       shouldn't sync when DB_TXN_WRITE_NOSYNC is used, but it still
-       fsync()s sometimes. */
-    db_env_set_func_fsync(my_fsync);
-
-    
-    if (removeOldEnv) {
-        printMsg(lvlError, "removing old Berkeley DB database environment...");
-        env->remove(path.c_str(), DB_FORCE);
-        return;
-    }
-
-    openEnv(env, path, DB_REGISTER | DB_RECOVER);
-
-    deleteEnv.release();
-    this->env = env;
-}
-
-
-void Database::open(const string & path)
-{
-    try {
-
-        open2(path, false);
-        
-    } catch (DbException e) {
-        
-        if (e.get_errno() == DB_VERSION_MISMATCH) {
-            /* Remove the environment while we are holding the global
-               lock.  If things go wrong there, we bail out.
-               !!! argh, we abolished the global lock :-( */
-            open2(path, true);
-
-            /* Try again. */
-            open2(path, false);
-
-            /* Force a checkpoint, as per the BDB docs. */
-            env->txn_checkpoint(DB_FORCE, 0, 0);
-
-            printMsg(lvlError, "database succesfully upgraded to new version");
-        }
-
-#if 0        
-        else if (e.get_errno() == DB_RUNRECOVERY) {
-            /* If recovery is needed, do it. */
-            printMsg(lvlError, "running recovery...");
-            open2(path, false, true);
-        }
-#endif        
-        
-        else
-            rethrow(e);
-    }
-}
-
-
-void Database::close()
-{
-    if (!env) return;
-
-    /* Close the database environment. */
-    debug(format("closing database environment"));
-
-    try {
-
-        for (std::map<TableId, Db *>::iterator i = tables.begin();
-             i != tables.end(); )
-        {
-            std::map<TableId, Db *>::iterator j = i;
-            ++j;
-            closeTable(i->first);
-            i = j;
-        }
-
-        /* Do a checkpoint every 128 kilobytes, or every 5 minutes. */
-        env->txn_checkpoint(128, 5, 0);
-        
-        env->close(0);
-
-    } catch (DbException e) { rethrow(e); }
-
-    delete env;
-
-    env = 0;
-}
-
-
-TableId Database::openTable(const string & tableName, bool sorted)
-{
-    requireEnv();
-    TableId table = nextId++;
-
-    try {
-
-        Db * db = new Db(env, 0);
-
-        try {
-            db->open(0, tableName.c_str(), 0, 
-                sorted ? DB_BTREE : DB_HASH,
-                DB_CREATE | DB_AUTO_COMMIT, 0666);
-        } catch (...) {
-            delete db;
-            throw;
-        }
-
-        tables[table] = db;
-
-    } catch (DbException e) { rethrow(e); }
-
-    return table;
-}
-
-
-void Database::closeTable(TableId table)
-{
-    try {
-        Db * db = getDb(table);
-        db->close(DB_NOSYNC);
-        delete db;
-        tables.erase(table);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::deleteTable(const string & table)
-{
-    try {
-        env->dbremove(0, table.c_str(), 0, DB_AUTO_COMMIT);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-bool Database::queryString(const Transaction & txn, TableId table, 
-    const string & key, string & data)
-{
-    checkInterrupt();
-
-    try {
-        Db * db = getDb(table);
-
-        Dbt kt((void *) key.c_str(), key.length());
-        Dbt dt;
-
-        int err = db->get(txn.txn, &kt, &dt, 0);
-        if (err) return false;
-
-        if (!dt.get_data())
-            data = "";
-        else
-            data = string((char *) dt.get_data(), dt.get_size());
-    
-    } catch (DbException e) { rethrow(e); }
-
-    return true;
-}
-
-
-bool Database::queryStrings(const Transaction & txn, TableId table, 
-    const string & key, Strings & data)
-{
-    string d;
-    if (!queryString(txn, table, key, d))
-        return false;
-    data = unpackStrings(d);
-    return true;
-}
-
-
-void Database::setString(const Transaction & txn, TableId table,
-    const string & key, const string & data)
-{
-    checkInterrupt();
-    try {
-        Db * db = getDb(table);
-        Dbt kt((void *) key.c_str(), key.length());
-        Dbt dt((void *) data.c_str(), data.length());
-        db->put(txn.txn, &kt, &dt, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::setStrings(const Transaction & txn, TableId table,
-    const string & key, const Strings & data, bool deleteEmpty)
-{
-    if (deleteEmpty && data.size() == 0)
-        delPair(txn, table, key);
-    else
-        setString(txn, table, key, packStrings(data));
-}
-
-
-void Database::delPair(const Transaction & txn, TableId table,
-    const string & key)
-{
-    checkInterrupt();
-    try {
-        Db * db = getDb(table);
-        Dbt kt((void *) key.c_str(), key.length());
-        db->del(txn.txn, &kt, 0);
-        /* Non-existence of a pair with the given key is not an
-           error. */
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-void Database::enumTable(const Transaction & txn, TableId table,
-    Strings & keys, const string & keyPrefix)
-{
-    try {
-        Db * db = getDb(table);
-
-        Dbc * dbc;
-        db->cursor(txn.txn, &dbc, 0);
-        DestroyDbc destroyDbc(dbc);
-
-        Dbt kt, dt;
-        u_int32_t flags = DB_NEXT;
-
-        if (!keyPrefix.empty()) {
-            flags = DB_SET_RANGE;
-            kt = Dbt((void *) keyPrefix.c_str(), keyPrefix.size());
-        }
-
-        while (dbc->get(&kt, &dt, flags) != DB_NOTFOUND) {
-            checkInterrupt();
-            string data((char *) kt.get_data(), kt.get_size());
-            if (!keyPrefix.empty() &&
-                string(data, 0, keyPrefix.size()) != keyPrefix)
-                break;
-            keys.push_back(data);
-            flags = DB_NEXT;
-        }
-
-    } catch (DbException e) { rethrow(e); }
-}
-
- 
-void Database::clearTable(const Transaction & txn, TableId table)
-{
-    try {
-        Db * db = getDb(table);
-        u_int32_t count;
-        db->truncate(txn.txn, &count, 0);
-    } catch (DbException e) { rethrow(e); }
-}
-
-
-}
-
-#endif

src/libstore/db.hh

@@ -1,107 +0,0 @@
-#ifndef __DB_H
-#define __DB_H
-
-#include "types.hh"
-
-#include <map>
-
-
-/* Defined externally. */
-class DbTxn;
-class DbEnv;
-class Db;
-
-
-namespace nix {
-
-
-class Database;
-
-
-class Transaction
-{
-    friend class Database;
-
-private:
-    DbTxn * txn;
-    
-public:
-    Transaction();
-    Transaction(Database & _db);
-    ~Transaction();
-
-    void begin(Database & db);
-    void abort();
-    void commit();
-
-    void moveTo(Transaction & t);
-};
-
-
-#define noTxn Transaction()
-
-
-typedef unsigned int TableId; /* table handles */
-
-
-class Database
-{
-    friend class Transaction;
-
-private:
-    DbEnv * env;
-
-    TableId nextId;
-    std::map<TableId, Db *> tables;
-
-    void requireEnv();
-
-    Db * getDb(TableId table);
-
-    void open2(const string & path, bool removeOldEnv);
-    
-public:
-    Database();
-    ~Database();
-    
-    void open(const string & path);
-    void close();
-
-    TableId openTable(const string & table, bool sorted = false);
-    void closeTable(TableId table);
-    void deleteTable(const string & table);
-
-    bool queryString(const Transaction & txn, TableId table, 
-        const string & key, string & data);
-
-    bool queryStrings(const Transaction & txn, TableId table, 
-        const string & key, Strings & data);
-
-    void setString(const Transaction & txn, TableId table,
-        const string & key, const string & data);
-
-    void setStrings(const Transaction & txn, TableId table,
-        const string & key, const Strings & data,
-        bool deleteEmpty = true);
-
-    void delPair(const Transaction & txn, TableId table,
-        const string & key);
-
-    void enumTable(const Transaction & txn, TableId table,
-        Strings & keys, const string & keyPrefix = "");
-
-    void clearTable(const Transaction & txn, TableId table);
-};
-
-
-class DbNoPermission : public Error
-{
-public:
-    DbNoPermission(const format & f) : Error(f) { };
-};
-
- 
-}
-
-
-#endif /* !__DB_H */

src/libstore/derivations-ast.def

@@ -1,10 +0,0 @@
-init initDerivationsHelpers
-
-Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
-
-| string string | ATerm | EnvBinding |
-| string ATermList | ATerm | DerivationInput |
-| string string string string | ATerm | DerivationOutput |
-
-Closure | ATermList ATermList | ATerm | OldClosure |
-| string ATermList | ATerm | OldClosureElem |

src/libstore/derivations.cc

@@ -1,22 +1,12 @@
 #include "derivations.hh"
 #include "store-api.hh"
-#include "aterm.hh"
 #include "globals.hh"
 #include "util.hh"
 
-#include "derivations-ast.hh"
-#include "derivations-ast.cc"
-
 
 namespace nix {
 
 
-Hash hashTerm(ATerm t)
-{
-    return hashString(htSHA256, atPrint(t));
-}
-
-
 Path writeDerivation(const Derivation & drv, const string & name)
 {
     PathSet references;
@@ -27,137 +17,151 @@ Path writeDerivation(const Derivation & drv, const string & name)
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
     string suffix = name + drvExtension;
-    string contents = atPrint(unparseDerivation(drv));
+    string contents = unparseDerivation(drv);
     return readOnlyMode
         ? computeStorePathForText(suffix, contents, references)
         : store->addTextToStore(suffix, contents, references);
 }
 
 
-static void checkPath(const string & s)
+static Path parsePath(std::istream & str)
 {
+    string s = parseString(str);
     if (s.size() == 0 || s[0] != '/')
         throw Error(format("bad path `%1%' in derivation") % s);
+    return s;
 }
     
 
-static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
+static StringSet parseStrings(std::istream & str, bool arePaths)
 {
-    for (ATermIterator i(paths); i; ++i) {
-        if (ATgetType(*i) != AT_APPL)
-            throw badTerm("not a path", *i);
-        string s = aterm2String(*i);
-        if (arePaths) checkPath(s);
-        out.insert(s);
-    }
+    StringSet res;
+    while (!endOfList(str))
+        res.insert(arePaths ? parsePath(str) : parseString(str));
+    return res;
 }
+    
 
-
-/* Shut up warnings. */
-void throwBadDrv(ATerm t) __attribute__ ((noreturn));
-
-void throwBadDrv(ATerm t) 
-{
-    throw badTerm("not a valid derivation", t);
-}
-
-
-Derivation parseDerivation(ATerm t)
+Derivation parseDerivation(const string & s)
 {
     Derivation drv;
-    ATermList outs, inDrvs, inSrcs, args, bnds;
-    ATerm builder, platform;
+    std::istringstream str(s);
+    expect(str, "Derive([");
 
-    if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds))
-        throwBadDrv(t);
-
-    for (ATermIterator i(outs); i; ++i) {
-        ATerm id, path, hashAlgo, hash;
-        if (!matchDerivationOutput(*i, id, path, hashAlgo, hash))
-            throwBadDrv(t);
+    /* Parse the list of outputs. */
+    while (!endOfList(str)) {
         DerivationOutput out;
-        out.path = aterm2String(path);
-        checkPath(out.path);
-        out.hashAlgo = aterm2String(hashAlgo);
-        out.hash = aterm2String(hash);
-        drv.outputs[aterm2String(id)] = out;
+        expect(str, "("); string id = parseString(str);
+        expect(str, ","); out.path = parsePath(str);
+        expect(str, ","); out.hashAlgo = parseString(str);
+        expect(str, ","); out.hash = parseString(str);
+        expect(str, ")");
+        drv.outputs[id] = out;
     }
 
-    for (ATermIterator i(inDrvs); i; ++i) {
-        ATerm drvPath;
-        ATermList ids;
-        if (!matchDerivationInput(*i, drvPath, ids))
-            throwBadDrv(t);
-        Path drvPath2 = aterm2String(drvPath);
-        checkPath(drvPath2);
-        StringSet ids2;
-        parseStrings(ids, ids2, false);
-        drv.inputDrvs[drvPath2] = ids2;
+    /* Parse the list of input derivations. */
+    expect(str, ",[");
+    while (!endOfList(str)) {
+        expect(str, "(");
+        Path drvPath = parsePath(str);
+        expect(str, ",[");
+        drv.inputDrvs[drvPath] = parseStrings(str, false);
+        expect(str, ")");
+    }
+
+    expect(str, ",["); drv.inputSrcs = parseStrings(str, true);
+    expect(str, ","); drv.platform = parseString(str);
+    expect(str, ","); drv.builder = parseString(str);
+
+    /* Parse the builder arguments. */
+    expect(str, ",[");
+    while (!endOfList(str))
+        drv.args.push_back(parseString(str));
+
+    /* Parse the environment variables. */
+    expect(str, ",[");
+    while (!endOfList(str)) {
+        expect(str, "("); string name = parseString(str);
+        expect(str, ","); string value = parseString(str);
+        expect(str, ")");
+        drv.env[name] = value;
     }
     
-    parseStrings(inSrcs, drv.inputSrcs, true);
-
-    drv.builder = aterm2String(builder);
-    drv.platform = aterm2String(platform);
-    
-    for (ATermIterator i(args); i; ++i) {
-        if (ATgetType(*i) != AT_APPL)
-            throw badTerm("string expected", *i);
-        drv.args.push_back(aterm2String(*i));
-    }
-
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm s1, s2;
-        if (!matchEnvBinding(*i, s1, s2))
-            throw badTerm("tuple of strings expected", *i);
-        drv.env[aterm2String(s1)] = aterm2String(s2);
-    }
-
+    expect(str, ")");
     return drv;
 }
 
 
-ATerm unparseDerivation(const Derivation & drv)
+static void printString(string & res, const string & s)
 {
-    ATermList outputs = ATempty;
-    for (DerivationOutputs::const_reverse_iterator i = drv.outputs.rbegin();
-         i != drv.outputs.rend(); ++i)
-        outputs = ATinsert(outputs,
-            makeDerivationOutput(
-                toATerm(i->first),
-                toATerm(i->second.path),
-                toATerm(i->second.hashAlgo),
-                toATerm(i->second.hash)));
+    res += '"';
+    for (const char * i = s.c_str(); *i; i++)
+        if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
+        else if (*i == '\n') res += "\\n";
+        else if (*i == '\r') res += "\\r";
+        else if (*i == '\t') res += "\\t";
+        else res += *i;
+    res += '"';
+}
 
-    ATermList inDrvs = ATempty;
-    for (DerivationInputs::const_reverse_iterator i = drv.inputDrvs.rbegin();
-         i != drv.inputDrvs.rend(); ++i)
-        inDrvs = ATinsert(inDrvs,
-            makeDerivationInput(
-                toATerm(i->first),
-                toATermList(i->second)));
+
+template<class ForwardIterator>
+static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
+{
+    res += '[';
+    bool first = true;
+    for ( ; i != j; ++i) {
+        if (first) first = false; else res += ',';
+        printString(res, *i);
+    }
+    res += ']';
+}
+
+
+string unparseDerivation(const Derivation & drv)
+{
+    string s;
+    s.reserve(65536);
+    s += "Derive([";
+
+    bool first = true;
+    foreach (DerivationOutputs::const_iterator, i, drv.outputs) {
+        if (first) first = false; else s += ',';
+        s += '('; printString(s, i->first);
+        s += ','; printString(s, i->second.path);
+        s += ','; printString(s, i->second.hashAlgo);
+        s += ','; printString(s, i->second.hash);
+        s += ')';
+    }
+
+    s += "],[";
+    first = true;
+    foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) {
+        if (first) first = false; else s += ',';
+        s += '('; printString(s, i->first);
+        s += ','; printStrings(s, i->second.begin(), i->second.end());
+        s += ')';
+    }
+
+    s += "],";
+    printStrings(s, drv.inputSrcs.begin(), drv.inputSrcs.end());
     
-    ATermList args = ATempty;
-    for (Strings::const_reverse_iterator i = drv.args.rbegin();
-         i != drv.args.rend(); ++i)
-        args = ATinsert(args, toATerm(*i));
+    s += ','; printString(s, drv.platform);
+    s += ','; printString(s, drv.builder);
+    s += ','; printStrings(s, drv.args.begin(), drv.args.end());
 
-    ATermList env = ATempty;
-    for (StringPairs::const_reverse_iterator i = drv.env.rbegin();
-         i != drv.env.rend(); ++i)
-        env = ATinsert(env,
-            makeEnvBinding(
-                toATerm(i->first),
-                toATerm(i->second)));
-
-    return makeDerive(
-        outputs,
-        inDrvs,
-        toATermList(drv.inputSrcs),
-        toATerm(drv.platform),
-        toATerm(drv.builder),
-        args,
-        env);
+    s += ",[";
+    first = true;
+    foreach (StringPairs::const_iterator, i, drv.env) {
+        if (first) first = false; else s += ',';
+        s += '('; printString(s, i->first);
+        s += ','; printString(s, i->second);
+        s += ')';
+    }
+    
+    s += "])";
+    
+    return s;
 }
 
 

src/libstore/derivations.hh

@@ -1,12 +1,10 @@
 #ifndef __DERIVATIONS_H
 #define __DERIVATIONS_H
 
-typedef struct _ATerm * ATerm;
-
-#include "hash.hh"
-
 #include <map>
 
+#include "types.hh"
+
 
 namespace nix {
 
@@ -53,17 +51,14 @@ struct Derivation
 };
 
 
-/* Hash an aterm. */
-Hash hashTerm(ATerm t);
-
 /* Write a derivation to the Nix store, and return its path. */
 Path writeDerivation(const Derivation & drv, const string & name);
 
 /* Parse a derivation. */
-Derivation parseDerivation(ATerm t);
+Derivation parseDerivation(const string & s);
 
-/* Parse a derivation. */
-ATerm unparseDerivation(const Derivation & drv);
+/* Print a derivation. */
+string unparseDerivation(const Derivation & drv);
 
 /* Check whether a file name ends with the extensions for
    derivations. */

src/libstore/gc.cc

@@ -1,12 +1,12 @@
 #include "globals.hh"
 #include "misc.hh"
-#include "pathlocks.hh"
 #include "local-store.hh"
 
 #include <boost/shared_ptr.hpp>
 
 #include <functional>
 #include <queue>
+#include <algorithm>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -14,11 +14,6 @@
 #include <fcntl.h>
 #include <unistd.h>
 
-#ifdef __CYGWIN__
-#include <windows.h>
-#include <sys/cygwin.h>
-#endif
-
 
 namespace nix {
 
@@ -35,7 +30,7 @@ static const int defaultGcLevel = 1000;
    read.  To be precise: when they try to create a new temporary root
    file, they will block until the garbage collector has finished /
    yielded the GC lock. */
-static int openGCLock(LockType lockType)
+int LocalStore::openGCLock(LockType lockType)
 {
     Path fnGCLock = (format("%1%/%2%")
         % nixStateDir % gcLockName).str();
@@ -131,7 +126,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
        Instead of reading all the roots, it would be more efficient to
        check if the root is in a directory in or linked from the
        gcroots directory. */
-    if (queryBoolSetting("gc-check-reachability", true)) {
+    if (queryBoolSetting("gc-check-reachability", false)) {
         Roots roots = store->findRoots();
         if (roots.find(gcRoot) == roots.end())
             printMsg(lvlError, 
@@ -140,7 +135,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
                     "therefore, `%2%' might be removed by the garbage collector")
                 % gcRoot % storePath);
     }
-        
+
     /* Grab the global GC root, causing us to block while a GC is in
        progress.  This prevents the set of permanent roots from
        increasing while a GC is in progress. */
@@ -178,15 +173,6 @@ void LocalStore::addTempRoot(const Path & path)
 
             fdGCLock.close();
       
-	    /* Note that on Cygwin a lot of the following complexity
-	       is unnecessary, since we cannot delete open lock
-	       files.  If we have the lock file open, then it's valid;
-	       if we can delete it, then it wasn't in use any more. 
-
-	       Also note that on Cygwin we cannot "upgrade" a lock
-	       from a read lock to a write lock. */
-
-#ifndef __CYGWIN__
             debug(format("acquiring read lock on `%1%'") % fnTempRoots);
             lockFile(fdTempRoots, ltRead, true);
 
@@ -200,10 +186,6 @@ void LocalStore::addTempRoot(const Path & path)
             /* The garbage collector deleted this file before we could
                get a lock.  (It won't delete the file after we get a
                lock.)  Try again. */
-
-#else
-            break;
-#endif
         }
 
     }
@@ -216,14 +198,9 @@ void LocalStore::addTempRoot(const Path & path)
     string s = path + '\0';
     writeFull(fdTempRoots, (const unsigned char *) s.c_str(), s.size());
 
-#ifndef __CYGWIN__
     /* Downgrade to a read lock. */
     debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
     lockFile(fdTempRoots, ltRead, true);
-#else
-    debug(format("releasing write lock on `%1%'") % fnTempRoots);
-    lockFile(fdTempRoots, ltNone, true);
-#endif
 }
 
 
@@ -251,19 +228,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
         Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
 
         debug(format("reading temporary root file `%1%'") % path);
-
-#ifdef __CYGWIN__
-	/* On Cygwin we just try to delete the lock file. */
-	char win32Path[MAX_PATH];
-	cygwin_conv_to_full_win32_path(path.c_str(), win32Path);
-	if (DeleteFile(win32Path)) {
-            printMsg(lvlError, format("removed stale temporary roots file `%1%'")
-                % path);
-            continue;
-        } else
-            debug(format("delete of `%1%' failed: %2%") % path % GetLastError());
-#endif
-
         FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
         if (*fd == -1) {
             /* It's okay if the file has disappeared. */
@@ -275,7 +239,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
         //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
         //if (*fd == -1) continue;
 
-#ifndef __CYGWIN__
         /* Try to acquire a write lock without blocking.  This can
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
@@ -286,7 +249,6 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
             writeFull(*fd, (const unsigned char *) "d", 1);
             continue;
         }
-#endif
 
         /* Acquire a read lock.  This will prevent the owning process
            from upgrading to a write lock, therefore it will block in
@@ -401,7 +363,7 @@ static void addAdditionalRoots(PathSet & roots)
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
             if (roots.find(path) == roots.end() && store->isValidPath(path)) {
-                debug(format("found additional root `%1%'") % path);
+                debug(format("got additional root `%1%'") % path);
                 roots.insert(path);
             }
         }
@@ -439,147 +401,179 @@ Paths topoSortPaths(const PathSet & paths)
 }
 
 
-static time_t lastFileAccessTime(const Path & path)
-{
-    checkInterrupt();
-    
-    struct stat st;
-    if (lstat(path.c_str(), &st) == -1)
-        throw SysError(format("statting `%1%'") % path);
-
-    if (S_ISDIR(st.st_mode)) {
-        time_t last = 0;
-	Strings names = readDirectory(path);
-	foreach (Strings::iterator, i, names) {
-            time_t t = lastFileAccessTime(path + "/" + *i);
-            if (t > last) last = t;
-        }
-        return last;
-    }
-
-    else if (S_ISLNK(st.st_mode)) return 0;
-
-    else return st.st_atime;
-}
-
-
 struct GCLimitReached { };
 
 
-void LocalStore::gcPath(const GCOptions & options, GCResults & results, 
-    const Path & path)
+struct LocalStore::GCState
 {
-    results.paths.insert(path);
-
-    if (!pathExists(path)) return;
-                
-    /* Okay, it's safe to delete. */
-    unsigned long long bytesFreed, blocksFreed;
-    deleteFromStore(path, bytesFreed, blocksFreed);
-    results.bytesFreed += bytesFreed;
-    results.blocksFreed += blocksFreed;
-
-    if (options.maxFreed && results.bytesFreed > options.maxFreed) {
-        printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % options.maxFreed);
-        throw GCLimitReached();
-    }
-
-    if (options.maxLinks) {
-        struct stat st;
-        if (stat(nixStore.c_str(), &st) == -1)
-            throw SysError(format("statting `%1%'") % nixStore);
-        if (st.st_nlink < options.maxLinks) {
-            printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % options.maxLinks);
-            throw GCLimitReached();
-        }
-    }
-}
-
-
-void LocalStore::gcPathRecursive(const GCOptions & options,
-    GCResults & results, PathSet & done, const Path & path)
-{
-    if (done.find(path) != done.end()) return;
-    done.insert(path);
-
-    startNest(nest, lvlDebug, format("looking at `%1%'") % path);
-        
-    /* Delete all the referrers first.  They must be garbage too,
-       since if they were live, then the current path would also be
-       live.  Note that deleteFromStore() below still makes sure that
-       the referrer set has become empty, just in case.  (However that
-       doesn't guard against deleting top-level paths that are only
-       reachable from GC roots.) */
-    PathSet referrers;
-    if (isValidPath(path))
-        queryReferrers(path, referrers);
-    foreach (PathSet::iterator, i, referrers)
-        if (*i != path) gcPathRecursive(options, results, done, *i);
-
-    printMsg(lvlInfo, format("deleting `%1%'") % path);
-            
-    gcPath(options, results, path);
-}
-
-
-struct CachingAtimeComparator : public std::binary_function<Path, Path, bool> 
-{
-    std::map<Path, time_t> cache;
-
-    time_t lookup(const Path & p)
+    GCOptions options;
+    GCResults & results;
+    PathSet roots;
+    PathSet tempRoots;
+    PathSet deleted;
+    PathSet live;
+    PathSet busy;
+    bool gcKeepOutputs;
+    bool gcKeepDerivations;
+    GCState(GCResults & results_) : results(results_)
     {
-        std::map<Path, time_t>::iterator i = cache.find(p);
-        if (i != cache.end()) return i->second;
-        debug(format("computing atime of `%1%'") % p);
-        cache[p] = lastFileAccessTime(p);
-        assert(cache.find(p) != cache.end());
-        return cache[p];
-    }
-        
-    bool operator () (const Path & p1, const Path & p2)
-    {
-        return lookup(p2) < lookup(p1);
     }
 };
 
 
-static string showTime(const string & format, time_t t)
+static bool shouldDelete(GCOptions::GCAction action)
 {
-    char s[128];
-    strftime(s, sizeof s, format.c_str(), localtime(&t));
-    return string(s);
+    return action == GCOptions::gcDeleteDead
+        || action == GCOptions::gcDeleteSpecific;
 }
 
 
-static bool isLive(const Path & path, const PathSet & livePaths,
-    const PathSet & tempRoots, const PathSet & tempRootsClosed)
+bool LocalStore::isActiveTempFile(const GCState & state,
+    const Path & path, const string & suffix)
 {
-    if (livePaths.find(path) != livePaths.end() ||
-        tempRootsClosed.find(path) != tempRootsClosed.end()) return true;
+    return hasSuffix(path, suffix)
+        && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end();
+}
 
-    /* A lock file belonging to a path that we're building right
-       now isn't garbage. */
-    if (hasSuffix(path, ".lock") && tempRoots.find(string(path, 0, path.size() - 5)) != tempRoots.end())
-        return true;
+    
+bool LocalStore::tryToDelete(GCState & state, const Path & path)
+{
+    checkInterrupt();
+    
+    if (!pathExists(path)) return true;
+    if (state.deleted.find(path) != state.deleted.end()) return true;
+    if (state.live.find(path) != state.live.end()) return false;
 
-    /* Don't delete .chroot directories for derivations that are
-       currently being built. */
-    if (hasSuffix(path, ".chroot") && tempRoots.find(string(path, 0, path.size() - 7)) != tempRoots.end())
-        return true;
+    startNest(nest, lvlDebug, format("considering whether to delete `%1%'") % path);
 
+    if (state.roots.find(path) != state.roots.end()) {
+        printMsg(lvlDebug, format("cannot delete `%1%' because it's a root") % path);
+        goto isLive;
+    }
+
+    if (isValidPath(path)) {
+
+        /* Recursively try to delete the referrers of this path.  If
+           any referrer can't be deleted, then this path can't be
+           deleted either. */
+        PathSet referrers;
+        queryReferrers(path, referrers);
+        foreach (PathSet::iterator, i, referrers)
+            if (*i != path && !tryToDelete(state, *i)) {
+                printMsg(lvlDebug, format("cannot delete `%1%' because it has live referrers") % path);
+                goto isLive;
+            }
+
+        /* If gc-keep-derivations is set and this is a derivation,
+           then don't delete the derivation if any of the outputs are
+           live. */
+        if (state.gcKeepDerivations && isDerivation(path)) {
+            PathSet outputs = queryDerivationOutputs(path);
+            foreach (PathSet::iterator, i, outputs)
+                if (!tryToDelete(state, *i)) {
+                    printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output `%2%' is alive") % path % *i);
+                    goto isLive;
+                }
+        }
+
+        /* If gc-keep-derivations and gc-keep-outputs are both set,
+           it's possible that the path has already been deleted (due
+           to the recursion below), so bail out. */
+        if (!pathExists(path)) return true;
+
+        /* If gc-keep-outputs is set, then don't delete this path if
+           there are derivers of this path that are not garbage. */
+        if (state.gcKeepOutputs) {
+            PathSet derivers = queryValidDerivers(path);
+            foreach (PathSet::iterator, deriver, derivers) {
+                /* Break an infinite recursion if gc-keep-derivations
+                   and gc-keep-outputs are both set by tentatively
+                   assuming that this path is garbage.  This is a safe
+                   assumption because at this point, the only thing
+                   that can prevent it from being garbage is the
+                   deriver.  Since tryToDelete() works "upwards"
+                   through the dependency graph, it won't encouter
+                   this path except in the call to tryToDelete() in
+                   the gc-keep-derivation branch. */
+                state.deleted.insert(path);
+                if (!tryToDelete(state, *deriver)) {
+                    state.deleted.erase(path);
+                    printMsg(lvlDebug, format("cannot delete `%1%' because its deriver `%2%' is alive") % path % *deriver);
+                    goto isLive;
+                }
+            }
+        }
+    }
+
+    else {
+
+        /* A lock file belonging to a path that we're building right
+           now isn't garbage. */
+        if (isActiveTempFile(state, path, ".lock")) return false;
+
+        /* Don't delete .chroot directories for derivations that are
+           currently being built. */
+        if (isActiveTempFile(state, path, ".chroot")) return false;
+
+    }
+
+    /* The path is garbage, so delete it. */
+    if (shouldDelete(state.options.action)) {
+        printMsg(lvlInfo, format("deleting `%1%'") % path);
+
+        unsigned long long bytesFreed, blocksFreed;
+        deleteFromStore(path, bytesFreed, blocksFreed);
+        state.results.bytesFreed += bytesFreed;
+        state.results.blocksFreed += blocksFreed;
+
+        if (state.options.maxFreed && state.results.bytesFreed > state.options.maxFreed) {
+            printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % state.options.maxFreed);
+            throw GCLimitReached();
+        }
+
+        if (state.options.maxLinks) {
+            struct stat st;
+            if (stat(nixStore.c_str(), &st) == -1)
+                throw SysError(format("statting `%1%'") % nixStore);
+            if (st.st_nlink < state.options.maxLinks) {
+                printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % state.options.maxLinks);
+                throw GCLimitReached();
+            }
+        }
+        
+    } else
+        printMsg(lvlTalkative, format("would delete `%1%'") % path);
+    
+    state.deleted.insert(path);
+    if (state.options.action != GCOptions::gcReturnLive)
+        state.results.paths.insert(path);
+    return true;
+
+ isLive:
+    state.live.insert(path);
+    if (state.options.action == GCOptions::gcReturnLive)
+        state.results.paths.insert(path);
     return false;
 }
 
 
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    bool gcKeepOutputs =
-        queryBoolSetting("gc-keep-outputs", false);
-    bool gcKeepDerivations =
-        queryBoolSetting("gc-keep-derivations", true);
-    int gcKeepOutputsThreshold = 
-        queryIntSetting ("gc-keep-outputs-threshold", defaultGcLevel);
+    GCState state(results);
+    state.options = options;    
+    
+    state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false);
+    state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true);
 
+    /* Using `--ignore-liveness' with `--delete' can have unintended
+       consequences if `gc-keep-outputs' or `gc-keep-derivations' are
+       true (the garbage collector will recurse into deleting the
+       outputs or derivers, respectively).  So disable them. */
+    if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
+        state.gcKeepOutputs = false;
+        state.gcKeepDerivations = false;
+    }
+    
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
        b) Processes from creating new temporary root files. */
@@ -590,198 +584,60 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printMsg(lvlError, format("finding garbage collector roots..."));
     Roots rootMap = options.ignoreLiveness ? Roots() : nix::findRoots(true);
 
-    PathSet roots;
-    foreach (Roots::iterator, i, rootMap) roots.insert(i->second);
+    foreach (Roots::iterator, i, rootMap) state.roots.insert(i->second);
 
     /* Add additional roots returned by the program specified by the
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
     if (!options.ignoreLiveness)
-        addAdditionalRoots(roots);
-
-    if (options.action == GCOptions::gcReturnRoots) {
-        results.paths = roots;
-        return;
-    }
-
-    /* Determine the live paths which is just the closure of the
-       roots under the `references' relation. */
-    printMsg(lvlError, format("computing live paths..."));
-    PathSet livePaths;
-    foreach (PathSet::const_iterator, i, roots)
-        computeFSClosure(canonPath(*i), livePaths);
-
-    if (gcKeepDerivations) {
-        foreach (PathSet::iterator, i, livePaths) {
-            /* Note that the deriver need not be valid (e.g., if we
-               previously ran the collector with `gcKeepDerivations'
-               turned off). */
-            Path deriver = queryDeriver(*i);
-            if (deriver != "" && isValidPath(deriver))
-                computeFSClosure(deriver, livePaths);
-        }
-    }
-
-    if (gcKeepOutputs) {
-        /* Hmz, identical to storePathRequisites in nix-store. */
-        foreach (PathSet::iterator, i, livePaths)
-            if (isDerivation(*i)) {
-                Derivation drv = derivationFromPath(*i);
-
-		string gcLevelStr = drv.env["__gcLevel"];
-		int gcLevel;
-		if (!string2Int(gcLevelStr, gcLevel))
-		    gcLevel = defaultGcLevel;
-		
-		if (gcLevel >= gcKeepOutputsThreshold)    
-		    foreach (DerivationOutputs::iterator, j, drv.outputs)
-			if (isValidPath(j->second.path))
-			    computeFSClosure(j->second.path, livePaths);
-            }
-    }
-
-    if (options.action == GCOptions::gcReturnLive) {
-        results.paths = livePaths;
-        return;
-    }
+        addAdditionalRoots(state.roots);
 
     /* Read the temporary roots.  This acquires read locks on all
        per-process temporary root files.  So after this point no paths
        can be added to the set of temporary roots. */
-    PathSet tempRoots;
     FDs fds;
-    readTempRoots(tempRoots, fds);
-
-    /* Close the temporary roots.  Note that we *cannot* do this in
-       readTempRoots(), because there we may not have all locks yet,
-       meaning that an invalid path can become valid (and thus add to
-       the references graph) after we have added it to the closure
-       (and computeFSClosure() assumes that the presence of a path
-       means that it has already been closed). */
-    PathSet tempRootsClosed;
-    foreach (PathSet::iterator, i, tempRoots)
-        if (isValidPath(*i))
-            computeFSClosure(*i, tempRootsClosed);
-        else
-            tempRootsClosed.insert(*i);
+    readTempRoots(state.tempRoots, fds);
+    state.roots.insert(state.tempRoots.begin(), state.tempRoots.end());
 
     /* After this point the set of roots or temporary roots cannot
        increase, since we hold locks on everything.  So everything
-       that is not currently in in `livePaths' or `tempRootsClosed'
-       can be deleted. */
-    
-    /* Read the Nix store directory to find all currently existing
-       paths and filter out all live paths. */
-    printMsg(lvlError, format("reading the Nix store..."));
-    PathSet storePaths;
-    
-    if (options.action != GCOptions::gcDeleteSpecific) {
-        Paths entries = readDirectory(nixStore);
-        foreach (Paths::iterator, i, entries) {
-            Path path = canonPath(nixStore + "/" + *i);
-            if (!isLive(path, livePaths, tempRoots, tempRootsClosed)) storePaths.insert(path);
-        }
-    }
+       that is not reachable from `roots'. */
+
+    /* Now either delete all garbage paths, or just the specified
+       paths (for gcDeleteSpecific). */
+
+    if (options.action == GCOptions::gcDeleteSpecific) {
 
-    else {
         foreach (PathSet::iterator, i, options.pathsToDelete) {
             assertStorePath(*i);
-            storePaths.insert(*i);
-            if (isLive(*i, livePaths, tempRoots, tempRootsClosed))
+            if (!tryToDelete(state, *i))
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
         }
-    }
-
-    if (options.action == GCOptions::gcReturnDead) {
-        results.paths.insert(storePaths.begin(), storePaths.end());
-        return;
-    }
-
-    /* Delete all dead store paths (or until one of the stop
-       conditions is reached). */
-
-    PathSet done;
-    try {
-
-        if (!options.useAtime) {
-            /* Delete the paths, respecting the partial ordering
-               determined by the references graph. */
-            printMsg(lvlError, format("deleting garbage..."));
-            foreach (PathSet::iterator, i, storePaths)
-                gcPathRecursive(options, results, done, *i);
-        }
-
-        else {
-
-            /* Delete in order of ascending last access time, still
-               maintaining the partial ordering of the reference
-               graph.  Note that we can't use a topological sort for
-               this because that takes time O(V+E), and in this case
-               E=O(V^2) (i.e. the graph is dense because of the edges
-               due to the atime ordering).  So instead we put all
-               deletable paths in a priority queue (ordered by atime),
-               and after deleting a path, add additional paths that
-               have become deletable to the priority queue. */
-
-            CachingAtimeComparator atimeComp;
-
-            /* Create a priority queue that orders paths by ascending
-               atime.  This is why C++ needs type inferencing... */
-            std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> > prioQueue =
-                std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> >(binary_function_ref_adapter<CachingAtimeComparator>(&atimeComp));
-
-           /* Initially put the paths that are invalid or have no
-              referrers into the priority queue. */
-            printMsg(lvlError, format("finding deletable paths..."));
-            foreach (PathSet::iterator, i, storePaths) {
-                checkInterrupt();
-                /* We can safely delete a path if it's invalid or
-                   it has no referrers.  Note that all the invalid
-                   paths will be deleted in the first round. */
-                if (isValidPath(*i)) {
-                    if (queryReferrersNoSelf(*i).empty()) prioQueue.push(*i);
-                } else prioQueue.push(*i);
-            }
-
-            debug(format("%1% initially deletable paths") % prioQueue.size());
-
-            /* Now delete everything in the order of the priority
-               queue until nothing is left. */
-            printMsg(lvlError, format("deleting garbage..."));
-            while (!prioQueue.empty()) {
-                checkInterrupt();
-                Path path = prioQueue.top(); prioQueue.pop();
-
-                if (options.maxAtime != (time_t) -1 &&
-                    atimeComp.lookup(path) > options.maxAtime)
-                    continue;
-                
-                printMsg(lvlInfo, format("deleting `%1%' (last accessed %2%)") % path % showTime("%F %H:%M:%S", atimeComp.lookup(path)));
-
-                PathSet references;
-                if (isValidPath(path)) references = queryReferencesNoSelf(path);
-
-                gcPath(options, results, path);
-
-                /* For each reference of the current path, see if the
-                   reference has now become deletable (i.e. is in the
-                   set of dead paths and has no referrers left).  If
-                   so add it to the priority queue. */
-                foreach (PathSet::iterator, i, references) {
-                    if (storePaths.find(*i) != storePaths.end() &&
-                        queryReferrersNoSelf(*i).empty())
-                    {
-                        debug(format("path `%1%' has become deletable") % *i);
-                        prioQueue.push(*i);
-                    }
-                }
-            }
-            
-        }
         
-    } catch (GCLimitReached & e) {
-    }
+    } else {
+        
+        printMsg(lvlError, format("reading the Nix store..."));
+        Paths entries = readDirectory(nixStore);
+
+        /* Randomise the order in which we delete entries to make the
+           collector less biased towards deleting paths that come
+           alphabetically first (e.g. /nix/store/000...).  This
+           matters when using --max-freed etc. */
+        vector<Path> entries_(entries.begin(), entries.end());
+        random_shuffle(entries_.begin(), entries_.end());
+
+        if (shouldDelete(state.options.action))
+            printMsg(lvlError, format("deleting garbage..."));
+        else
+            printMsg(lvlError, format("determining live/dead paths..."));
+    
+        try {
+            foreach (vector<Path>::iterator, i, entries_)
+                tryToDelete(state, canonPath(nixStore + "/" + *i));
+        } catch (GCLimitReached & e) {
+        }
+    }        
 }
 
 

src/libstore/globals.cc

@@ -13,7 +13,6 @@ string nixDataDir = "/UNINIT";
 string nixLogDir = "/UNINIT";
 string nixStateDir = "/UNINIT";
 string nixDBPath = "/UNINIT";
-string nixCacheFile = "/UNINIT";
 string nixConfDir = "/UNINIT";
 string nixLibexecDir = "/UNINIT";
 string nixBinDir = "/UNINIT";
@@ -21,8 +20,9 @@ string nixBinDir = "/UNINIT";
 bool keepFailed = false;
 bool keepGoing = false;
 bool tryFallback = false;
-Verbosity buildVerbosity = lvlInfo;
+Verbosity buildVerbosity = lvlError;
 unsigned int maxBuildJobs = 1;
+unsigned int buildCores = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
 time_t maxSilentTime = 0;

src/libstore/globals.hh

@@ -24,9 +24,6 @@ extern string nixStateDir;
 /* nixDBPath is the path name of our Berkeley DB environment. */
 extern string nixDBPath;
 
-/* nixCacheFile is the path name of the normal form cache. */
-extern string nixCacheFile;
-
 /* nixConfDir is the directory where configuration files are
    stored. */
 extern string nixConfDir;
@@ -58,6 +55,11 @@ extern Verbosity buildVerbosity;
 /* Maximum number of parallel build jobs.  0 means unlimited. */
 extern unsigned int maxBuildJobs;
 
+/* Number of CPU cores to utilize in parallel within a build, i.e. by passing
+   this number to Make via '-j'. 0 means that the number of actual CPU cores on
+   the local host ought to be auto-detected. */
+extern unsigned int buildCores;
+
 /* Read-only mode.  Don't copy stuff to the store, don't change the
    database. */
 extern bool readOnlyMode;

src/libstore/local-store.hh

@@ -5,6 +5,11 @@
 
 #include "store-api.hh"
 #include "util.hh"
+#include "pathlocks.hh"
+
+
+class sqlite3;
+class sqlite3_stmt;
 
 
 namespace nix {
@@ -12,8 +17,9 @@ namespace nix {
 
 /* Nix store and database schema version.  Version 1 (or 0) was Nix <=
    0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
-   Version 4 is Nix 0.11.  Version 5 is Nix 0.12*/
-const int nixSchemaVersion = 5;
+   Version 4 is Nix 0.11.  Version 5 is Nix 0.12-0.16.  Version 6 is
+   Nix 1.0. */
+const int nixSchemaVersion = 6;
 
 
 extern string drvsLogDir;
@@ -41,6 +47,34 @@ struct RunningSubstituter
 };
 
 
+/* Wrapper object to close the SQLite database automatically. */
+struct SQLite
+{
+    sqlite3 * db;
+    SQLite() { db = 0; }
+    ~SQLite();
+    operator sqlite3 * () { return db; }
+};
+
+
+/* Wrapper object to create and destroy SQLite prepared statements. */
+struct SQLiteStmt
+{
+    sqlite3 * db;
+    sqlite3_stmt * stmt;
+    unsigned int curArg;
+    SQLiteStmt() { stmt = 0; }
+    void create(sqlite3 * db, const string & s);
+    void reset();
+    ~SQLiteStmt();
+    operator sqlite3_stmt * () { return stmt; }
+    void bind(const string & value);
+    void bind(int value);
+    void bind64(long long value);
+    void bind();
+};
+    
+
 class LocalStore : public StoreAPI
 {
 private:
@@ -64,6 +98,8 @@ public:
 
     PathSet queryValidPaths();
     
+    ValidPathInfo queryPathInfo(const Path & path);
+
     Hash queryPathHash(const Path & path);
 
     void queryReferences(const Path & path, PathSet & references);
@@ -71,6 +107,14 @@ public:
     void queryReferrers(const Path & path, PathSet & referrers);
 
     Path queryDeriver(const Path & path);
+
+    /* Return all currently valid derivations that have `path' as an
+       output.  (Note that the result of `queryDeriver()' is the
+       derivation that was actually used to produce `path', which may
+       not exist anymore.) */
+    PathSet queryValidDerivers(const Path & path);
+
+    PathSet queryDerivationOutputs(const Path & path);
     
     PathSet querySubstitutablePaths();
     
@@ -132,8 +176,7 @@ public:
        execution of the derivation (or something equivalent).  Also
        register the hash of the file system contents of the path.  The
        hash must be a SHA-256 hash. */
-    void registerValidPath(const Path & path,
-        const Hash & hash, const PathSet & references, const Path & deriver);
+    void registerValidPath(const ValidPathInfo & info);
 
     void registerValidPaths(const ValidPathInfos & infos);
 
@@ -144,6 +187,10 @@ public:
     /* Query whether `path' previously failed to build. */
     bool hasPathFailed(const Path & path);
 
+    PathSet queryFailedPaths();
+
+    void clearFailedPaths(const PathSet & paths);
+
 private:
 
     Path schemaPath;
@@ -151,43 +198,66 @@ private:
     /* Lock file used for upgrading. */
     AutoCloseFD globalLock;
 
-    /* !!! The cache can grow very big.  Maybe it should be pruned
-       every once in a while. */
-    std::map<Path, ValidPathInfo> pathInfoCache;
+    /* The SQLite database object. */
+    SQLite db;
 
-    /* Store paths for which the referrers file must be purged. */
-    PathSet delayedUpdates;
+    /* Some precompiled SQLite statements. */
+    SQLiteStmt stmtRegisterValidPath;
+    SQLiteStmt stmtUpdatePathInfo;
+    SQLiteStmt stmtAddReference;
+    SQLiteStmt stmtQueryPathInfo;
+    SQLiteStmt stmtQueryReferences;
+    SQLiteStmt stmtQueryReferrers;
+    SQLiteStmt stmtInvalidatePath;
+    SQLiteStmt stmtRegisterFailedPath;
+    SQLiteStmt stmtHasPathFailed;
+    SQLiteStmt stmtQueryFailedPaths;
+    SQLiteStmt stmtClearFailedPath;
+    SQLiteStmt stmtAddDerivationOutput;
+    SQLiteStmt stmtQueryValidDerivers;
+    SQLiteStmt stmtQueryDerivationOutputs;
 
     int getSchema();
 
-    void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false);
+    void openDB(bool create);
 
-    ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false);
+    unsigned long long queryValidPathId(const Path & path);
 
+    unsigned long long addValidPath(const ValidPathInfo & info);
+        
+    void addReference(unsigned long long referrer, unsigned long long reference);
+    
+    void appendReferrer(const Path & from, const Path & to, bool lock);
+    
     void rewriteReferrers(const Path & path, bool purge, PathSet referrers);
 
-    void flushDelayedUpdates();
-    
-    bool queryReferrersInternal(const Path & path, PathSet & referrers);
-    
     void invalidatePath(const Path & path);
+
+    void verifyPath(const Path & path, const PathSet & store,
+        PathSet & done, PathSet & validPaths);
+
+    void updatePathInfo(const ValidPathInfo & info);
+
+    void upgradeStore6();
+    PathSet queryValidPathsOld();
+    ValidPathInfo queryPathInfoOld(const Path & path);
+
+    struct GCState;
+
+    bool tryToDelete(GCState & state, const Path & path);
+    
+    bool isActiveTempFile(const GCState & state,
+        const Path & path, const string & suffix);
+        
+    int openGCLock(LockType lockType);
     
-    void upgradeStore12();
-
-    void gcPath(const GCOptions & options, GCResults & results,
-        const Path & path);
-
-    void gcPathRecursive(const GCOptions & options,
-        GCResults & results, PathSet & done, const Path & path);
-
     void startSubstituter(const Path & substituter,
         RunningSubstituter & runningSubstituter);
+
+    Path createTempDirInStore();
 };
 
 
-/* Copy a path recursively. */
-void copyPath(const Path & src, const Path & dst);
-
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
    - the last modification date on each file is set to 1 (i.e.,