Fix typo.

useful for fields like meta.maintainers, meta.priority (which can be
  a proper integer now) and even meta.license (if there are multiple
  licenses).

Makefile.am

@@ -1,23 +1,19 @@
 SUBDIRS = externals src scripts corepkgs doc misc tests
 EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
-  svn-revision nix.conf.example NEWS
+  nix.conf.example NEWS version
 
 include ./substitute.mk
 
 nix.spec: nix.spec.in
 
-rpm: nix.spec dist
-	rpm $(EXTRA_RPM_FLAGS) -ta $(distdir).tar.gz
-
-relname:
-	echo -n $(distdir) > relname
-
 install-data-local: init-state
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 	$(INSTALL_DATA) $(srcdir)/nix.conf.example $(DESTDIR)$(sysconfdir)/nix
 	if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
 		$(INSTALL_DATA) $(srcdir)/nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
 	fi
+	$(INSTALL) -d $(DESTDIR)$(docdir)
+	$(INSTALL_DATA) README $(DESTDIR)$(docdir)/
 
 if INIT_STATE
 
@@ -37,25 +33,16 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
 	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
-	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(prefix)/store
+	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(storedir)
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
 	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
-#	$(bindir)/nix-store --init
 
 else
+
 init-state:
+
 endif
 
-init-ext3cow-header-hack:
-	@echo "Symlinking ext3cow header file into src"
-	ln -sf $(ext3cowheader) src/libext3cow/
-
-svn-revision:
-	svnversion . > svn-revision
-
-all: init-ext3cow-header-hack
-
-all-local: NEWS
-
-NEWS: doc/manual/NEWS.txt
+NEWS:
+	$(MAKE) -C doc/manual NEWS.txt
 	cp $(srcdir)/doc/manual/NEWS.txt NEWS

README

@@ -1,9 +1,10 @@
-For installation and usage instructions, please read the manual, which
-can be found in `docs/manual/manual.html', and additionally at the Nix
-website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
+Nix is a purely functional package manager.  For installation and
+usage instructions, please read the manual, which can be found in
+`docs/manual/manual.html', and additionally at the Nix website at
+<http://nixos.org/>.
 
 
 Acknowledgments
 
 This product includes software developed by the OpenSSL Project for
-use in the OpenSSL Toolkit (http://www.OpenSSL.org/)
+use in the OpenSSL Toolkit (http://www.OpenSSL.org/).

TODO

@@ -1,3 +0,0 @@
-- runtimeStateArgs now must be set to someting (or it will see it as a hardcoded path)
-- import and export of state paths
-- 

aterm-gc.supp

@@ -1,184 +1,117 @@
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:mark_memory_young
-   fun:mark_phase_young
+   fun:*
+   fun:AT_collect_minor
 }
 
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm_young
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_markTerm_young
-   fun:mark_memory_young
-   fun:mark_phase_young
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Value4
-   fun:AT_markTerm
-   fun:mark_memory
-   fun:mark_phase
-}
-
-{
-   ATerm library conservatively scans for GC roots
-   Memcheck:Cond
-   fun:mark_phase_young
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
    ATerm library conservatively scans for GC roots
    Memcheck:Value4
-   fun:mark_phase_young
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
-   Memcheck:Cond
-   fun:AT_isValidSymbol
-   fun:mark_phase_young
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
+   ATerm library conservatively scans for GC roots
    Memcheck:Value4
-   fun:AT_isValidSymbol
-   fun:mark_phase_young
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
-   Memcheck:Value4
-   fun:AT_isInsideValidTerm
-   fun:mark_phase_young
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:*
    fun:AT_collect_minor
 }
 
 {
-   <insert a suppression name here>
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr4
+   fun:*
+   fun:AT_collect_minor
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr8
+   fun:*
+   fun:AT_collect_minor
+}
+
+{
+   ATerm library conservatively scans for GC roots
    Memcheck:Cond
-   fun:AT_isInsideValidTerm
-   fun:mark_phase_young
-   fun:AT_collect_minor
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr4
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Addr8
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value4
+   fun:*
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Value8
+   fun:*
+   fun:*
+   fun:AT_collect
+}
+
+{
+   ATerm library conservatively scans for GC roots
+   Memcheck:Cond
+   fun:*
+   fun:*
+   fun:AT_collect
 }

bootstrap.sh

@@ -1,6 +1,6 @@
 #! /bin/sh -e
 mkdir -p config
-libtoolize --force --copy
+libtoolize --copy
 aclocal
 autoheader
 automake --add-missing --copy

configure.ac

@@ -1,24 +1,9 @@
-AC_INIT(nix, 0.12)
+AC_INIT(nix, m4_esyscmd([echo -n $(cat ./version)$VERSION_SUFFIX]))
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
 AM_INIT_AUTOMAKE([dist-bzip2 foreign])
-        
-# Change to `1' to produce a `stable' release (i.e., the `preREVISION'
-# suffix is not added).
-STABLE=0
 
-# Put the revision number in the version.
-if test "$STABLE" != "1"; then
-    if REVISION=`test -d $srcdir/.svn && svnversion -n $srcdir 2> /dev/null`; then
-        VERSION=${VERSION}pre${REVISION}
-    elif REVISION=`cat $srcdir/svn-revision 2> /dev/null`; then
-        VERSION=${VERSION}pre${REVISION}
-    fi
-fi
-
-AC_DEFINE_UNQUOTED(NIX_VERSION, ["$VERSION"], [version])
-
-AC_PREFIX_DEFAULT(/nix)
+AC_DEFINE_UNQUOTED(NIX_VERSION, ["$VERSION"], [Nix version.])
 
 AC_CANONICAL_HOST
 
@@ -54,21 +39,41 @@ case $sys_name in
 esac
 
 AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM],
-  [platform identifier (e.g., `i686-linux')]),
+  [Platform identifier (e.g., `i686-linux').]),
   system=$withval, system="${machine_name}-${sys_name}")
 AC_MSG_RESULT($system)
 AC_SUBST(system)
 AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')])
 
 
+# State should be stored in /nix/var, unless the user overrides it explicitly.
+test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
+
+
+# Whether to produce a statically linked binary.  On Cygwin, this is
+# the default: dynamically linking against the ATerm DLL does work,
+# except that it requires the ATerm "lib" directory to be in $PATH, as
+# Windows doesn't have anything like an RPATH embedded in executable.
+# Since this is kind of annoying, we use static libraries for now.
+    
+AC_ARG_ENABLE(static-nix, AC_HELP_STRING([--enable-static-nix],
+  [produce statically linked binaries]),
+  static_nix=$enableval, static_nix=no)
+
+if test "$sys_name" = cygwin; then
+   static_nix=yes
+fi
+  
+if test "$static_nix" = yes; then
+    AC_DISABLE_SHARED
+    AC_ENABLE_STATIC
+fi
+
+  
 # Windows-specific stuff.
 if test "$sys_name" = "cygwin"; then
     # We cannot delete open files.
     AC_DEFINE(CANNOT_DELETE_OPEN_FILES, 1, [Whether it is impossible to delete open files.])
-
-    # Shared libraries don't work, currently.
-    AC_DISABLE_SHARED
-    AC_ENABLE_STATIC
 fi
 
 
@@ -94,13 +99,15 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <iostream>
 using namespace std;
 static char buf[1024];]],
     [[cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));]])],
-    [AC_MSG_RESULT(yes) AC_DEFINE(HAVE_PUBSETBUF, 1, [whether pubsetbuf is available])],
+    [AC_MSG_RESULT(yes) AC_DEFINE(HAVE_PUBSETBUF, 1, [Whether pubsetbuf is available.])],
     AC_MSG_RESULT(no))
 AC_LANG_POP(C++)
 
 
 # Check for chroot support (requires chroot() and bind mounts).
 AC_CHECK_FUNCS([chroot])
+AC_CHECK_FUNCS([unshare])
+AC_CHECK_HEADERS([sched.h], [], [], [])
 AC_CHECK_HEADERS([sys/param.h], [], [], [])
 AC_CHECK_HEADERS([sys/mount.h], [], [],
 [#ifdef HAVE_SYS_PARAM_H
@@ -109,12 +116,17 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
 ])
 
 
-# Check for <locale>
+# Check for <locale>.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([locale], [], [], [])
 AC_LANG_POP(C++)
 
 
+# Check whether we have the personality() syscall, which allows us to
+# do i686-linux builds on x86_64-linux machines.
+AC_CHECK_HEADERS([sys/personality.h])
+
+
 AC_DEFUN([NEED_PROG],
 [
 AC_PATH_PROG($1, $2)
@@ -124,7 +136,7 @@ fi
 ])
 
 NEED_PROG(curl, curl)
-NEED_PROG(shell, bash)
+NEED_PROG(bash, bash)
 NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
@@ -136,6 +148,7 @@ NEED_PROG(perl, perl)
 NEED_PROG(tar, tar)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(dblatex, dblatex)
+AC_PATH_PROG(gzip, gzip)
 
 AC_PATH_PROG(openssl_prog, openssl, openssl) # if not found, call openssl in $PATH
 AC_SUBST(openssl_prog)
@@ -173,16 +186,16 @@ AC_SUBST(xmlflags)
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
   [path of the Nix store]),
-  storedir=$withval, storedir='${prefix}/store')
+  storedir=$withval, storedir='/nix/store')
 AC_SUBST(storedir)
 
-AC_ARG_WITH(store-state-dir, AC_HELP_STRING([--with-store-state-dir=PATH],
-  [path of the Nix state store]),
-  storestatedir=$withval, storestatedir='${prefix}/state')
-AC_SUBST(storestatedir)
+AC_ARG_ENABLE(old-db-compat, AC_HELP_STRING([--disable-old-db-compat],
+  [disable support for converting from old Berkeley DB-based Nix stores]),
+  old_db_compat=$enableval, old_db_compat=yes)
+AM_CONDITIONAL(OLD_DB_COMPAT, test "$old_db_compat" = "yes")
 
 AC_ARG_WITH(bdb, AC_HELP_STRING([--with-bdb=PATH],
-  [prefix of Berkeley DB]),
+  [prefix of Berkeley DB (for Nix <= 0.11 compatibility)]),
   bdb=$withval, bdb=)
 AM_CONDITIONAL(HAVE_BDB, test -n "$bdb")
 if test -z "$bdb"; then
@@ -192,21 +205,15 @@ else
   bdb_lib="-L$bdb/lib -ldb_cxx"
   bdb_include="-I$bdb/include"
 fi
+if test "$old_db_compat" = "no"; then
+  bdb_lib=
+  bdb_include=
+else
+  AC_DEFINE(OLD_DB_COMPAT, 1, [Whether to support converting from old Berkeley DB-based Nix stores.])
+fi
 AC_SUBST(bdb_lib)
 AC_SUBST(bdb_include)
 
-AC_ARG_WITH(ext3cow-header, AC_HELP_STRING([--with-ext3cow-header=PATH],
-  [path of the ext3cow header ext3cow_fs.h]),
-  ext3cowheader=$withval, ext3cowheader=)
-AC_SUBST(ext3cowheader)
-AC_CHECK_HEADER(${ext3cowheader})
-
-NEED_PROG(rsync, rsync)
-AC_ARG_WITH(rsync, AC_HELP_STRING([--with-rsync=PATH],
-  [path to the rsync binary.]),
-  rsync=$withval)
-AC_SUBST(rsync)
-
 AC_ARG_WITH(aterm, AC_HELP_STRING([--with-aterm=PATH],
   [prefix of CWI ATerm library]),
   aterm=$withval, aterm=)
@@ -232,7 +239,7 @@ if test -n "$openssl"; then
   LDFLAGS="-L$openssl/lib -lcrypto $LDFLAGS"
   CFLAGS="-I$openssl/include $CFLAGS"
   CXXFLAGS="-I$openssl/include $CXXFLAGS"
-  AC_DEFINE(HAVE_OPENSSL, 1, [whether to use OpenSSL])
+  AC_DEFINE(HAVE_OPENSSL, 1, [Whether to use OpenSSL.])
 fi
 
 AC_ARG_WITH(bzip2, AC_HELP_STRING([--with-bzip2=PATH],
@@ -245,7 +252,7 @@ if test -z "$bzip2"; then
   bzip2_lib='-L${top_builddir}/externals/inst-bzip2/lib -lbz2'
   bzip2_include='-I${top_builddir}/externals/inst-bzip2/include'
   # The binary will be copied to $libexecdir.
-  bzip2_bin='${libexecdir}'
+  bzip2_bin='${libexecdir}/nix'
   # But for testing, we have to use the temporary copy :-(
   bzip2_bin_test='${top_builddir}/externals/inst-bzip2/bin'
 else
@@ -275,6 +282,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown])
 
 # Nice to have, but not essential.
 AC_CHECK_FUNCS([strsignal])
+AC_CHECK_FUNCS([posix_fallocate])
 
 
 # This is needed if ATerm, Berkeley DB or bzip2 are static libraries,
@@ -284,6 +292,14 @@ if test "$(uname)" = "Darwin"; then
 fi
 
 
+if test "$static_nix" = yes; then
+    # `-all-static' has to be added at the end of configure, because
+    # the C compiler doesn't know about -all-static (it's filtered out
+    # by libtool, but configure doesn't use libtool).
+    LDFLAGS="-all-static $LDFLAGS"
+fi
+
+
 AM_CONFIG_HEADER([config.h])
 AC_CONFIG_FILES([Makefile
    externals/Makefile
@@ -294,9 +310,7 @@ AC_CONFIG_FILES([Makefile
    src/libutil/Makefile
    src/libstore/Makefile
    src/libmain/Makefile
-   src/libext3cow/Makefile
    src/nix-store/Makefile
-   src/nix-state/Makefile
    src/nix-hash/Makefile
    src/libexpr/Makefile
    src/nix-instantiate/Makefile

corepkgs/buildenv/builder.pl.in

@@ -3,39 +3,24 @@
 use strict;
 use Cwd;
 use IO::Handle;
-use Fcntl;
 
 STDOUT->autoflush(1);
 
 my $out = $ENV{"out"};
 mkdir "$out", 0755 || die "error creating $out";
 
-sub readlink_or_StateWrapper;
 
 my $symlinks = 0;
-my %path_state_identifier = ();
 
 my %priorities;
 
-my $nixBinDir = $ENV{"nixBinDir"};
-my $nixStore = $ENV{"nixStore"};
 
 # For each activated package, create symlinks.
 
 sub createLinks {
     my $srcDir = shift;
-    
-    #Lookup each $stateIdentifiers in $path_state_identifier
-    #we strip $srcDir to its rootdir e.g. /nix/store/......./
-    my @srcDirParts = split /\// , substr($srcDir, length ($nixStore), length ($srcDir));
-    my $srcDirRoot = $nixStore . "/" . $srcDirParts[1];
-	# print "srcDirRoot $srcDirRoot \n";
-    
     my $dstDir = shift;
     my $priority = shift;
-	my $pkgStateIdentifier = $path_state_identifier{$srcDirRoot};		# We have to look it up each time since recursion can change the $srcDir, but not the identifier
-
-	#print "createLinks $srcDir to $dstDir with iden $pkgStateIdentifier \n";
 
     my @srcFiles = glob("$srcDir/*");
 
@@ -44,21 +29,20 @@ sub createLinks {
         $baseName =~ s/^.*\///g; # strip directory
         my $dstFile = "$dstDir/$baseName";
 
-    	# Urgh, hacky...
-		if ($srcFile =~ /\/propagated-build-inputs$/ ||
+        # Urgh, hacky...
+	if ($srcFile =~ /\/propagated-build-inputs$/ ||
             $srcFile =~ /\/nix-support$/ ||
             $srcFile =~ /\/perllocal.pod$/ ||
             $srcFile =~ /\/info\/dir$/ ||
             $srcFile =~ /\/log$/)
         {
             # Do nothing.
-		}
+	}
 
         elsif (-d $srcFile) {
 
             lstat $dstFile;
             
-            #go recursive on directorys
             if (-d _) {
                 createLinks($srcFile, $dstFile, $priority);
             }
@@ -69,15 +53,13 @@ sub createLinks {
                     die "collission between directory `$srcFile' and non-directory `$target'";
                 }
                 unlink $dstFile or die "error unlinking `$dstFile': $!";
-                mkdir $dstFile, 0755 || die "error creating directory `$dstFile': $!";
+                mkdir $dstFile, 0755 || 
+                    die "error creating directory `$dstFile': $!";
                 createLinks($target, $dstFile, $priorities{$dstFile});
                 createLinks($srcFile, $dstFile, $priority);
             }
 
             else {
-                
-                #print "1ST DIR LINK $srcFile to $dstFile with iden $pkgStateIdentifier \n";
-                
                 symlink($srcFile, $dstFile) ||
                     die "error creating link `$dstFile': $!";
                 $priorities{$dstFile} = $priority;
@@ -87,67 +69,23 @@ sub createLinks {
 
         else {
 
-			# print "ELSE LINK $srcFile to $dstFile with iden $pkgStateIdentifier \n";
-
-			# if we have a state component with a identifier different then ""
-			if($pkgStateIdentifier ne "__NOSTATE__" && $pkgStateIdentifier ne ""){
-	            
-	            my @pathparts = split /\// , $srcFile;
-	            my $parentDir = $pathparts[scalar(@pathparts) - 2];
-	            
-	            if( $parentDir eq "bin" || $parentDir eq "sbin"){		#hacky....
-		            
-		            print "STATELINK $srcFile to $dstFile - $pkgStateIdentifier \n";
-		            
-		            my $new_dstFile;
-		            my $new_stateIdentifier;		            
-		            if($pkgStateIdentifier eq "__EMTPY__"){
-		            	$new_dstFile = $dstFile;
-		            	$new_stateIdentifier = "";
-		            }
-		            else{
-		            	$new_dstFile = "$dstFile-$pkgStateIdentifier";
-		            	$new_stateIdentifier = $pkgStateIdentifier;
-		            }
-		            
-		            # We also check with -e if the wrapperscript-file exists, and if is it a symlink (with -l)
-		            if (-l $new_dstFile || -e $new_dstFile) {
-			                my $target = readlink_or_StateWrapper $new_dstFile;
-			                die "(state) collission between `$srcFile' and `$target' (over $new_dstFile)";
-			        }
-		            
-		            sysopen (DSTFILEHANDLE, $new_dstFile, O_RDWR|O_EXCL|O_CREAT, 0755);
-					printf DSTFILEHANDLE "#! @shell@ \n";
-					printf DSTFILEHANDLE "$nixBinDir/nix-state --run --identifier=$new_stateIdentifier $srcFile \"\$@\" \n"; 
-					close (DSTFILEHANDLE);
-	            }
-	            
-			}
-			elsif($pkgStateIdentifier ne "__NOSTATE__" && $pkgStateIdentifier eq ""){		#TODO we now dont create symlinks for state packages with a empty identifier
-																							#TODO but we must do it if there is no normal non-state pacakge
-			}
-			
-	        else {
-
-	            if (-l $dstFile || -e $dstFile) {
-	                my $target = readlink_or_StateWrapper $dstFile;
-	                my $prevPriority = $priorities{$dstFile};
-	                die ( "Collission between `$srcFile' and `$target'. "
-	                    . "Suggested solution: use `nix-env --set-flag "
-	                    . "priority NUMBER PKGNAME' to change the priority of "
-	                    . "one of the conflicting packages.\n" )
-	                    if $prevPriority == $priority;
-	                next if $prevPriority < $priority;
-	                unlink $dstFile or die;
-	            }
-	
-				# print "2ND LINK $srcFile to $dstFile with iden $pkgStateIdentifier \n";
-	            symlink($srcFile, $dstFile) ||
-	                die "error creating link `$dstFile': $!";
-	            $priorities{$dstFile} = $priority;
-	            $symlinks++;
-	        }
-		}
+            if (-l $dstFile) {
+                my $target = readlink $dstFile;
+                my $prevPriority = $priorities{$dstFile};
+                die ( "Collission between `$srcFile' and `$target'. "
+                    . "Suggested solution: use `nix-env --set-flag "
+                    . "priority NUMBER PKGNAME' to change the priority of "
+                    . "one of the conflicting packages.\n" )
+                    if $prevPriority == $priority;
+                next if $prevPriority < $priority;
+                unlink $dstFile or die;
+            }
+            
+            symlink($srcFile, $dstFile) ||
+                die "error creating link `$dstFile': $!";
+            $priorities{$dstFile} = $priority;
+            $symlinks++;
+        }
     }
 }
 
@@ -178,24 +116,6 @@ sub addPkg {
     }
 }
 
-sub readlink_or_StateWrapper {
-
-	my $src = shift;
-	my $target;
-
-    if (-l $src)
-    {   $target = readlink $src;    }
-    else{
-    	open(DAT, $src) || die("Could not open file!");
-        my @raw_data=<DAT>;
-        close(DAT);
-        $target = $raw_data[1];
-    }
-    return $target
-}
-
-my @stateIdentifiers = split ' ', $ENV{"stateIdentifiers"};
-my $si_counter = 0;
 
 # Convert the stuff we get from the environment back into a coherent
 # data type.
@@ -211,23 +131,15 @@ my %pkgs;
 for (my $n = 0; $n < scalar @paths; $n++) {
     $pkgs{$paths[$n]} =
         { active => $active[$n]
-        , priority => $priority[$n] 
-        , stateidentifier => $stateIdentifiers[$n]
-        };
-        
-    $path_state_identifier{$paths[$n]} = $stateIdentifiers[$n];
+        , priority => $priority[$n] };
 }
 
+
 # Symlink to the packages that have been installed explicitly by the
 # user.
 foreach my $pkg (sort (keys %pkgs)) {
-
-	#print "SP: $pkg \n";
-	#print "SI: $pkgs{$pkg}->{stateidentifier} \n";
-    #print "PR: $pkgs{$pkg}->{priority} \n";	
-    
+    #print $pkg, " ", $pkgs{$pkg}->{priority}, "\n";
     addPkg($pkg, $pkgs{$pkg}->{priority}) if $pkgs{$pkg}->{active} ne "false";
-    $si_counter++;
 }
 
 
@@ -237,7 +149,6 @@ foreach my $pkg (sort (keys %pkgs)) {
 # priority in case of collisions.
 my $priorityCounter = 1000; # don't care about collisions
 while (scalar(keys %postponed) > 0) {
-    
     my @pkgDirs = keys %postponed;
     %postponed = ();
     foreach my $pkgDir (sort @pkgDirs) {
@@ -245,7 +156,8 @@ while (scalar(keys %postponed) > 0) {
     }
 }
 
+
 print STDERR "created $symlinks symlinks in user environment\n";
 
-symlink($ENV{"manifest"}, "$out/manifest") or die "cannot create manifest";
 
+symlink($ENV{"manifest"}, "$out/manifest") or die "cannot create manifest";

corepkgs/buildenv/default.nix

@@ -1,13 +1,11 @@
-{system, derivations, stateIdentifiers, manifest, nixBinDir, nixStore}:
+{system, derivations, manifest}:
 
 derivation { 
   name = "user-environment";
   system = system;
   builder = ./builder.pl;
   
-  stateIdentifiers = stateIdentifiers;
   manifest = manifest;
-  inherit nixBinDir nixStore;
 
   # !!! grmbl, need structured data for passing this in a clean way.
   paths = derivations;

corepkgs/channels/unpack.sh.in

@@ -8,9 +8,15 @@ inputs=($inputs)
 for ((n = 0; n < ${#inputs[*]}; n += 2)); do
     channelName=${inputs[n]}
     channelTarball=${inputs[n+1]}
+    
     echo "unpacking channel $channelName"
+    
     @bunzip2@ < $channelTarball | @tar@ xf -
 
+    if test -e */channel-name; then
+        channelName="$(@coreutils@/cat */channel-name)"
+    fi
+
     nr=1
     attrName=$(echo $channelName | @tr@ -- '- ' '__')
     dirName=$attrName

createRelease.sh

@@ -1,21 +0,0 @@
-#! /bin/sh -e
-
-dir1=releases
-dir2=nix-state
-mkdir -p $dir1
-cd $dir1
-rm -rf $dir2
-mkdir -p $dir2
-cd $dir2
-svn co https://svn.cs.uu.nl:12443/repos/trace/nix/branches/state ./
-revision=`svn info | grep ^Revision | sed 's/Revision: //g'`
-cd ..
-date=`date +%Y%m%d`
-tarfile=snix-${date}-rev${revision}.tar.gz
-tar -cvf $tarfile \
-    --preserve-permissions \
-    --atime-preserve \
-    --gzip \
-    --verbose \
-    --no-ignore-command-error \
-    $dir2/

doc/Makefile.am

@@ -1 +1 @@
-SUBDIRS = 
+SUBDIRS = manual

doc/manual/Makefile.am

@@ -14,15 +14,16 @@ XSLTPROC = $(xsltproc) $(xmlflags) \
 man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
  nix-prefetch-url.1 nix-channel.1 \
- nix-pack-closure.1 nix-unpack-closure.1 \
  nix-install-package.1 nix-hash.1 nix-copy-closure.1
 
+man8_MANS = nix-worker.8
+
 FIGURES = figures/user-environments.png
 
 MANUAL_SRCS = manual.xml introduction.xml installation.xml \
  package-management.xml writing-nix-expressions.xml builtins.xml \
  build-farm.xml \
- $(man1_MANS:.1=.xml) \
+ $(man1_MANS:.1=.xml) $(man8_MANS:.8=.xml) \
  troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
  env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
  conf-file.xml release-notes.xml \
@@ -76,12 +77,16 @@ NEWS.txt: release-notes.xml
 all-local: manual.html NEWS.html NEWS.txt
 
 install-data-local: manual.html
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/manual
-	$(INSTALL_DATA) manual.html $(DESTDIR)$(datadir)/nix/manual
-	$(INSTALL_DATA) style.css $(DESTDIR)$(datadir)/nix/manual
-	cp -r images $(DESTDIR)$(datadir)/nix/manual/images
-	$(INSTALL) -d $(DESTDIR)$(datadir)/nix/manual/figures
-	$(INSTALL_DATA) $(FIGURES) $(DESTDIR)$(datadir)/nix/manual/figures
+	$(INSTALL) -d $(DESTDIR)$(docdir)/manual
+	$(INSTALL_DATA) manual.html $(DESTDIR)$(docdir)/manual
+	ln -sf manual.html $(DESTDIR)$(docdir)/manual/index.html
+	$(INSTALL_DATA) style.css $(DESTDIR)$(docdir)/manual
+	cp -r images $(DESTDIR)$(docdir)/manual/images
+	$(INSTALL) -d $(DESTDIR)$(docdir)/manual/figures
+	$(INSTALL_DATA) $(FIGURES) $(DESTDIR)$(docdir)/manual/figures
+	$(INSTALL) -d $(DESTDIR)$(docdir)/release-notes
+	$(INSTALL_DATA) NEWS.html $(DESTDIR)$(docdir)/release-notes/index.html
+	$(INSTALL_DATA) style.css $(DESTDIR)$(docdir)/release-notes/
 
 images:
 	mkdir images

doc/manual/NEWS.txt

@@ -1 +0,0 @@
-New state nix version by wouter ...

doc/manual/build-farm.xml

@@ -56,7 +56,7 @@ build farm, since:
 <para>TODO</para>
 
 <para>The sources of the Nix build farm are at <link
-xlink:href='https://svn.cs.uu.nl:12443/repos/trace/release/trunk'/>.</para>
+xlink:href='https://svn.nixos.org/repos/nix/release/trunk'/>.</para>
 
 </section>
 
@@ -85,7 +85,7 @@ nix@scratchy.labs.cs.uu.nl  i686-linux      /home/nix/.ssh/id_scratchy_auto
 
 <para>An example build hook can be found in the Nix build farm
 sources: <link
-xlink:href='https://svn.cs.uu.nl:12443/repos/trace/release/trunk/common/distributed/build-remote.pl'
+xlink:href='https://svn.nixos.org/repos/nix/release/trunk/common/distributed/build-remote.pl'
 />.  It should be suitable for most purposes, with maybe some minor
 adjustments.  It uses <command>ssh</command> and
 <command>rsync</command> to copy the build inputs and outputs and

doc/manual/builtins.xml

@@ -77,18 +77,25 @@ attrValues = attrs: map (name: builtins.getAttr name attrs) (builtins.attrNames
 if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
     This allows a Nix expression to fall back gracefully on older Nix
-    installations that don’t have the desired built-in function.
-    However, in that case you should not write
+    installations that don’t have the desired built-in
+    function.</para></listitem>
 
-<programlisting>
-if builtins ? getEnv then __getEnv "PATH" else ""</programlisting>
+  </varlistentry>
 
-    This Nix expression will trigger an “undefined variable” error on
-    older Nix versions since <function>__getEnv</function> doesn’t
-    exist.  <literal>builtins.getEnv</literal>, on the other hand, is
-    safe since <literal>builtins</literal> always exists and attribute
-    selection is lazy, so it’s only performed if the test
-    succeeds.</para></listitem>
+  
+  <varlistentry><term><function>builtins.compareVersions</function>
+  <replaceable>s1</replaceable> <replaceable>s2</replaceable></term>
+
+    <listitem><para>Compare two strings representing versions and
+    return <literal>-1</literal> if version
+    <replaceable>s1</replaceable> is older than version
+    <replaceable>s2</replaceable>, <literal>0</literal> if they are
+    the same, and <literal>1</literal> if
+    <replaceable>s1</replaceable> is newer than
+    <replaceable>s2</replaceable>.  The version comparison algorithm
+    is the same as the one used by <link
+    linkend="ssec-version-comparisons"><command>nix-env
+    -u</command></link>.</para></listitem>
 
   </varlistentry>
 
@@ -146,6 +153,16 @@ if builtins ? getEnv then __getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.div</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the quotient of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.filterSource</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
@@ -337,6 +354,36 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.isString</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a string, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
+
+  
+  <varlistentry><term><function>builtins.isInt</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a int, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
+
+  
+  <varlistentry><term><function>builtins.isBool</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return <literal>true</literal> if
+    <replaceable>e</replaceable> evaluates to a bool, and
+    <literal>false</literal> otherwise.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>isNull</function>
   <replaceable>e</replaceable></term>
 
@@ -352,6 +399,15 @@ x: x + 456</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.length</function>
+  <replaceable>e</replaceable></term>
+
+    <listitem><para>Return the length of the list
+    <replaceable>e</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.lessThan</function>
   <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
@@ -408,6 +464,31 @@ map (x: "foo" + x) ["bar" "bla" "abc"]</programlisting>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.mul</function>
+  <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
+
+    <listitem><para>Return the product of the integers
+    <replaceable>e1</replaceable> and
+    <replaceable>e2</replaceable>.</para></listitem>
+
+  </varlistentry>
+
+  
+  <varlistentry><term><function>builtins.parseDrvName</function>
+  <replaceable>s</replaceable></term>
+
+    <listitem><para>Split the string <replaceable>s</replaceable> into
+    a package name and version.  The package name is everything up to
+    but not including the first dash followed by a digit, and the
+    version is everything following that dash.  The result is returned
+    in an attribute set <literal>{name, version}</literal>.  Thus,
+    <literal>builtins.parseDrvName "nix-0.12pre12876"</literal>
+    returns <literal>{name = "nix"; version =
+    "0.12pre12876";}</literal>.</para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>builtins.pathExists</function>
   <replaceable>path</replaceable></term>
 

doc/manual/conf-file.xml

@@ -60,27 +60,6 @@ env-keep-derivations = false
   </varlistentry>
 
   
-  <varlistentry xml:id="conf-gc-reserved-space"><term><literal>gc-reserved-space</literal></term>
-
-    <listitem><para>This option specifies how much space should be
-    reserved in normal use so that the garbage collector can run
-    succesfully.  Since the garbage collector must perform Berkeley DB
-    transactions, it needs some disk space for itself.  However, when
-    the disk is full, this space is not available, so the collector
-    would not be able to run precisely when it is most needed.</para>
-
-    <para>For this reason, when Nix is run, it allocates a file
-    <filename>/nix/var/nix/db/reserved</filename> of the size
-    specified by this option.  When the garbage collector is run, this
-    file is deleted before the Berkeley DB environment is opened.
-    This should give it enough room to proceed.</para>
-
-    <para>The default is <literal>1048576</literal> (1
-    MiB).</para></listitem>
-
-  </varlistentry>
-
-  
   <varlistentry><term><literal>env-keep-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations

doc/manual/env-common.xml

@@ -151,12 +151,12 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
   <para>On the basis of this information, and whatever persistent
   state the build hook keeps about other machines and their current
   load, it has to decide what to do with the build.  It should print
-  out on file descriptor 3 one of the following responses (terminated
-  by a newline, <literal>"\n"</literal>):
+  out on standard error one of the following responses (terminated by
+  a newline, <literal>"\n"</literal>):
 
   <variablelist>
 
-    <varlistentry><term><literal>decline</literal></term>
+    <varlistentry><term><literal># decline</literal></term>
 
       <listitem><para>The build hook is not willing or able to perform
       the build; the calling Nix process should do the build itself,
@@ -164,7 +164,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
     </varlistentry>
 
-    <varlistentry><term><literal>postpone</literal></term>
+    <varlistentry><term><literal># postpone</literal></term>
 
       <listitem><para>The build hook cannot perform the build now, but
       can do so in the future (e.g., because all available build slots
@@ -174,7 +174,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
     </varlistentry>
 
-    <varlistentry><term><literal>accept</literal></term>
+    <varlistentry><term><literal># accept</literal></term>
 
       <listitem><para>The build hook has accepted the
       build.</para></listitem>
@@ -185,37 +185,12 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 
   </para>
 
-  <para>If the build hook accepts the build, it is possible that it is
-  no longer necessary to do the build because some other process has
-  performed the build in the meantime.  To prevent races, the hook
-  must read from file descriptor 4 a single line that tells it whether
-  to continue:
-
-  <variablelist>
-
-    <varlistentry><term><literal>cancel</literal></term>
-
-      <listitem><para>The build has already been done, so the hook
-      should exit.</para></listitem>
-
-    </varlistentry>
-  
-    <varlistentry><term><literal>okay</literal></term>
-
-      <listitem><para>The hook should proceed with the build.  At this
-      point, the calling Nix process has acquired locks on the output
-      path, so no other Nix process will perform the
-      build.</para></listitem>
-
-    </varlistentry>
-
-  </variablelist>
-
-  </para>
-
-  <para>If the hook has been told to proceed, Nix will store in the
-  hook’s current directory a number of text files that contain
-  information about the derivation:
+  <para>After sending <literal># accept</literal>, the hook should
+  read one line from standard input, which will be the string
+  <literal>okay</literal>.  It can then proceed with the build.
+  Before sending <literal>okay</literal>, Nix will store in the hook’s
+  current directory a number of text files that contain information
+  about the derivation:
 
   <variablelist>
 
@@ -255,7 +230,9 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
   <para>The hook should copy the inputs to the remote machine,
   register the validity of the inputs, perform the remote build, and
   copy the outputs back to the local machine.  An exit code other than
-  <literal>0</literal> indicates that the hook has failed.</para>
+  <literal>0</literal> indicates that the hook has failed.  An exit
+  code equal to 100 means that the remote build failed (as opposed to,
+  e.g., a network error).</para>
 
   </listitem>
 
@@ -274,6 +251,27 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 </varlistentry>
 
     
+<varlistentry xml:id="envar-other-stores"><term><envar>NIX_OTHER_STORES</envar></term>
+
+  <listitem><para>This variable contains the paths of remote Nix
+  installations from whichs paths can be copied, separated by colons.
+  See <xref linkend="sec-sharing-packages" /> for details.  Each path
+  should be the <filename>/nix</filename> directory of a remote Nix
+  installation (i.e., not the <filename>/nix/store</filename>
+  directory).  The paths are subject to globbing, so you can set it so
+  something like <literal>/var/run/nix/remote-stores/*/nix</literal>
+  and mount multiple remote filesystems in
+  <literal>/var/run/nix/remote-stores</literal>.</para>
+
+  <para>Note that if you’re building through the <link
+  linkend="sec-nix-worker">Nix daemon</link>, the only setting for
+  this variable that matters is the one that the
+  <command>nix-worker</command> process uses.  So if you want to
+  change it, you have to restart the daemon.</para></listitem>
+
+</varlistentry>
+
+    
 </variablelist>
 
 

doc/manual/installation.xml

@@ -42,25 +42,22 @@ platforms as well.</para>
 <section><title>Obtaining Nix</title>
 
 <para>The easiest way to obtain Nix is to download a <link
-xlink:href="http://nix.cs.uu.nl/">source distribution</link>.  RPMs
+xlink:href="http://nixos.org/">source distribution</link>.  RPMs
 for Red Hat, SuSE, and Fedora Core are also available.</para>
 
 <para>Alternatively, the most recent sources of Nix can be obtained
 from its <link
-xlink:href="https://svn.cs.uu.nl:12443/repos/trace/nix/trunk">Subversion
+xlink:href="https://svn.nixos.org/repos/nix/nix/trunk">Subversion
 repository</link>.  For example, the following command will check out
 the latest revision into a directory called
 <filename>nix</filename>:</para>
 
 <screen>
-$ svn checkout https://svn.cs.uu.nl:12443/repos/trace/nix/trunk nix</screen>
+$ svn checkout https://svn.nixos.org/repos/nix/nix/trunk nix</screen>
 
 <para>Likewise, specific releases can be obtained from the <link
-xlink:href="https://svn.cs.uu.nl:12443/repos/trace/nix/tags">tags
-directory</link> of the repository.  If you don't have Subversion, you
-can also download an automatically generated <link
-xlink:href="https://svn.cs.uu.nl:12443/dist/trace/">compressed
-tar-file</link> of the head revision of the trunk.</para>
+xlink:href="https://svn.nixos.org/repos/nix/nix/tags">tags
+directory</link> of the repository.</para>
 
 </section>
 
@@ -99,18 +96,23 @@ ubiquitous 2.5.4a won't.  Note that these are only required if you
 modify the parser or when you are building from the Subversion
 repository.</para>
 
-<para>Nix uses Sleepycat's Berkeley DB, CWI's ATerm library and the
-bzip2 compressor (including the bzip2 library).  These are included in
-the Nix source distribution.  If you build from the Subversion
-repository, you must download them yourself and place them in the
+<para>Nix uses CWI's ATerm library and the bzip2 compressor (including
+the bzip2 library).  These are included in the Nix source
+distribution.  If you build from the Subversion repository, you must
+download them yourself and place them in the
 <filename>externals/</filename> directory.  See
 <filename>externals/Makefile.am</filename> for the precise URLs of
 these packages.  Alternatively, if you already have them installed,
 you can use <command>configure</command>'s
-<option>--with-bdb</option>, <option>--with-aterm</option> and
-<option>--with-bzip2</option> options to point to their respective
-locations.  Note that Berkeley DB <emphasis>must</emphasis> be version
-4.5; other versions may not have compatible database formats.</para>
+<option>--with-aterm</option> and <option>--with-bzip2</option>
+options to point to their respective locations.</para>
+
+<para>If you want to be able to upgrade Nix stores from before version
+0.12pre12020, you need Sleepycat's Berkeley DB version version 4.5.
+(Other versions may not have compatible database formats.).  Berkeley
+DB 4.5 is included in the Nix source distribution.  If you do not need
+this ability, you can build Nix with the
+<option>--disable-old-db-compat</option> configure option.</para>
 
 </section>
 
@@ -161,7 +163,7 @@ options.</para>
 <section><title>Installing from RPMs</title>
 
 <para>RPM packages of Nix can be downloaded from <link
-xlink:href="http://nix.cs.uu.nl/" />.  These RPMs should work for most
+xlink:href="http://nixos.org/" />.  These RPMs should work for most
 fairly recent releases of SuSE and Red Hat Linux.  They have been
 known to work work on SuSE Linux 8.1 and 9.0, and Red Hat 9.0.  In
 fact, it should work on any RPM-based Linux distribution based on
@@ -193,12 +195,12 @@ $ rm -rf /nix/var</screen>
 
 <para>You can install the latest stable version of Nix through Nix
 itself by subscribing to the channel <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-stable" />,
+xlink:href="http://nixos.org/releases/nix/channels/nix-stable" />,
 or the latest unstable version by subscribing to the channel <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-unstable" />.
+xlink:href="http://nixos.org/releases/nix/channels/nix-unstable" />.
 You can also do a <link linkend="sec-one-click">one-click
 installation</link> by clicking on the package links at <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/" />.</para>
+xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para>
 
 </section>
 
@@ -334,8 +336,8 @@ $ chown -R root /nix/store /nix/var/nix</screen>
 
 </para>
 
-<para>The Nix daemon should be started as follows (as
-<literal>root</literal>):
+<para>The <link linkend="sec-nix-worker">Nix daemon</link> should be
+started as follows (as <literal>root</literal>):
 
 <screen>
 $ nix-worker --daemon</screen>

doc/manual/introduction.xml

@@ -240,7 +240,7 @@ configuration (e.g., to build configuration files in
 possible to easily roll back the entire configuration of the system to
 an earlier state.  Also, users can install software without root
 privileges.  For more information and downloads, see the <link
-xlink:href="http://nix.cs.uu.nl/nixos/">NixOS homepage</link>.</para>
+xlink:href="http://nixos.org/">NixOS homepage</link>.</para>
 
 </simplesect>
 
@@ -257,14 +257,15 @@ xlink:href="http://nix.cs.uu.nl/nixos/">NixOS homepage</link>.</para>
 
 <section><title>About us</title>
 
-<para>Nix was developed at the <link
+<para>Nix was originally developed at the <link
 xlink:href="http://www.cs.uu.nl/">Department of Information and
 Computing Sciences</link>, Utrecht University by the <link
 xlink:href="http://www.cs.uu.nl/wiki/Trace/WebHome">TraCE
-project</link>.  The project is funded by the Software Engineering
-Research Program <link
+project</link> (2003-2008).  The project was funded by the Software
+Engineering Research Program <link
 xlink:href="http://www.jacquard.nl/">Jacquard</link> to improve the
-support for variability in software systems.</para>
+support for variability in software systems.  Further funding is now
+provided by the NIRICT LaQuSo Build Farm project.</para>
 
 </section>
 
@@ -299,24 +300,24 @@ Lesser General Public License for more details.</para>
 
 <para>Some background information on Nix can be found in a number of
 papers.  The ICSE 2004 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'>Imposing
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/immdsd-icse2004-final.pdf'>Imposing
 a Memory Management Discipline on Software Deployment</citetitle>
 discusses the hashing mechanism used to ensure reliable dependency
 identification and non-interference between different versions and
 variants of packages.  The LISA 2004 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'>Nix:
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/nspfssd-lisa2004-final.pdf'>Nix:
 A Safe and Policy-Free System for Software Deployment</citetitle>
 gives a more general discussion of Nix from a system-administration
 perspective.  The CBSE 2005 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'>Efficient
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/eupfcdm-cbse2005-final.pdf'>Efficient
 Upgrading in a Purely Functional Component Deployment Model
 </citetitle> is about transparent patch deployment in Nix.  The SCM-12
 paper <citetitle
-xlink:href='http://www.cs.uu.nl/~eelco/pubs/servicecm-scm12-final.pdf'>
+xlink:href='http://www.st.ewi.tudelft.nl/~dolstra/pubs/servicecm-scm12-final.pdf'>
 Service Configuration Management</citetitle> shows how services (e.g.,
 web servers) can be deployed and managed through Nix.  A short
 overview of NixOS is given in the HotOS XI paper <citetitle
-xlink:href="http://www.cs.uu.nl/~eelco/pubs/hotos-final.pdf">Purely
+xlink:href="http://www.st.ewi.tudelft.nl/~dolstra/pubs/hotos-final.pdf">Purely
 Functional System Configuration Management</citetitle>.  The Nix
 homepage has <link
 xlink:href="http://nix.cs.uu.nl/docs/papers.html">an up-to-date list
@@ -328,7 +329,7 @@ Purely Functional Software Deployment Model</citetitle>, which
 contains most of the papers listed above.</para>
 
 <para>Nix has a homepage at <link
-xlink:href="http://nix.cs.uu.nl/"/>.</para>
+xlink:href="http://nixos.org/"/>.</para>
 
 </section>
 

doc/manual/manual.xml

@@ -14,8 +14,8 @@
         <surname>Dolstra</surname>
       </personname>
       <affiliation>
-        <orgname>Utrecht University</orgname>
-        <orgdiv>Faculty of Science, Department of Information and Computing Sciences</orgdiv>
+        <orgname>Delft University of Technology</orgname>
+        <orgdiv>Department of Software Technology</orgdiv>
       </affiliation>
     </author>
 
@@ -24,10 +24,11 @@
       <year>2005</year>
       <year>2006</year>
       <year>2007</year>
+      <year>2008</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
 
-    <date>September 2007</date>
+    <date>November 2008</date>
     
   </info>
 
@@ -88,10 +89,6 @@
         <title>nix-install-package</title>
         <xi:include href="nix-install-package.xml" />
       </section>
-      <section xml:id="sec-nix-pack-closure">
-        <title>nix-pack-closure</title>
-        <xi:include href="nix-pack-closure.xml" />
-      </section>
       <section xml:id="sec-nix-prefetch-url">
         <title>nix-prefetch-url</title>
         <xi:include href="nix-prefetch-url.xml" />
@@ -104,16 +101,16 @@
         <title>nix-push</title>
         <xi:include href="nix-push.xml" />
       </section>
-      <section xml:id="sec-nix-unpack-closure">
-        <title>nix-unpack-closure</title>
-        <xi:include href="nix-unpack-closure.xml" />
+      <section xml:id="sec-nix-worker">
+        <title>nix-worker</title>
+        <xi:include href="nix-worker.xml" />
       </section>
     </section>
     
   </appendix>
 
   <xi:include href="troubleshooting.xml" />
-  <xi:include href="bugs.xml" />
+  <!-- <xi:include href="bugs.xml" /> -->
   <xi:include href="glossary.xml" />
 
   <appendix>

doc/manual/nix-copy-closure.xml

@@ -128,7 +128,7 @@ those paths.  If this bothers you, use
 <para>Copy Firefox with all its dependencies to a remote machine:
 
 <screen>
-$ nix-copy-closure alice@itchy.labs $(type -tP firefox)</screen>
+$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)</screen>
 
 </para>
 

doc/manual/nix-env.xml

@@ -526,7 +526,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4'</screen>
 
 </refsection>
     
-<refsection><title>Versions</title>
+<refsection xml:id="ssec-version-comparisons"><title>Versions</title>
 
 <para>The upgrade operation determines whether a derivation
 <varname>y</varname> is an upgrade of a derivation

doc/manual/nix-pack-closure.xml

@@ -1,82 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<refmeta>
-  <refentrytitle>nix-pack-closure</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-pack-closure</refname>
-  <refpurpose>pack the closure of a store path into a single file that
-  can be unpacked with
-  <command>nix-unpack-closure</command></refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-pack-closure</command>
-    <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-pack-closure</command> packs the
-contents of the store paths <replaceable>paths</replaceable> and
-<emphasis>all their dependencies</emphasis> into a single file, which
-is written to standard output.  (That is, it
-<emphasis>serialises</emphasis> <replaceable>paths</replaceable>.)
-The output can then be unpacked into the Nix store of another machine
-using <command>nix-unpack-closure</command>.</para>
-
-<para>Together, <command>nix-pack-closure</command> and
-<command>nix-unpack-closure</command> provide a quick and easy way to
-deploy a package to a different machine.  However, as the output of
-<command>nix-pack-closure</command> tends to be rather large (since it
-contains all dependencies), it’s not very efficient.
-<command>nix-push</command> and <command>nix-pull</command> are more
-efficient, but are also a bit more cumbersome to use.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<para>To copy some instance of Subversion with all its dependencies to
-another machine:
-
-<screen>
-$ nix-pack-closure /nix/store/hj232g1r...-subversion-1.3.0 > svn.closure
-
-<lineannotation>Copy <!-- !!! <filename> -->svn.closure to the remote machine, then on the remote machine do:</lineannotation>
-$ nix-unpack-closure &lt; svn.closure</screen>
-
-</para>
-
-<para>Copy the program <command>azureus</command> with all its
-dependencies to the machine <literal>scratchy</literal>:
-
-<screen>
-$ nix-pack-closure $(which azureus) | ssh scratchy nix-unpack-closure</screen>
-    
-</para>
-
-<para>As a variation on the previous example, copy
-<command>azureus</command>, and also install it in the user’s profile
-on the target machine:
-
-<screen>
-$ nix-pack-closure $(which azureus) | ssh scratchy 'nix-env -i $(nix-unpack-closure)'</screen>
-
-</para>
- 
-
-</refsection>
-
-
-</refentry>

doc/manual/nix-store.xml

@@ -128,6 +128,7 @@ lrwxrwxrwx    1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134
     <arg choice='plain'><option>-r</option></arg>
   </group>
   <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
+  <arg><option>--dry-run</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -166,6 +167,11 @@ the specified store paths.  Realisation is a somewhat overloaded term:
 output.  (For non-derivations argument, the argument itself is
 printed.)</para>
 
+<para>If the <option>--dry-run</option> option is used, then
+<command>nix-store</command> will print on standard error a
+description of what packages would be built or downloaded, and then
+quit.</para>
+
 </refsection>
             
 
@@ -204,6 +210,10 @@ linkend="sec-nix-build"><command>nix-build</command></link> does.</para>
     <arg choice='plain'><option>--print-dead</option></arg>
     <arg choice='plain'><option>--delete</option></arg>
   </group>
+  <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
+  <arg><option>--max-links</option> <replaceable>nrlinks</replaceable></arg>
+  <arg><option>--max-atime</option> <replaceable>atime</replaceable></arg>
+  <arg><option>--use-atime</option></arg>
 </cmdsynopsis>
 
 </refsection>
@@ -258,7 +268,70 @@ the Nix store not reachable via file system references from a set of
 
 </variablelist>
 
-<para>The behaviour of the collector is influenced by the <link
+<para>By default, all unreachable paths are deleted.  The following
+options control what gets deleted and in what order:
+
+<variablelist>
+
+  <varlistentry><term><option>--max-freed</option> <replaceable>bytes</replaceable></term>
+  
+    <listitem><para>Keep deleting paths until at least
+    <replaceable>bytes</replaceable> bytes have been
+    deleted, then stop.</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--max-links</option> <replaceable>nrlinks</replaceable></term>
+  
+    <listitem><para>Keep deleting paths until the hard link count on
+    <filename>/nix/store</filename> is less than
+    <replaceable>nrlinks</replaceable>, then stop.  This is useful for
+    very large Nix stores on filesystems with a 32000 subdirectories
+    limit (like <literal>ext3</literal>).</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--max-atime</option> <replaceable>atime</replaceable></term>
+  
+    <listitem><para>Only delete a store path if its last-accessed time
+    is less than <replaceable>atime</replaceable>.  This allows you to
+    garbage-collect only packages that haven’t been used recently.
+    The time is expressed as the number of seconds in the Unix epoch,
+    i.e., since 1970-01-01 00:00:00 UTC.  An easy way to convert to
+    this format is <literal>date +%s -d "<replaceable>date
+    specification</replaceable>"</literal>.</para>
+
+    <para>For directories, the last-accessed time is the highest
+    last-accessed time of any regular file in the directory (or in any
+    of its subdirectories).  That is, the <literal>atime</literal>
+    field maintained by the filesystem is ignored for directories.
+    This is because operations such as rebuilding the
+    <command>locate</command> database tend to update the
+    <literal>atime</literal> values of all directories, so they’re not
+    a useful indicator of whether a package was recently used.</para>
+
+    <para>Note that <command>nix-store --optimise</command> reads all
+    regular files in the Nix store, and so causes all last-accessed
+    times to be set to the present time.  This makes
+    <option>--max-atime</option> ineffective (for a while at
+    least).</para></listitem>
+    
+  </varlistentry>
+
+  <varlistentry><term><option>--use-atime</option></term>
+
+    <listitem><para>Delete store paths in order of ascending
+    last-accessed time.  This is useful in conjunction with the other
+    options to delete only the least recently used
+    packages.</para></listitem>
+
+  </varlistentry>
+  
+</variablelist>
+
+</para>
+
+<para>The behaviour of the collector is also influenced by the <link
 linkend="conf-gc-keep-outputs"><literal>gc-keep-outputs</literal></link>
 and <link
 linkend="conf-gc-keep-derivations"><literal>gc-keep-derivations</literal></link>
@@ -284,6 +357,20 @@ deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv'
 
 </para>
 
+<para>To delete unreachable paths not accessed in the last two months:
+
+<screen>
+$ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
+
+</para>
+
+<para>To delete at least 100 MiBs of unreachable paths:
+
+<screen>
+$ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen>
+
+</para>
+
 </refsection>
 
 
@@ -950,7 +1037,51 @@ ktorrent-2.2.1/NEWS
 </refsection>
 
 
-<!-- TODO: export, import operations -->
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--dump-db</option></title>
+
+<refsection>
+  <title>Synopsis</title>
+  <cmdsynopsis>
+    <command>nix-store</command>
+    <arg choice='plain'><option>--dump-db</option></arg>
+  </cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+            
+<para>The operation <option>--dump-db</option> writes a dump of the
+Nix database to standard output.  It can be loaded into an empty Nix
+store using <option>--load-db</option>.  This is useful for making
+backups and when migrating to different database schemas.</para>
+
+</refsection>
+            
+</refsection>
+
+
+<!--######################################################################-->
+
+<refsection><title>Operation <option>--dump-db</option></title>
+
+<refsection>
+  <title>Synopsis</title>
+  <cmdsynopsis>
+    <command>nix-store</command>
+    <arg choice='plain'><option>--load-db</option></arg>
+  </cmdsynopsis>
+</refsection>
+
+<refsection><title>Description</title>
+            
+<para>The operation <option>--load-db</option> reads a dump of the Nix
+database created by <option>--dump-db</option> from standard input and
+loads it into the Nix database.</para>
+
+</refsection>
+            
+</refsection>
 
 
 </refentry>

doc/manual/nix-unpack-closure.xml

@@ -1,42 +0,0 @@
-<refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink"
-          xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<refmeta>
-  <refentrytitle>nix-unpack-closure</refentrytitle>
-  <manvolnum>1</manvolnum>
-  <refmiscinfo class="source">Nix</refmiscinfo>
-  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
-</refmeta>
-
-<refnamediv>
-  <refname>nix-unpack-closure</refname>
-  <refpurpose>unpack the closure of a store path created by <command>nix-pack-closure</command> into the Nix store</refpurpose>
-</refnamediv>
-
-<refsynopsisdiv>
-  <cmdsynopsis>
-    <command>nix-unpack-closure</command>
-  </cmdsynopsis>
-</refsynopsisdiv>
-
-
-<refsection><title>Description</title>
-
-<para>The command <command>nix-unpack-closure</command> unpacks the
-closure of a set of store paths created by
-<command>nix-pack-closure</command> into the local Nix store.  The
-closure is a single file read from standard input.  See the
-description of <command>nix-pack-closure</command> for details and
-examples.</para>
-
-<para>The top-level paths in the closure (i.e., the paths passed to
-the original <command>nix-pack-closure</command> call that created the
-closure) are printed on standard output.  These paths can be passed,
-for instance, to <literal>nix-env -i</literal> to install them into a
-user environment on the target machine.</para>
-
-</refsection>
-
-
-</refentry>

doc/manual/nix-worker.xml

@@ -0,0 +1,34 @@
+<refentry xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-worker</refentrytitle>
+  <manvolnum>8</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
+<refnamediv>
+  <refname>nix-worker</refname>
+  <refpurpose>Nix multi-user support daemon</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-worker</command>
+    <arg choice="plain"><option>--daemon</option></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+
+<refsection><title>Description</title>
+
+<para>The Nix daemon is necessary in multi-user Nix installations.  It
+performs build actions and other operations on the Nix store on behalf
+of unprivileged users.</para>
+
+
+</refsection>
+
+</refentry>

doc/manual/opt-common-syn.xml

@@ -24,6 +24,7 @@
 <arg><option>--fallback</option></arg>
 <arg><option>--readonly-mode</option></arg>
 <arg><option>--log-type</option> <replaceable>type</replaceable></arg>
+<arg><option>--show-trace</option></arg>
 <sbr />
 
 </nop>

doc/manual/opt-common.xml

@@ -251,14 +251,14 @@
 
 <programlisting>
 { # The system (e.g., `i686-linux') for which to build the packages.
-  system ? __currentSystem
+  system ? builtins.currentSystem
   <replaceable>...</replaceable>
 }: <replaceable>...</replaceable></programlisting>
 
   So if you call this Nix expression (e.g., when you do
   <literal>nix-env -i <replaceable>pkgname</replaceable></literal>),
   the function will be called automatically using the value <link
-  linkend='builtin-currentSystem'><literal>__currentSystem</literal></link>
+  linkend='builtin-currentSystem'><literal>builtins.currentSystem</literal></link>
   for the <literal>system</literal> argument.  You can override this
   using <option>--arg</option>, e.g., <literal>nix-env -i
   <replaceable>pkgname</replaceable> --arg system
@@ -305,6 +305,14 @@
 </varlistentry>
 
 
+<varlistentry><term><option>--show-trace</option></term>
+  
+  <listitem><para>Causes Nix to print out a stack trace in case of Nix
+  expression evaluation errors.</para></listitem>
+  
+</varlistentry>
+
+
 </variablelist>
 
 

doc/manual/package-management.xml

@@ -38,7 +38,7 @@ to end-user applications like Mozilla Firefox.  (Nix is however not
 tied to the Nix Package collection; you could write your own Nix
 expressions based on it, or completely new ones.)  You can download
 the latest version from <link
-xlink:href='http://nix.cs.uu.nl/dist/nix' />.</para>
+xlink:href='http://nixos.org/releases/full-index-nixpkgs.html' />.</para>
 
 <para>Assuming that you have downloaded and unpacked a release of Nix
 Packages, you can view the set of available packages in the release:
@@ -118,15 +118,15 @@ available somewhere.  This is done using the
 containing a <emphasis>manifest</emphasis> describing what binaries
 are available.  This URL should correspond to the Nix Packages release
 that you’re using.  For instance, if you obtained a release from <link
-xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />, then
-you should do:
+xlink:href='http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x'
+/>, then you should do:
 
 <screen>
-$ nix-pull http://nix.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/MANIFEST</screen>
+$ nix-pull http://nixos.org/releases/nixpkgs/nixpkgs-0.12pre11712-4lrp7j8x/MANIFEST</screen>
 
 If you then issue the installation command, it should start
 downloading binaries from <systemitem
-class='fqdomainname'>nix.cs.uu.nl</systemitem>, instead of building
+class='fqdomainname'>nixos.org</systemitem>, instead of building
 them from source.  This might still take a while since all
 dependencies must be downloaded, but on a reasonably fast connection
 such as an DSL line it’s on the order of a few minutes.</para>
@@ -458,7 +458,7 @@ URL.</para>
 <command>nix-channel --add</command>, e.g.,
 
 <screen>
-$ nix-channel --add http://nix.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable</screen>
+$ nix-channel --add http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable</screen>
 
 subscribes you to a channel that always contains that latest version
 of the Nix Packages collection.  (Instead of
@@ -496,20 +496,19 @@ available in the subscribed channels.</para>
 
 <para>Often, when you want to install a specific package (e.g., from
 the <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/">Nix
-Packages collection</link> or from our <link
-xlink:href='http://nix.cs.uu.nl/dist/'>release server</link>),
-subscribing to a channel is a bit cumbersome.  And channels don’t help
-you at all if you want to install an older version of a package than
-the one provided by the current contents of the channel, or a package
-that has been removed from the channel.  That’s when
-<emphasis>one-click installs</emphasis> come in handy: you can just go
-to the web page that contains the package, click on it, and it will be
-installed with all the necessary dependencies.</para>
+xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/">Nix
+Packages collection</link>), subscribing to a channel is a bit
+cumbersome.  And channels don’t help you at all if you want to install
+an older version of a package than the one provided by the current
+contents of the channel, or a package that has been removed from the
+channel.  That’s when <emphasis>one-click installs</emphasis> come in
+handy: you can just go to the web page that contains the package,
+click on it, and it will be installed with all the necessary
+dependencies.</para>
 
 <para>For instance, you can go to <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/nixpkgs-unstable-latest/" />
-— or to any older release of Nix Packages — and click on any link for
+xlink:href="http://nixos.org/releases/nixpkgs/nixpkgs-unstable/" /> —
+or to any older release of Nix Packages — and click on any link for
 the individual packages for your platform (say, <link
 xlink:href='http://nix.cs.uu.nl/dist/nix/nixpkgs-0.10pre6622/pkgs/subversion-1.4.0-i686-linux.nixpkg'><literal>subversion-1.4.0</literal>
 for <literal>i686-linux</literal></link>).  The first time you do
@@ -529,4 +528,82 @@ linkend='sec-nix-install-package' /> for details.</para>
 </section>
 
 
+<section xml:id="sec-sharing-packages"><title>Sharing packages between machines</title>
+
+<para>Sometimes you want to copy a package from one machine to
+another.  Or, you want to install some packages and you know that
+another machine already has some or all of those packages or their
+dependencies.  In that case there are mechanisms to quickly copy
+packages between machines.</para>
+
+<para>The command <command
+linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix
+store path along with all its dependencies to or from another machine
+via the SSH protocol.  It doesn’t copy store paths that are already
+present on the target machine.  For example, the following command
+copies Firefox with all its dependencies:
+
+<screen>
+$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)</screen>
+
+See <xref linkend='sec-nix-copy-closure' /> for details.</para>
+
+<para>With <command linkend='refsec-nix-store-export'>nix-store
+--export</command> and <command
+linkend='refsec-nix-store-import'>nix-store --import</command> you can
+write the closure of a store path (that is, the path and all its
+dependencies) to a file, and then unpack that file into another Nix
+store.  For example,
+
+<screen>
+$ nix-store --export $(type -p firefox) > firefox.closure</screen>
+
+writes the closure of Firefox to a file.  You can then copy this file
+to another machine and install the closure:
+
+<screen>
+$ nix-store --import &lt; firefox.closure</screen>
+
+Any store paths in the closure that are already present in the target
+store are ignored.  It is also possible to pipe the export into
+another command, e.g. to copy and install a closure directly to/on
+another machine:
+
+<screen>
+$ nix-store --export $(type -p firefox) | bzip2 | \
+    ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen>
+
+But note that <command>nix-copy-closure</command> is generally more
+efficient in this example because it only copies paths that are not
+already present in the target Nix store.</para>
+
+<para>Finally, if you can mount the Nix store of a remote machine in
+your local filesystem, Nix can copy paths from the remote Nix store to
+the local Nix store <emphasis>on demand</emphasis>.  For instance,
+suppose that you mount a remote machine containing a Nix store via
+<command
+xlink:href="http://fuse.sourceforge.net/sshfs.html">sshfs</command>:
+
+<screen>
+$ sshfs alice@itchy.example.org:/ /mnt</screen>
+
+You should then set the <envar>NIX_OTHER_STORES</envar> environment
+variable to tell Nix about this remote Nix store:
+
+<screen>
+$ export NIX_OTHER_STORES=/mnt/nix</screen>
+
+Then if you do any Nix operation, e.g.
+
+<screen>
+$ nix-env -i firefox</screen>
+
+and Nix has to build a path that it sees is already present in
+<filename>/mnt/nix</filename>, then it will just copy from there
+instead of building it from source.</para>
+ 
+
+</section>
+
+
 </chapter>

doc/manual/quick-start.xml

@@ -11,7 +11,7 @@ to the following chapters.</para>
 <orderedlist>
 
 <listitem><para>Download a source tarball or RPM from <link
-xlink:href='http://nix.cs.uu.nl/'/>.  Build source
+xlink:href='http://nixos.org/'/>.  Build source
 distributions using the regular sequence:
         
 <screen>
@@ -33,7 +33,7 @@ file).</para></listitem>
 
 <screen>
 $ nix-channel --add \
-    http://nix.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable</screen>
+    http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable</screen>
 
 </para></listitem>
 

doc/manual/release-notes.xml

@@ -8,11 +8,259 @@
 
 <!--==================================================================-->
 
-<section xml:id="ssec-relnotes-0.12"><title>Release 0.12 (TBA)</title>
+<section xml:id="ssec-relnotes-0.13"><title>Release 0.13 (April NN,
+2009)</title>
+
+<para>This is primarily a bug fix release.  It has some new
+features:</para>
 
 <itemizedlist>
 
-  <listitem><para><command>nix-store --dump-db / --load-db</command>.</para></listitem>
+  <listitem>
+    <para>Syntactic sugar for writing nested attribute sets.  Instead of
+
+<programlisting>
+{
+  foo = {
+    bar = 123;
+    xyzzy = true;
+  };
+  a = { b = { c = "d"; }; };
+}
+</programlisting>
+
+    you can write
+
+<programlisting>
+{
+  foo.bar = 123;
+  foo.xyzzy = true;
+  a.b.c = "d";
+}
+</programlisting>
+
+    This is useful, for instance, in NixOS configuration files.</para>
+    
+  </listitem>
+
+  <listitem>
+    <para>Support for Nix channels generated by Hydra, the Nix-based
+    continuous build system.  (Hydra generates NAR archives on the
+    fly, so the size and hash of these archives isn’t known in
+    advance.)</para>
+  </listitem>
+
+  <listitem>
+    <para>Support <literal>i686-linux</literal> builds directly on
+    <literal>x86_64-linux</literal> Nix installations.  This is
+    implemented using the <function>personality()</function> syscall,
+    which causes <command>uname</command> to return
+    <literal>i686</literal> in child processes.</para>
+  </listitem>
+  
+  <listitem>
+    <para>Various improvements to the <literal>chroot</literal>
+    support.  Building in a <literal>chroot</literal> works quite well
+    now.</para>
+  </listitem>
+
+  <listitem>
+    <para>Nix no longer blocks if it tries to build a path and another
+    process is already building the same path.  Instead it tries to
+    build another buildable path first.  This improves
+    parallelism.</para>
+  </listitem>
+
+  <listitem>
+    <para>Support for large (> 4 GiB) files in NAR archives.</para>
+  </listitem>
+    
+  <listitem>
+    <para>Various (performance) improvements to the remote build
+    mechanism.</para>
+  </listitem>
+    
+  <listitem>
+    <para>New primops: <varname>builtins.addErrorContext</varname> (to
+    add a string to stack traces — useful for debugging),
+    <varname>builtins.isBool</varname>,
+    <varname>builtins.isString</varname>,
+    <varname>builtins.isInt</varname>.</para>
+  </listitem>
+
+</itemizedlist>
+
+</section>
+
+
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.12"><title>Release 0.12 (November 20,
+2008)</title>
+
+<itemizedlist>
+
+  <listitem>
+    <para>Nix no longer uses Berkeley DB to store Nix store metadata.
+    The principal advantages of the new storage scheme are: it works
+    properly over decent implementations of NFS (allowing Nix stores
+    to be shared between multiple machines); no recovery is needed
+    when a Nix process crashes; no write access is needed for
+    read-only operations; no more running out of Berkeley DB locks on
+    certain operations.</para>
+
+    <para>You still need to compile Nix with Berkeley DB support if
+    you want Nix to automatically convert your old Nix store to the
+    new schema.  If you don’t need this, you can build Nix with the
+    <filename>configure</filename> option
+    <option>--disable-old-db-compat</option>.</para>
+
+    <para>After the automatic conversion to the new schema, you can
+    delete the old Berkeley DB files:
+
+    <screen>
+$ cd /nix/var/nix/db
+$ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG</screen>
+
+    The new metadata is stored in the directories
+    <filename>/nix/var/nix/db/info</filename> and
+    <filename>/nix/var/nix/db/referrer</filename>.  Though the
+    metadata is stored in human-readable plain-text files, they are
+    not intended to be human-editable, as Nix is rather strict about
+    the format.</para>
+
+    <para>The new storage schema may or may not require less disk
+    space than the Berkeley DB environment, mostly depending on the
+    cluster size of your file system.  With 1 KiB clusters (which
+    seems to be the <literal>ext3</literal> default nowadays) it
+    usually takes up much less space.</para>
+  </listitem>
+
+  <listitem><para>There is a new substituter that copies paths
+  directly from other (remote) Nix stores mounted somewhere in the
+  filesystem.  For instance, you can speed up an installation by
+  mounting some remote Nix store that already has the packages in
+  question via NFS or <literal>sshfs</literal>.  The environment
+  variable <envar>NIX_OTHER_STORES</envar> specifies the locations of
+  the remote Nix directories,
+  e.g. <literal>/mnt/remote-fs/nix</literal>.</para></listitem>
+
+  <listitem><para>New <command>nix-store</command> operations
+  <option>--dump-db</option> and <option>--load-db</option> to dump
+  and reload the Nix database.</para></listitem>
+
+  <listitem><para>The garbage collector has a number of new options to
+  allow only some of the garbage to be deleted.  The option
+  <option>--max-freed <replaceable>N</replaceable></option> tells the
+  collector to stop after at least <replaceable>N</replaceable> bytes
+  have been deleted.  The option <option>--max-links
+  <replaceable>N</replaceable></option> tells it to stop after the
+  link count on <filename>/nix/store</filename> has dropped below
+  <replaceable>N</replaceable>.  This is useful for very large Nix
+  stores on filesystems with a 32000 subdirectories limit (like
+  <literal>ext3</literal>).  The option <option>--use-atime</option>
+  causes store paths to be deleted in order of ascending last access
+  time.  This allows non-recently used stuff to be deleted.  The
+  option <option>--max-atime <replaceable>time</replaceable></option>
+  specifies an upper limit to the last accessed time of paths that may
+  be deleted.  For instance,
+
+    <screen>
+    $ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen>
+
+  deletes everything that hasn’t been accessed in two months.</para></listitem>
+
+  <listitem><para><command>nix-env</command> now uses optimistic
+  profile locking when performing an operation like installing or
+  upgrading, instead of setting an exclusive lock on the profile.
+  This allows multiple <command>nix-env -i / -u / -e</command>
+  operations on the same profile in parallel.  If a
+  <command>nix-env</command> operation sees at the end that the profile
+  was changed in the meantime by another process, it will just
+  restart.  This is generally cheap because the build results are
+  still in the Nix store.</para></listitem>
+
+  <listitem><para>The option <option>--dry-run</option> is now
+  supported by <command>nix-store -r</command> and
+  <command>nix-build</command>.</para></listitem>
+
+  <listitem><para>The information previously shown by
+  <option>--dry-run</option> (i.e., which derivations will be built
+  and which paths will be substituted) is now always shown by
+  <command>nix-env</command>, <command>nix-store -r</command> and
+  <command>nix-build</command>.  The total download size of
+  substitutable paths is now also shown.  For instance, a build will
+  show something like
+
+    <screen>
+the following derivations will be built:
+  /nix/store/129sbxnk5n466zg6r1qmq1xjv9zymyy7-activate-configuration.sh.drv
+  /nix/store/7mzy971rdm8l566ch8hgxaf89x7lr7ik-upstart-jobs.drv
+  ...
+the following paths will be downloaded/copied (30.02 MiB):
+  /nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4
+  /nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6
+  ...</screen>      
+  
+  </para></listitem>
+
+  <listitem><para>Language features:
+
+    <itemizedlist>
+
+      <listitem><para>@-patterns as in Haskell.  For instance, in a
+      function definition
+
+      <programlisting>f = args @ {x, y, z}: <replaceable>...</replaceable>;</programlisting>
+
+      <varname>args</varname> refers to the argument as a whole, which
+      is further pattern-matched against the attribute set pattern
+      <literal>{x, y, z}</literal>.</para></listitem>
+
+      <listitem><para>“<literal>...</literal>” (ellipsis) patterns.
+      An attribute set pattern can now say <literal>...</literal>  at
+      the end of the attribute name list to specify that the function
+      takes <emphasis>at least</emphasis> the listed attributes, while
+      ignoring additional attributes.  For instance,
+
+      <programlisting>{stdenv, fetchurl, fuse, ...}: <replaceable>...</replaceable></programlisting>
+
+      defines a function that accepts any attribute set that includes
+      at least the three listed attributes.</para></listitem>
+
+      <listitem><para>New primops:
+      <varname>builtins.parseDrvName</varname> (split a package name
+      string like <literal>"nix-0.12pre12876"</literal> into its name
+      and version components, e.g. <literal>"nix"</literal> and
+      <literal>"0.12pre12876"</literal>),
+      <varname>builtins.compareVersions</varname> (compare two version
+      strings using the same algorithm that <command>nix-env</command>
+      uses), <varname>builtins.length</varname> (efficiently compute
+      the length of a list), <varname>builtins.mul</varname> (integer
+      multiplication), <varname>builtins.div</varname> (integer
+      division).
+      <!-- <varname>builtins.genericClosure</varname> -->
+      </para></listitem>
+      
+    </itemizedlist>
+
+  </para></listitem>
+
+  <listitem><para><command>nix-prefetch-url</command> now supports
+  <literal>mirror://</literal> URLs, provided that the environment
+  variable <envar>NIXPKGS_ALL</envar> points at a Nixpkgs
+  tree.</para></listitem>
+
+  <listitem><para>Removed the commands
+  <command>nix-pack-closure</command> and
+  <command>nix-unpack-closure</command>.   You can do almost the same
+  thing but much more efficiently by doing <literal>nix-store --export
+  $(nix-store -qR <replaceable>paths</replaceable>) > closure</literal> and
+  <literal>nix-store --import &lt;
+  closure</literal>.</para></listitem>
+
+  <listitem><para>Lots of bug fixes, including a big performance bug in
+  the handling of <literal>with</literal>-expressions.</para></listitem>
 
 </itemizedlist>
 

doc/manual/troubleshooting.xml

@@ -5,121 +5,10 @@
 
 
 <para>This section provides solutions for some common problems.  See
-the <link xlink:href="https://bugs.cs.uu.nl/browse/NIX">Nix
+the <link xlink:href="http://bugs.strategoxt.org/browse/NIX">Nix
 bug tracker</link> for a list of currently known issues.</para>
 
 
-<section><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
-
-<para>Symptom: Nix operations (in particular the
-<command>nix-store</command> operations <option>--gc</option>,
-<option>--verify</option>, and <option>--clear-substitutes</option> —
-the latter being called by <command>nix-channel --update</command>)
-failing:
-
-<screen>
-$ nix-store --verify
-error: Db::del: Cannot allocate memory</screen>
-
-</para>
-
-<para>Possible solution: make sure that no Nix processes are running,
-then do:
-
-<screen>
-$ cd /nix/var/nix/db
-$ rm __db.00*</screen>
-
-</para>
-
-</section>
-
-
-<section><title>Berkeley DB gives weird error messages</title>
-
-<para>Symptom: you get error messages such as
-
-<screen>
-Berkeley DB message: Finding last valid log LSN: file: 1 offset 28
-Berkeley DB error: file validpaths (meta pgno = 0) has LSN [483][34721].
-Berkeley DB error: end of log is [1][28]
-Berkeley DB error: /nix/var/nix/db/validpaths: unexpected file type or format</screen>
-
-or other weird Berkeley DB errors, and they don’t go away (i.e.,
-automatic recovery doesn’t work).  This may be the case after a system
-crash.</para>
-
-<para>Solution: first try to run <command>db_recover</command> and
-then <link linkend='refsec-nix-store-verify'><command>nix-store
---verify</command></link>:
-
-<screen>
-$ db_recover -h /nix/var/nix/db
-$ nix-store --verify</screen>
-
-(Make sure that you have the right version of
-<command>db_recover</command>, namely, Berkeley DB 4.4 for Nix 0.10,
-and 4.5 for Nix 0.11.)</para>
-
-<para>If that doesn’t work, it’s time to bring out the big guns:
-
-<screen>
-$ cd /nix/var/nix
-$ cp -pr db db-backup  <lineannotation>(making a backup just in case)</lineannotation>
-$ cd db
-$ rm __db.* log*  <lineannotation>(removing the Berkeley DB environment)</lineannotation>
-$ mkdir tmp
-$ for i in *; do db_dump $i | (cd tmp &amp;&amp; db_load $i); done 
-<lineannotation>(ignore error messages about non-database files like “reserved”)</lineannotation>
-$ mv tmp/* .
-$ nix-store --verify</screen>
-
-</para>
-
-</section>
-
-
-<section><title>Berkeley DB out of locks</title>
-
-<para>It is possible, especially in <command>nix-store
---verify</command> or when running the garbage collector, to run out
-of Berkeley DB locks, like this:
-
-<screen>
-$ nix-store --verify
-checking path existence
-checking path realisability
-checking the derivers table
-checking the references table
-Berkeley DB error: Lock table is out of available object entries
-error: Db::get: Cannot allocate memory</screen>
-
-</para>
-
-<para>A workaround is to increase the number of locks that Berkeley DB
-allocates.  (The real solution would be for Nix to not use so many
-locks.)  This can be done by putting the following in the file
-<filename>/nix/var/nix/db/<link
-xlink:href="http://www.oracle.com/technology/documentation/berkeley-db/db/ref/env/db_config.html">DB_CONFIG</link></filename>:
-
-<programlisting>
-set_lk_max_locks   100000
-set_lk_max_lockers 100000
-set_lk_max_objects 100000
-</programlisting>
-
-(Increase these numbers if necessary.)  Then make sure that there are
-no running Nix processes and delete the Berkeley DB environment:
-
-<screen>
-$ rm /nix/var/nix/db/__db.*</screen>
-
-The Berkeley DB environment is automatically recreated with the new
-limits when you run any Nix command.</para>
-
-</section>
-
-
 <section><title>Collisions in <command>nix-env</command></title>
 
 <para>Symptom: when installing or upgrading, you get an error message such as
@@ -187,7 +76,8 @@ Furthermore, the <literal>st_nlink</literal> field of the
 <para>This only happens on very large Nix installations (such as build
 machines).</para>
 
-<para>Quick solution: run the garbage collector.</para>
+<para>Quick solution: run the garbage collector.  You may want to use
+the <option>--max-links</option> option.</para>
 
 <para>Real solution: put the Nix store on a file system that supports
 more than 32,000 subdirectories per directory, such as ReiserFS.

doc/manual/writing-nix-expressions.xml

@@ -731,9 +731,9 @@ stdenv.mkDerivation {
     xlink:href='http://www.ietf.org/rfc/rfc2396.txt'>RFC 2396</link>
     can be written <emphasis>as is</emphasis>, without quotes.  For
     instance, the string
-    <literal>"https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2"</literal>
+    <literal>"http://example.org/foo.tar.bz2"</literal>
     can also be written as
-    <literal>https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2</literal>.</para>
+    <literal>http://example.org/foo.tar.bz2</literal>.</para>
 
   </listitem>
 
@@ -937,17 +937,84 @@ set.</para>
 <para>Functions have the following form:
 
 <programlisting>
-{<replaceable>params</replaceable>}: <replaceable>body</replaceable></programlisting>
+<replaceable>pattern</replaceable>: <replaceable>body</replaceable></programlisting>
 
-This defines a function that must be called with an attribute set
-containing the attributes listed in <replaceable>params</replaceable>,
-which is a comma-separated list of attribute names.  Optionally, for
-each parameter a <emphasis>default value</emphasis> may be specified
-by writing <literal><replaceable>param</replaceable> ?
-<replaceable>e</replaceable></literal>, where
-<replaceable>e</replaceable> is an arbitrary expression.  If a
-parameter has a default, the corresponding attribute may be omitted in
-function calls.</para>
+The pattern specifies what the argument of the function must look
+like, and binds variables in the body to (parts of) the
+argument.  There are three kinds of patterns:</para>
+
+<itemizedlist>
+
+  
+  <listitem><para>If a pattern is a single identifier, then the
+  function matches any argument.  Example:
+
+  <programlisting>
+let negate = x: !x;
+    concat = x: y: x + y;
+in if negate true then concat "foo" "bar" else ""</programlisting>
+
+  Note that <function>concat</function> is a function that takes one
+  argument and returns a function that takes another argument.  This
+  allows partial parameterisation (i.e., only filling some of the
+  arguments of a function); e.g.,
+
+  <programlisting>
+map (concat "foo") ["bar" "bla" "abc"]</programlisting>
+
+  evaluates to <literal>["foobar" "foobla"
+  "fooabc"]</literal>.</para></listitem>
+
+  
+  <listitem><para>An <emphasis>attribute set pattern</emphasis> of the
+  form <literal>{name1, name2, …, nameN}</literal>
+  matches an attribute set containing the listed attributes, and binds
+  the values of those attributes to variables in the function body.
+  For example, the function
+
+<programlisting>
+{x, y, z}: z + y + x</programlisting>
+
+  can only be called with a set containing exactly the attributes
+  <varname>x</varname>, <varname>y</varname> and
+  <varname>z</varname>.  No other attributes are allowed.  If you want
+  to allow additional arguments, you can use an ellipsis
+  (<literal>...</literal>): 
+
+<programlisting>
+{x, y, z, ....}: z + y + x</programlisting>
+
+  This works on any set that contains at least the three named
+  attributes.</para>
+
+  <para>It is possible to provide <emphasis>default values</emphasis>
+  for attributes, in which case they are allowed to be missing.  A
+  default value is specified by writing
+  <literal><replaceable>name</replaceable> ?
+  <replaceable>e</replaceable></literal>, where
+  <replaceable>e</replaceable> is an arbitrary expression.  For example,
+
+<programlisting>
+{x, y ? "foo", z ? "bar"}: z + y + x</programlisting>
+
+  specifies a function that only requires an attribute named
+  <varname>x</varname>, but optionally accepts <varname>y</varname>
+  and <varname>z</varname>.</para></listitem>
+
+
+  <listitem><para>An <literal>@</literal>-pattern requires that the
+  argument matches with the patterns on the left- and right-hand side
+  of the <literal>@</literal>-sign.  For example:
+
+<programlisting>
+args@{x, y, z, ...}: z + y + x + args.a</programlisting>
+
+  Here <varname>args</varname> is bound to the entire argument, which
+  is further matches against the pattern <literal>{x, y, z,
+  ...}</literal>.</para></listitem>
+
+
+</itemizedlist>
 
 <para>Note that functions do not have names.  If you want to give them
 a name, you can bind them to an attribute, e.g.,
@@ -958,31 +1025,6 @@ in concat {x = "foo"; y = "bar";}</programlisting>
 
 </para>
 
-<para>It is also possible to define a function that takes a single
-argument and that does not need to be called with an attribute set as
-argument.  The syntax is
-
-<programlisting>
-<replaceable>var</replaceable>: <replaceable>body</replaceable></programlisting>
-
-where <replaceable>var</replaceable> is the name of the argument.  It
-is not possible to define a default.  Example:
-
-<programlisting>
-let negate = x: !x;
-    concat = x: y: x + y;
-in if negate true then concat "foo" "bar" else ""</programlisting>
-
-Note that <function>concat</function> is a function that takes one
-arguments and returns a function that takes another argument.  This
-allows partial parameterisation (i.e., only filling some of the
-arguments of a function); e.g.,
-
-<programlisting>
-map (concat "foo") ["bar" "bla" "abc"]</programlisting>
-
-evaluates to <literal>["foobar" "foobla" "fooabc"]</literal>.</para>
-
 </simplesect>
 
 
@@ -1390,7 +1432,7 @@ command-line argument.  See <xref linkend='sec-standard-environment'
   inputs.</para></listitem>
 
   <listitem><para>After the build, Nix sets the last-modified
-  timestamp on all files in the build result to 0 (00:00:00 1/1/1970
+  timestamp on all files in the build result to 1 (00:00:01 1/1/1970
   UTC), sets the group to the default group, and sets the mode of the
   file to 0444 or 0555 (i.e., read-only, with execute permission
   enabled if the file was originally executable).  Note that possible
@@ -1644,59 +1686,6 @@ impureEnvVars = ["http_proxy" "https_proxy" <replaceable>...</replaceable>];
 
 <section xml:id='sec-standard-environment'><title>The standard environment</title>
 
-<para>The standard build environment in the Nix Packages collection
-provides a basic environment for building Unix packages.  It consists
-of the following packages:
-
-<itemizedlist>
-
-  <listitem><para>The GNU C Compiler, configured with C and C++
-  support.  On Linux, the compiler has been patched to provide greater
-  <quote>purity</quote> assurance.  For instance, the compiler doesn't
-  search in locations such as <filename>/usr/include</filename>.  In
-  fact, attempts to add such directories through the
-  <option>-I</option> flag are filtered out.  Likewise, the linker
-  (from GNU binutils) doesn't search in standard locations such as
-  <filename>/usr/lib</filename>.  Programs built on Linux are linked
-  against a GNU C Library that likewise doesn't search in the default
-  system locations.</para></listitem>
-
-  <listitem><para>GNU coreutils (contains a few dozen standard Unix
-  commands).</para></listitem>
-
-  <listitem><para>GNU findutils (contains
-  <command>find</command>).</para></listitem>
-
-  <listitem><para>GNU diffutils (contains <command>diff</command>,
-  <command>cmp</command>).</para></listitem>
-
-  <listitem><para>GNU <command>sed</command>.</para></listitem>
-
-  <listitem><para>GNU <command>grep</command>.</para></listitem>
-
-  <listitem><para>GNU <command>awk</command>.</para></listitem>
-
-  <listitem><para>GNU <command>tar</command>.</para></listitem>
-
-  <listitem><para><command>gzip</command> and
-  <command>bzip2</command>.</para></listitem>
-
-  <listitem><para>GNU Make.  It has been patched to provide
-  <quote>nested</quote> output that can be fed into the
-  <command>nix-log2xml</command> command and
-  <command>log2html</command> stylesheet to create a structured,
-  readable output of the build steps performed by
-  Make.</para></listitem>
-
-  <listitem><para>Bash.  This is the shell used for all builders in
-  the Nix Packages collection.  Not using <command>/bin/sh</command>
-  removes a large source of portability problems.</para></listitem>
-
-  <listitem><para>Patch.</para></listitem>
-
-</itemizedlist>
-
-</para>
 
 <para>The standard environment is used by passing it as an input
 called <envar>stdenv</envar> to the derivation, and then doing
@@ -1765,115 +1754,6 @@ myPostInstall() {
 
 </para>
 
-<para>The generic builder has a number of <emphasis>phases</emphasis>,
-each of which can be override in its entirety by setting the indicated
-variable.  The phases are:
-
-<itemizedlist>
-
-  <listitem>
-
-    <para><function>unpackPhase</function> unpacks the source files
-    listed in the <envar>src</envar> environment variable to the
-    current directory.  It supports <filename>tar</filename> files,
-    optionally compressed with <command>gzip</command> or
-    <command>bzip2</command>; Zip files (but note that the
-    <command>unzip</command> command is not a part of the standard
-    environment; you should add it as a build input yourself); and
-    unpacked source trees (i.e., directories; they are copied
-    verbatim).  You can add support for other file types by setting
-    the <varname>findUnpacker</varname> hook.  This hook should set
-    the variable <varname>unpackCmd</varname> to contain the command
-    to be executed to unpack the file.</para>
-
-    <para>After unpacking all source files,
-    <function>unpackPhase</function> changes the current directory to
-    the directory created by unpacking the sources.  If there are
-    multiple source directories, you should set
-    <varname>sourceRoot</varname> to the name of the intended
-    directory.</para>
-
-    <para>It also calls the hook <varname>postUnpack</varname> after
-    unpacking.</para>
-
-  </listitem>
-
-  <listitem><para><function>patchPhase</function> calls the
-  <command>patch</command> command with the <option>-p1</option>
-  option for each patch file listed in the <envar>patches</envar>
-  variable.</para></listitem>
-
-  <listitem>
-
-    <para><function>configurePhase</function> runs the script called
-    <filename>configure</filename> in the current directory with a
-    <option>--prefix</option> set to the output path.  You can add
-    additional flags through the <varname>configureFlags</varname>
-    variable.  If <filename>configure</filename> does not exist,
-    nothing happens.</para>
-
-    <para>Before and after running <filename>configure</filename>, the
-    hooks <varname>preConfigure</varname> and
-    <varname>postConfigure</varname> are called, respectively.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para><function>buildPhase</function> calls
-    <command>make</command>.  You can set flags for
-    <command>make</command> through the <varname>makeFlags</varname>
-    variable.</para>
-
-    <para>Before and after running <command>make</command>, the hooks
-    <varname>preBuild</varname> and <varname>postBuild</varname> are
-    called, respectively.</para>
-    
-  </listitem>
-
-  <listitem><para><function>checkPhase</function> calls <command>make
-  check</command>, but only if the <varname>doCheck</varname> variable
-  is set to <literal>1</literal>.  Additional flags can be set through
-  the <varname>checkFlags</varname> variable.</para></listitem>
-
-  <listitem>
-
-    <para><function>installPhase</function> calls <command>make
-    install</command>.  Additional flags can be set through the
-    <varname>installFlags</varname> variable.  It also strips any
-    static libraries in the output path of debug information unless
-    <varname>dontStrip</varname> is set to
-    <literal>1</literal>.</para>
-
-    <para>Before and after running <command>make install</command>,
-    the hooks <varname>preInstall</varname> and
-    <varname>postInstall</varname> are called, respectively.</para>
-
-  </listitem>
-
-  <listitem>
-
-    <para><function>distPhase</function> calls <command>make
-    dist</command>, but only if the <varname>doDist</varname> variable
-    is set to <literal>1</literal>.  Additional flags can be set
-    through the <varname>distFlags</varname> variable.  The resulting
-    tarball is copied to the <filename>/tarballs</filename>
-    subdirectory of the output path.</para>
-
-    <para>Before and after running <command>make dist</command>, the
-    hooks <varname>preDist</varname> and <varname>postDist</varname>
-    are called, respectively.</para>
-    
-  </listitem>
-
-</itemizedlist>
-
-</para>
-
-<para>You can change the order in which phases are executed, or add
-new phases, by setting the <varname>phases</varname> variable.  The
-default is <literal>patchPhase configurePhase buildPhase checkPhase
-installPhase distPhase</literal>.</para>
 
 </section>
 

externals/Makefile.am

@@ -2,6 +2,8 @@
 
 DB = db-4.5.20
 
+if OLD_DB_COMPAT
+
 $(DB).tar.gz:
 	@echo "Nix requires Berkeley DB to build."
 	@echo "Please download version 4.5.20 from"
@@ -32,6 +34,12 @@ build-db: have-db
 	touch build-db
 endif
 
+else
+
+build-db:
+
+endif
+
 
 # CWI ATerm
 
@@ -40,7 +48,7 @@ ATERM = aterm-2.4.2-fixes-r2
 $(ATERM).tar.bz2:
 	@echo "Nix requires the CWI ATerm library to build."
 	@echo "Please download version 2.4.2-fixes-r2 from"
-	@echo "  http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes-r2.tar.bz2"
+	@echo "  http://nixos.org/tarballs/aterm-2.4.2-fixes-r2.tar.bz2"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -67,12 +75,12 @@ endif
 
 # bzip2
 
-BZIP2 = bzip2-1.0.4
+BZIP2 = bzip2-1.0.5
 
 $(BZIP2).tar.gz:
 	@echo "Nix requires bzip2 to build."
-	@echo "Please download version 1.0.4 from"
-	@echo "  http://www.bzip.org/1.0.4/bzip2-1.0.4.tar.gz"
+	@echo "Please download version 1.0.5 from"
+	@echo "  http://www.bzip.org/1.0.5/bzip2-1.0.5.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -105,5 +113,5 @@ EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.bz2 $(BZIP2).tar.gz \
   bdb-cygwin.patch
 
 ext-clean:
-	$(RM) -f have-db build-db have-aterm build-aterm
+	$(RM) -f have-db build-db have-aterm build-aterm have-bzip2 build-bzip2
 	$(RM) -rf $(DB) $(ATERM) $(BZIP2)

install_full.sh

@@ -1,67 +0,0 @@
-#! /bin/sh -e
-
-make clean				# comment this out when needed !!!
-
-export nixstatepath=/nixstate2/nix
-export ACLOCAL_PATH=/home/wouterdb/.nix-profile/share/aclocal
-
-if [ "$1" = "full" ]; then
-  nix-env-all-pkgs.sh -i gcc
-  nix-env-all-pkgs.sh -i gnum4
-  nix-env-all-pkgs.sh -i autoconf
-  nix-env-all-pkgs.sh -i automake
-  nix-env-all-pkgs.sh -i gnused
-  nix-env-all-pkgs.sh -i db4
-  nix-env-all-pkgs.sh -i aterm
-  nix-env-all-pkgs.sh -i bzip2
-  nix-env-all-pkgs.sh -i flex
-  nix-env-all-pkgs.sh -i bsdiff
-  nix-env-all-pkgs.sh -i libtool
-  nix-env-all-pkgs.sh -i docbook5
-  nix-env-all-pkgs.sh -i docbook5-xsl
-  nix-env-all-pkgs.sh -i bison
-  nix-env-all-pkgs.sh -i gdb			#optional for debugging
-  nix-env-all-pkgs.sh -i gnupatch
-  nix-env-all-pkgs.sh -i gnumake
-  nix-env-all-pkgs.sh -i ext3cow-tools
-  nix-env-all-pkgs.sh -i e3cfsprogs
-  nix-env-all-pkgs.sh -i rsync  
-fi
-
-if [ "$1" = "full" ] || [ "$1" = "auto" ]; then
-  export AUTOCONF=autoconf
-  export AUTOHEADER=autoheader
-  export AUTOMAKE=automake
-  autoconf
-  autoreconf -f
-  aclocal
-  autoheader
-  automake
-fi
-
-./bootstrap.sh
-
-./configure --with-aterm=$HOME/.nix-profile \
-            --with-bzip2=$HOME/.nix-profile \
-            --with-bdb=$HOME/.nix-profile \
-            --with-docbook-xsl=$HOME/.nix-profile \
-            --with-docbook-rng=/home/wouterdb/.nix-profile/xml/rng/docbook \
-            --with-docbook-xsl=/home/wouterdb/.nix-profile/xml/xsl/docbook \
-            --prefix=$nixstatepath \
-            --with-store-dir=/nix/store \
-	    	--with-store-state-dir=/nix/state \
-            --with-ext3cow-header=/nix/store/2sm0h2xd1zsm5had53q1pvzmnsn8fy8k-linux-2.6.21/lib/modules/2.6.21-ck1-default/build/include/linux/ext3cow_fs.h \
-            --localstatedir=/nix/var
-
-#Options from the nix expr
-#--disable-init-state
-#--with-store-dir=/nix/store 
-#--localstatedir=/nix/var
-#--with-aterm=/nix/store/pkmzbb613wa8cwngx8jjb5jaic8yhyzs-aterm-2.4.2-fixes
-#--with-bdb=/nix/store/4yv4j1cd7i5j3mhs5wpc1kzlz1cj8n82-db4-4.5.20
-#--with-bzip2=/nix/store/dh0mdgkvhv3pwrf8zp58phpzn9rcm49r-bzip2-1.0.3
-#--disable-init-state
-
-
-echo "New state nix version by wouter ..." > doc/manual/NEWS.txt
-make

install_install_d.sh

@@ -1,16 +0,0 @@
-#! /bin/sh -e
-
-if [ $(whoami) = "root" ]
-then
-
-su - wouterdb -c "cd /home/wouterdb/dev/nix-state/; make"
-make install
-chown -R wouterdb.wouterdb /nixstate2/nix/
-
-./restartDaemon.sh
-
-else
-  echo "You must be ROOT to run this script."
-  exit 0
-fi
-

install_make.sh

@@ -1,5 +0,0 @@
-#! /bin/sh -e
-
-make
-
-

mergeTrunkBackIn.sh

@@ -1,36 +0,0 @@
-svn merge -r 10855:10943 https://svn.cs.uu.nl:12443/repos/trace/nix/trunk
-
-#already done:
-# 8628 
-# 8632
-# 8634
-# 8636
-# 8655
-# 8691
-# 8698
-# 8711
-# 8864
-# 9063
-# 9105 
-# 9207
-# 9217
-# 9332
-# 9429
-# 9433
-# 9435
-# 9437
-# 9439
-# 9445
-# 9476
-# 9506
-# 9536
-# 9549
-# 9561
-# 9584
-# 9751
-# 10133
-# 10154
-# 10531
-# 10692
-# 10855
-# 10943

misc/emacs/nix-mode.el

@@ -76,10 +76,10 @@ The hook `nix-mode-hook' is run when Nix mode is started.
     ("\\<baseNameOf\\>" . font-lock-builtin-face)
     ("\\<toString\\>" . font-lock-builtin-face)
     ("\\<isNull\\>" . font-lock-builtin-face)
-    ("\\<\\([a-zA-Z_][a-zA-Z0-9_']*\\)[ \t]*="
-     (1 font-lock-variable-name-face nil nil))
     ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+"
      . font-lock-constant-face)
+    ("\\<\\([a-zA-Z_][a-zA-Z0-9_'\.]*\\)[ \t]*="
+     (1 font-lock-variable-name-face nil nil))
     ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+"
      . font-lock-constant-face)
     ))

misc/vim/syntax/nix.vim

@@ -21,12 +21,14 @@ syn match nixFuncArg "\zs\w\+\ze\s*:"
 syn region nixStringParam start=+\${+ end=+}+
 syn region nixMultiLineComment start=+/\*+ skip=+\\"+ end=+\*/+
 syn match  nixEndOfLineComment "#.*$"
-syn region nixString      start=+"+ skip=+\\"+ end=+"+ contains=nixStringParam
+syn region nixStringIndented start=+''+ skip=+'''\|''${\|"+ end=+''+ contains=nixStringParam
+syn region nixString         start=+"+ skip=+\\"+ end=+"+ contains=nixStringParam
 
 hi def link nixKeyword       Keyword
 hi def link nixConditional   Conditional
 hi def link nixBrace         Special
 hi def link nixString        String
+hi def link nixStringIndented String
 hi def link nixBuiltin       Special
 hi def link nixStringParam   Macro
 hi def link nixMultiLineComment Comment

nix.conf.example

@@ -29,25 +29,6 @@
 #gc-keep-derivations = true
 
 
-### Option `gc-reserved-space'
-#
-# This option specifies how much space should be reserved in normal
-# use so that the garbage collector can run succesfully.  Since the
-# garbage collector must perform Berkeley DB transactions, it needs
-# some disk space for itself.  However, when the disk is full, this
-# space is not available, so the collector would not be able to run
-# precisely when it is most needed.
-#
-# For this reason, when Nix is run, it allocates a file
-# /nix/var/nix/db/reserved of the size specified by this option.  When
-# the garbage collector is run, this file is deleted before the
-# Berkeley DB environment is opened.  This should give it enough room
-# to proceed.
-#
-# The default is "1048576" (1 MiB).
-#gc-reserved-space = 1048576
-
-
 ### Option `env-keep-derivations'
 #
 # If `false' (default), derivations are not stored in Nix user
@@ -164,30 +145,26 @@
 # (using `mount --bind' on Linux) some directories from the normal
 # file system hierarchy inside the chroot.  These are the Nix store,
 # the temporary build directory (usually /tmp/nix-<pid>-<number>) and
-# the directories listed here.  The default is "/dev /proc".  Files
-# in /dev (such as /dev/null) are needed by many builds, and some
-# files in /proc may also be needed occasionally.
+# the directories listed here.  The default is "/dev /dev/pts /proc".
+# Files in /dev (such as /dev/null) are needed by many builds, and
+# some files in /proc may also be needed occasionally.
 #
 # Example:
 #   build-use-chroot = /dev /proc /bin
-#build-chroot-dirs = /dev /proc
+#build-chroot-dirs = /dev /dev/pts /proc
 
 
-### Option `system'
+### Option `build-cache-failure'
 #
-# This option specifies the canonical Nix system name of the current
-# installation, such as `i686-linux' or `powerpc-darwin'.  Nix can
-# only build derivations whose `system' attribute equals the value
-# specified here.  In general, it never makes sense to modify this
-# value from its default, since you can use it to `lie' about the
-# platform you are building on (e.g., perform a Mac OS build on a
-# Linux machine; the result would obviously be wrong).  It only makes
-# sense if the Nix binaries can run on multiple platforms, e.g.,
-# `universal binaries' that run on `powerpc-darwin' and `i686-darwin'.
-#
-# It defaults to the canonical Nix system name detected by `configure'
-# at build time.
+# If this option is enabled, Nix will do negative caching; that is, it
+# will remember failed builds, and won't attempt to try to build them
+# again if you ask for it.  Negative caching is disabled by default
+# because Nix cannot distinguish between permanent build errors (e.g.,
+# a syntax error in a source file) and transient build errors (e.g., a
+# full disk), as they both cause the builder to return a non-zero exit
+# code.  You can clear the cache by doing `rm -f
+# /nix/var/nix/db/failed/*'.
 #
 # Example:
-#   system = i686-darwin
-#system =
+#   build-cache-failure = true
+#build-cache-failure = false

nix.spec.in

@@ -13,11 +13,10 @@ Version: @version@
 Release: 1
 License: GPL
 Group: Software Deployment
-URL: http://nix.cs.uu.nl/
+URL: http://nixos.org/
 Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
-%define _prefix /nix
-Prefix: %{_prefix}
+Prefix: /usr
 Requires: /usr/bin/perl
 Requires: curl
 
@@ -47,14 +46,14 @@ if test -n "%{enable_setuid}"; then
         extraFlags="$extraFlags --with-nix-group=%{nix_group}"
     fi
 fi
-./configure --prefix=%{_prefix} $extraFlags
+./configure --prefix=%{_prefix} --sysconfdir=/etc $extraFlags
 make
 make check
 
 %install
 rm -rf $RPM_BUILD_ROOT
 make DESTDIR=$RPM_BUILD_ROOT install
-rm $RPM_BUILD_ROOT/%{_prefix}/etc/nix/nix.conf
+rm $RPM_BUILD_ROOT/etc/nix/nix.conf
 strip $RPM_BUILD_ROOT/%{_prefix}/bin/* || true
 
 %clean
@@ -76,10 +75,9 @@ fi
 %{_prefix}/lib
 %{_prefix}/libexec
 %{_prefix}/include
-%{_prefix}/var
 %{_prefix}/share
-%{_prefix}/store
+/etc/profile.d/nix.sh
+/nix/var
+/nix/store
 %config
-%{_prefix}/etc
-#%doc
-#%{_prefix}/share/nix/manual
+/etc/nix

release.nix

@@ -0,0 +1,196 @@
+{ nixpkgs ? ../nixpkgs }:
+
+let
+
+  jobs = rec {
+
+
+    tarball =
+      { nix ? {outPath = ./.; rev = 1234;}
+      , officialRelease ? false
+      }:
+
+      with import nixpkgs {};
+
+      releaseTools.sourceTarball {
+        name = "nix-tarball";
+        version = builtins.readFile ./version;
+        src = nix;
+        inherit officialRelease;
+
+        buildInputs = [curl bison flex2533 perl libxml2 libxslt w3m bzip2 jing_tools tetex dblatex];
+
+        configureFlags = ''
+          --with-docbook-rng=${docbook5}/xml/rng/docbook
+          --with-docbook-xsl=${docbook5_xsl}/xml/xsl/docbook
+          --with-xml-flags=--nonet
+        '';
+
+        # Include the BDB, ATerm and Bzip2 tarballs in the distribution.
+        preConfigure = ''
+          stripHash ${db45.src}
+          # Remove unnecessary stuff from the Berkeley DB tarball.
+          ( mkdir bdb-temp
+            cd bdb-temp
+            tar xfz ${db45.src}
+            cd *
+            rm -rf docs test tcl perl libdb_java java rpc_server build_vxworks \
+              examples_java examples_c examples_cxx dist/tags
+            mkdir test
+            touch test/include.tcl
+            cd ..
+            tar cvfz ../externals/$strippedName *
+          )
+
+          stripHash ${aterm242fixes.src}
+          cp -pv ${aterm242fixes.src} externals/$strippedName
+
+          stripHash ${bzip2.src}
+          cp -pv ${bzip2.src} externals/$strippedName
+        '';
+
+        preDist = ''
+          make -C doc/manual install prefix=$out
+          
+          make -C doc/manual manual.pdf prefix=$out
+          cp doc/manual/manual.pdf $out/manual.pdf
+          
+          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
+          echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products
+          echo "doc release-notes $out/share/doc/nix/release-notes" >> $out/nix-support/hydra-build-products
+        '';
+      };
+
+
+    build =
+      { tarball ? jobs.tarball {}
+      , system ? "i686-linux"
+      }:
+
+      with import nixpkgs {inherit system;};
+
+      releaseTools.nixBuild {
+        name = "nix";
+        src = tarball;
+
+        buildInputs = [curl perl bzip2 openssl];
+
+        configureFlags = ''
+          --disable-init-state
+          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+        '';
+      };
+
+      
+    static =
+      { tarball ? jobs.tarball {}
+      , system ? "i686-linux"
+      }:
+
+      with import nixpkgs {inherit system;};
+
+      releaseTools.binaryTarball {
+        name = "nix-static-tarball";
+        src = tarball;
+
+        buildInputs = [curl perl bzip2];
+
+        configureFlags = ''
+          --disable-init-state
+          --disable-old-db-compat --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+          --enable-static-nix
+        '';
+      };
+
+      
+    coverage =
+      { tarball ? jobs.tarball {}
+      }:
+
+      with import nixpkgs {};
+
+      releaseTools.coverageAnalysis {
+        name = "nix-build";
+        src = tarball;
+
+        buildInputs = [
+          curl perl bzip2 openssl
+          # These are for "make check" only:
+          graphviz libxml2 libxslt
+        ];
+
+        configureFlags = ''
+          --disable-init-state --disable-shared
+          --with-bdb=${db45} --with-aterm=${aterm242fixes} --with-bzip2=${bzip2}
+        '';
+
+        lcovFilter = ["*/boost/*" "*-tab.*"];
+
+        # We call `dot', and even though we just use it to
+        # syntax-check generated dot files, it still requires some
+        # fonts.  So provide those.
+        FONTCONFIG_FILE = texFunctions.fontsConf;
+      };
+
+      
+    rpm_fedora5i386 = makeRPM_i686 (diskImages: diskImages.fedora5i386) 20;
+    rpm_fedora9i386 = makeRPM_i686 (diskImages: diskImages.fedora9i386) 50;
+    rpm_fedora9x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora9x86_64) 50;
+    rpm_fedora10i386 = makeRPM_i686 (diskImages: diskImages.fedora10i386) 40;
+    rpm_fedora10x86_64 = makeRPM_x86_64 (diskImages: diskImages.fedora10x86_64) 40;
+    rpm_opensuse103i386 = makeRPM_i686 (diskImages: diskImages.opensuse103i386) 40;
+
+    
+    deb_debian40i386 = makeDeb_i686 (diskImages: diskImages.debian40i386) 40;
+    deb_debian40x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian40x86_64) 40;
+    deb_debian50i386 = makeDeb_i686 (diskImages: diskImages.debian50i386) 30;
+    deb_debian50x86_64 = makeDeb_x86_64 (diskImages: diskImages.debian50x86_64) 30;
+    deb_ubuntu804i386 = makeDeb_i686 (diskImages: diskImages.ubuntu804i386) 50;
+    deb_ubuntu804x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu804x86_64) 50;
+    deb_ubuntu810i386 = makeDeb_i686 (diskImages: diskImages.ubuntu810i386) 40;
+    deb_ubuntu810x86_64 = makeDeb_x86_64 (diskImages: diskImages.ubuntu810x86_64) 40;
+
+
+  };
+
+
+  makeRPM_i686 = makeRPM "i686-linux";
+  makeRPM_x86_64 = makeRPM "x86_64-linux";
+
+  makeRPM = 
+    system: diskImageFun: prio:
+    { tarball ? jobs.tarball {}
+    }:
+
+    with import nixpkgs {inherit system;};
+
+    releaseTools.rpmBuild rec {
+      name = "nix-rpm-${diskImage.name}";
+      src = tarball;
+      diskImage = diskImageFun vmTools.diskImages;
+      memSize = 1024;
+      meta = { schedulingPriority = toString prio; };
+    };
+
+
+  makeDeb_i686 = makeDeb "i686-linux";
+  makeDeb_x86_64 = makeDeb "x86_64-linux";
+  
+  makeDeb =
+    system: diskImageFun: prio:
+    { tarball ? jobs.tarball {}
+    }:
+
+    with import nixpkgs {inherit system;};
+
+    releaseTools.debBuild {
+      name = "nix-deb";
+      src = tarball;
+      diskImage = diskImageFun vmTools.diskImages;
+      memSize = 1024;
+      meta = { schedulingPriority = toString prio; };
+      configureFlags = "--sysconfdir=/etc";
+    };
+
+
+in jobs

restartDaemon.sh

@@ -1,9 +0,0 @@
-#! /bin/sh
-
-initctl stop nix-daemon
-killproc.sh nix-worker
-sleep 2
-
-#/nixstate2/nix/bin/nix-worker --daemon > /dev/null 2>&1 &
-/nixstate2/nix/bin/nix-worker --daemon
-#gdb --args /nixstate2/nix/bin/nix-worker --daemon

scripts/Makefile.am

@@ -1,24 +1,26 @@
 bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build \
-  nix-pack-closure nix-unpack-closure \
   nix-copy-closure 
 
 noinst_SCRIPTS = nix-profile.sh generate-patches.pl \
-  find-runtime-roots.pl build-remote.pl nix-reduce-build
+  find-runtime-roots.pl build-remote.pl nix-reduce-build \
+  copy-from-other-stores.pl nix-http-export.cgi
 
 nix-pull nix-push: readmanifest.pm readconfig.pm download-using-manifests.pl
 
-install-exec-local: readmanifest.pm download-using-manifests.pl find-runtime-roots.pl
+install-exec-local: readmanifest.pm download-using-manifests.pl copy-from-other-stores.pl find-runtime-roots.pl
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
 	$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
 	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
 	$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
-	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) build-remote.pl $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix/substituters
+	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix/substituters
+	$(INSTALL_PROGRAM) copy-from-other-stores.pl $(DESTDIR)$(libexecdir)/nix/substituters
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 
 include ../substitute.mk
@@ -31,9 +33,10 @@ EXTRA_DIST = nix-collect-garbage.in \
   readconfig.pm.in \
   nix-build.in \
   download-using-manifests.pl.in \
+  copy-from-other-stores.pl.in \
   generate-patches.pl.in \
-  nix-pack-closure.in nix-unpack-closure.in \
   nix-copy-closure.in \
   find-runtime-roots.pl.in \
   build-remote.pl.in \
-  nix-reduce-build.in
+  nix-reduce-build.in \
+  nix-http-export.cgi.in 

scripts/build-remote.pl.in

@@ -3,6 +3,7 @@
 use strict;
 use Fcntl ':flock';
 use English '-no_match_vars';
+use IO::Handle;
 
 # General operation:
 #
@@ -23,16 +24,12 @@ use English '-no_match_vars';
 
 my $loadIncreased = 0;
 
-my $amWilling = shift @ARGV;
-my $localSystem = shift @ARGV;
-my $neededSystem = shift @ARGV;
-my $drvPath = shift @ARGV;
+my ($amWilling, $localSystem, $neededSystem, $drvPath, $maxSilentTime) = @ARGV;
+$maxSilentTime = 0 unless defined $maxSilentTime;
 
 sub sendReply {
     my $reply = shift;
-    open OUT, ">&3" or die;
-    print OUT "$reply\n";
-    close OUT;
+    print STDERR "# $reply\n";
 }
 
 sub decline {
@@ -47,15 +44,11 @@ mkdir $currentLoad, 0777 or die unless -d $currentLoad;
 my $conf = $ENV{"NIX_REMOTE_SYSTEMS"};
 decline if !defined $conf || ! -e $conf;
 
-# Decline if the local system can do the build.
-decline if $amWilling && ($localSystem eq $neededSystem);
+my $canBuildLocally = $amWilling && ($localSystem eq $neededSystem);
 
 
 # Otherwise find a willing remote machine.
-my %machines;
-my %systemTypes;
-my %sshKeys;
-my %maxJobs;
+my @machines;
 my %curJobs;
 
 
@@ -67,10 +60,12 @@ while (<CONF>) {
     s/\#.*$//g;
     next if /^\s*$/;
     /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\d+)\s*$/ or die;
-    $machines{$1} = "";
-    $systemTypes{$1} = $2;
-    $sshKeys{$1} = $3;
-    $maxJobs{$1} = $4;
+    push @machines,
+        { hostName => $1
+        , systemType => $2
+        , sshKeys => $3
+        , maxJobs => $4
+        };
 }
 
 close CONF;
@@ -85,21 +80,26 @@ flock(MAINLOCK, LOCK_EX) or die;
 # Find a suitable system.
 my $rightType = 0;
 my $machine;
-LOOP: foreach my $cur (keys %machines) {
-    if ($neededSystem eq $systemTypes{$cur}) {
+my $slotLock;
+LOOP: foreach my $cur (@machines) {
+    if ($neededSystem eq $cur->{systemType}
+        || ($neededSystem eq "i686-linux" && $cur->{systemType} eq "x86_64-linux"))
+    {
         $rightType = 1;
 
         # We have a machine of the right type.  Try to get a lock on
         # one of the machine's lock files.
         my $slot = 0;
-        while ($slot < $maxJobs{$cur}) {
-            my $slotLock = "$currentLoad/$cur-$slot";
-            open SLOTLOCK, ">>$slotLock" or die;
-            if (flock(SLOTLOCK, LOCK_EX | LOCK_NB)) {
+        while ($slot < $cur->{maxJobs}) {
+            my $slotLockFn = "$currentLoad/" . $cur->{systemType} . "-" . $cur->{hostName} . "-$slot";
+            $slotLock = new IO::Handle;
+            open $slotLock, ">>$slotLockFn" or die;
+            if (flock($slotLock, LOCK_EX | LOCK_NB)) {
+                utime undef, undef, $slotLock;
                 $machine = $cur;
                 last LOOP;
-            }                
-            close SLOTLOCK;
+            }
+            close $slotLock;
             $slot++;
         }
     }
@@ -108,9 +108,11 @@ LOOP: foreach my $cur (keys %machines) {
 close MAINLOCK;
 
 
-# Didn't find one?
+# Didn't find one?  Then decline or postpone.
 if (!defined $machine) {
-    if ($rightType) {
+    # Postpone if we have a machine of the right type, except if the
+    # local system can and wants to do the build.
+    if ($rightType && !$canBuildLocally) {
         sendReply "postpone";
         exit 0;
     } else {
@@ -120,11 +122,8 @@ if (!defined $machine) {
 
 # Yes we did, accept.
 sendReply "accept";
-open IN, "<&4" or die;
-my $x = <IN>;
+my $x = <STDIN>;
 chomp $x;
-#print "got $x\n";  
-close IN;
 
 if ($x ne "okay") {
     exit 0;
@@ -132,7 +131,8 @@ if ($x ne "okay") {
 
 
 # Do the actual job.
-print "BUILDING REMOTE: $drvPath on $machine\n";
+my $hostName = $machine->{hostName};
+print STDERR "building `$drvPath' on `$hostName'\n";
 
 # Make sure that we don't get any SSH passphrase or host key popups -
 # if there is any problem it should fail, not do something
@@ -141,10 +141,10 @@ $ENV{"DISPLAY"} = "";
 $ENV{"SSH_PASSWORD_FILE="} = "";
 $ENV{"SSH_ASKPASS="} = "";
 
-my $sshOpts = "-i $sshKeys{$machine} -x";
+my $sshOpts = "-i " . $machine->{sshKeys} . " -x";
 
 # Hack to support Cygwin: if we login without a password, we don't
-# have exactly the same right as when we do.  This causes the
+# have exactly the same rights as when we do.  This causes the
 # Microsoft C compiler to fail with certain flags:
 #
 #   http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99676
@@ -154,8 +154,8 @@ my $sshOpts = "-i $sshKeys{$machine} -x";
 # password.  (It only calls this command when there is no controlling
 # terminal, but Nix ensures that is is the case.  When doing this
 # manually, use setsid(1).)
-if ($sshKeys{$machine} =~ /^password:/) {
-    my $passwordFile = $sshKeys{$machine};
+if ($machine->{sshKeys} =~ /^password:/) {
+    my $passwordFile = $machine->{sshKeys};
     $passwordFile =~ s/^password://;
     $sshOpts = "ssh -x";
     $ENV{"SSH_PASSWORD_FILE"} = $passwordFile;
@@ -173,36 +173,40 @@ $inputs =~ s/\n/ /g;
 my $outputs = `cat outputs`; die if ($? != 0);
 $outputs =~ s/\n/ /g;
 
-print "COPYING INPUTS...\n";
+print "copying inputs...\n";
 
 my $maybeSign = "";
 $maybeSign = "--sign" if -e "/nix/etc/nix/signing-key.sec";
 
-system("NIX_SSHOPTS=\"$sshOpts\" nix-copy-closure $machine $maybeSign $drvPath $inputs") == 0
-    or die "cannot copy inputs to $machine: $?";
+system("NIX_SSHOPTS=\"$sshOpts\" @bindir@/nix-copy-closure --gzip $hostName $maybeSign $drvPath $inputs") == 0
+    or die "cannot copy inputs to $hostName: $?";
 
-print "BUILDING...\n";
+print "building...\n";
 
-system("ssh $sshOpts $machine 'nix-store -rvvK $drvPath'") == 0
-    or die "remote build on $machine failed: $?";
+my $buildFlags = "--max-silent-time $maxSilentTime";
 
-print "REMOTE BUILD DONE: $drvPath on $machine\n";
+# `-tt' forces allocation of a pseudo-terminal.  This is required to
+# make the remote nix-store process receive a signal when the
+# connection dies.  Without it, the remote process might continue to
+# run indefinitely (that is, until it next tries to write to
+# stdout/stderr).
+if (system("ssh -tt $sshOpts $hostName 'nix-store --realise -K $buildFlags $drvPath > /dev/null'") != 0) {
+    # If we couldn't run ssh or there was an ssh problem (indicated by
+    # exit code 255), then we return exit code 1; otherwise we assume
+    # that the builder failed, which we indicated to Nix using exit
+    # code 100.  It's important to distinguish between the two because
+    # the first is a transient failure and the latter is permanent.
+    my $res = $? == -1 || ($? >> 8) == 255 ? 1 : 100;
+    print STDERR "build of `$drvPath' on `$hostName' failed with exit code $?\n";
+    exit $res;
+}
+
+print "build of `$drvPath' on `$hostName' succeeded\n";
 
 foreach my $output (split '\n', $outputs) {
     my $maybeSignRemote = "";
     $maybeSignRemote = "--sign" if $UID != 0;
     
-    system("ssh $sshOpts $machine 'nix-store --export $maybeSignRemote $output' > dump") == 0
-	or die "cannot copy $output from $machine: $?";
-
-    # This doesn't work yet, since the caller has a lock on the output
-    # path.  We should move towards lock-free invocation of build
-    # hooks and substitutes.
-    #system("nix-store --import < dump") == 0
-    #    or die "cannot import $output: $?";
-
-    # Hack: skip the first 8 bytes (the nix-store --export next
-    # archive marker).  The archive follows.
-    system("(dd bs=1 count=8 of=/dev/null && cat) < dump | nix-store --restore $output") == 0
-	or die "cannot restore $output: $?";
+    system("ssh $sshOpts $hostName 'nix-store --export $maybeSignRemote $output | gzip' | gunzip | @bindir@/nix-store --import > /dev/null") == 0
+	or die "cannot copy $output from $hostName: $?";
 }

scripts/copy-from-other-stores.pl.in

@@ -0,0 +1,98 @@
+#! @perl@ -w
+
+use strict;
+use File::Basename;
+use IO::Handle;
+
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
+
+STDOUT->autoflush(1);
+
+my @remoteStoresAll = split ':', ($ENV{"NIX_OTHER_STORES"} or "");
+
+my @remoteStores;
+foreach my $dir (@remoteStoresAll) {
+    push @remoteStores, glob($dir);
+}
+
+
+sub findStorePath {
+    my $storePath = shift;
+    
+    my $storePathName = basename $storePath;
+    
+    foreach my $store (@remoteStores) {
+        # Determine whether $storePath exists by looking for the
+        # existence of the info file, and if so, get store path info
+        # from that file.  This rather breaks abstraction: we should
+        # be using `nix-store' for that.  But right now there is no
+        # good way to tell nix-store to access a store mounted under a
+        # different location (there's $NIX_STORE, but that only works
+        # if the remote store is mounted under its "real" location).
+        my $infoFile = "$store/var/nix/db/info/$storePathName";
+        my $storePath2 = "$store/store/$storePathName";
+        if (-f $infoFile && -e $storePath2) {
+            return ($infoFile, $storePath2);
+        }
+    }
+}
+
+
+if ($ARGV[0] eq "--query") {
+
+    while (<STDIN>) {
+        my $cmd = $_; chomp $cmd;
+
+        if ($cmd eq "have") {
+            my $storePath = <STDIN>; chomp $storePath;
+            (my $infoFile) = findStorePath $storePath;
+            print STDOUT ($infoFile ? "1\n" : "0\n");
+        }
+
+        elsif ($cmd eq "info") {
+            my $storePath = <STDIN>; chomp $storePath;
+            (my $infoFile) = findStorePath $storePath;
+            if (!$infoFile) {
+                print "0\n";
+                next; # not an error
+            }
+            print "1\n";
+
+            my $deriver = "";
+            my @references = ();
+
+            open INFO, "<$infoFile" or die "cannot read info file $infoFile\n";
+            while (<INFO>) {
+                chomp;
+                /^([\w-]+): (.*)$/ or die "bad info file";
+                my $key = $1;
+                my $value = $2;
+                if ($key eq "Deriver") { $deriver = $value; }
+                elsif ($key eq "References") { @references = split ' ', $value; }
+            }
+            close INFO;
+
+            print "$deriver\n";
+            print scalar @references, "\n";
+            print "$_\n" foreach @references;
+            print "0\n"; # !!! showing size not supported (yet)
+        }
+
+        else { die "unknown command `$cmd'"; }
+    }
+}
+
+
+elsif ($ARGV[0] eq "--substitute") {
+    die unless scalar @ARGV == 2;
+    my $storePath = $ARGV[1];
+    (my $infoFile, my $sourcePath) = findStorePath $storePath;
+    die unless $infoFile;
+    print "\n*** Copying `$storePath' from `$sourcePath'\n\n";
+    system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $storePath") == 0
+        or die "cannot copy `$sourcePath' to `$storePath'";
+}
+
+
+else { die; }

scripts/download-using-manifests.pl.in

@@ -5,20 +5,13 @@ use readmanifest;
 use POSIX qw(strftime);
 use File::Temp qw(tempdir);
 
-my $manifestDir = "@localstatedir@/nix/manifests";
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
+STDOUT->autoflush(1);
+
+my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "@localstatedir@/nix/manifests");
 my $logFile = "@localstatedir@/log/nix/downloads";
 
-open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
-
-# Create a temporary directory.
-my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-chdir $tmpDir or die "cannot change to `$tmpDir': $!";
-
-my $tmpNar = "$tmpDir/nar";
-my $tmpNar2 = "$tmpDir/nar2";
-
 
 # Load all manifests.
 my %narFiles;
@@ -26,61 +19,56 @@ my %localPaths;
 my %patches;
 
 for my $manifest (glob "$manifestDir/*.nixmanifest") {
-#    print STDERR "reading $manifest\n";
-    if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
+    my $version = readManifest($manifest, \%narFiles, \%localPaths, \%patches);
+    if ($version < 3) {
         print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
         exit 1;
     }
+    if ($version >= 10) {
+        print STDERR "manifest `$manifest' is too new; please delete it or upgrade Nix\n";
+        exit 1;
+    }
 }
 
 
 # Parse the arguments.
 
-if ($ARGV[0] eq "--query-paths") {
-    foreach my $storePath (keys %narFiles) { print "$storePath\n"; }
-    foreach my $storePath (keys %localPaths) { print "$storePath\n"; }
-    exit 0;
-}
+if ($ARGV[0] eq "--query") {
 
-elsif ($ARGV[0] eq "--query-info") {
-    shift @ARGV;
-    foreach my $storePath (@ARGV) {
-        my $info;
-        if (defined $narFiles{$storePath}) {
-            $info = @{$narFiles{$storePath}}[0];
+    while (<STDIN>) {
+        my $cmd = $_; chomp $cmd;
+
+        if ($cmd eq "have") {
+            my $storePath = <STDIN>; chomp $storePath;
+            print STDOUT ((defined $narFiles{$storePath} or defined $localPaths{$storePath})
+                ? "1\n" : "0\n");
         }
-        elsif (defined $localPaths{$storePath}) {
-            $info = @{$localPaths{$storePath}}[0];
+
+        elsif ($cmd eq "info") {
+            my $storePath = <STDIN>; chomp $storePath;
+            my $info;
+            if (defined $narFiles{$storePath}) {
+                $info = @{$narFiles{$storePath}}[0];
+            }
+            elsif (defined $localPaths{$storePath}) {
+                $info = @{$localPaths{$storePath}}[0];
+            }
+            else {
+                print "0\n";
+                next; # not an error
+            }
+            print "1\n";
+            print "$info->{deriver}\n";
+            my @references = split " ", $info->{references};
+            print scalar @references, "\n";
+            print "$_\n" foreach @references;
+            my $size = $info->{size} || 0;
+            print "$size\n";
         }
-        else {
-            next; # not an error
-        }
-        print "$storePath\n";
-        print "$info->{deriver}\n";
         
-        my @references = split " ", $info->{references};
-        my $count = scalar @references;
-        print "$count\n";
-        foreach my $reference (@references) {
-            print "$reference\n";
-		}
-	
-		my $isStateStorePath = `@bindir@/nix-state --is-state-store-path-download-using-manifests $storePath`;
-		if($isStateStorePath ne "true" && $isStateStorePath ne "false"){
-			die "The call for isStateStorePath must return true or false.....";
-		}
-	
-		if($isStateStorePath eq "true"){
-	        my @stateReferences = split " ", $info->{stateReferences};
-	        my $scount = scalar @stateReferences;
-	        print "$scount\n";
-	        foreach my $stateReference (@stateReferences) {
-	            print "$stateReference\n";
-	        }
-	        
-	    	print "$info->{revision}\n";
-	    }
+        else { die "unknown command `$cmd'"; }
     }
+
     exit 0;
 }
 
@@ -93,6 +81,18 @@ die unless scalar @ARGV == 2;
 my $targetPath = $ARGV[1];
 
 
+# Create a temporary directory.
+my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
+    or die "cannot create a temporary directory";
+
+chdir $tmpDir or die "cannot change to `$tmpDir': $!";
+
+my $tmpNar = "$tmpDir/nar";
+my $tmpNar2 = "$tmpDir/nar2";
+
+
+open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
+
 my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
 print LOGFILE "$$ get $targetPath $date\n";
 
@@ -105,7 +105,7 @@ foreach my $localPath (@{$localPathList}) {
     my $sourcePath = $localPath->{copyFrom};
     if (-e $sourcePath) {
         print "\n*** Step 1/1: copying from $sourcePath\n";
-        system("@bindir@/nix-store --dump $sourcePath | @bindir@/nix-store --restore $targetPath") == 0
+        system("$binDir/nix-store --dump $sourcePath | $binDir/nix-store --restore $targetPath") == 0
             or die "cannot copy `$sourcePath' to `$targetPath'";
         exit 0;
     }
@@ -154,7 +154,7 @@ addToQueue $targetPath;
 
 sub isValidPath {
     my $p = shift;
-    return system("@bindir@/nix-store --check-validity '$p' 2> /dev/null") == 0;
+    return system("$binDir/nix-store --check-validity '$p' 2> /dev/null") == 0;
 }
 
 sub parseHash {
@@ -188,15 +188,13 @@ while ($queueFront < scalar @queue) {
                 my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
                 my $format = "--base32";
                 $format = "" if $baseHashAlgo eq "md5";
-                my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
+                my $hash = `$binDir/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
                 chomp $hash;
-#                print "  MY HASH is $hash\n";
                 if ($hash ne $baseHash) {
                     print LOGFILE "$$ rejecting $patch->{basePath}\n";
                     next;
                 }
             }
-#            print "  PATCH from $patch->{basePath}\n";
             addToQueue $patch->{basePath};
             addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
         }
@@ -204,10 +202,12 @@ while ($queueFront < scalar @queue) {
         # Add NAR file edges to the start node.
         my $narFileList = $narFiles{$u};
         foreach my $narFile (@{$narFileList}) {
-#            print "  NAR from $narFile->{url}\n";
-            addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
+            # !!! how to handle files whose size is not known in advance?
+            # For now, assume some arbitrary size (1 MB).
+            addEdge "start", $u, ($narFile->{size} || 1000000), "narfile", $narFile;
             if ($u eq $targetPath) {
-                print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
+                my $size = $narFile->{size} || -1;
+                print LOGFILE "$$ full-download-would-be $size\n";
             }
         }
 
@@ -233,8 +233,6 @@ while (scalar @todo > 0) {
 
     my $u_ = $graph{$u};
 
-#    print "IN $u $u_->{d}\n";
-
     foreach my $edge (@{$u_->{edges}}) {
         my $v_ = $graph{$edge->{end}};
         if ($v_->{d} > $u_->{d} + $edge->{weight}) {
@@ -242,7 +240,6 @@ while (scalar @todo > 0) {
             # Store the edge; to edge->start is actually the
             # predecessor.
             $v_->{pred} = $edge; 
-#            print "  RELAX $edge->{end} $v_->{d}\n";
         }
     }
 }
@@ -264,20 +261,18 @@ while ($cur ne "start") {
 my $curStep = 1;
 my $maxStep = scalar @path;
 
-sub downloadFile {
-    my $url = shift;
-    my ($hashAlgo, $hash) = parseHash(shift);
+sub downloadFile { 
+    my $url = shift; 
     $ENV{"PRINT_PATH"} = 1;
     $ENV{"QUIET"} = 1;
-    $ENV{"NIX_HASH_ALGO"} = $hashAlgo;
-    my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
+    my ($hash, $path) = `$binDir/nix-prefetch-url '$url'`;
     die "download of `$url' failed" unless $? == 0;
-    chomp $hash2;
     chomp $path;
-    die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
     return $path;
 }
 
+my $finalNarHash;
+
 while (scalar @path > 0) {
     my $edge = pop @path;
     my $u = $edge->{start};
@@ -294,7 +289,7 @@ while (scalar @path > 0) {
             # as a base to one or more patches.  So turn the base path
             # into a NAR archive, to which we can apply the patch.
             print "  packing base path...\n";
-            system("@bindir@/nix-store --dump $v > $tmpNar") == 0
+            system("$binDir/nix-store --dump $v > $tmpNar") == 0
                 or die "cannot dump `$v'";
         }
     }
@@ -307,7 +302,7 @@ while (scalar @path > 0) {
 
         # Download the patch.
         print "  downloading patch...\n";
-        my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
+        my $patchPath = downloadFile "$patch->{url}";
 
         # Apply the patch to the NAR archive produced in step 1 (for
         # the already present path) or a later step (for patch sequences).
@@ -322,20 +317,23 @@ while (scalar @path > 0) {
             # This was the last patch.  Unpack the final NAR archive
             # into the target path.
             print "  unpacking patched archive...\n";
-            system("@bindir@/nix-store --restore $v < $tmpNar2") == 0
+            system("$binDir/nix-store --restore $v < $tmpNar2") == 0
                 or die "cannot unpack $tmpNar2 into `$v'";
         }
+
+        $finalNarHash = $patch->{narHash};
     }
 
     elsif ($edge->{type} eq "narfile") {
         my $narFile = $edge->{info};
         print "downloading `$narFile->{url}' into `$v'\n";
 
-        print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
+        my $size = $narFile->{size} || -1;
+        print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
         
         # Download the archive.
         print "  downloading archive...\n";
-        my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
+        my $narFilePath = downloadFile "$narFile->{url}";
 
         if ($curStep < $maxStep) {
             # The archive will be used a base to a patch.
@@ -344,14 +342,39 @@ while (scalar @path > 0) {
         } else {
             # Unpack the archive into the target path.
             print "  unpacking archive...\n";
-            system("@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'") == 0
+            system("@bunzip2@ < '$narFilePath' | $binDir/nix-store --restore '$v'") == 0
                 or die "cannot unpack `$narFilePath' into `$v'";
         }
+
+        $finalNarHash = $narFile->{narHash};
     }
 
     $curStep++;
 }
 
 
+# Make sure that the hash declared in the manifest matches what we
+# downloaded and unpacked.
+
+if (defined $finalNarHash) {
+    my ($hashAlgo, $hash) = parseHash $finalNarHash;
+
+    # The hash in the manifest can be either in base-16 or base-32.
+    # Handle both.
+    my $extraFlag =
+        ($hashAlgo eq "sha256" && length($hash) != 64)
+        ? "--base32" : "";
+    
+    my $hash2 = `@bindir@/nix-hash --type $hashAlgo $extraFlag $targetPath`
+        or die "cannot compute hash of path `$targetPath'";
+    chomp $hash2;
+    
+    die "hash mismatch in downloaded path $targetPath; expected $hash, got $hash2"
+        if $hash ne $hash2;
+} else {
+    die "cannot check integrity of the downloaded path since its hash is not known";
+}
+
+
 print LOGFILE "$$ success\n";
 close LOGFILE;

scripts/nix-build.in

@@ -2,6 +2,8 @@
 
 use strict;
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 my $addDrvLink = 0;
 my $addOutLink = 1;
@@ -9,6 +11,8 @@ my $addOutLink = 1;
 my $outLink;
 my $drvLink;
 
+my $dryRun = 0;
+
 my @instArgs = ();
 my @buildArgs = ();
 my @exprs = ();
@@ -73,7 +77,7 @@ EOF
 
     elsif ($arg eq "--attr" or $arg eq "-A") {
         $n++;
-        die "$0: `--attr' requires an argument\n" unless $n < scalar @ARGV;
+        die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @instArgs, ("--attr", $ARGV[$n]);
     }
 
@@ -83,12 +87,31 @@ EOF
         $n += 2;
     }
 
-    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time") {
+    elsif ($arg eq "--log-type") {
+        $n++;
+        die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
+        push @instArgs, ($arg, $ARGV[$n]);
+        push @buildArgs, ($arg, $ARGV[$n]);
+    }
+
+    elsif ($arg eq "--option") {
+        die "$0: `$arg' requires two arguments\n" unless $n + 2 < scalar @ARGV;
+        push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]);
+        push @buildArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]);
+        $n += 2;
+    }
+
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type") {
         $n++;
         die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
         push @buildArgs, ($arg, $ARGV[$n]);
     }
 
+    elsif ($arg eq "--dry-run") {
+        push @buildArgs, "--dry-run";
+        $dryRun = 1;
+    }
+    
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -117,9 +140,12 @@ foreach my $expr (@exprs) {
     # Instantiate.
     my @drvPaths;
     # !!! would prefer the perl 5.8.0 pipe open feature here.
-    my $pid = open(DRVPATHS, "-|") || exec "@bindir@/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
+    my $pid = open(DRVPATHS, "-|") || exec "$binDir/nix-instantiate", "--add-root", $drvLink, "--indirect", @instArgs, $expr;
     while (<DRVPATHS>) {chomp; push @drvPaths, $_;}
-    close DRVPATHS or exit 1;
+    if (!close DRVPATHS) {
+        die "nix-instantiate killed by signal " . ($? & 127) . "\n" if ($? & 127);
+        exit 1;
+    }
 
     foreach my $drvPath (@drvPaths) {
         my $target = readlink $drvPath or die "cannot read symlink `$drvPath'";
@@ -128,10 +154,15 @@ foreach my $expr (@exprs) {
 
     # Build.
     my @outPaths;
-    $pid = open(OUTPATHS, "-|") || exec "@bindir@/nix-store", "--add-root", $outLink, "--indirect", "-rv",
+    $pid = open(OUTPATHS, "-|") || exec "$binDir/nix-store", "--add-root", $outLink, "--indirect", "-rv",
         @buildArgs, @drvPaths;
     while (<OUTPATHS>) {chomp; push @outPaths, $_;}
-    close OUTPATHS or exit 1;
+    if (!close OUTPATHS) {
+        die "nix-store killed by signal " . ($? & 127) . "\n" if ($? & 127);
+        exit 1;
+    }
+
+    next if $dryRun;
 
     foreach my $outPath (@outPaths) {
         my $target = readlink $outPath or die "cannot read symlink `$outPath'";

scripts/nix-channel.in

@@ -125,15 +125,13 @@ sub update {
 
     my $rootFile = "$rootsDir/per-user/$userName/channels";
     
-    # Instantiate the Nix expression.
+    # Build the Nix expression.
     print "unpacking channel Nix expressions...\n";
-    my $storeExpr = `@bindir@/nix-instantiate --add-root '$rootFile'.tmp @datadir@/nix/corepkgs/channels/unpack.nix --argstr system @system@ --arg inputs '$inputs'`
-        or die "cannot instantiate Nix expression";
-    chomp $storeExpr;
-
-    # Build the resulting derivation.
-    my $outPath = `@bindir@/nix-store --add-root '$rootFile' -r '$storeExpr'`
-        or die "cannot realise store expression";
+    my $outPath = `\\
+        @bindir@/nix-build --out-link '$rootFile' --drv-link '$rootFile'.tmp \\
+        @datadir@/nix/corepkgs/channels/unpack.nix \\
+        --argstr system @system@ --arg inputs '$inputs'`
+        or die "cannot unpack the channels";
     chomp $outPath;
 
     unlink "$rootFile.tmp";

scripts/nix-collect-garbage.in

@@ -4,6 +4,8 @@ use strict;
 
 my $profilesDir = "@localstatedir@/nix/profiles";
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 # Process the command line arguments.
 my @args = ();
@@ -34,7 +36,7 @@ sub removeOldGenerations {
         $name = $dir . "/" . $name;
         if (-l $name && (readlink($name) =~ /link/)) {
             print STDERR "removing old generations of profile $name\n";
-            system("@bindir@/nix-env", "-p", $name, "--delete-generations", "old");
+            system("$binDir/nix-env", "-p", $name, "--delete-generations", "old");
         }
         elsif (! -l $name && -d $name) {
             removeOldGenerations $name;
@@ -48,4 +50,4 @@ removeOldGenerations $profilesDir if $removeOld;
 
 
 # Run the actual garbage collector.
-exec "@bindir@/nix-store", "--gc", @args;
+exec "$binDir/nix-store", "--gc", @args;

scripts/nix-copy-closure.in

@@ -1,7 +1,6 @@
 #! @perl@ -w
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 
 if (scalar @ARGV < 1) {
@@ -56,30 +55,18 @@ while (@ARGV) {
 if ($toMode) { # Copy TO the remote machine.
 
     my @allStorePaths;
-    my %storePathsSeen;
 
-    foreach my $storePath (@storePaths) {
-        # $arg might be a symlink to the store, so resolve it.
-        my $storePath2 = (`$binDir/nix-store --query --resolve '$storePath'`
-            or die "cannot resolve `$storePath'");
-        chomp $storePath2;
-
-        # Get the closure of this path.
-        my $pid = open(READ,
-            "$binDir/nix-store --query --requisites '$storePath2'|") or die;
+    # Get the closure of this path.
+    my $pid = open(READ, "$binDir/nix-store --query --requisites @storePaths|") or die;
     
-        while (<READ>) {
-            chomp;
-            die "bad: $_" unless /^\//;
-            if (!defined $storePathsSeen{$_}) {
-                push @allStorePaths, $_;
-                $storePathsSeen{$_} = 1;
-            }
-        }
-
-        close READ or die "nix-store failed: $?";
+    while (<READ>) {
+        chomp;
+        die "bad: $_" unless /^\//;
+        push @allStorePaths, $_;
     }
 
+    close READ or die "nix-store failed: $?";
+
 
     # Ask the remote host which paths are invalid.
     open(READ, "ssh @sshOpts $sshHost nix-store --check-validity --print-invalid @allStorePaths|");
@@ -112,15 +99,11 @@ else { # Copy FROM the remote machine.
         "ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
     
     my @allStorePaths;
-    my %storePathsSeen;
 
     while (<READ>) {
         chomp;
         die "bad: $_" unless /^\//;
-        if (!defined $storePathsSeen{$_}) {
-            push @allStorePaths, $_;
-            $storePathsSeen{$_} = 1;
-        }
+        push @allStorePaths, $_;
     }
 
     close READ or die "nix-store on remote machine `$sshHost' failed: $?";

scripts/nix-http-export.cgi.in

@@ -0,0 +1,51 @@
+#! /bin/sh
+
+export HOME=/tmp
+export NIX_REMOTE=daemon
+
+TMP_DIR="${TMP_DIR:-/tmp/nix-export}"
+
+@coreutils@/mkdir -p "$TMP_DIR" || true
+@coreutils@/chmod a+r "$TMP_DIR"
+
+needed_path="?$QUERY_STRING"
+needed_path="${needed_path#*[?&]needed_path=}"
+needed_path="${needed_path%%&*}"
+#needed_path="$(echo $needed_path  | ./unhttp)"
+needed_path="${needed_path//%2B/+}"
+needed_path="${needed_path//%3D/=}"
+
+echo needed_path: "$needed_path" >&2
+
+NIX_STORE="${NIX_STORE_DIR:-/nix/store}"
+
+echo NIX_STORE: "${NIX_STORE}" >&2
+
+full_path="${NIX_STORE}"/"$needed_path"
+
+if [ "$needed_path" != "${needed_path%.drv}" ]; then
+	echo "Status: 403 You should create the derivation file yourself"
+	echo "Content-Type: text/plain"
+	echo
+	echo "Refusing to disclose derivation contents"
+	exit
+fi
+
+if @bindir@/nix-store --check-validity "$full_path"; then
+	if ! [ -e nix-export/"$needed_path".nar.gz ]; then
+		@bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz
+		@coreutils@/ln -fs  "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz 
+	fi;
+	echo "Status: 301 Moved"
+	echo "Location: nix-export/"$needed_path".nar.gz"
+	echo
+else 
+	echo "Status: 404 No such path found"
+	echo "Content-Type: text/plain"
+	echo
+	echo "Path not found:"
+	echo "$needed_path"
+	echo "checked:"
+	echo "$full_path"
+fi
+

scripts/nix-install-package.in

@@ -3,6 +3,8 @@
 use strict;
 use File::Temp qw(tempdir);
 
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+
 
 sub usageError {
     print STDERR <<EOF;
@@ -59,7 +61,7 @@ if ($interactive && !defined $ENV{"NIX_HAVE_TERMINAL"}) {
     $ENV{"NIX_HAVE_TERMINAL"} = "1";
     $ENV{"LD_LIBRARY_PATH"} = "";
     foreach my $term ("xterm", "konsole", "gnome-terminal", "xterm") {
-        exec($term, "-e", "@bindir@/nix-install-package", @ARGV);
+        exec($term, "-e", "$binDir/nix-install-package", @ARGV);
     }
     die "cannot execute `xterm'";
 }
@@ -121,13 +123,18 @@ if ($interactive) {
 }
 
 
+# Store the manifest in the temporary directory so that we don't
+# pollute /nix/var/nix/manifests.
+$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
+
+
 print "\nPulling manifests...\n";
-system("@bindir@/nix-pull", $manifestURL) == 0
+system("$binDir/nix-pull", $manifestURL) == 0
     or barf "nix-pull failed: $?";
 
 
 print "\nInstalling package...\n";
-system("@bindir@/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0
+system("$binDir/nix-env", "--install", $outPath, "--force-name", $drvName, @extraNixEnvArgs) == 0
     or barf "nix-env failed: $?";
 
 

scripts/nix-pack-closure.in

@@ -1,75 +0,0 @@
-#! @perl@ -w
-
-# This tool computes the closure of a path (using "nix-store --query
-# --requisites") and puts the contents of each path in the closure in
-# a big NAR archive that can be installed on another Nix installation
-# using "nix-unpack-closure".
-
-# TODO: make this program "streamy", i.e., don't use a temporary
-# directory.
-
-use strict;
-use File::Temp qw(tempdir);
-
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
-
-my $tmpDir = tempdir("nix-pack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-mkdir "$tmpDir/contents", 0755 or die;
-mkdir "$tmpDir/references", 0755 or die;
-mkdir "$tmpDir/derivers", 0755 or die;
-
-open TOPLEVEL, ">$tmpDir/top-level" or die;
-
-
-my %storePaths;
-
-
-while (@ARGV) {
-    my $storePath = shift @ARGV;
-
-    # $storePath might be a symlink to the store, so resolve it.
-    $storePath = (`$binDir/nix-store --query --resolve '$storePath'`
-        or die "cannot resolve `$storePath'");
-    chomp $storePath;
-    print TOPLEVEL $storePath, "\n";
-
-    # Get the closure of this path.
-    my $pid = open(READ,
-        "$binDir/nix-store --query --requisites '$storePath'|") or die;
-    
-    while (<READ>) {
-        chomp;
-        die "bad: $_" unless /^\//;
-        $storePaths{$_} = "";
-    }
-
-    close READ or die "nix-store failed: $?";
-}
-
-
-close TOPLEVEL or die;
-
-
-foreach my $storePath (sort(keys %storePaths)) {
-    print STDERR "packing `$storePath'...\n";
-
-    $storePath =~ /\/([^\/]+)$/;
-    my $name = $1;
-
-    system("$binDir/nix-store --dump '$storePath' > $tmpDir/contents/$name") == 0
-        or die "nix-store --dump failed on `$storePath': $?";
-
-    system("$binDir/nix-store --query --references '$storePath' > $tmpDir/references/$name") == 0
-        or die "nix-store --query --references failed on `$storePath': $?";
-
-    system("$binDir/nix-store --query --deriver '$storePath' > $tmpDir/derivers/$name") == 0
-        or die "nix-store --query --deriver failed on `$storePath': $?";
-}
-
-
-# Write a NAR archive of everything to standard output.
-system("nix-store --dump '$tmpDir'") == 0
-    or die "nix-store --dump failed";

scripts/nix-prefetch-url.in

@@ -37,6 +37,7 @@ fi
 
 
 mkTempDir() {
+    if test -n "$tmpPath"; then return; fi
     local i=0
     while true; do
         if test -z "$TMPDIR"; then TMPDIR=/tmp; fi
@@ -59,11 +60,32 @@ removeTempDir() {
 
 
 doDownload() {
-    @curl@ $cacheFlags --fail -# --location --max-redirs 20 --disable-epsv \
+    @curl@ $cacheFlags --fail --location --max-redirs 20 --disable-epsv \
         --cookie-jar $tmpPath/cookies "$url" -o $tmpFile
 }
 
 
+# Hack to support the mirror:// scheme from Nixpkgs.
+if test "${url:0:9}" = "mirror://"; then
+    if test -z "$NIXPKGS_ALL"; then
+        echo "Resolving mirror:// URLs requires Nixpkgs.  Please point \$NIXPKGS_ALL at a Nixpkgs tree." >&2
+        exit 1
+    fi
+
+    mkTempDir
+    nix-build "$NIXPKGS_ALL" -A resolveMirrorURLs --argstr url "$url" -o $tmpPath/urls > /dev/null
+
+    expanded=($(cat $tmpPath/urls))
+    if test "${#expanded[*]}" = 0; then
+        echo "$0: cannot resolve $url." >&2
+        exit 1
+    fi
+
+    echo "$url expands to ${expanded[*]} (using ${expanded[0]})" >&2
+    url="${expanded[0]}"
+fi
+
+
 # If we don't know the hash or a file with that hash doesn't exist,
 # download the file and add it to the store.
 if test -z "$finalPath"; then
@@ -78,7 +100,7 @@ if test -z "$finalPath"; then
     # garbage-collected independently.
     if test -n "$NIX_DOWNLOAD_CACHE"; then
         echo -n "$url" > $tmpPath/url
-        urlHash=$(nix-hash --type sha256 --base32 --flat $tmpPath/url)
+        urlHash=$(@bindir@/nix-hash --type sha256 --base32 --flat $tmpPath/url)
         echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url"
         cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType"
         cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp"

scripts/nix-profile.sh.in

@@ -7,5 +7,12 @@ if test -n "$HOME"; then
         @coreutils@/ln -s "$_NIX_DEF_LINK" "$NIX_LINK"
     fi
 
-    export PATH=$NIX_LINK/bin:@prefix@/bin:$PATH
+    export PATH=$NIX_LINK/bin:$PATH
+fi
+
+# This is a quick hack to make fontconfig-based packages in Nixpkgs
+# work out of the box on non-NixOS systems.  Of course, we should
+# really fix fontconfig...
+if test -z "$FONTCONFIG_FILE" -a -e /etc/fonts/fonts.conf; then
+    export FONTCONFIG_FILE=/etc/fonts/fonts.conf
 fi

scripts/nix-pull.in

@@ -7,23 +7,12 @@ use readmanifest;
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
+my $libexecDir = ($ENV{"NIX_LIBEXEC_DIR"} or "@libexecdir@");
+my $storeDir = ($ENV{"NIX_STORE_DIR"} or "@storedir@");
+my $stateDir = ($ENV{"NIX_STATE_DIR"} or "@localstatedir@/nix");
+my $manifestDir = ($ENV{"NIX_MANIFESTS_DIR"} or "$stateDir/manifests");
 
-my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
-$libexecDir = "@libexecdir@" unless defined $libexecDir;
-
-my $stateDir = $ENV{"NIX_STATE_DIR"};
-$stateDir = "@localstatedir@/nix" unless defined $stateDir;
-
-my $storeDir = $ENV{"NIX_STORE_DIR"};
-$storeDir = "@storedir@" unless defined $storeDir;
-
-my $storeStateDir = $ENV{"NIX_STORE_STATE_DIR"};
-$storeStateDir = "@storestatedir@" unless defined $storeStateDir;
-
-my $ext3cowheader = $ENV{"NIX_EXT3_COW_HEADER"};
-$ext3cowheader = "@ext3cowheader@" unless defined $ext3cowheader;
 
 # Prevent access problems in shared-stored installations.
 umask 0022;
@@ -74,10 +63,11 @@ sub processURL {
         print "obtaining list of Nix archives at `$url'...\n";
         $manifest = downloadFile $url;
     }
+
+    my $version = readManifest($manifest, \%narFiles, \%localPaths, \%patches);
     
-    if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
-        die "`$url' is not manifest or it is too old (i.e., for Nix <= 0.7)\n";
-    }
+    die "`$url' is not a manifest or it is too old (i.e., for Nix <= 0.7)\n" if $version < 3;
+    die "manifest `$url' is too new\n" if $version >= 5;
 
     if ($skipWrongStore) {
         foreach my $path (keys %narFiles) {
@@ -97,7 +87,7 @@ sub processURL {
         or die "cannot hash `$manifest'";
     chomp $hash;
     
-    my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
+    my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest";
     
     system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
         or die "cannot link `$finalPath to `$manifest'";

scripts/nix-push.in

@@ -16,8 +16,7 @@ my $curl = "@curl@ --fail --silent";
 my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
 $curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
 
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
+my $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
 
 my $dataDir = $ENV{"NIX_DATA_DIR"};
 $dataDir = "@datadir@" unless defined $dataDir;
@@ -107,7 +106,7 @@ foreach my $storePath (@storePaths) {
     # Construct a Nix expression that creates a Nix archive.
     my $nixexpr = 
         "((import $dataDir/nix/corepkgs/nar/nar.nix) " .
-        "{storePath = builtins.toPath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
+        "{storePath = builtins.storePath \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
     
     print NIX $nixexpr;
 }
@@ -129,7 +128,7 @@ while (<READ>) {
 close READ or die "nix-instantiate failed: $?";
 
 
-# Realise the store expressions.
+# Build the derivations.
 print STDERR "creating archives...\n";
 
 my @narPaths;
@@ -141,12 +140,7 @@ while (scalar @tmp > 0) {
     my @tmp2 = @tmp[0..$n - 1];
     @tmp = @tmp[$n..scalar @tmp - 1];
 
-    # Note: we disable build hooks because of the impure path
-    # reference (see above).  Even if that is fixed, using a hook
-    # probably wouldn't make that much sense; pumping lots of data
-    # around just to compress them won't gain that much.
-    $ENV{"NIX_BUILD_HOOK"} = "";
-    my $pid = open(READ, "$binDir/nix-store --no-build-hook --realise @tmp2|")
+    my $pid = open(READ, "$binDir/nix-store --realise @tmp2|")
         or die "cannot run nix-store";
     while (<READ>) {
         chomp;
@@ -224,13 +218,14 @@ writeManifest $manifest, \%narFiles, \%patches;
 sub copyFile {
     my $src = shift;
     my $dst = shift;
-    system("@coreutils@/cp", $src, "$dst.tmp") == 0 or die "cannot copy file";
-    rename("$dst.tmp", "$dst") or die "cannot rename file";
+    my $tmp = "$dst.tmp.$$";
+    system("@coreutils@/cp", $src, $tmp) == 0 or die "cannot copy file";
+    rename($tmp, $dst) or die "cannot rename file: $!";
 }
 
 
-# Upload the archives.
-print STDERR "uploading archives...\n";
+# Upload/copy the archives.
+print STDERR "uploading/copying archives...\n";
 
 sub archiveExists {
     my $name = shift;
@@ -244,9 +239,12 @@ foreach my $narArchive (@narArchives) {
     my $basename = $1;
 
     if ($localCopy) {
+        # Since nix-push creates $dst atomically, if it exists we
+        # don't have to copy again.
+        my $dst = "$localArchivesDir/$basename";
         if (! -f "$localArchivesDir/$basename") {
             print STDERR "  $narArchive\n";
-            copyFile $narArchive, "$localArchivesDir/$basename";
+            copyFile $narArchive, $dst;
         }
     }
     else {

scripts/nix-reduce-build.in

@@ -3,10 +3,30 @@
 WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX);
 cd "$WORKING_DIRECTORY";
 
-if test -z "$1" ; then
-	echo 'nix-reduce-build (paths or Nix expressions) -- (logins at remote computers)' >&2
+if test -z "$1" || test "a--help" = "a$1" ; then
+	echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2
 	echo As in: >&2
-	echo nix-reduce-build /etc/nixos/nixos -- user@somewhere.nowhere.example.org >&2
+	echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2
+	echo nix-reduce-build /etc/nixos/nixos -- \\
+	echo "   " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2
+	echo "  store path name will be added into the end of the URL" >&2
+	echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2
+	echo "  that should be a directory where gzipped 'nix-store --export' ">&2
+	echo "  files are located (they should have .nar.gz extension)"  >&2
+	echo "        Or all together: " >&2
+	echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\
+	"    ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2
+	echo "        Also supports best-effort local builds of failing expression set:" >&2
+	echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2
+	echo "  nix-daemon:// builds using daemon"
+	echo "  nix-self:// builds directly using nix-store from current installation" >&2
+	echo "  nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2;
+	echo "derivations with specified output hash (sha256, sha1 or md5)." >&2
+	echo "  nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2;
+	echo "maximum amount of paths" >&2;
+	echo "  nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2;
+	echo "maximum amount of paths" >&2;
+	echo "        If no package sources are specified, required paths are listed." >&2;
 	exit;
 fi;
 
@@ -19,11 +39,14 @@ echo Will work on $(cat initial | wc -l) targets. >&2
 
 while read ; do
 	case "$REPLY" in 
-		${NIX_STORE_PATH:-/nix/store}/*)
+		${NIX_STORE_DIR:-/nix/store}/*)
 			echo "$REPLY" >> paths; >&2
 			;;
 		*)
-			nix-instantiate "$REPLY" >> paths;
+			(
+				IFS=: ;
+				nix-instantiate $REPLY >> paths;
+			);
 			;;
 	esac;
 done < initial;
@@ -51,14 +74,94 @@ echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2
 cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
 echo We need $(cat needed-paths | wc -l) paths. >&2
 
+egrep '[.]drv$' derivers-closure > critical-derivers;
+
 if test -z "$1" ; then
 	cat needed-paths;	
 fi;
 
+refresh_critical_derivers() {
+    echo "Finding needed derivers..." >&2;
+    cat critical-derivers | while read; do
+        if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then
+            echo "$REPLY";
+        fi;
+    done > new-critical-derivers;
+    mv new-critical-derivers critical-derivers;
+    echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2
+}
+
+build_here() {
+    cat critical-derivers | while read; do 
+        echo "Realising $REPLY using nix-daemon" >&2
+        @bindir@/nix-store -r "${REPLY}"
+    done;
+}
+
+try_to_substitute(){
+    cat needed-paths | while read ; do 
+        echo "Building $REPLY using nix-daemon" >&2
+        @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}"
+    done;
+}
+
 for i in "$@"; do 
-	cat needed-paths | while read; do 
-		nix-copy-closure --from "$i" --gzip "$REPLY" </dev/null || true; 
-	done;
+	sshHost="${i#ssh://}";
+	httpHost="${i#http://}";
+	httpsHost="${i#https://}";
+	filePath="${i#file:/}";
+	if [ "$i" != "$sshHost" ]; then
+		cat needed-paths | while read; do 
+			echo "Getting $REPLY and its closure over ssh" >&2
+			nix-copy-closure --from "$sshHost" --gzip "$REPLY" </dev/null || true; 
+		done;
+	elif [ "$i" != "$httpHost" ] || [ "$i" != "$httpsHost" ]; then
+		cat needed-paths | while read; do
+			echo "Getting $REPLY over http/https" >&2
+			curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import;
+		done;
+	elif [ "$i" != "$filePath" ] ; then
+		cat needed-paths | while read; do 
+			echo "Installing $REPLY from file" >&2
+			gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import;
+		done;
+	elif [ "$i" = "nix-daemon://" ] ; then
+		NIX_REMOTE=daemon try_to_substitute;
+		refresh_critical_derivers;
+		NIX_REMOTE=daemon build_here;
+	elif [ "$i" = "nix-self://" ] ; then
+		NIX_REMOTE= try_to_substitute;
+		refresh_critical_derivers;
+		NIX_REMOTE= build_here;
+	elif [ "$i" = "nix-daemon-fixed://" ] ; then
+		refresh_critical_derivers;
+
+		cat critical-derivers | while read; do 
+			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
+				echo "Realising $REPLY using nix-daemon" >&2
+				NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}"
+			fi;
+		done;
+	elif [ "$i" = "nix-self-fixed://" ] ; then
+		refresh_critical_derivers;
+
+		cat critical-derivers | while read; do 
+			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
+				echo "Realising $REPLY using direct Nix build" >&2
+				NIX_REMOTE= @bindir@/nix-store -r "${REPLY}"
+			fi;
+		done;
+	elif [ "$i" = "nix-daemon-substitute://" ] ; then
+		NIX_REMOTE=daemon try_to_substitute;
+	elif [ "$i" = "nix-self-substitute://" ] ; then
+		NIX_REMOTE= try_to_substitute;
+	elif [ "$i" = "nix-daemon-build://" ] ; then
+		refresh_critical_derivers;
+		NIX_REMOTE=daemon build_here;
+	elif [ "$i" = "nix-self-build://" ] ; then
+		refresh_critical_derivers;
+		NIX_REMOTE= build_here;
+	fi;
 	mv needed-paths wanted-paths;
 	cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
 	echo We still need $(cat needed-paths | wc -l) paths. >&2

scripts/nix-unpack-closure.in

@@ -1,88 +0,0 @@
-#! @perl@ -w
-
-# This tool unpacks the closures created by "nix-pack-closure" and
-# adds them to the Nix store.
-
-# TODO: make this program "streamy", i.e., don't use a temporary
-# directory.
-
-use strict;
-use File::Temp qw(tempdir);
-
-my $binDir = $ENV{"NIX_BIN_DIR"};
-$binDir = "@bindir@" unless defined $binDir;
-
-my $tmpDir = tempdir("nix-unpack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
-    or die "cannot create a temporary directory";
-
-
-# Unpack the NAR archive on standard input.
-system("nix-store --restore '$tmpDir/unpacked'") == 0
-    or die "nix-store --restore failed";
-
-
-open VALID, ">$tmpDir/validity" or die;
-
-
-# For each path in the closure that is not yet valid, add it to the
-# store.  TODO: use proper locking.  Or even better, let nix-store do
-# this.
-opendir(DIR, "$tmpDir/unpacked/contents") or die "cannot open directory: $!";
-
-foreach my $name (sort(readdir DIR)) {
-    next if $name eq "." or $name eq "..";
-
-    my $storePath = "@storedir@/$name"; # !!!
-
-    # !!! this really isn't a good validity check!
-    system "$binDir/nix-store --check-validity '$storePath' 2> /dev/null";
-    if ($? != 0) {
-        print STDERR "unpacking `$storePath'...\n";
-
-        # !!! race
-        system("@coreutils@/rm -rf '$storePath'") == 0
-            or die "cannot remove `$storePath': $?";
-
-        system("$binDir/nix-store --restore '$storePath' < '$tmpDir/unpacked/contents/$name'") == 0
-            or die "nix-store --dump failed on `$storePath': $?";
-        
-        print VALID "$storePath\n";
-
-        open DRV, "<$tmpDir/unpacked/derivers/$name" or die;
-        my $deriver = <DRV>;
-        chomp $deriver;
-        $deriver = "" if $deriver eq "unknown-deriver";
-        close DRV;
-
-        my @refs;
-        open REFS, "<$tmpDir/unpacked/references/$name" or die;
-        while (<REFS>) {
-            chomp;
-            push @refs, $_;
-        }
-        close REFS;
-
-        print VALID "$deriver\n";
-        
-        print VALID (scalar @refs), "\n";
-        foreach my $ref (@refs) {
-            print VALID "$ref\n";
-        }
-    }
-}
-
-closedir(DIR) or die;
-
-
-# Register the invalid paths as valid.
-system("nix-store --register-validity <'$tmpDir/validity'") == 0
-    or die "nix-store --register-validity failed";
-
-
-# Show the top-level paths so that something useful can be done with
-# them, e.g., passing them to `nix-env -i'.
-if (-e "$tmpDir/unpacked/top-level") {
-    open TOPLEVEL, "<$tmpDir/unpacked/top-level" or die;
-    while (<TOPLEVEL>) { print "$_"; }
-    close TOPLEVEL;
-}

scripts/readmanifest.pm.in

@@ -2,10 +2,7 @@ use strict;
 
 
 sub addPatch {
-    my $patches = shift;
-    my $storePath = shift;
-    my $patch = shift;
-    my $allowConflicts = shift;
+    my ($patches, $storePath, $patch) = @_;
 
     $$patches{$storePath} = []
         unless defined $$patches{$storePath};
@@ -14,15 +11,9 @@ sub addPatch {
 
     my $found = 0;
     foreach my $patch2 (@{$patchList}) {
-        if ($patch2->{url} eq $patch->{url}) {
-            if ($patch2->{hash} eq $patch->{hash}) {
-                $found = 1 if ($patch2->{basePath} eq $patch->{basePath});
-            } else {
-                die "conflicting hashes for URL $patch->{url}, " .
-                    "namely $patch2->{hash} and $patch->{hash}"
-                    unless $allowConflicts;
-            }
-        }
+        $found = 1 if
+            $patch2->{url} eq $patch->{url} &&
+            $patch2->{basePath} eq $patch->{basePath};
     }
     
     push @{$patchList}, $patch if !$found;
@@ -32,12 +23,7 @@ sub addPatch {
 
 
 sub readManifest {
-    my $manifest = shift;
-    my $narFiles = shift;
-    my $localPaths = shift;
-    my $patches = shift;
-    my $allowConflicts = shift;
-    $allowConflicts = 0 unless defined $allowConflicts;
+    my ($manifest, $narFiles, $localPaths, $patches) = @_;
 
     open MANIFEST, "<$manifest"
         or die "cannot open `$manifest': $!";
@@ -98,15 +84,7 @@ sub readManifest {
 
                     my $found = 0;
                     foreach my $narFile (@{$narFileList}) {
-                        if ($narFile->{url} eq $url) {
-                            if ($narFile->{hash} eq $hash) {
-                                $found = 1;
-                            } else {
-                                die "conflicting hashes for URL $url, " .
-                                    "namely $narFile->{hash} and $hash"
-                                    unless $allowConflicts;
-                            }
-                        }
+                        $found = 1 if $narFile->{url} eq $url;
                     }
                     if (!$found) {
                         push @{$narFileList},
@@ -124,7 +102,7 @@ sub readManifest {
                         , basePath => $basePath, baseHash => $baseHash
                         , narHash => $narHash, patchType => $patchType
                         , hashAlgo => $hashAlgo
-                        }, $allowConflicts;
+                        };
                 }
 
                 elsif ($type eq "localPath") {
@@ -171,12 +149,8 @@ sub readManifest {
 }
 
 
-sub writeManifest
-{
-    my $manifest = shift;
-    my $narFiles = shift;
-    my $patches = shift;
-    my $copySources = shift;
+sub writeManifest {
+    my ($manifest, $narFiles, $patches) = @_;
 
     open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
 
@@ -190,9 +164,9 @@ sub writeManifest
             print MANIFEST "{\n";
             print MANIFEST "  StorePath: $storePath\n";
             print MANIFEST "  NarURL: $narFile->{url}\n";
-            print MANIFEST "  Hash: $narFile->{hash}\n";
+            print MANIFEST "  Hash: $narFile->{hash}\n" if defined $narFile->{hash};
             print MANIFEST "  NarHash: $narFile->{narHash}\n";
-            print MANIFEST "  Size: $narFile->{size}\n";
+            print MANIFEST "  Size: $narFile->{size}\n" if defined $narFile->{size};
             print MANIFEST "  References: $narFile->{references}\n"
                 if defined $narFile->{references} && $narFile->{references} ne "";
             print MANIFEST "  Deriver: $narFile->{deriver}\n"

src/Makefile.am

@@ -1,5 +1,5 @@
-SUBDIRS = bin2c boost libutil libext3cow libstore libmain nix-store nix-hash \
+SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
  libexpr nix-instantiate nix-env nix-worker nix-setuid-helper \
- nix-log2xml bsdiff-4.3 nix-state
+ nix-log2xml bsdiff-4.3
 
 EXTRA_DIST = aterm-helper.pl

src/boost/Makefile.am

@@ -1,6 +1,6 @@
 SUBDIRS = format
 
-pkginclude_HEADERS = assert.hpp checked_delete.hpp format.hpp \
+nobase_pkginclude_HEADERS = assert.hpp checked_delete.hpp format.hpp \
  shared_ptr.hpp weak_ptr.hpp throw_exception.hpp \
  enable_shared_from_this.hpp \
  detail/shared_count.hpp detail/workaround.hpp

src/libexpr/Makefile.am

@@ -2,11 +2,13 @@ pkglib_LTLIBRARIES = libexpr.la
 
 libexpr_la_SOURCES = \
  nixexpr.cc eval.cc primops.cc lexer-tab.cc parser-tab.cc \
- get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc
+ get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc \
+ names.cc normal-forms.cc
 
 pkginclude_HEADERS = \
  nixexpr.hh eval.hh parser.hh lexer-tab.hh parser-tab.hh \
- get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh
+ get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh \
+ names.hh nixexpr-ast.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
  ../boost/format/libformat.la

src/libexpr/eval.cc

@@ -16,13 +16,23 @@ namespace nix {
     
 
 EvalState::EvalState()
-    : normalForms(32768), primOps(128)
+    : sessionNormalForms(32768), normalForms(32768), primOps(128)
 {
-    nrEvaluated = nrCached = 0;
+    nrEvaluated = nrCached = nrDepthAfterReset = 0;
 
     initNixExprHelpers();
 
     addPrimOps();
+
+    allowUnsafeEquality = getEnv("NIX_NO_UNSAFE_EQ", "") == "";
+    safeCache = true;
+
+    loadNormalForms();
+}
+
+EvalState::~EvalState()
+{
+    saveNormalForms();
 }
 
 
@@ -48,6 +58,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
     throw EvalError(format(s) % s2);
 }
 
+LocalNoInlineNoReturn(void throwTypeError(const char * s))
+{
+    throw TypeError(s);
+}
+
 LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s2))
 {
     throw TypeError(format(s) % s2);
@@ -69,77 +84,91 @@ LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2,
 }
 
 
+/* Pattern-match `pat' against `arg'.  The result is a set of
+   substitutions (`subs') and a set of recursive substitutions
+   (`subsRecursive').  The latter can refer to the variables bound by
+   both `subs' and `subsRecursive'. */
+static void patternMatch(EvalState & state,
+    Pattern pat, Expr arg, ATermMap & subs, ATermMap & subsRecursive)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    
+    if (matchVarPat(pat, name)) 
+        subs.set(name, arg);
+
+    else if (matchAttrsPat(pat, formals, ellipsis)) {
+
+        arg = evalExpr(state, arg);
+
+        /* Get the actual arguments. */
+        ATermMap attrs;
+        queryAllAttrs(arg, attrs);
+        unsigned int nrAttrs = attrs.size();
+
+        /* For each formal argument, get the actual argument.  If
+           there is no matching actual argument but the formal
+           argument has a default, use the default. */
+        unsigned int attrsUsed = 0;
+        for (ATermIterator i(formals); i; ++i) {
+            Expr name, def;
+            DefaultValue def2;
+            if (!matchFormal(*i, name, def2)) abort(); /* can't happen */
+
+            Expr value = attrs[name];
+
+            if (value == 0) {
+                if (!matchDefaultValue(def2, def)) def = 0;
+                if (def == 0) throw TypeError(format("the argument named `%1%' required by the function is missing")
+                    % aterm2String(name));
+                subsRecursive.set(name, def);
+            } else {
+                attrsUsed++;
+                attrs.remove(name);
+                subs.set(name, value);
+            }
+
+        }
+
+        /* Check that each actual argument is listed as a formal
+           argument (unless the attribute match specifies a `...'). */
+        if (ellipsis == eFalse && attrsUsed != nrAttrs)
+            throw TypeError(format("the function does not expect an argument named `%1%'")
+                % aterm2String(attrs.begin()->key));
+    }
+
+    else if (matchAtPat(pat, pat1, pat2)) {
+        patternMatch(state, pat1, arg, subs, subsRecursive);
+        patternMatch(state, pat2, arg, subs, subsRecursive);
+    }
+
+    else abort();
+}
+
+
 /* Substitute an argument set into the body of a function. */
 static Expr substArgs(EvalState & state,
-    Expr body, ATermList formals, Expr arg)
+    Expr body, Pattern pat, Expr arg)
 {
-    unsigned int nrFormals = ATgetLength(formals);
-    ATermMap subs(nrFormals);
-
-    /* Get the actual arguments and put them in the substitution. */
-    ATermMap args;
-    queryAllAttrs(arg, args);
-    for (ATermMap::const_iterator i = args.begin(); i != args.end(); ++i)
-        subs.set(i->key, i->value);
+    ATermMap subs(16), subsRecursive(16);
     
-    /* Get the formal arguments. */
-    ATermVector defsUsed;
-    ATermList recAttrs = ATempty;
-    for (ATermIterator i(formals); i; ++i) {
-        Expr name, def;
-        ValidValues valids2;
-        DefaultValue def2;
-        if (!matchFormal(*i, name, valids2, def2)) abort(); /* can't happen */
+    patternMatch(state, pat, arg, subs, subsRecursive);
 
-        Expr value = subs[name];
-        
-        if (value == 0) {
-            if (!matchDefaultValue(def2, def)) def = 0;
-            if (def == 0) throw TypeError(format("the argument named `%1%' required by the function is missing")
-                % aterm2String(name));
-            value = def;
-            defsUsed.push_back(name);
-            recAttrs = ATinsert(recAttrs, makeBind(name, def, makeNoPos()));
-        }
-
-        ATermList valids;
-        if (matchValidValues(valids2, valids)) {
-            value = evalExpr(state, value);
-            bool found = false;
-            for (ATermIterator j(valids); j; ++j) {
-                Expr v = evalExpr(state, *j);
-                if (value == v) {
-                    found = true;
-                    break;
-                }
-            }
-            if (!found) throw TypeError(format("the argument named `%1%' has an illegal value")
-                % aterm2String(name));            
-        }
-    }
-
-    /* Make a recursive attribute set out of the (argument-name,
-       value) tuples.  This is so that we can support default
-       parameters that refer to each other, e.g.  ({x, y ? x + x}: y)
-       {x = "foo";} evaluates to "foofoo". */
-    if (defsUsed.size() != 0) {
-        for (ATermMap::const_iterator i = args.begin(); i != args.end(); ++i)
+    /* If we used any default values, make a recursive attribute set
+       out of the (argument-name, value) tuples.  This is so that we
+       can support default values that refer to each other, e.g.  ({x,
+       y ? x + x}: y) {x = "foo";} evaluates to "foofoo". */
+    if (subsRecursive.size() != 0) {
+        ATermList recAttrs = ATempty;
+        foreach (ATermMap::const_iterator, i, subs)
+            recAttrs = ATinsert(recAttrs, makeBind(i->key, i->value, makeNoPos()));
+        foreach (ATermMap::const_iterator, i, subsRecursive)
             recAttrs = ATinsert(recAttrs, makeBind(i->key, i->value, makeNoPos()));
         Expr rec = makeRec(recAttrs, ATempty);
-        for (ATermVector::iterator i = defsUsed.begin(); i != defsUsed.end(); ++i)
-            subs.set(*i, makeSelect(rec, *i));
-    }
-    
-    if (subs.size() != nrFormals) {
-        /* One or more actual arguments were not declared as formal
-           arguments.  Find out which. */
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            subs.remove(name);
-        }
-        throw TypeError(format("the function does not expect an argument named `%1%'")
-            % aterm2String(subs.begin()->key));
+        foreach (ATermMap::const_iterator, i, subsRecursive)
+            subs.set(i->key, makeSelect(rec, i->key));
     }
 
     return substitute(Substitution(0, &subs), body);
@@ -151,11 +180,12 @@ static Expr substArgs(EvalState & state,
    to attributes substituted with selection expressions on the
    original set.  E.g., e = `rec {x = f x y; y = x;}' becomes `{x = f
    (e.x) (e.y); y = e.x;}'. */
-LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
+LocalNoInline(ATerm expandRec(EvalState & state, ATerm e, ATermList rbnds, ATermList nrbnds))
 {
     ATerm name;
     Expr e2;
     Pos pos;
+    Expr eOverrides = 0;
 
     /* Create the substitution list. */
     ATermMap subs(ATgetLength(rbnds) + ATgetLength(nrbnds));
@@ -165,9 +195,26 @@ LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
     }
     for (ATermIterator i(nrbnds); i; ++i) {
         if (!matchBind(*i, name, e2, pos)) abort(); /* can't happen */
+        if (name == sOverrides) eOverrides = e2;
         subs.set(name, e2);
     }
 
+    /* If the rec contains an attribute called `__overrides', then
+       evaluate it, and add the attributes in that set to the rec.
+       This allows overriding of recursive attributes, which is
+       otherwise not possible.  (You can use the // operator to
+       replace an attribute, but other attributes in the rec will
+       still reference the original value, because that value has been
+       substituted into the bodies of the other attributes.  Hence we
+       need __overrides.) */
+    ATermMap overrides;
+    if (eOverrides) {
+        eOverrides = evalExpr(state, eOverrides);
+        queryAllAttrs(eOverrides, overrides, false);
+        foreach (ATermMap::const_iterator, i, overrides)
+            subs.set(i->key, i->value);
+    }
+
     Substitution subs_(0, &subs);
 
     /* Create the non-recursive set. */
@@ -177,6 +224,10 @@ LocalNoInline(ATerm expandRec(ATerm e, ATermList rbnds, ATermList nrbnds))
         as.set(name, makeAttrRHS(substitute(subs_, e2), pos));
     }
 
+    if (eOverrides)
+        foreach (ATermMap::const_iterator, i, overrides)
+            as.set(i->key, makeAttrRHS(i->value, makeNoPos()));
+
     /* Copy the non-recursive bindings.  !!! inefficient */
     for (ATermIterator i(nrbnds); i; ++i) {
         if (!matchBind(*i, name, e2, pos)) abort(); /* can't happen */
@@ -305,9 +356,11 @@ string coerceToString(EvalState & state, Expr e, PathSet & context,
     }
         
     ATermList es;
-    if (matchAttrs(e, es))
-        return coerceToString(state, makeSelect(e, toATerm("outPath")),
-            context, coerceMore, copyToStore);
+    if (matchAttrs(e, es)) {
+        Expr e2 = queryAttr(e, "outPath");
+        if (!e2) throwTypeError("cannot coerce an attribute set (except a derivation) to a string");
+        return coerceToString(state, e2, context, coerceMore, copyToStore);
+    }
 
     if (coerceMore) {
 
@@ -379,15 +432,18 @@ Path coerceToPath(EvalState & state, Expr e, PathSet & context)
 
 Expr autoCallFunction(Expr e, const ATermMap & args)
 {
-    ATermList formals;
+    Pattern pat;
     ATerm body, pos;
+    ATermList formals;
+    ATermBool ellipsis;
     
-    if (matchFunction(e, formals, body, pos)) {
+    /* !!! this should be more general */
+    if (matchFunction(e, pat, body, pos) && matchAttrsPat(pat, formals, ellipsis)) {
         ATermMap actualArgs(ATgetLength(formals));
         
         for (ATermIterator i(formals); i; ++i) {
-            Expr name, def, value; ATerm values, def2;
-            if (!matchFormal(*i, name, values, def2)) abort();
+            Expr name, def, value; ATerm def2;
+            if (!matchFormal(*i, name, def2)) abort();
             if ((value = args.get(name)))
                 actualArgs.set(name, makeAttrRHS(value, makeNoPos()));
             else if (!matchDefaultValue(def2, def))
@@ -420,15 +476,17 @@ LocalNoInline(Expr evalVar(EvalState & state, ATerm name))
     if (arity == 0)
         /* !!! backtrace for primop call */
         return ((PrimOp) ATgetBlobData(fun)) (state, ATermVector());
-    else
+    else {
+        state.safeCache = false;
         return makePrimOp(arity, fun, ATempty);
+    }
 }
 
 
 LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
 {
-    ATermList formals;
-    ATerm pos, name;
+    Pattern pat;
+    ATerm pos;
     Expr body;
         
     /* Evaluate the left-hand side. */
@@ -447,6 +505,7 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
             for (ATermIterator i(args); i; ++i)
                 args2[--arity] = *i;
             /* !!! backtrace for primop call */
+            state.safeCache = true;
             return ((PrimOp) ATgetBlobData(funBlob))
                 (state, args2);
         } else
@@ -454,10 +513,9 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
             return makePrimOp(arity, funBlob, args);
     }
 
-    else if (matchFunction(fun, formals, body, pos)) {
-        arg = evalExpr(state, arg);
+    else if (matchFunction(fun, pat, body, pos)) {
         try {
-            return evalExpr(state, substArgs(state, body, formals, arg));
+            return evalExpr(state, substArgs(state, body, pat, arg));
         } catch (Error & e) {
             addErrorPrefix(e, "while evaluating the function at %1%:\n",
                 showPos(pos));
@@ -465,18 +523,6 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg))
         }
     }
         
-    else if (matchFunction1(fun, name, body, pos)) {
-        try {
-            ATermMap subs(1);
-            subs.set(name, arg);
-            return evalExpr(state, substitute(Substitution(0, &subs), body));
-        } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating the function at %1%:\n",
-                showPos(pos));
-            throw;
-        }
-    }
-
     else throwTypeError(
         "attempt to call something which is neither a function nor a primop (built-in operation) but %1%",
         showType(fun));
@@ -611,6 +657,48 @@ LocalNoInline(Expr evalOpConcat(EvalState & state, Expr e1, Expr e2))
 }
 
 
+/* Implementation of the `==' and `!=' operators. */
+LocalNoInline(bool areEqual(EvalState & state, Expr e1, Expr e2))
+{
+    e1 = evalExpr(state, e1);
+    e2 = evalExpr(state, e2);
+
+    /* We cannot test functions/primops for equality, and we currently
+       don't support testing equality between attribute sets or lists
+       - that would have to be a deep equality test to be sound. */
+    AFun sym1 = ATgetAFun(e1);
+    AFun sym2 = ATgetAFun(e2);
+
+    if (sym1 != sym2) return false;
+
+    /* Functions are incomparable. */
+    if (sym1 == symFunction || sym1 == symPrimOp) return false;
+
+    if (!state.allowUnsafeEquality && sym1 == symAttrs)
+        throw EvalError("comparison of attribute sets is not implemented");
+
+    /* !!! This allows comparisons of infinite data structures to
+       succeed, such as `let x = [x]; in x == x'.  This is
+       undesirable, since equivalent (?) terms such as `let x = [x]; y
+       = [y]; in x == y' don't terminate. */
+    if (e1 == e2) return true;
+    
+    if (sym1 == symList) {
+        ATermList es1; matchList(e1, es1);
+        ATermList es2; matchList(e2, es2);
+        if (ATgetLength(es1) != ATgetLength(es2)) return false;
+        ATermIterator i(es1), j(es2);
+        while (*i) {
+            if (!areEqual(state, *i, *j)) return false;
+            ++i; ++j;
+        }
+        return true;
+    }
+    
+    return false;
+}
+
+
 static char * deepestStack = (char *) -1; /* for measuring stack usage */
 
 
@@ -633,7 +721,6 @@ Expr evalExpr2(EvalState & state, Expr e)
         sym == symInt ||
         sym == symBool ||
         sym == symFunction ||
-        sym == symFunction1 ||
         sym == symAttrs ||
         sym == symList ||
         sym == symPrimOp)
@@ -657,7 +744,7 @@ Expr evalExpr2(EvalState & state, Expr e)
     /* Mutually recursive sets. */
     ATermList rbnds, nrbnds;
     if (matchRec(e, rbnds, nrbnds))
-        return expandRec(e, rbnds, nrbnds);
+        return expandRec(state, e, rbnds, nrbnds);
 
     /* Conditionals. */
     if (matchIf(e, e1, e2, e3))
@@ -675,12 +762,10 @@ Expr evalExpr2(EvalState & state, Expr e)
        However, we don't want to make (==) strict, because that would
        make operations like `big_derivation == null' very slow (unless
        we were to evaluate them side-by-side). */
-    if (matchOpEq(e, e1, e2))
-        return makeBool(evalExpr(state, e1) == evalExpr(state, e2));
-
-    if (matchOpNEq(e, e1, e2))
-        return makeBool(evalExpr(state, e1) != evalExpr(state, e2));
-
+    if (matchOpEq(e, e1, e2)) return makeBool(areEqual(state, e1, e2));
+        
+    if (matchOpNEq(e, e1, e2)) return makeBool(!areEqual(state, e1, e2));
+        
     /* Negation. */
     if (matchOpNot(e, e1))
         return makeBool(!evalBool(state, e1));
@@ -728,11 +813,19 @@ Expr evalExpr(EvalState & state, Expr e)
         format("evaluating expression: %1%") % e);
 #endif
 
-    state.nrEvaluated++;
+    const unsigned int hugeEvalExpr = 100;
+    unsigned int nrEvaluated = state.nrEvaluated++;
+    state.nrDepthAfterReset++;
 
     /* Consult the memo table to quickly get the normal form of
        previously evaluated expressions. */
     Expr nf = state.normalForms.get(e);
+    if (nf) {
+        state.nrCached++;
+        return nf;
+    }
+
+    nf = state.sessionNormalForms.get(e);
     if (nf) {
         if (nf == makeBlackHole())
             throwEvalError("infinite recursion encountered");
@@ -741,14 +834,25 @@ Expr evalExpr(EvalState & state, Expr e)
     }
 
     /* Otherwise, evaluate and memoize. */
-    state.normalForms.set(e, makeBlackHole());
+    state.sessionNormalForms.set(e, makeBlackHole());
     try {
         nf = evalExpr2(state, e);
     } catch (Error & err) {
-        state.normalForms.remove(e);
+        state.sessionNormalForms.remove(e);
         throw;
     }
-    state.normalForms.set(e, nf);
+
+    // session independent condition
+    if (state.nrDepthAfterReset && state.safeCache &&
+        // heuristic condition
+        state.nrEvaluated - nrEvaluated > hugeEvalExpr) {
+        state.sessionNormalForms.remove(e);
+        state.normalForms.set(e, nf);
+        state.nrDepthAfterReset--;
+    } else {
+        state.sessionNormalForms.set(e, nf);
+    }
+
     return nf;
 }
 
@@ -756,6 +860,7 @@ Expr evalExpr(EvalState & state, Expr e)
 Expr evalFile(EvalState & state, const Path & path)
 {
     startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path);
+    state.nrDepthAfterReset = 0;
     Expr e = parseExprFromFile(state, path);
     try {
         return evalExpr(state, e);
@@ -793,29 +898,6 @@ static Expr strictEvalExpr_(EvalState & state, Expr e, ATermMap & nfs)
         return makeList(ATreverse(es2));
     }
     
-    ATermList formals;
-    ATerm body, pos;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermList formals2 = ATempty;
-        
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ValidValues valids; ATerm dummy;
-            if (!matchFormal(*i, name, valids, dummy)) abort();
-
-            ATermList valids2;
-            if (matchValidValues(valids, valids2)) {
-                ATermList valids3 = ATempty;
-                for (ATermIterator j(valids2); j; ++j)
-                    valids3 = ATinsert(valids3, strictEvalExpr(state, *j, nfs));
-                valids = makeValidValues(ATreverse(valids3));
-            }
-
-            formals2 = ATinsert(formals2, makeFormal(name, valids, dummy));
-        }
-        
-        return makeFunction(ATreverse(formals2), body, pos);
-    }
-    
     return e;
 }
 
@@ -850,11 +932,13 @@ void printEvalStats(EvalState & state)
     char x;
     bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
     printMsg(showStats ? lvlInfo : lvlDebug,
-        format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space")
+             format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space, %6% cached normal reduction, %7% session dependent reduction.")
         % state.nrEvaluated % state.nrCached
         % ((float) state.nrCached / (float) state.nrEvaluated * 100)
         % AT_calcAllocatedSize()
-        % (&x - deepestStack));
+        % (&x - deepestStack)
+        % state.normalForms.size()
+        % state.sessionNormalForms.size());
     if (showStats)
         printATermMapStats();
 }

src/libexpr/eval.hh

@@ -16,7 +16,7 @@ class Hash;
 typedef std::map<Path, PathSet> DrvRoots;
 typedef std::map<Path, Hash> DrvHashes;
 
-/* Cache for calls to addToStore(); maps source paths to the store				//THIS OK ????
+/* Cache for calls to addToStore(); maps source paths to the store
    paths. */
 typedef std::map<Path, Path> SrcToStore;
 
@@ -30,6 +30,7 @@ typedef Expr (* PrimOp) (EvalState &, const ATermVector & args);
 struct EvalState 
 {
     ATermMap normalForms;
+    ATermMap sessionNormalForms;
     ATermMap primOps;
     DrvRoots drvRoots;
     DrvHashes drvHashes; /* normalised derivation hashes */
@@ -37,12 +38,19 @@ struct EvalState
 
     unsigned int nrEvaluated;
     unsigned int nrCached;
+    unsigned int nrDepthAfterReset;
+
+    bool allowUnsafeEquality;
+    bool safeCache;
 
     EvalState();
+    ~EvalState();
 
     void addPrimOps();
     void addPrimOp(const string & name,
         unsigned int arity, PrimOp primOp);
+    void loadNormalForms();
+    void saveNormalForms();
 };
 
 

src/libexpr/expr-to-xml.cc

@@ -4,6 +4,8 @@
 #include "aterm.hh"
 #include "util.hh"
 
+#include <cstdlib>
+
 
 namespace nix {
 
@@ -38,6 +40,31 @@ static void showAttrs(const ATermMap & attrs, XMLWriter & doc,
 }
 
 
+static void printPatternAsXML(Pattern pat, XMLWriter & doc)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    if (matchVarPat(pat, name))
+        doc.writeEmptyElement("varpat", singletonAttrs("name", aterm2String(name)));
+    else if (matchAttrsPat(pat, formals, ellipsis)) {
+        XMLOpenElement _(doc, "attrspat");
+        for (ATermIterator i(formals); i; ++i) {
+            Expr name; ATerm dummy;
+            if (!matchFormal(*i, name, dummy)) abort();
+            doc.writeEmptyElement("attr", singletonAttrs("name", aterm2String(name)));
+        }
+        if (ellipsis == eTrue) doc.writeEmptyElement("ellipsis");
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        XMLOpenElement _(doc, "at");
+        printPatternAsXML(pat1, doc);
+        printPatternAsXML(pat2, doc);
+    }
+}
+
+
 static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
     ExprSet & drvsSeen)
 {
@@ -45,8 +72,8 @@ static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
     string s;
     ATerm s2;
     int i;
-    ATermList as, es, formals;
-    ATerm body, pos;
+    ATermList as, es;
+    ATerm pat, body, pos;
 
     checkInterrupt();
 
@@ -107,22 +134,9 @@ static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
             printTermAsXML(*i, doc, context, drvsSeen);
     }
 
-    else if (matchFunction(e, formals, body, pos)) {
+    else if (matchFunction(e, pat, body, pos)) {
         XMLOpenElement _(doc, "function");
-        
-        for (ATermIterator i(formals); i; ++i) {
-            Expr name; ValidValues valids; ATerm dummy;
-            if (!matchFormal(*i, name, valids, dummy)) abort();
-            XMLOpenElement _(doc, "arg", singletonAttrs("name", aterm2String(name)));
-
-            ATermList valids2;
-            if (matchValidValues(valids, valids2)) {
-                for (ATermIterator j(valids2); j; ++j) {
-                    XMLOpenElement _(doc, "value");
-                    printTermAsXML(*j, doc, context, drvsSeen);
-                }
-            }
-        }
+        printPatternAsXML(pat, doc);
     }
 
     else

src/libexpr/get-drvs.cc

@@ -35,53 +35,6 @@ string DrvInfo::queryOutPath(EvalState & state) const
     return outPath;
 }
 
-string DrvInfo::queryStateIdentifier(EvalState & state) const
-{
-	if (stateIdentifier == "") {
-        ATermMap attrs2 = *attrs;
-        
-        for (ATermMap::const_iterator i = attrs2.begin(); i != attrs2.end(); ++i) {
-        	
-        	string key = (string)aterm2String(i->key);		//cast because aterm2String returns a char*
-        	//printMsg(lvlError, format("ATTR2: '%1%'") % key);
-	       	if(key == "stateIdentifier"){		  
-        		PathSet context;
-            	string value = coerceToString(state, i->value, context);
-        		(string &) stateIdentifier = value;
-        	}
-        }
-        
-        //If still empty
-        if (trim(stateIdentifier) == "")
-        	(string &) stateIdentifier = "__NOSTATE__";
-    }
-    
-    return stateIdentifier;
-}
-
-string DrvInfo::queryRuntimeStateArgs(EvalState & state) const
-{
-	if (runtimeStateArgs == "") {
-        ATermMap attrs2 = *attrs;
-        
-        for (ATermMap::const_iterator i = attrs2.begin(); i != attrs2.end(); ++i) {
-        	
-        	string key = (string)aterm2String(i->key);		//cast because aterm2String returns a char*
-	       	if(key == "runtimeStateArgs"){		  
-        		PathSet context;
-            	string value = coerceToString(state, i->value, context);
-        		(string &) runtimeStateArgs = value;
-        	}
-        }
-        
-        //If still empty
-        if (trim(runtimeStateArgs) == "")
-        	(string &) runtimeStateArgs = "__NOARGS__";
-    }
-    
-    return runtimeStateArgs;
-}
-
 
 MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
 {
@@ -97,31 +50,55 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
         Expr e = evalExpr(state, i->value);
         string s;
         PathSet context;
-        if (matchStr(e, s, context))
-            meta[aterm2String(i->key)] = s;
-        /* For future compatibility, ignore attribute values that are
-           not strings. */
+        MetaValue value;
+        int n;
+        ATermList es;
+        if (matchStr(e, s, context)) {
+            value.type = MetaValue::tpString;
+            value.stringValue = s;
+            meta[aterm2String(i->key)] = value;
+        } else if (matchInt(e, n)) {
+            value.type = MetaValue::tpInt;
+            value.intValue = n;
+            meta[aterm2String(i->key)] = value;
+        } else if (matchList(e, es)) {
+            value.type = MetaValue::tpStrings;
+            for (ATermIterator j(es); j; ++j)
+                value.stringValues.push_back(evalStringNoCtx(state, *j));
+            meta[aterm2String(i->key)] = value;
+        }
     }
 
     return meta;
 }
 
 
-string DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
+MetaValue DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
 {
     /* !!! evaluates all meta attributes => inefficient */
-    MetaInfo meta = queryMetaInfo(state);
-    MetaInfo::iterator i = meta.find(name);
-    return i == meta.end() ? "" : i->second;
+    return queryMetaInfo(state)[name];
 }
 
 
 void DrvInfo::setMetaInfo(const MetaInfo & meta)
 {
     ATermMap metaAttrs;
-    for (MetaInfo::const_iterator i = meta.begin(); i != meta.end(); ++i)
-        metaAttrs.set(toATerm(i->first),
-            makeAttrRHS(makeStr(i->second), makeNoPos()));
+    foreach (MetaInfo::const_iterator, i, meta) {
+        Expr e;
+        switch (i->second.type) {
+            case MetaValue::tpInt: e = makeInt(i->second.intValue); break;
+            case MetaValue::tpString: e = makeStr(i->second.stringValue); break;
+            case MetaValue::tpStrings: {
+                ATermList es = ATempty;
+                foreach (Strings::const_iterator, j, i->second.stringValues)
+                    es = ATinsert(es, makeStr(*j));
+                e = makeList(ATreverse(es));
+                break;
+            }
+            default: abort();
+        }
+        metaAttrs.set(toATerm(i->first), makeAttrRHS(e, makeNoPos()));
+    }
     attrs->set(toATerm("meta"), makeAttrs(metaAttrs));
 }
 
@@ -149,23 +126,20 @@ static bool getDerivation(EvalState & state, Expr e,
 
         boost::shared_ptr<ATermMap> attrs(new ATermMap());
         queryAllAttrs(e, *attrs, false);
-
+        
         Expr a = attrs->get(toATerm("type"));
-        if (!a || evalStringNoCtx(state, a) != "derivation") 
-        	return true;
+        if (!a || evalStringNoCtx(state, a) != "derivation") return true;
 
         /* Remove spurious duplicates (e.g., an attribute set like
            `rec { x = derivation {...}; y = x;}'. */
-        if (doneExprs.find(e) != doneExprs.end()) 
-        	return false;
+        if (doneExprs.find(e) != doneExprs.end()) return false;
         doneExprs.insert(e);
 
         DrvInfo drv;
     
         a = attrs->get(toATerm("name"));
         /* !!! We really would like to have a decent back trace here. */
-        if (!a) 
-        	throw TypeError("derivation name missing");
+        if (!a) throw TypeError("derivation name missing");
         drv.name = evalStringNoCtx(state, a);
 
         a = attrs->get(toATerm("system"));
@@ -175,9 +149,7 @@ static bool getDerivation(EvalState & state, Expr e,
             drv.system = evalStringNoCtx(state, a);
 
         drv.attrs = attrs;
-        
-        //printMsg(lvlError, format("TEST '%1%'") % evalStringNoCtx(state, a) );
-        
+
         drv.attrPath = attrPath;
 
         drvs.push_back(drv);
@@ -194,8 +166,7 @@ bool getDerivation(EvalState & state, Expr e, DrvInfo & drv)
     Exprs doneExprs;
     DrvInfos drvs;
     getDerivation(state, e, "", drvs, doneExprs);
-    if (drvs.size() != 1) 
-    	return false;
+    if (drvs.size() != 1) return false;
     drv = drvs.front();
     return true;
 }
@@ -226,24 +197,30 @@ static void getDerivations(EvalState & state, Expr e,
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        Expr e2 = drvMap.get(toATerm("_combineChannels"));
-        bool combineChannels = e2 && evalBool(state, e2);
-        
-        for (ATermMap::const_iterator i = drvMap.begin(); i != drvMap.end(); ++i) {
-            startNest(nest, lvlDebug,
-                format("evaluating attribute `%1%'") % aterm2String(i->key));
-            string pathPrefix2 = addToPath(pathPrefix, aterm2String(i->key));
-            if (combineChannels) {
-                if (((string) aterm2String(i->key)) != "_combineChannels")
-                    getDerivations(state, i->value, pathPrefix2, autoArgs, drvs, doneExprs);
-            }
-            else if (getDerivation(state, i->value, pathPrefix2, drvs, doneExprs)) {
+        bool combineChannels = drvMap.get(toATerm("_combineChannels"));
+
+        /* Consider the attributes in sorted order to get more
+           deterministic behaviour in nix-env operations (e.g. when
+           there are names clashes between derivations, the derivation
+           bound to the attribute with the "lower" name should take
+           precedence). */
+        typedef std::map<string, Expr> AttrsSorted;
+        AttrsSorted attrsSorted;
+        foreach (ATermMap::const_iterator, i, drvMap)
+            attrsSorted[aterm2String(i->key)] = i->value;
+
+        foreach (AttrsSorted::iterator, i, attrsSorted) {
+            startNest(nest, lvlDebug, format("evaluating attribute `%1%'") % i->first);
+            string pathPrefix2 = addToPath(pathPrefix, i->first);
+            if (combineChannels)
+                getDerivations(state, i->second, pathPrefix2, autoArgs, drvs, doneExprs);
+            else if (getDerivation(state, i->second, pathPrefix2, drvs, doneExprs)) {
                 /* If the value of this attribute is itself an
                    attribute set, should we recurse into it?  => Only
                    if it has a `recurseForDerivations = true'
                    attribute. */
                 ATermList es;
-                Expr e = evalExpr(state, i->value);
+                Expr e = evalExpr(state, i->second), e2;
                 if (matchAttrs(e, es)) {
                     ATermMap attrs(ATgetLength(es));
                     queryAllAttrs(e, attrs, false);

src/libexpr/get-drvs.hh

@@ -12,7 +12,16 @@
 namespace nix {
 
 
-typedef std::map<string, string> MetaInfo;
+struct MetaValue
+{
+    enum { tpNone, tpString, tpStrings, tpInt } type;
+    string stringValue;
+    Strings stringValues;
+    int intValue;
+};
+
+
+typedef std::map<string, MetaValue> MetaInfo;
 
 
 struct DrvInfo
@@ -20,8 +29,6 @@ struct DrvInfo
 private:
     string drvPath;
     string outPath;
-    string stateIdentifier;
-    string runtimeStateArgs;
     
 public:
     string name;
@@ -35,10 +42,8 @@ public:
 
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
-    string queryStateIdentifier(EvalState & state) const;
-    string queryRuntimeStateArgs(EvalState & state) const;
     MetaInfo queryMetaInfo(EvalState & state) const;
-    string queryMetaInfo(EvalState & state, const string & name) const;
+    MetaValue queryMetaInfo(EvalState & state, const string & name) const;
 
     void setDrvPath(const string & s)
     {
@@ -49,17 +54,7 @@ public:
     {
         outPath = s;
     }
-    
-    void setStateIdentifier(const string & s)
-    {
-    	stateIdentifier = s;	
-    }
-    
-    void setRuntimeStateArgs(const string & s)
-    {
-    	runtimeStateArgs = s;	
-    }
-    
+
     void setMetaInfo(const MetaInfo & meta);
 };
 

src/libexpr/lexer.l

@@ -95,6 +95,7 @@ let         { return LET; }
 in          { return IN; }
 rec         { return REC; }
 inherit     { return INHERIT; }
+\.\.\.      { return ELLIPSIS; }
 
 \=\=        { return EQ; }
 \!\=        { return NEQ; }

src/libexpr/names.cc

@@ -92,25 +92,6 @@ int compareVersions(const string & v1, const string & v2)
 }
 
 
-static void testCompareVersions()
-{
-#define TEST(v1, v2, n) assert( \
-    compareVersions(v1, v2) == n && compareVersions(v2, v1) == -n)
-    TEST("1.0", "2.3", -1);
-    TEST("2.1", "2.3", -1);
-    TEST("2.3", "2.3", 0);
-    TEST("2.5", "2.3", 1);
-    TEST("3.1", "2.3", 1);
-    TEST("2.3.1", "2.3", 1);
-    TEST("2.3.1", "2.3a", 1);
-    TEST("2.3pre1", "2.3", -1);
-    TEST("2.3pre3", "2.3pre12", -1);
-    TEST("2.3a", "2.3c", -1);
-    TEST("2.3pre1", "2.3c", -1);
-    TEST("2.3pre1", "2.3q", -1);
-}
-
-
 DrvNames drvNamesFromArgs(const Strings & opArgs)
 {
     DrvNames result;

src/libexpr/nix.sdf

@@ -1,10 +1,13 @@
+%% Note: this SDF grammar is no longer used in the Nix expression
+%% parser and may not be up to date.
+
 definition
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Top level syntax.
 
-module Nix
+module Main
 imports Nix-Exprs Nix-Layout
 
 
@@ -38,6 +41,7 @@ exports
     "with" Expr ";" Expr -> Expr {cons("With")}
 
     "rec" "{" Bind* "}" -> Expr {cons("Rec")}
+    "let" Bind* "in" Expr -> Expr {cons("Let")}
     "let" "{" Bind* "}" -> Expr {cons("LetRec")}
     "{" Bind* "}" -> Expr {cons("Attrs")}
 
@@ -102,7 +106,8 @@ exports
 
     [0-9]+ -> Int
 
-    "\"" ~[\n\"]* "\"" -> Str
+    "\"" (~[\"\\] | ("\\" ~[]) )* "\"" -> Str
+    "''" (~[\"\\] | ("\\" ~[]) )* "''" -> Str
 
     [a-zA-Z0-9\.\_\-\+]* ("/"[a-zA-Z0-9\.\_\-\+]+)+ -> Path
 

src/libexpr/nixexpr-ast.def

@@ -3,8 +3,7 @@ init initNixExprHelpers
 Pos | string int int | Pos |
 NoPos | | Pos |
 
-Function | ATermList Expr Pos | Expr |
-Function1 | string Expr Pos | Expr |
+Function | Pattern Expr Pos | Expr |
 Assert | Expr Expr Pos | Expr |
 With | Expr Expr Pos | Expr |
 If | Expr Expr Expr | Expr |
@@ -67,25 +66,27 @@ PrimOp | int ATermBlob ATermList | Expr |
 Attrs | ATermList | Expr |
 Closed | Expr | Expr |
 Rec | ATermList ATermList | Expr |
-Bool | ATerm | Expr |
+Bool | ATermBool | Expr |
 Null | | Expr |
 
 Bind | string Expr Pos | ATerm |
+BindAttrPath | ATermList Expr Pos | ATerm | # desugared during parsing
 Bind | string Expr | ATerm | ObsoleteBind
 Inherit | Expr ATermList Pos | ATerm |
 
 Scope | | Expr |
 
-Formal | string ValidValues DefaultValue | ATerm |
+VarPat | string | Pattern |
+AttrsPat | ATermList ATermBool | Pattern | # bool = `...'
+AtPat | Pattern Pattern | Pattern |
 
-ValidValues | ATermList | ValidValues |
-UnrestrictedValues | | ValidValues |
+Formal | string DefaultValue | ATerm |
 
 DefaultValue | Expr | DefaultValue |
 NoDefaultValue | | DefaultValue |
 
-True | | ATerm |
-False | | ATerm |
+True | | ATermBool |
+False | | ATermBool |
 
 PrimOpDef | int ATermBlob | ATerm |
 
@@ -93,3 +94,4 @@ AttrRHS | Expr Pos | ATerm |
 
 eTrue = makeBool(makeTrue())
 eFalse = makeBool(makeFalse())
+sOverrides = toATerm("__overrides")

src/libexpr/nixexpr.cc

@@ -6,6 +6,8 @@
 #include "nixexpr-ast.hh"
 #include "nixexpr-ast.cc"
 
+#include <cstdlib>
+
 
 namespace nix {
     
@@ -17,7 +19,7 @@ string showPos(ATerm pos)
     if (matchNoPos(pos)) return "undefined position";
     if (!matchPos(pos, path, line, column))
         throw badTerm("position expected", pos);
-    return (format("`%1%', line %2%") % aterm2String(path) % line).str();
+    return (format("`%1%:%2%:%3%'") % aterm2String(path) % line % column).str();
 }
     
 
@@ -108,6 +110,31 @@ Expr makeAttrs(const ATermMap & attrs)
 }
 
 
+static void varsBoundByPattern(ATermMap & map, Pattern pat)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    /* Use makeRemoved() so that it can be used directly in
+       substitute(). */
+    if (matchVarPat(pat, name))
+        map.set(name, makeRemoved());
+    else if (matchAttrsPat(pat, formals, ellipsis)) { 
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm d1;
+            if (!matchFormal(*i, name, d1)) abort();
+            map.set(name, makeRemoved());
+        }
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        varsBoundByPattern(map, pat1);
+        varsBoundByPattern(map, pat2);
+    }
+    else abort();
+}
+
+
 Expr substitute(const Substitution & subs, Expr e)
 {
     checkInterrupt();
@@ -133,27 +160,17 @@ Expr substitute(const Substitution & subs, Expr e)
 
     /* In case of a function, filter out all variables bound by this
        function. */
-    ATermList formals;
+    Pattern pat;
     ATerm body;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermMap map(ATgetLength(formals));
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            map.set(name, makeRemoved());
-        }
+    if (matchFunction(e, pat, body, pos)) {
+        ATermMap map(16);
+        varsBoundByPattern(map, pat);
         Substitution subs2(&subs, &map);
         return makeFunction(
-            (ATermList) substitute(subs2, (ATerm) formals),
+            (Pattern) substitute(subs2, (Expr) pat),
             substitute(subs2, body), pos);
     }
 
-    if (matchFunction1(e, name, body, pos)) {
-        ATermMap map(1);
-        map.set(name, makeRemoved());
-        return makeFunction1(name, substitute(Substitution(&subs, &map), body), pos);
-    }
-        
     /* Idem for a mutually recursive attribute set. */
     ATermList rbnds, nrbnds;
     if (matchRec(e, rbnds, nrbnds)) {
@@ -211,9 +228,9 @@ static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
     done.insert(e);
     
     ATerm name, pos, value;
-    ATermList formals;
     ATerm with, body;
     ATermList rbnds, nrbnds;
+    Pattern pat;
 
     /* Closed terms don't have free variables, so we don't have to
        check by definition. */
@@ -225,28 +242,11 @@ static void checkVarDefs2(set<Expr> & done, const ATermMap & defs, Expr e)
                 % aterm2String(name));
     }
 
-    else if (matchFunction(e, formals, body, pos)) {
+    else if (matchFunction(e, pat, body, pos)) {
         ATermMap defs2(defs);
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            defs2.set(name, (ATerm) ATempty);
-        }
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm valids, deflt;
-            set<Expr> done2;
-            if (!matchFormal(*i, name, valids, deflt)) abort();
-            checkVarDefs2(done, defs, valids);
-            checkVarDefs2(done2, defs2, deflt);
-        }
-        set<Expr> done2;
-        checkVarDefs2(done2, defs2, body);
-    }
-        
-    else if (matchFunction1(e, name, body, pos)) {
-        ATermMap defs2(defs);
-        defs2.set(name, (ATerm) ATempty);
+        varsBoundByPattern(defs2, pat);
         set<Expr> done2;
+        checkVarDefs2(done2, defs2, pat);
         checkVarDefs2(done2, defs2, body);
     }
         
@@ -360,17 +360,17 @@ Expr makeStr(const string & s, const PathSet & context)
 
 string showType(Expr e)
 {
-    ATerm t1, t2, t3;
+    ATerm t1, t2;
     ATermList l1;
     ATermBlob b1;
     int i1;
+    Pattern p1;
     if (matchStr(e, t1, l1)) return "a string";
     if (matchPath(e, t1)) return "a path";
     if (matchNull(e)) return "null";
     if (matchInt(e, i1)) return "an integer";
     if (matchBool(e, t1)) return "a boolean";
-    if (matchFunction(e, l1, t1, t2)) return "a function";
-    if (matchFunction1(e, t1, t2, t3)) return "a function";
+    if (matchFunction(e, p1, t1, t2)) return "a function";
     if (matchAttrs(e, l1)) return "an attribute set";
     if (matchList(e, l1)) return "a list";
     if (matchPrimOp(e, i1, b1, l1)) return "a partially applied built-in function";

src/libexpr/nixexpr.hh

@@ -11,6 +11,7 @@ namespace nix {
 
 
 MakeError(EvalError, Error)
+MakeError(ParseError, Error)
 MakeError(AssertionError, EvalError)
 MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
@@ -21,11 +22,10 @@ MakeError(TypeError, EvalError)
    property of the ATerm library allows us to implement caching of
    normals forms efficiently. */
 typedef ATerm Expr;
-
 typedef ATerm DefaultValue;
-typedef ATerm ValidValues;
-
 typedef ATerm Pos;
+typedef ATerm Pattern;
+typedef ATerm ATermBool;
 
 
 /* A STL vector of ATerms.  Should be used with great care since it's

src/libexpr/normal-forms.cc

@@ -0,0 +1,64 @@
+#include "eval.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "serialise.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+namespace nix {
+
+
+void EvalState::loadNormalForms()
+{
+    //    nixCacheFile = getEnv("NIX_CACHE_FILE", nixStateDir + "/reduce-cache");
+    nixCacheFile = getEnv("NIX_CACHE_FILE");
+    string loadFlag = getEnv("NIX_CACHE_FILE_LOAD", "1");
+
+    if(nixCacheFile == "" || loadFlag == "")
+        return;
+    if(!pathExists(nixCacheFile))
+        return;
+
+    printMsg(lvlInfo, format("Load cache: ..."));
+    try {
+        int fd = open(nixCacheFile.c_str(), O_RDONLY);
+        if (fd == -1)
+            throw SysError(format("opening file `%1%'") % nixCacheFile);
+        AutoCloseFD auto_fd = fd;
+
+        FdSource source = fd;
+        normalForms = readATermMap(source);
+    } catch (Error & e) {
+        e.addPrefix(format("Cannot load cached reduce operations from %1%:\n") % nixCacheFile);
+        throw;
+    }
+    printMsg(lvlInfo, format("Load cache: end"));
+}
+
+void EvalState::saveNormalForms()
+{
+    string saveFlag = getEnv("NIX_CACHE_FILE_SAVE", "");
+
+    if(nixCacheFile == "" || saveFlag == "")
+        return;
+
+    printMsg(lvlInfo, format("Save cache: ..."));
+    try {
+        int fd = open(nixCacheFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
+        if (fd == -1)
+            throw SysError(format("opening file `%1%'") % nixCacheFile);
+        AutoCloseFD auto_fd = fd;
+
+        FdSink sink = fd;
+        writeATermMap(normalForms, sink);
+    } catch (Error & e) {
+        e.addPrefix(format("Cannot save cached reduce operations to %1%:\n") % nixCacheFile);
+        throw;
+    }
+    printMsg(lvlInfo, format("Save cache: end"));
+}
+
+
+}

src/libexpr/parser.y

@@ -25,6 +25,7 @@
     
 #include "parser-tab.hh"
 #include "lexer-tab.hh"
+#define YYSTYPE YYSTYPE // workaround a bug in Bison 2.4
 
 #include "nixexpr.hh"
 #include "nixexpr-ast.hh"
@@ -43,28 +44,128 @@ struct ParseData
     Path path;
     string error;
 };
+ 
 
-
-static Expr fixAttrs(int recursive, ATermList as)
+static string showAttrPath(ATermList attrPath)
 {
-    ATermList bs = ATempty, cs = ATempty;
-    ATermList * is = recursive ? &cs : &bs;
+    string s;
+    for (ATermIterator i(attrPath); i; ++i) {
+        if (!s.empty()) s += '.';
+        s += aterm2String(*i);
+    }
+    return s;
+}
+ 
+
+struct Tree
+{
+    Expr leaf; ATerm pos; bool recursive;
+    typedef std::map<ATerm, Tree> Children;
+    Children children;
+    Tree() { leaf = 0; recursive = true; }
+};
+
+
+static ATermList buildAttrs(const Tree & t, ATermList & nonrec)
+{
+    ATermList res = ATempty;
+    for (Tree::Children::const_reverse_iterator i = t.children.rbegin();
+         i != t.children.rend(); ++i)
+        if (!i->second.recursive)
+            nonrec = ATinsert(nonrec, makeBind(i->first, i->second.leaf, i->second.pos));
+        else
+            res = ATinsert(res, i->second.leaf
+                ? makeBind(i->first, i->second.leaf, i->second.pos)
+                : makeBind(i->first, makeAttrs(buildAttrs(i->second, nonrec)), makeNoPos()));
+    return res;
+}
+ 
+
+static Expr fixAttrs(bool recursive, ATermList as)
+{
+    Tree attrs;
+
     for (ATermIterator i(as); i; ++i) {
-        ATermList names;
-        Expr src;
-        ATerm pos;
+        ATermList names, attrPath; Expr src, e; ATerm name, pos;
+
         if (matchInherit(*i, src, names, pos)) {
             bool fromScope = matchScope(src);
             for (ATermIterator j(names); j; ++j) {
-                Expr rhs = fromScope ? makeVar(*j) : makeSelect(src, *j);
-                *is = ATinsert(*is, makeBind(*j, rhs, pos));
+                if (attrs.children.find(*j) != attrs.children.end()) 
+                    throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
+                        % showAttrPath(ATmakeList1(*j)) % showPos(pos));
+                Tree & t(attrs.children[*j]);
+                t.leaf = fromScope ? makeVar(*j) : makeSelect(src, *j);
+                t.pos = pos;
+                if (recursive && fromScope) t.recursive = false;
             }
-        } else bs = ATinsert(bs, *i);
+        }
+
+        else if (matchBindAttrPath(*i, attrPath, e, pos)) {
+
+            Tree * t(&attrs);
+            
+            for (ATermIterator j(attrPath); j; ) {
+                name = *j; ++j;
+                if (t->leaf) throw ParseError(format("attribute set containing `%1%' at %2% already defined at %3%")
+                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
+                t = &(t->children[name]);
+            }
+
+            if (t->leaf)
+                throw ParseError(format("duplicate definition of attribute `%1%' at %2% and %3%")
+                    % showAttrPath(attrPath) % showPos(pos) % showPos (t->pos));
+            if (!t->children.empty())
+                throw ParseError(format("duplicate definition of attribute `%1%' at %2%")
+                    % showAttrPath(attrPath) % showPos(pos));
+
+            t->leaf = e; t->pos = pos;
+        }
+
+        else abort(); /* can't happen */
     }
-    if (recursive)
-        return makeRec(bs, cs);
-    else
-        return makeAttrs(bs);
+
+    ATermList nonrec = ATempty;
+    ATermList rec = buildAttrs(attrs, nonrec);
+        
+    return recursive ? makeRec(rec, nonrec) : makeAttrs(rec);
+}
+
+
+static void checkPatternVars(ATerm pos, ATermMap & map, Pattern pat)
+{
+    ATerm name;
+    ATermList formals;
+    Pattern pat1, pat2;
+    ATermBool ellipsis;
+    if (matchVarPat(pat, name)) {
+        if (map.get(name))
+            throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+                % aterm2String(name) % showPos(pos));
+        map.set(name, name);
+    }
+    else if (matchAttrsPat(pat, formals, ellipsis)) { 
+        for (ATermIterator i(formals); i; ++i) {
+            ATerm d1;
+            if (!matchFormal(*i, name, d1)) abort();
+            if (map.get(name))
+                throw ParseError(format("duplicate formal function argument `%1%' at %2%")
+                    % aterm2String(name) % showPos(pos));
+            map.set(name, name);
+        }
+    }
+    else if (matchAtPat(pat, pat1, pat2)) {
+        checkPatternVars(pos, map, pat1);
+        checkPatternVars(pos, map, pat2);
+    }
+    else abort();
+}
+
+
+static void checkPatternVars(ATerm pos, Pattern pat)
+{
+    ATermMap map;
+    checkPatternVars(pos, map, pat);
 }
 
 
@@ -146,7 +247,7 @@ static Expr stripIndentation(ATermList es)
         /* Remove the last line if it is empty and consists only of
            spaces. */
         if (n == 1) {
-            unsigned int p = s2.find_last_of('\n');
+            string::size_type p = s2.find_last_of('\n');
             if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos)
                 s2 = string(s2, 0, p + 1);
         }
@@ -208,15 +309,22 @@ static void freeAndUnprotect(void * p)
 %union {
   ATerm t;
   ATermList ts;
+  struct {
+    ATermList formals;
+    bool ellipsis;
+  } formals;
 }
 
 %type <t> start expr expr_function expr_if expr_op
 %type <t> expr_app expr_select expr_simple bind inheritsrc formal
-%type <ts> binds ids expr_list formals string_parts ind_string_parts
+%type <t> pattern pattern2
+%type <ts> binds ids attrpath expr_list string_parts ind_string_parts
+%type <formals> formals
 %token <t> ID INT STR IND_STR PATH URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL
 %token DOLLAR_CURLY /* == ${ */
 %token IND_STRING_OPEN IND_STRING_CLOSE
+%token ELLIPSIS
 
 %nonassoc IMPL
 %left OR
@@ -236,16 +344,14 @@ start: expr { data->result = $1; };
 expr: expr_function;
 
 expr_function
-  : '{' formals '}' ':' expr_function
-    { $$ = makeFunction($2, $5, CUR_POS); }
-  | ID ':' expr_function
-    { $$ = makeFunction1($1, $3, CUR_POS); }
+  : pattern ':' expr_function
+    { checkPatternVars(CUR_POS, $1); $$ = makeFunction($1, $3, CUR_POS); }
   | ASSERT expr ';' expr_function
     { $$ = makeAssert($2, $4, CUR_POS); }
   | WITH expr ';' expr_function
     { $$ = makeWith($2, $4, CUR_POS); }
   | LET binds IN expr_function
-    { $$ = makeSelect(fixAttrs(1, ATinsert($2, makeBind(toATerm("<let-body>"), $4, CUR_POS))), toATerm("<let-body>")); }
+    { $$ = makeSelect(fixAttrs(true, ATinsert($2, makeBindAttrPath(ATmakeList1(toATerm("<let-body>")), $4, CUR_POS))), toATerm("<let-body>")); }
   | expr_if
   ;
 
@@ -300,12 +406,12 @@ expr_simple
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
   | LET '{' binds '}'
-    { $$ = makeSelect(fixAttrs(1, $3), toATerm("body")); }
+    { $$ = makeSelect(fixAttrs(true, $3), toATerm("body")); }
   | REC '{' binds '}'
-    { $$ = fixAttrs(1, $3); }
+    { $$ = fixAttrs(true, $3); }
   | '{' binds '}'
-    { $$ = fixAttrs(0, $2); }
-  | '[' expr_list ']' { $$ = makeList($2); }
+    { $$ = fixAttrs(false, $2); }
+  | '[' expr_list ']' { $$ = makeList(ATreverse($2)); }
   ;
 
 string_parts
@@ -320,14 +426,24 @@ ind_string_parts
   | { $$ = ATempty; }
   ;
 
+pattern
+  : pattern2 '@' pattern { $$ = makeAtPat($1, $3); }
+  | pattern2
+  ;
+
+pattern2
+  : ID { $$ = makeVarPat($1); }
+  | '{' formals '}' { $$ = makeAttrsPat($2.formals, $2.ellipsis ? eTrue : eFalse); }
+  ;
+
 binds
   : binds bind { $$ = ATinsert($1, $2); }
   | { $$ = ATempty; }
   ;
 
 bind
-  : ID '=' expr ';'
-    { $$ = makeBind($1, $3, CUR_POS); }
+  : attrpath '=' expr ';'
+    { $$ = makeBindAttrPath(ATreverse($1), $3, CUR_POS); }
   | INHERIT inheritsrc ids ';'
     { $$ = makeInherit($2, $3, CUR_POS); }
   ;
@@ -339,24 +455,30 @@ inheritsrc
 
 ids: ids ID { $$ = ATinsert($1, $2); } | { $$ = ATempty; };
 
+attrpath
+  : attrpath '.' ID { $$ = ATinsert($1, $3); }
+  | ID { $$ = ATmakeList1($1); }
+  ;
+
 expr_list
-  : expr_select expr_list { $$ = ATinsert($2, $1); }
-    /* yes, this is right-recursive, but it doesn't matter since
-       otherwise we would need ATreverse which requires unbounded
-       stack space */
+  : expr_list expr_select { $$ = ATinsert($1, $2); }
   | { $$ = ATempty; }
   ;
 
 formals
-  : formal ',' formals { $$ = ATinsert($3, $1); } /* idem - right recursive */
-  | formal { $$ = ATinsert(ATempty, $1); }
-  | { $$ = ATempty; }
+  : formal ',' formals /* !!! right recursive */
+    { $$.formals = ATinsert($3.formals, $1); $$.ellipsis = $3.ellipsis; }
+  | formal
+    { $$.formals = ATinsert(ATempty, $1); $$.ellipsis = false; }
+  |
+    { $$.formals = ATempty; $$.ellipsis = false; }
+  | ELLIPSIS
+    { $$.formals = ATempty; $$.ellipsis = true; }
   ;
 
 formal
-  : ID { $$ = makeFormal($1, makeUnrestrictedValues(), makeNoDefaultValue()); }
-  | ID ':' '[' expr_list ']' { $$ = makeFormal($1, makeValidValues($4), makeNoDefaultValue()); }
-  | ID '?' expr { $$ = makeFormal($1, makeUnrestrictedValues(), makeDefaultValue($3)); }
+  : ID { $$ = makeFormal($1, makeNoDefaultValue()); }
+  | ID '?' expr { $$ = makeFormal($1, makeDefaultValue($3)); }
   ;
   
 %%
@@ -373,63 +495,6 @@ formal
 namespace nix {
       
 
-static void checkAttrs(ATermMap & names, ATermList bnds)
-{
-    for (ATermIterator i(bnds); i; ++i) {
-        ATerm name;
-        Expr e;
-        ATerm pos;
-        if (!matchBind(*i, name, e, pos)) abort(); /* can't happen */
-        if (names.get(name))
-            throw EvalError(format("duplicate attribute `%1%' at %2%")
-                % aterm2String(name) % showPos(pos));
-        names.set(name, name);
-    }
-}
-
-
-static void checkAttrSets(ATerm e)
-{
-    ATermList formals;
-    ATerm body, pos;
-    if (matchFunction(e, formals, body, pos)) {
-        ATermMap names(ATgetLength(formals));
-        for (ATermIterator i(formals); i; ++i) {
-            ATerm name;
-            ATerm d1, d2;
-            if (!matchFormal(*i, name, d1, d2)) abort();
-            if (names.get(name))
-                throw EvalError(format("duplicate formal function argument `%1%' at %2%")
-                    % aterm2String(name) % showPos(pos));
-            names.set(name, name);
-        }
-    }
-
-    ATermList bnds;
-    if (matchAttrs(e, bnds)) {
-        ATermMap names(ATgetLength(bnds));
-        checkAttrs(names, bnds);
-    }
-    
-    ATermList rbnds, nrbnds;
-    if (matchRec(e, rbnds, nrbnds)) {
-        ATermMap names(ATgetLength(rbnds) + ATgetLength(nrbnds));
-        checkAttrs(names, rbnds);
-        checkAttrs(names, nrbnds);
-    }
-    
-    if (ATgetType(e) == AT_APPL) {
-        int arity = ATgetArity(ATgetAFun(e));
-        for (int i = 0; i < arity; ++i)
-            checkAttrSets(ATgetArgument(e, i));
-    }
-
-    else if (ATgetType(e) == AT_LIST)
-        for (ATermIterator i((ATermList) e); i; ++i)
-            checkAttrSets(*i);
-}
-
-
 static Expr parse(EvalState & state,
     const char * text, const Path & path,
     const Path & basePath)
@@ -444,16 +509,14 @@ static Expr parse(EvalState & state,
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
     
-    if (res) throw EvalError(data.error);
+    if (res) throw ParseError(data.error);
 
     try {
         checkVarDefs(state.primOps, data.result);
     } catch (Error & e) {
-        throw EvalError(format("%1%, in `%2%'") % e.msg() % path);
+        throw ParseError(format("%1%, in `%2%'") % e.msg() % path);
     }
     
-    checkAttrSets(data.result);
-
     return data.result;
 }
 

src/libexpr/primops.cc

@@ -1,14 +1,13 @@
 #include "misc.hh"
 #include "eval.hh"
-#include "db.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
 #include "archive.hh"
 #include "expr-to-xml.hh"
 #include "nixexpr-ast.hh"
-#include "local-store.hh"
 #include "parser.hh"
+#include "names.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -75,12 +74,14 @@ static Expr prim_null(EvalState & state, const ATermVector & args)
    platforms. */
 static Expr prim_currentSystem(EvalState & state, const ATermVector & args)
 {
+    state.nrDepthAfterReset = 0;
     return makeStr(thisSystem);
 }
 
 
 static Expr prim_currentTime(EvalState & state, const ATermVector & args)
 {
+    state.nrDepthAfterReset = 0;
     return ATmake("Int(<int>)", time(0));
 }
 
@@ -121,67 +122,34 @@ static Expr prim_isNull(EvalState & state, const ATermVector & args)
 static Expr prim_isFunction(EvalState & state, const ATermVector & args)
 {
     Expr e = evalExpr(state, args[0]);
-    ATermList formals;
-    ATerm name, body, pos;
-    return makeBool(
-        matchFunction(e, formals, body, pos) ||
-        matchFunction1(e, name, body, pos));
+    Pattern pat;
+    ATerm body, pos;
+    return makeBool(matchFunction(e, pat, body, pos));
 }
 
-
-static Path findDependency(Path dir, string dep)
+/* Determine whether the argument is an Int. */
+static Expr prim_isInt(EvalState & state, const ATermVector & args)
 {
-    if (dep[0] == '/') throw EvalError(
-        format("illegal absolute dependency `%1%'") % dep);
-
-    Path p = canonPath(dir + "/" + dep);
-
-    if (pathExists(p))
-        return p;
-    else
-        return "";
+    int i;
+    return makeBool(matchInt(evalExpr(state, args[0]), i));
 }
 
-
-/* Make path `p' relative to directory `pivot'.  E.g.,
-   relativise("/a/b/c", "a/b/x/y") => "../x/y".  Both input paths
-   should be in absolute canonical form. */
-static string relativise(Path pivot, Path p)
+/* Determine whether the argument is an String. */
+static Expr prim_isString(EvalState & state, const ATermVector & args)
 {
-    assert(pivot.size() > 0 && pivot[0] == '/');
-    assert(p.size() > 0 && p[0] == '/');
-        
-    if (pivot == p) return ".";
-
-    /* `p' is in `pivot'? */
-    Path pivot2 = pivot + "/";
-    if (p.substr(0, pivot2.size()) == pivot2) {
-        return p.substr(pivot2.size());
-    }
-
-    /* Otherwise, `p' is in a parent of `pivot'.  Find up till which
-       path component `p' and `pivot' match, and add an appropriate
-       number of `..' components. */
-    string::size_type i = 1;
-    while (1) {
-        string::size_type j = pivot.find('/', i);
-        if (j == string::npos) break;
-        j++;
-        if (pivot.substr(0, j) != p.substr(0, j)) break;
-        i = j;
-    }
-
-    string prefix;
-    unsigned int slashes = count(pivot.begin() + i, pivot.end(), '/') + 1;
-    while (slashes--) {
-        prefix += "../";
-    }
-
-    return prefix + p.substr(i);
+    string s;
+    PathSet l;
+    return makeBool(matchStr(evalExpr(state, args[0]), s, l));
 }
 
+/* Determine whether the argument is an Bool. */
+static Expr prim_isBool(EvalState & state, const ATermVector & args)
+{
+    ATermBool b;
+    return makeBool(matchBool(evalExpr(state, args[0]), b));
+}
 
-static Expr prim_dependencyClosure(EvalState & state, const ATermVector & args)
+static Expr prim_genericClosure(EvalState & state, const ATermVector & args)
 {
     startNest(nest, lvlDebug, "finding dependencies");
 
@@ -192,87 +160,40 @@ static Expr prim_dependencyClosure(EvalState & state, const ATermVector & args)
     if (!startSet) throw EvalError("attribute `startSet' required");
     ATermList startSet2 = evalList(state, startSet);
 
-    Path pivot;
-    PathSet workSet;
-    for (ATermIterator i(startSet2); i; ++i) {
-        PathSet context; /* !!! what to do? */
-        Path p = coerceToPath(state, *i, context);
-        workSet.insert(p);
-        pivot = dirOf(p);
-    }
+    set<Expr> workSet; // !!! gc roots
+    for (ATermIterator i(startSet2); i; ++i) workSet.insert(*i);
 
-    /* Get the search path. */
-    PathSet searchPath;
-    Expr e = queryAttr(attrs, "searchPath");
-    if (e) {
-        ATermList list = evalList(state, e);
-        for (ATermIterator i(list); i; ++i) {
-            PathSet context; /* !!! what to do? */
-            Path p = coerceToPath(state, *i, context);
-            searchPath.insert(p);
-        }
-    }
-
-    Expr scanner = queryAttr(attrs, "scanner");
-    if (!scanner) throw EvalError("attribute `scanner' required");
+    /* Get the operator. */
+    Expr op = queryAttr(attrs, "operator");
+    if (!op) throw EvalError("attribute `operator' required");
     
-    /* Construct the dependency closure by querying the dependency of
-       each path in `workSet', adding the dependencies to
-       `workSet'. */
-    PathSet doneSet;
+    /* Construct the closure by applying the operator to element of
+       `workSet', adding the result to `workSet', continuing until
+       no new elements are found. */
+    ATermList res = ATempty;
+    set<Expr> doneKeys; // !!! gc roots
     while (!workSet.empty()) {
-	Path path = *(workSet.begin());
-	workSet.erase(path);
+	Expr e = *(workSet.begin());
+	workSet.erase(e);
 
-	if (doneSet.find(path) != doneSet.end()) continue;
-        doneSet.insert(path);
+        e = strictEvalExpr(state, e);
 
-        try {
-            
-            /* Call the `scanner' function with `path' as argument. */
-            debug(format("finding dependencies in `%1%'") % path);
-            ATermList deps = evalList(state, makeCall(scanner, makeStr(path)));
+        Expr key = queryAttr(e, "key");
+        if (!key) throw EvalError("attribute `key' required");
 
-            /* Try to find the dependencies relative to the `path'. */
-            for (ATermIterator i(deps); i; ++i) {
-                string s = evalStringNoCtx(state, *i);
-                
-                Path dep = findDependency(dirOf(path), s);
+	if (doneKeys.find(key) != doneKeys.end()) continue;
+        doneKeys.insert(key);
+        res = ATinsert(res, e);
+        
+        /* Call the `operator' function with `e' as argument. */
+        ATermList res = evalList(state, makeCall(op, e));
 
-                if (dep == "") {
-                    for (PathSet::iterator j = searchPath.begin();
-                         j != searchPath.end(); ++j)
-                    {
-                        dep = findDependency(*j, s);
-                        if (dep != "") break;
-                    }
-                }
-                
-                if (dep == "")
-                    debug(format("did NOT find dependency `%1%'") % s);
-                else {
-                    debug(format("found dependency `%1%'") % dep);
-                    workSet.insert(dep);
-                }
-            }
-
-        } catch (Error & e) {
-            e.addPrefix(format("while finding dependencies in `%1%':\n")
-                % path);
-            throw;
-        }
+        /* Try to find the dependencies relative to the `path'. */
+        for (ATermIterator i(res); i; ++i)
+            workSet.insert(evalExpr(state, *i));
     }
 
-    /* Return a list of the dependencies we've just found. */
-    ATermList deps = ATempty;
-    for (PathSet::iterator i = doneSet.begin(); i != doneSet.end(); ++i) {
-        deps = ATinsert(deps, makeStr(relativise(pivot, *i)));
-        deps = ATinsert(deps, makeStr(*i));
-    }
-
-    debug(format("dependency list is `%1%'") % makeList(deps));
-    
-    return makeList(deps);
+    return makeList(res);
 }
 
 
@@ -287,15 +208,29 @@ static Expr prim_abort(EvalState & state, const ATermVector & args)
 static Expr prim_throw(EvalState & state, const ATermVector & args)
 {
     PathSet context;
-    throw ThrownError(format("user-thrown exception: `%1%'") %
+    throw ThrownError(format("user-thrown exception: %1%") %
         evalString(state, args[0], context));
 }
 
 
+static Expr prim_addErrorContext(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    try {
+        return evalExpr(state, args[1]);
+    } catch (Error & e) {
+        e.addPrefix(format("%1%\n") %
+            evalString(state, args[0], context));
+        throw;
+    }
+}
+
+
 /* Return an environment variable.  Use with care. */
 static Expr prim_getEnv(EvalState & state, const ATermVector & args)
 {
     string name = evalStringNoCtx(state, args[0]);
+    state.nrDepthAfterReset = 0;
     return makeStr(getEnv(name));
 }
 
@@ -312,20 +247,19 @@ static Expr prim_trace(EvalState & state, const ATermVector & args)
 }
 
 
-static Expr prim_relativise(EvalState & state, const ATermVector & args)
-{
-    PathSet context; /* !!! what to do? */
-    Path pivot = coerceToPath(state, args[0], context);
-    Path path = coerceToPath(state, args[1], context);
-    return makeStr(relativise(pivot, path));
-}
-
-
 /*************************************************************
  * Derivations
  *************************************************************/
 
 
+static bool isFixedOutput(const Derivation & drv)
+{
+    return drv.outputs.size() == 1 &&
+        drv.outputs.begin()->first == "out" &&
+        drv.outputs.begin()->second.hash != "";
+}
+
+
 /* Returns the hash of a derivation modulo fixed-output
    subderivations.  A fixed-output derivation is a derivation with one
    output (`out') for which an expected hash and hash algorithm are
@@ -338,77 +272,43 @@ static Expr prim_relativise(EvalState & state, const ATermVector & args)
    function, we do not want to rebuild everything depending on it
    (after all, (the hash of) the file being downloaded is unchanged).
    So the *output paths* should not change.  On the other hand, the
-   *derivation store expression paths* should change to reflect the
-   new dependency graph.
+   *derivation paths* should change to reflect the new dependency
+   graph.
 
    That's what this function does: it returns a hash which is just the
-   of the derivation ATerm, except that any input store expression
+   hash of the derivation ATerm, except that any input derivation
    paths have been replaced by the result of a recursive call to this
-   function, and that for fixed-output derivations we return
-   (basically) its outputHash. */
+   function, and that for fixed-output derivations we return a hash of
+   its output path. */
 static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 {
-    //printMsg(lvlError, format("DRV: %1% %2%") % drv.env.find("name")->second % (drv.outputs.size() == 1));
-    
-    /* Return a fixed hash for fixed-output derivations. 
-     * E.g. derivations that have a hash  
-     */
-    if (drv.outputs.size() == 1) {
+    /* Return a fixed hash for fixed-output derivations. */
+    if (isFixedOutput(drv)) {
         DerivationOutputs::const_iterator i = drv.outputs.begin();
-
-        if (i->first == "out" &&
-            i->second.hash != "")
-        {
-            //printMsg(lvlError, format("FIXED OUTPUT: %1% - %2% - %3%") % i->second.hashAlgo % i->second.hash % i->second.path);
-            
-            return hashString(htSHA256, "fixed:out:"
-                + i->second.hashAlgo + ":"
-                + i->second.hash + ":"
-                + i->second.path);
-        }
+        return hashString(htSHA256, "fixed:out:"
+            + i->second.hashAlgo + ":"
+            + i->second.hash + ":"
+            + i->second.path);
     }
-    
-    /* If we have a state derivation, we clear state paramters because they (sometimes) can affect the outPath: 
-     * If this drv has runtime paramters: 			The state indentifier and statepath may change, but the componentPath (outPath) can stay the same
-     * If this drv doesnt have runtime paramters: 	The state indentifier and statepath may change, but the componentPath changes since it is build with another identifier
-     * In both cases: Other runtime state parameters like stateDirs, synchronisation and shareState never change the out or statepath so always need to be out of the hash 
-     */ 
-    if(isStateDrv(drv)){
-	    
-	    if(drv.stateOutputs.size() != 1)
-	    	throw EvalError(format("There are more then one stateOutputs in the derviation....."));
-	    
-	    DerivationStateOutput drvso = drv.stateOutputs["state"];
-	    
-	    if(drvso.runtimeStateArgs != ""){			//Has runtime parameters		-->  Clear all state parameters
-			drv.stateOutputs.clear();
-			drv.stateOutputDirs.clear();
-			drv.env["statePath"] = "";
-	    }
-	    else{										//Has NO runtime parameters		-->  Clear state parameters selectively
-	    	drvso.clearAllRuntimeParamters();
-	    	drv.stateOutputDirs.clear();
-	    }
-	}
 
     /* For other derivations, replace the inputs paths with recursive
        calls to this function.*/
     DerivationInputs inputs2;
-    for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i)
-    {
+    foreach (DerivationInputs::const_iterator, i, drv.inputDrvs) {
         Hash h = state.drvHashes[i->first];
         if (h.type == htUnknown) {
-            Derivation drv2 = derivationFromPathTxn(noTxn, i->first);
+            Derivation drv2 = derivationFromPath(i->first);
             h = hashDerivationModulo(state, drv2);
             state.drvHashes[i->first] = h;
         }
         inputs2[printHash(h)] = i->second;
     }
     drv.inputDrvs = inputs2;
-    //printMsg(lvlError, format("%1% with %2% --> %3%") % drv.env.find("name")->second % printHash(hashTerm(unparseDerivation(drv))) % unparseDerivation(drv));
-	return hashTerm(unparseDerivation(drv));
+    
+    return hashTerm(unparseDerivation(drv));
 }
 
+
 /* Construct (as a unobservable side effect) a Nix derivation
    expression that performs the derivation described by the argument
    set.  Returns the original set extended with the following
@@ -419,6 +319,7 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
 static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
 {
     startNest(nest, lvlVomit, "evaluating derivation");
+    state.nrDepthAfterReset = 0;
 
     ATermMap attrs;
     queryAllAttrs(evalExpr(state, args[0]), attrs, true);
@@ -443,22 +344,9 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
     
     PathSet context;
 
-    string outputHash;
-    string outputHashAlgo;
+    string outputHash, outputHashAlgo;
     bool outputHashRecursive = false;
 
-  	//state vars
-    bool enableState = false;		//We dont do state by default, but if a user defines stateDirs for example, than this becomes true.
-    bool disableState = false;		//Becomes true if the user explicitly says: no state
-    string shareType = "none";
-    string syncState = "none";
-    string stateIdentifier = "";
-    bool createDirsBeforeInstall = false;
-    string runtimeStateArgs = "";
-    string sharedState = "";
-    string externalState = "";
-    vector<DerivationStateOutputDir> stateDirs;
-
     for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i) {
         string key = aterm2String(i->key);
         ATerm value;
@@ -484,73 +372,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
                     drv.args.push_back(s);
                 }
             }
-			//Add specific state variables
-   	        else if(key == "stateDirs") {
-              
-              enableState = true;
-              value = evalExpr(state, value);
-              ATermList list = evalList(state, value);
-              
-              for (ATermIterator j(list); j; ++j){
-				Expr v = evalExpr(state, *j);
-                ATermMap stateDirattrs;
-			    queryAllAttrs(evalExpr(state, v), stateDirattrs, true);
-                DerivationStateOutputDir dir = DerivationStateOutputDir();
-                
-                for (ATermMap::const_iterator k = stateDirattrs.begin(); k != stateDirattrs.end(); ++k) {
-			        
-			        string statekey = aterm2String(k->key);
-			        ATerm statevalue;
-			        Expr statepos;
-			        ATerm staterhs = k->value;
-			        if (!matchAttrRHS(staterhs, statevalue, statepos)) abort();
-			        startNest(nest, lvlVomit, format("processing statedir attribute `%1%'") % statekey);
-			        try {
-			            string s = trim(coerceToString(state, statevalue, context, true));
-			            if (statekey == "dir") { 
-			            	
-			            	//Add a / to the end if it's not there
-							if(s[s.length() - 1] != '/')
-				            	s = s + "/";
-				            
-				            //Remove the / at the beginning if it's there
-							if(s[0] == '/' && s.length() != 1)
-				            	s = s.substr(1, s.length());
-				            
-				            dir.path = s;
-			            }
-			            else if (statekey == "file") { dir.path = s; }
-			            else if (statekey == "type") { dir.type = s; }
-			            else if (statekey == "interval") { 
-			            	if(s == "")
-			            		continue;
-			            	int n;
-			            	if (!string2Int(s, n)) throw Error("interval is not a number"); 
-			            	if(n == 0) throw Error("Interval cannot be 0");
-			            	dir.interval = s; 
-			            }
-			            else throw EvalError(format("invalid sub-attirbute `%1%' for attribute dirs") % statekey);
-            	    }
-			        catch (Error & e) {
-			            e.addPrefix(format("while evaluating the state derivation attribute `%1%' at %2%:\n") % key % showPos(statepos));
-			            e.addPrefix(format("while instantiating the derivation named `%1%' at %2%:\n") % drvName % showPos(posDrvName));
-			            throw;
-			        }
-            	}
-            	
-            	stateDirs.push_back(dir);
-              }
-                 	         
-   	        }
-   	        else if(key == "solidStateDependency"){ externalState = coerceToString(state, value, context, true); }
-   	        else if(key == "shareType") { shareType = coerceToString(state, value, context, true); }
-   	        else if(key == "synchronization") { syncState = coerceToString(state, value, context, true); }
-   	        else if(key == "disableState") { disableState = evalBool(state, value); }
-            else if(key == "identifier"){ stateIdentifier = coerceToString(state, value, context, true); }
-            else if(key == "createDirsBeforeInstall"){ createDirsBeforeInstall = evalBool(state, value); }
-            else if(key == "runtimeStateArgs"){ runtimeStateArgs = coerceToString(state, value, context, true); }
-            else if(key == "shareStateFrom"){ sharedState = coerceToString(state, value, context, true);  }
-            
+
             /* All other attributes are passed to the builder through
                the environment. */
             else {
@@ -575,18 +397,38 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
                 % drvName % showPos(posDrvName));
             throw;
         }
+
     }
     
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
-    for (PathSet::iterator i = context.begin(); i != context.end(); ++i) {
-        debug(format("derivation uses `%1%'") % *i);
-        assert(isStorePath(*i));
-        if (isDerivation(*i))
-            drv.inputDrvs[*i] = singleton<StringSet>("out");
+    foreach (PathSet::iterator, i, context) {
+        Path path = *i;
+        
+        /* Paths marked with `=' denote that the path of a derivation
+           is explicitly passed to the builder.  Since that allows the
+           builder to gain access to every path in the dependency
+           graph of the derivation (including all outputs), all paths
+           in the graph must be added to this derivation's list of
+           inputs to ensure that they are available when the builder
+           runs. */
+        if (path.at(0) == '=') {
+            path = string(path, 1);
+            PathSet refs; computeFSClosure(path, refs);
+            foreach (PathSet::iterator, j, refs) {
+                drv.inputSrcs.insert(*j);
+                if (isDerivation(*j))
+                    drv.inputDrvs[*j] = singleton<StringSet>("out");
+            }
+        }
+        
+        debug(format("derivation uses `%1%'") % path);
+        assert(isStorePath(path));
+        if (isDerivation(path))
+            drv.inputDrvs[path] = singleton<StringSet>("out");
         else
-            drv.inputSrcs.insert(*i);
+            drv.inputSrcs.insert(path);
     }
             
     /* Do we have all required attributes? */
@@ -596,6 +438,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
         throw EvalError("required attribute `system' missing");
 
     /* If an output hash was given, check it. */
+    Path outPath;
     if (outputHash == "")
         outputHashAlgo = "";
     else {
@@ -614,6 +457,7 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
                 % outputHash % outputHashAlgo);
         string s = outputHash;
         outputHash = printHash(h);
+        outPath = makeFixedOutputPath(outputHashRecursive, ht, h, drvName);
         if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
     }
 
@@ -630,54 +474,22 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
        output names do get reflected in the hash. */
     drv.env["out"] = "";
     drv.outputs["out"] = DerivationOutput("", outputHashAlgo, outputHash);
-	
-	/* If there are no runtime paratermers, then we need to take the the stateIdentifier into account for the hash calcaulation below: hashDerivationModulo(...) 
-	 * We also add enableState to make it parse the drv to a state-drv
-	 * We also add runtimeStateArgs for the hash calc in hashDerivationModulo(...) to check if its needs to take the stateIdentiefier into account in the hash
-	 */
-	
-	if(enableState && !disableState){    
-		if(runtimeStateArgs == ""){
-			string enableStateS = bool2string("true");
-			drv.stateOutputs["state"] = DerivationStateOutput("", "", "", "", stateIdentifier, enableStateS, "", "", "", runtimeStateArgs, queryCurrentUsername(), "", "", false);
-		}	
-	}
         
     /* Use the masked derivation expression to compute the output
        path. */
-    Hash componentHash = hashDerivationModulo(state, drv);
-    Path outPath = makeStorePath("output:out", componentHash, drvName);
+    if (outPath == "")
+        outPath = makeStorePath("output:out", hashDerivationModulo(state, drv), drvName);
 
     /* Construct the final derivation store expression. */
     drv.env["out"] = outPath;
-    drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, outputHash);
-    
-    //printMsg(lvlError, format("DerivationOutput %1% %2% %3%") % outPath % outputHashAlgo % outputHash);
-	//only add state when we have to to keep compitibilty with the 'old' format.
-	//We add state when it's enbaled by the keywords, and not excplicitly disabled by the user
-	Path stateOutPath;
-	if(enableState && !disableState){    
-		/* Add the state path based on the outPath
-		 * 
-		 * NOTE: we do not include the username into the hash calculation of the statepath yet, multiple different users can use the same dervation 
-		 * but need different state paths. Thats why we keep a 'dummy' value e.g. global hash for everyone, and later at build time recalculate the real state path 
-		 */
-	    stateOutPath = makeStatePath(printHash(componentHash), drvName, stateIdentifier, queryCurrentUsername());		//State path
-    	drv.env["statePath"] = stateOutPath;
-
-    	string enableStateS = bool2string("true");
-    	string createDirsBeforeInstallS = bool2string(createDirsBeforeInstall);
-    	drv.stateOutputs["state"] = DerivationStateOutput(stateOutPath, printHash(componentHash), outputHashAlgo, outputHash, stateIdentifier, enableStateS, 
-    	                                                  shareType, syncState, createDirsBeforeInstallS, runtimeStateArgs, queryCurrentUsername(), sharedState, externalState);
-    	
-    	for(vector<DerivationStateOutputDir>::iterator i = stateDirs.begin(); i != stateDirs.end(); ++i)
-    		drv.stateOutputDirs[(*i).path] = *(i);
-	}
+    drv.outputs["out"] =
+        DerivationOutput(outPath, outputHashAlgo, outputHash);
 
     /* Write the resulting term into the Nix store directory. */
     Path drvPath = writeDerivation(drv, drvName);
 
-    printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'") % drvName % drvPath);
+    printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'")
+        % drvName % drvPath);
 
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store expressions, so we can't
@@ -686,17 +498,10 @@ static Expr prim_derivationStrict(EvalState & state, const ATermVector & args)
 
     /* !!! assumes a single output */
     ATermMap outAttrs(2);
-    outAttrs.set(toATerm("outPath"), makeAttrRHS(makeStr(outPath, singleton<PathSet>(drvPath)), makeNoPos()));
-    outAttrs.set(toATerm("drvPath"), makeAttrRHS(makeStr(drvPath, singleton<PathSet>(drvPath)), makeNoPos()));
-    
-    /* TODO !!!!!!!!!!!!!!!!!!!! Recheck this !!!!!!!!!!!!!!!!!!!!
-     * We now always set the statePath since someone might 'convert' and old non-state expression into a state expression like this:
-     * let oldDrv = import ../../applications/networking/mailreaders/thunderbird-2.x;
-     * 	   newDrv = stdenv.mkDerivation( oldDrv { ... } // { name = "x-state-v"; stateDirs = [ ... ]; }  );
-	 * in newDrv
-     */ 
-    //if(enableState && !disableState)		
-    	outAttrs.set(toATerm("statePath"), makeAttrRHS(makeStr(stateOutPath, singleton<PathSet>(drvPath)), makeNoPos()));
+    outAttrs.set(toATerm("outPath"),
+        makeAttrRHS(makeStr(outPath, singleton<PathSet>(drvPath)), makeNoPos()));
+    outAttrs.set(toATerm("drvPath"),
+        makeAttrRHS(makeStr(drvPath, singleton<PathSet>("=" + drvPath)), makeNoPos()));
 
     return makeAttrs(outAttrs);
 }
@@ -717,8 +522,6 @@ static Expr prim_derivationLazy(EvalState & state, const ATermVector & args)
         makeAttrRHS(makeSelect(drvStrict, toATerm("outPath")), makeNoPos()));
     attrs.set(toATerm("drvPath"),
         makeAttrRHS(makeSelect(drvStrict, toATerm("drvPath")), makeNoPos()));
-    attrs.set(toATerm("statePath"),
-        makeAttrRHS(makeSelect(drvStrict, toATerm("statePath")), makeNoPos()));
     
     return makeAttrs(attrs);
 }
@@ -738,12 +541,36 @@ static Expr prim_toPath(EvalState & state, const ATermVector & args)
 }
 
 
+/* Allow a valid store path to be used in an expression.  This is
+   useful in some generated expressions such as in nix-push, which
+   generates a call to a function with an already existing store path
+   as argument.  You don't want to use `toPath' here because it copies
+   the path to the Nix store, which yields a copy like
+   /nix/store/newhash-oldhash-oldname.  In the past, `toPath' had
+   special case behaviour for store paths, but that created weird
+   corner cases. */
+static Expr prim_storePath(EvalState & state, const ATermVector & args)
+{
+    PathSet context;
+    Path path = canonPath(coerceToPath(state, args[0], context));
+    state.nrDepthAfterReset = 0;
+    if (!isInStore(path))
+        throw EvalError(format("path `%1%' is not in the Nix store") % path);
+    Path path2 = toStorePath(path);
+    if (!store->isValidPath(path2))
+        throw EvalError(format("store path `%1%' is not valid") % path2);
+    context.insert(path2);
+    return makeStr(path, context);
+}
+
+
 static Expr prim_pathExists(EvalState & state, const ATermVector & args)
 {
     PathSet context;
     Path path = coerceToPath(state, args[0], context);
     if (!context.empty())
         throw EvalError(format("string `%1%' cannot refer to other paths") % path);
+    state.nrDepthAfterReset = 0;
     return makeBool(pathExists(path));
 }
 
@@ -754,9 +581,9 @@ static Expr prim_baseNameOf(EvalState & state, const ATermVector & args)
 {
     PathSet context;
     return makeStr(baseNameOf(coerceToString(state, args[0], context)), context);
-
 }
 
+
 /* Return the directory of the given path, i.e., everything before the
    last slash.  Return either a path or a string depending on the type
    of the argument. */
@@ -777,6 +604,7 @@ static Expr prim_readFile(EvalState & state, const ATermVector & args)
     Path path = coerceToPath(state, args[0], context);
     if (!context.empty())
         throw EvalError(format("string `%1%' cannot refer to other paths") % path);
+    state.nrDepthAfterReset = 0;
     return makeStr(readFile(path));
 }
 
@@ -807,14 +635,16 @@ static Expr prim_toFile(EvalState & state, const ATermVector & args)
     string contents = evalString(state, args[1], context);
 
     PathSet refs;
-    PathSet stateRefs;		//refs refers to the file references, there are no state references in this case.
 
     for (PathSet::iterator i = context.begin(); i != context.end(); ++i) {
-        if (isDerivation(*i))
+        Path path = *i;
+        if (path.at(0) == '=') path = string(path, 1);
+        if (isDerivation(path))
             throw EvalError(format("in `toFile': the file `%1%' cannot refer to derivation outputs") % name);
-        refs.insert(*i);
+        refs.insert(path);
     }
     
+    state.nrDepthAfterReset = 0;
     Path storePath = readOnlyMode
         ? computeStorePathForText(name, contents, refs)
         : store->addTextToStore(name, contents, refs);
@@ -867,9 +697,10 @@ static Expr prim_filterSource(EvalState & state, const ATermVector & args)
 
     FilterFromExpr filter(state, args[0]);
 
+    state.nrDepthAfterReset = 0;
     Path dstPath = readOnlyMode
-        ? computeStorePathForPath(path, false, false, "", filter).first
-        : store->addToStore(path, false, false, "", filter);
+        ? computeStorePathForPath(path, true, htSHA256, filter).first
+        : store->addToStore(path, true, htSHA256, filter);
 
     return makeStr(dstPath, singleton<PathSet>(dstPath));
 }
@@ -965,7 +796,7 @@ static Expr prim_removeAttrs(EvalState & state, const ATermVector & args)
 }
 
 
-/* Determine whether the argument is a list. */
+/* Determine whether the argument is an attribute set. */
 static Expr prim_isAttrs(EvalState & state, const ATermVector & args)
 {
     ATermList list;
@@ -1021,6 +852,14 @@ static Expr prim_map(EvalState & state, const ATermVector & args)
 }
 
 
+/* Return the length of a list.  This is an O(1) time operation. */
+static Expr prim_length(EvalState & state, const ATermVector & args)
+{
+    ATermList list = evalList(state, args[0]);
+    return makeInt(ATgetLength(list));
+}
+
+
 /*************************************************************
  * Integer arithmetic
  *************************************************************/
@@ -1042,6 +881,23 @@ static Expr prim_sub(EvalState & state, const ATermVector & args)
 }
 
 
+static Expr prim_mul(EvalState & state, const ATermVector & args)
+{
+    int i1 = evalInt(state, args[0]);
+    int i2 = evalInt(state, args[1]);
+    return makeInt(i1 * i2);
+}
+
+
+static Expr prim_div(EvalState & state, const ATermVector & args)
+{
+    int i1 = evalInt(state, args[0]);
+    int i2 = evalInt(state, args[1]);
+    if (i2 == 0) throw EvalError("division by zero");
+    return makeInt(i1 / i2);
+}
+
+
 static Expr prim_lessThan(EvalState & state, const ATermVector & args)
 {
     int i1 = evalInt(state, args[0]);
@@ -1098,23 +954,54 @@ static Expr prim_unsafeDiscardStringContext(EvalState & state, const ATermVector
     return makeStr(s, PathSet());
 }
 
-/* Expression serialization/deserialization */ 
 
-static Expr prim_ExprToString ( EvalState & state, const ATermVector & args)
+/* Expression serialization/deserialization */
+
+
+static Expr prim_exprToString(EvalState & state, const ATermVector & args)
 {
-	return makeStr ( atPrint ( evalExpr ( state, args [ 0 ] ) ) );
+    /* !!! this disregards context */
+    return makeStr(atPrint(evalExpr(state, args[0])));
 }
 
-static Expr prim_StringToExpr ( EvalState & state, const ATermVector & args)
+
+static Expr prim_stringToExpr(EvalState & state, const ATermVector & args)
 {
-	string s;
-	PathSet l;
-	if (! matchStr ( evalExpr ( state, args[0] ), s, l )) {
-		throw EvalError("__stringToExpr needs string argument!");
-	}
-	return ATreadFromString(s.c_str());
+    /* !!! this can introduce arbitrary garbage terms in the
+       evaluator! */;
+    string s;
+    PathSet l;
+    if (!matchStr(evalExpr(state, args[0]), s, l))
+        throw EvalError("stringToExpr needs string argument!");
+    return ATreadFromString(s.c_str());
 }
 
+
+/*************************************************************
+ * Versions
+ *************************************************************/
+
+
+static Expr prim_parseDrvName(EvalState & state, const ATermVector & args)
+{
+    string name = evalStringNoCtx(state, args[0]);
+    DrvName parsed(name);
+    ATermMap attrs(2);
+    attrs.set(toATerm("name"), makeAttrRHS(makeStr(parsed.name), makeNoPos()));
+    attrs.set(toATerm("version"), makeAttrRHS(makeStr(parsed.version), makeNoPos()));
+    return makeAttrs(attrs);
+}
+
+
+static Expr prim_compareVersions(EvalState & state, const ATermVector & args)
+{
+    string version1 = evalStringNoCtx(state, args[0]);
+    string version2 = evalStringNoCtx(state, args[1]);
+    int d = compareVersions(version1, version2);
+    return makeInt(d);
+}
+
+
 /*************************************************************
  * Primop registration
  *************************************************************/
@@ -1135,17 +1022,20 @@ void EvalState::addPrimOps()
     addPrimOp("import", 1, prim_import);
     addPrimOp("isNull", 1, prim_isNull);
     addPrimOp("__isFunction", 1, prim_isFunction);
-    addPrimOp("dependencyClosure", 1, prim_dependencyClosure);
+    addPrimOp("__isString", 1, prim_isString);
+    addPrimOp("__isInt", 1, prim_isInt);
+    addPrimOp("__isBool", 1, prim_isBool);
+    addPrimOp("__genericClosure", 1, prim_genericClosure);
     addPrimOp("abort", 1, prim_abort);
     addPrimOp("throw", 1, prim_throw);
+    addPrimOp("__addErrorContext", 2, prim_addErrorContext);
     addPrimOp("__getEnv", 1, prim_getEnv);
     addPrimOp("__trace", 2, prim_trace);
+
     
     // Expr <-> String
-    addPrimOp("__exprToString", 1, prim_ExprToString);
-    addPrimOp("__stringToExpr", 1, prim_StringToExpr);
-
-    addPrimOp("relativise", 2, prim_relativise);
+    addPrimOp("__exprToString", 1, prim_exprToString);
+    addPrimOp("__stringToExpr", 1, prim_stringToExpr);
 
     // Derivations
     addPrimOp("derivation!", 1, prim_derivationStrict);
@@ -1153,6 +1043,7 @@ void EvalState::addPrimOps()
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
+    addPrimOp("__storePath", 1, prim_storePath);
     addPrimOp("__pathExists", 1, prim_pathExists);
     addPrimOp("baseNameOf", 1, prim_baseNameOf);
     addPrimOp("dirOf", 1, prim_dirOf);
@@ -1176,10 +1067,13 @@ void EvalState::addPrimOps()
     addPrimOp("__head", 1, prim_head);
     addPrimOp("__tail", 1, prim_tail);
     addPrimOp("map", 2, prim_map);
+    addPrimOp("__length", 1, prim_length);
 
     // Integer arithmetic
     addPrimOp("__add", 2, prim_add);
     addPrimOp("__sub", 2, prim_sub);
+    addPrimOp("__mul", 2, prim_mul);
+    addPrimOp("__div", 2, prim_div);
     addPrimOp("__lessThan", 2, prim_lessThan);
 
     // String manipulation
@@ -1187,7 +1081,10 @@ void EvalState::addPrimOps()
     addPrimOp("__substring", 3, prim_substring);
     addPrimOp("__stringLength", 1, prim_stringLength);
     addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
-    
+
+    // Versions
+    addPrimOp("__parseDrvName", 1, prim_parseDrvName);
+    addPrimOp("__compareVersions", 2, prim_compareVersions);
 }
 
 

src/libext3cow/Makefile.am

@@ -1,17 +0,0 @@
-pkglib_LTLIBRARIES = libext3cow.la
-
-libext3cow_la_SOURCES = epoch2date.c snapshot.cc tt.c ext3cow_tools.h ext3cow_fs.h
-
-pkginclude_HEADERS = snapshot.hh
-
-#TODO linux kernel header
-
-libext3cow_la_LIBADD = ../libutil/libutil.la \
-                       ../boost/format/libformat.la 
-
-AM_CXXFLAGS = -Wall \
- -I$(srcdir)/.. ${aterm_include} \
- -I$(srcdir)/../libutil
-
-AM_CFLAGS = \
- ${aterm_include}

src/libext3cow/epoch2date.c

@@ -1,24 +0,0 @@
-
-/* epoch2date.c - an ext3cow tool for turning epoch numbers into dates.
- * Copyright (C) 2003-2007 Zachary N. J. Peterson
- */
-
-
-#include "ext3cow_tools.h"
-
-int main(int argc, char** argv){
-
-  time_t time;
-
-  if(argc < 2){
-    fprintf(stderr, "usage: %s seconds\n", argv[0]);
-    exit(1);
-  }
-
-  time = atoi(argv[1]);
-
-  printf("%s", ctime(&time));
-
-  return 0;
-
-}

src/libext3cow/ext3cow_tools.h

@@ -1,10 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <sys/ioctl.h>
-#include <time.h>
-#include <sys/time.h>
-#include "ext3cow_fs.h"

src/libext3cow/snapshot.cc

@@ -1,97 +0,0 @@
-
-/* snapshot.c - an ext3cow tool for taking file system snapshots
- * Copyright (C) 2003-2007 Zachary N. J. Peterson
- */
-
-
-#include "ext3cow_tools.h"
-#include "snapshot.hh"
-#include "types.hh"
-
-//using namespace nix;
-
-namespace nix {
-
-/////////////////////Original functions commented out: 
-
-/*
-void snapshot_usage(void){
-  
-  fprintf(stderr, "usage: snapshot <mountpoint>\n");
-
-}
-
-int snapshot_main(int argc, char** argv){
-
-  int fd;
-  int ret;
-  unsigned int epoch = 0;
-  char path[256] = ".\0";
-  
-  if(argc > 1)
-    strcpy(path, argv[1]);
-
-  printf("Snapshot on %s: ", path);
-
-  fd = open(path, O_RDONLY);
-  
-  // test for ext3cow fs type
-
-  if(fd < 0){
-    printf("failed.\n");
-    perror(argv[0]);
-    exit(1);
-  }
-
-  epoch = (int)ioctl(fd, EXT3COW_IOC_TAKESNAPSHOT, &epoch);
-  if((int)epoch < 0){
-    printf("failed.\n");
-    perror(argv[0]);
-    exit(1);
-  }
-
-  printf("%u\n", (unsigned int)epoch);
-
-
-  return 0;
-
-}
-*/
-
-/////////////////////End original functions
-
-
-unsigned int take_snapshot(const string & dir2) //const string & file_or_dir)
-{
-  const char* dir = dir2.c_str();
-
-  int fd;
-  unsigned int epoch = 0;
-  
-  //char path[256] = ".\0";
-  //TODO 256 length check ???
-  //strcpy(path, dir);
-
-  fd = open(dir, O_RDONLY);
-  
-  /* test for ext3cow fs type */
-
-  if(fd < 0){
-    printf("failed.\n");
-    perror(dir);
-    exit(1);
-  }
-
-  epoch = (int)ioctl(fd, EXT3COW_IOC_TAKESNAPSHOT, &epoch);
-  if((int)epoch < 0){
-    printf("failed.\n");
-    perror(dir);
-    exit(1);
-  }
-  
-  //printf("%u\n", (unsigned int)epoch);
-
-  return epoch;
-}
-
-}

src/libext3cow/snapshot.hh

@@ -1,13 +0,0 @@
-#ifndef SNAPSHOT_H_
-#define SNAPSHOT_H_
-
-#include "types.hh"
-
-namespace nix {
-
-//unsigned int take_snapshot(const string & file_or_dir);
-unsigned int take_snapshot(const string & dir);
-
-}
-
-#endif /*SNAPSHOT_H_*/

src/libext3cow/tt.c

@@ -1,46 +0,0 @@
-
-/* tt.c - an ext3cow tool for retreiving the current file system epoch
- * Copyright (C) 2003-2007 Zachary N. J. Peterson
- */
-
-#include "ext3cow_tools.h"
-
-void tt_usage(void){
-  
-  fprintf(stderr, "usage: tt <mountpoint>\n");
-
-}
-
-int tt_main(int argc, char** argv){
-
-  int fd;
-  int ret;
-  unsigned int epoch = 0;
-  char path[255] = ".\0";
-  
-  if(argc > 1)
-    strcpy(path, argv[1]);
-
-
-  fd = open(path, O_RDONLY);
-  
-  /* test for ext3cow fs type */
-
-  if(fd < 0){
-    fprintf(stderr, "Couldn't open %s\n", path);
-    exit(1);
-  }
-
-
-  ret = ioctl(fd, EXT3COW_IOC_GETEPOCH, &epoch);
-  if(ret < 0){
-    printf("tt on %s failed.\n", path);
-    perror(argv[0]);
-    exit(1);
-  }
-  printf("Epoch: %d\n", ret);
-  close(fd);
-  
-  return 0;
-
-}

src/libmain/Makefile.am

@@ -1,12 +1,13 @@
 pkglib_LTLIBRARIES = libmain.la
 
-libmain_la_SOURCES = shared.cc shared.hh
+libmain_la_SOURCES = shared.cc
+
+libmain_la_LIBADD = ../libstore/libstore.la
+
+pkginclude_HEADERS = shared.hh
 
 AM_CXXFLAGS = \
  -DNIX_STORE_DIR=\"$(storedir)\" \
- -DNIX_STORE_STATE_DIR=\"$(storestatedir)\" \
- -DNIX_EXT3_COW_HEADER=\"$(ext3cowheader)\" \
- -DNIX_RSYNC=\"$(rsync)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \
  -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \
  -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \

src/libmain/shared.cc

@@ -4,6 +4,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "util.hh"
+#include "misc.hh"
 
 #include <iostream>
 #include <cctype>
@@ -49,6 +50,34 @@ void printGCWarning()
 }
 
 
+void printMissing(const PathSet & paths)
+{
+    unsigned long long downloadSize;
+    PathSet willBuild, willSubstitute, unknown;
+    queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize);
+
+    if (!willBuild.empty()) {
+        printMsg(lvlInfo, format("the following derivations will be built:"));
+        foreach (PathSet::iterator, i, willBuild)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+
+    if (!willSubstitute.empty()) {
+        printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") %
+            (downloadSize / (1024.0 * 1024.0)));
+        foreach (PathSet::iterator, i, willSubstitute)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+
+    if (!unknown.empty()) {
+        printMsg(lvlInfo, format("don't know how to build the following paths%1%:")
+            % (readOnlyMode ? " (may be caused by read-only store access)" : ""));
+        foreach (PathSet::iterator, i, unknown)
+            printMsg(lvlInfo, format("  %1%") % *i);
+    }
+}
+
+
 static void setLogType(string lt)
 {
     if (lt == "pretty") logType = ltPretty;
@@ -58,27 +87,18 @@ static void setLogType(string lt)
 }
 
 
-static unsigned int getIntArg(const string & opt,
+unsigned long long getIntArg(const string & opt,
     Strings::iterator & i, const Strings::iterator & end)
 {
     ++i;
     if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
-    int n;
+    long long n;
     if (!string2Int(*i, n) || n < 0)
         throw UsageError(format("`%1%' requires a non-negative integer") % opt);
     return n;
 }
 
 
-struct RemoveTempRoots 
-{
-    ~RemoveTempRoots()
-    {
-        removeTempRoots();
-    }
-};
-
-
 void initDerivationsHelpers();
 
 
@@ -99,27 +119,34 @@ static void closeStore()
 }
 
 
+RemoveTempRoots::~RemoveTempRoots()
+{
+    removeTempRoots();
+}
+
+
+static bool showTrace = false;
+
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
 {
     /* Setup Nix paths. */
     nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
-    nixStoreState = canonPath(getEnv("NIX_STORE_STATE_DIR", NIX_STORE_STATE_DIR));						//store state dir usually /nix/state
     nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
     nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
-    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));									//nix global state dir
+    nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
     nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
-    nixExt3CowHeader = getEnv("NIX_EXT3_COW_HEADER", NIX_EXT3_COW_HEADER);
-    nixRsync = getEnv("NIX_RSYNC", NIX_RSYNC);
     nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
     nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
     nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
-    if (subs == "default")
-        substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl");
-    else
+    if (subs == "default") {
+        substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+        substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+    } else
         substituters = tokenizeString(subs, ":");
 
     /* Get some settings from the configuration file. */
@@ -163,29 +190,23 @@ static void initAndRun(int argc, char * * argv)
     while (argc--) args.push_back(*argv++);
     args.erase(args.begin());
     
-    /* We dont expand for nix-state since we need to pass arguments to other 
-     * programs that can decide for themselves if they want expansion or not 
-     */
-    if(programId != "nix-state")	 
-    {
-	    /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f'), and
-	       ignore options for the ATerm library. */
-	    for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
-	        string arg = *i;
-	        if (string(arg, 0, 4) == "-at-") ;
-	        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-') {
-	            for (unsigned int j = 1; j < arg.length(); j++)
-	                if (isalpha(arg[j]))
-	                    remaining.push_back((string) "-" + arg[j]);
-	                else {
-	                    remaining.push_back(string(arg, j));
-	                    break;
-	                }
-	        } else remaining.push_back(arg);
-	    }
-	    args = remaining;
-	    remaining.clear();
+    /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f'), and
+       ignore options for the ATerm library. */
+    for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
+        string arg = *i;
+        if (string(arg, 0, 4) == "-at-") ;
+        else if (arg.length() > 2 && arg[0] == '-' && arg[1] != '-') {
+            for (unsigned int j = 1; j < arg.length(); j++)
+                if (isalpha(arg[j]))
+                    remaining.push_back((string) "-" + arg[j]);
+                else {
+                    remaining.push_back(string(arg, j));
+                    break;
+                }
+        } else remaining.push_back(arg);
     }
+    args = remaining;
+    remaining.clear();
 
     /* Process default options. */
     for (Strings::iterator i = args.begin(); i != args.end(); ++i) {
@@ -201,15 +222,15 @@ static void initAndRun(int argc, char * * argv)
             ; /* !!! obsolete - remove eventually */
         else if (arg == "--no-build-output" || arg == "-Q")
             buildVerbosity = lvlVomit;
-        
-        //we need to push back since arguments need to be passed on in the state wrapper script
-        else if (arg == "--help" && programId != "nix-state") {
+        else if (arg == "--print-build-trace")
+            printBuildTrace = true;
+        else if (arg == "--help") {
             printHelp();
-            remaining.push_back(arg);
+            return;
         }
         else if (arg == "--version") {
             std::cout << format("%1% (Nix) %2%") % programId % NIX_VERSION << std::endl;
-            remaining.push_back(arg);
+            return;
         }
         else if (arg == "--keep-failed" || arg == "-K")
             keepFailed = true;
@@ -225,12 +246,21 @@ static void initAndRun(int argc, char * * argv)
             maxSilentTime = getIntArg(arg, i, args.end());
         else if (arg == "--no-build-hook")
             useBuildHook = false;
+        else if (arg == "--show-trace")
+            showTrace = true;
+        else if (arg == "--option") {
+            ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
+            string name = *i;
+            ++i; if (i == args.end()) throw UsageError("`--option' requires two arguments");
+            string value = *i;
+            overrideSetting(name, tokenizeString(value));
+        }
         else remaining.push_back(arg);
     }
 
     /* Automatically clean up the temporary roots file when we
        exit. */
-    RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
+    RemoveTempRoots removeTempRoots __attribute__((unused));
 
     /* Make sure that the database gets closed properly, even if
        terminate() is called (which happens sometimes due to bugs in
@@ -340,7 +370,9 @@ int main(int argc, char * * argv)
             % e.what() % programId);
         return 1;
     } catch (BaseError & e) {
-        printMsg(lvlError, format("error: %1%") % e.msg());
+        printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg());
+        if (e.prefix() != "" && !showTrace)
+            printMsg(lvlError, "(use `--show-trace' to show detailed location information)");
         return 1;
     } catch (std::exception & e) {
         printMsg(lvlError, format("error: %1%") % e.what());

src/libmain/shared.hh

@@ -26,6 +26,11 @@ namespace nix {
 Path makeRootName(const Path & gcRoot, int & counter);
 void printGCWarning();
 
+void printMissing(const PathSet & paths);
+
+unsigned long long getIntArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end);
+
 /* Whether we're running setuid. */
 extern bool setuidMode;
 
@@ -33,6 +38,11 @@ extern volatile ::sig_atomic_t blockInt;
 
 MakeError(UsageError, nix::Error);
 
+struct RemoveTempRoots 
+{
+    ~RemoveTempRoots();    
+};
+
 }
 
 

src/libstore/Makefile.am

@@ -1,24 +1,23 @@
 pkglib_LTLIBRARIES = libstore.la
 
 libstore_la_SOURCES = \
- store-api.cc local-store.cc remote-store.cc derivations.cc build.cc misc.cc \
- globals.cc db.cc references.cc pathlocks.cc gc.cc store-state.cc
+  store-api.cc local-store.cc remote-store.cc derivations.cc build.cc misc.cc \
+  globals.cc db.cc references.cc pathlocks.cc gc.cc upgrade-schema.cc \
+  optimise-store.cc
 
 pkginclude_HEADERS = \
- store-api.hh local-store.hh remote-store.hh derivations.hh misc.hh \
- globals.hh db.hh references.hh pathlocks.hh \
- worker-protocol.hh store-state.hh build.hh
+  store-api.hh local-store.hh remote-store.hh derivations.hh misc.hh \
+  globals.hh db.hh references.hh pathlocks.hh \
+  worker-protocol.hh
 
-libstore_la_LIBADD = ../libutil/libutil.la \
- ../libext3cow/libext3cow.la \
- ../boost/format/libformat.la
+libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la
 
 BUILT_SOURCES = derivations-ast.cc derivations-ast.hh
 
 EXTRA_DIST = derivations-ast.def derivations-ast.cc
 
 AM_CXXFLAGS = -Wall \
- -I$(srcdir)/.. ${bdb_include} ${aterm_include} -I$(srcdir)/../libutil -I$(srcdir)/../nix-state -I$(srcdir)/../libext3cow
+ -I$(srcdir)/.. ${bdb_include} ${aterm_include} -I$(srcdir)/../libutil
 
 derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
 	$(perl) $(srcdir)/../aterm-helper.pl derivations-ast.hh derivations-ast.cc < $(srcdir)/derivations-ast.def

src/libstore/build.hh

@@ -1,12 +0,0 @@
-#ifndef BUILD_HH_
-#define BUILD_HH_
-
-#include "db.hh"
-
-namespace nix {
-
-	void ensurePathTxn(const Transaction & txn, const Path & path);
-
-}
-
-#endif /*BUILD_HH_*/

src/libstore/db.cc

@@ -1,9 +1,10 @@
+#include "config.h"
+
+#ifdef OLD_DB_COMPAT
+
 #include "db.hh"
 #include "util.hh"
 #include "pathlocks.hh"
-#include "derivations.hh"
-#include "local-store.hh"
-#include "misc.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -71,7 +72,7 @@ Transaction::~Transaction()
 void Transaction::begin(Database & db)
 {
     assert(txn == 0);
-    db.requireEnv("begin transaction");
+    db.requireEnv();
     try {
         db.env->txn_begin(0, &txn, 0);
     } catch (DbException e) { rethrow(e); }
@@ -110,11 +111,11 @@ void Transaction::moveTo(Transaction & t)
 }
 
 
-void Database::requireEnv(string debug)
+void Database::requireEnv()
 {
     checkInterrupt();
-    if (!env) throw Error(format("database environment is not open while trying '%1%'"
-                          "(maybe you don't have sufficient permission?)") % debug);
+    if (!env) throw Error("database environment is not open "
+        "(maybe you don't have sufficient permission?)");
 }
 
 
@@ -216,9 +217,9 @@ void Database::open2(const string & path, bool removeOldEnv)
        run db_recover on the database to remove the existing DB
        environment (since changes only take effect on new
        environments). */
-    env->set_lk_max_locks(100000);
-    env->set_lk_max_lockers(100000);
-    env->set_lk_max_objects(100000);
+    env->set_lk_max_locks(10000);
+    env->set_lk_max_lockers(10000);
+    env->set_lk_max_objects(10000);
     env->set_lk_detect(DB_LOCK_DEFAULT);
     
     /* Dangerous, probably, but from the docs it *seems* that BDB
@@ -310,7 +311,7 @@ void Database::close()
 
 TableId Database::openTable(const string & tableName, bool sorted)
 {
-    requireEnv(tableName);
+    requireEnv();
     TableId table = nextId++;
 
     try {
@@ -457,7 +458,6 @@ void Database::enumTable(const Transaction & txn, TableId table,
     } catch (DbException e) { rethrow(e); }
 }
 
-
  
 void Database::clearTable(const Transaction & txn, TableId table)
 {
@@ -470,3 +470,5 @@ void Database::clearTable(const Transaction & txn, TableId table)
 
 
 }
+
+#endif

src/libstore/db.hh

@@ -54,14 +54,12 @@ private:
     TableId nextId;
     std::map<TableId, Db *> tables;
 
-    void requireEnv(string debug);
+    void requireEnv();
 
     Db * getDb(TableId table);
 
     void open2(const string & path, bool removeOldEnv);
     
-    
-    
 public:
     Database();
     ~Database();

src/libstore/derivations-ast.def

@@ -1,22 +1,10 @@
 init initDerivationsHelpers
 
-Derive | ATermList ATermList ATermList ATermList ATermList string string ATermList ATermList | ATerm |
+Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
+
 | string string | ATerm | EnvBinding |
 | string ATermList | ATerm | DerivationInput |
 | string string string string | ATerm | DerivationOutput |
-| string string string string string string string string string string string string string string | ATerm | DerivationStateOutput |
-| string string string | ATerm | DerivationStateOutputDir |
-
-#We use DeriveWithOutState to create derivations that dont use state, and thus dont have the stateDerivationStateOutput and DerivationStateOutputDir in their derivation
-#Ive put this in becuase its easy to stay backwards compatible, but maybe it should be removed somtime to prevent confusion & duplication
-#The function will be called matchDerivateWithOutState, but it will match the Derive term to remain backwards compatible
-
-Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm | DeriveWithOutState
-| string string | ATerm | EnvBindingWithOutState |
-| string ATermList | ATerm | DerivationInputWithOutState |
-| string string string string | ATerm | DerivationOutputWithOutState |
-
-#end drv without state
 
 Closure | ATermList ATermList | ATerm | OldClosure |
 | string ATermList | ATerm | OldClosureElem |

src/libstore/derivations.cc

@@ -21,12 +21,11 @@ Path writeDerivation(const Derivation & drv, const string & name)
 {
     PathSet references;
     references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
-    for (DerivationInputs::const_iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i)
+    foreach (DerivationInputs::const_iterator, i, drv.inputDrvs)
         references.insert(i->first);
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-       
     string suffix = name + drvExtension;
     string contents = atPrint(unparseDerivation(drv));
     return readOnlyMode
@@ -67,14 +66,9 @@ Derivation parseDerivation(ATerm t)
 {
     Derivation drv;
     ATermList outs, inDrvs, inSrcs, args, bnds;
-    ATermList stateOuts = ATempty, stateOutDirs = ATempty;
-    
     ATerm builder, platform;
 
-	bool withState;
-    if (matchDerive(t, outs, stateOuts, stateOutDirs, inDrvs, inSrcs, platform, builder, args, bnds) ) { withState = true; }
-    else if (matchDeriveWithOutState(t, outs, inDrvs, inSrcs, platform, builder, args, bnds) ) { withState = false; }
-    else
+    if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds))
         throwBadDrv(t);
 
     for (ATermIterator i(outs); i; ++i) {
@@ -88,46 +82,6 @@ Derivation parseDerivation(ATerm t)
         out.hash = aterm2String(hash);
         drv.outputs[aterm2String(id)] = out;
     }
-    
-    if(withState)
-    {
-	    //parse state part
-	    for (ATermIterator i(stateOuts); i; ++i) {
-	        ATerm id, statepath, componentHash, hashAlgo, hash, stateIdentifier, enabled, shareType, synchronization, createDirsBeforeInstall, runtimeStateArgs, username, sharedState, externalState;		//TODO unitialized warning
-	        if (!matchDerivationStateOutput(*i, id, statepath, componentHash, hashAlgo, hash, stateIdentifier, enabled, shareType, synchronization, createDirsBeforeInstall, runtimeStateArgs, username, sharedState, externalState))
-	            throwBadDrv(t);
-	        DerivationStateOutput stateOut;
-	        stateOut.statepath = aterm2String(statepath);
-	        stateOut.componentHash = aterm2String(componentHash);
-	        //checkPath(stateOut.path);									//should we check the statpath .... ???
-	        stateOut.hashAlgo = aterm2String(hashAlgo);
-	        stateOut.hash = aterm2String(hash);
-	        stateOut.stateIdentifier = aterm2String(stateIdentifier);
-	        stateOut.enabled = aterm2String(enabled);
-	        stateOut.shareType = aterm2String(shareType);
-	        stateOut.synchronization = aterm2String(synchronization);
-	        stateOut.createDirsBeforeInstall = aterm2String(createDirsBeforeInstall);
-	        stateOut.runtimeStateArgs = aterm2String(runtimeStateArgs);
-	        stateOut.username = aterm2String(username);
-	        stateOut.sharedState = aterm2String(sharedState);
-	        stateOut.externalState = aterm2String(externalState);
-	        drv.stateOutputs[aterm2String(id)] = stateOut;
-	    }
-
-	    //parse state dirs part
-	    for (ATermIterator i(stateOutDirs); i; ++i) {
-	        ATerm id, path, type, interval;
-	        if (!matchDerivationStateOutputDir(*i, id, type, interval))
-	            throwBadDrv(t);
-	        path = id;    								//We prevent duplication since the key is also the path
-	        DerivationStateOutputDir stateOutDirs;
-	        stateOutDirs.path = aterm2String(path);
-	        stateOutDirs.type = aterm2String(type);
-	        stateOutDirs.interval = aterm2String(interval);
-	        drv.stateOutputDirs[aterm2String(id)] = stateOutDirs;
-	    }
-	    
-	}
 
     for (ATermIterator i(inDrvs); i; ++i) {
         ATerm drvPath;
@@ -166,7 +120,8 @@ Derivation parseDerivation(ATerm t)
 ATerm unparseDerivation(const Derivation & drv)
 {
     ATermList outputs = ATempty;
-    for (DerivationOutputs::const_reverse_iterator i = drv.outputs.rbegin(); i != drv.outputs.rend(); ++i)
+    for (DerivationOutputs::const_reverse_iterator i = drv.outputs.rbegin();
+         i != drv.outputs.rend(); ++i)
         outputs = ATinsert(outputs,
             makeDerivationOutput(
                 toATerm(i->first),
@@ -174,42 +129,6 @@ ATerm unparseDerivation(const Derivation & drv)
                 toATerm(i->second.hashAlgo),
                 toATerm(i->second.hash)));
 
-	//decide if we need to add state to the derivation
-    bool createState = false;
-        
-    ATermList stateOutputs = ATempty;
-    for (DerivationStateOutputs::const_reverse_iterator i = drv.stateOutputs.rbegin(); i != drv.stateOutputs.rend(); ++i){
-    	if(i->second.enabled == "true"){
-    		createState = true;
-    	}
-        stateOutputs = ATinsert(stateOutputs,
-            makeDerivationStateOutput(
-                toATerm(i->first),
-                toATerm(i->second.statepath),
-                toATerm(i->second.componentHash),
-                toATerm(i->second.hashAlgo),
-                toATerm(i->second.hash),
-                toATerm(i->second.stateIdentifier),
-                toATerm(i->second.enabled),
-                toATerm(i->second.shareType),
-                toATerm(i->second.synchronization),
-                toATerm(i->second.createDirsBeforeInstall),
-                toATerm(i->second.runtimeStateArgs),
-                toATerm(i->second.username),
-                toATerm(i->second.sharedState),
-                toATerm(i->second.externalState)
-                ));
-    }
-                
-    ATermList stateOutputDirs = ATempty;
-    for (DerivationStateOutputDirs::const_reverse_iterator i = drv.stateOutputDirs.rbegin(); i != drv.stateOutputDirs.rend(); ++i)
-        stateOutputDirs = ATinsert(stateOutputDirs,
-            makeDerivationStateOutputDir(
-                toATerm(i->first),
-                toATerm(i->second.type),
-                toATerm(i->second.interval)
-                ));
-	
     ATermList inDrvs = ATempty;
     for (DerivationInputs::const_reverse_iterator i = drv.inputDrvs.rbegin();
          i != drv.inputDrvs.rend(); ++i)
@@ -231,36 +150,20 @@ ATerm unparseDerivation(const Derivation & drv)
                 toATerm(i->first),
                 toATerm(i->second)));
 
-	if(createState){	
-	    return makeDerive(
-	        outputs,
-	        stateOutputs,
-	        stateOutputDirs,
-	        inDrvs,
-	        toATermList(drv.inputSrcs),
-	        toATerm(drv.platform),
-	        toATerm(drv.builder),
-	        args,
-	        env);
-	}
-	else{
-		return makeDeriveWithOutState(
-	        outputs,
-	        inDrvs,
-	        toATermList(drv.inputSrcs),
-	        toATerm(drv.platform),
-	        toATerm(drv.builder),
-	        args,
-	        env);
-	}
+    return makeDerive(
+        outputs,
+        inDrvs,
+        toATermList(drv.inputSrcs),
+        toATerm(drv.platform),
+        toATerm(drv.builder),
+        args,
+        env);
 }
 
 
 bool isDerivation(const string & fileName)
 {
-    return
-        fileName.size() >= drvExtension.size() &&
-        string(fileName, fileName.size() - drvExtension.size()) == drvExtension;
+    return hasSuffix(fileName, drvExtension);
 }
 
  

src/libstore/derivations.hh

@@ -4,7 +4,6 @@
 typedef struct _ATerm * ATerm;
 
 #include "hash.hh"
-#include "util.hh"
 
 #include <map>
 
@@ -21,8 +20,8 @@ const string drvExtension = ".drv";
 struct DerivationOutput
 {
     Path path;
-    string hashAlgo; 	/* hash used for expected hash computation */
-    string hash; 		/* expected hash, may be null */
+    string hashAlgo; /* hash used for expected hash computation */
+    string hash; /* expected hash, may be null */
     DerivationOutput()
     {
     }
@@ -34,147 +33,17 @@ struct DerivationOutput
     }
 };
 
-struct DerivationStateOutput
-{
-    Path statepath;
-    string componentHash;
-    string hashAlgo;
-    string hash;
-    string stateIdentifier;				//the identifier
-    string enabled;						//enable or disable state
-    string shareType;						//none, full, group
-    string synchronization;				//none (no locks), exclusive-lock, recursive-exclusive-lock
-    
-    string commitReferences;			//TODO none, direct, recursive-all
-    string commitBinaries;				//TODO list of binaries that need (or not) to be committed when these binaries are called
-         
-    string createDirsBeforeInstall;		//if true: creates state dirs before installation
-	string runtimeStateArgs;			//if not empty: these are the runtime parameters where state can be found (you can use $statepath here)
-    
-    string username;	
-    
-    string sharedState;					//Path to share state From		
-    
-    string externalState;				//a statePath not not in the stateStore (Official hack)
-    
-    DerivationStateOutput()
-    {
-    }
-    
-    //TODO add const ??
-    DerivationStateOutput(Path statepath, string componentHash, string hashAlgo, string hash, string stateIdentifier, string enabled, string shareType, string synchronization, string createDirsBeforeInstall, string runtimeStateArgs, string username, string sharedState, string externalState, bool check=true)
-    {
-        if(check){
-	        if(shareType != "none" && shareType != "full" && shareType != "group")
-	        	throw Error(format("shareType '%1%' is not a correct type") % shareType);
-	        if(synchronization != "none" && synchronization != "exclusive-lock" && synchronization != "recursive-exclusive-lock")
-	        	throw Error(format("synchronization '%1%' is not a correct type") % synchronization);
-	        if(username == "")
-	        	throw Error(format("Username cannot be empty"));
-	        if(stateIdentifier == "__EMTPY__" || stateIdentifier == "__NOSTATE__")
-	        	throw Error(format("the stateIdenfier cannot be this value '%1%'") % stateIdentifier);
-	        if(runtimeStateArgs == "__NOARGS__")
-	        	throw Error(format("the runtimeStateArgs cannot be this value '%1%'") % runtimeStateArgs);
-			if(externalState != "" && sharedState != "")
-				throw Error(format("You cannot have an externalState and sharedState at the same time"));	        
-        }
-        
-        //TODO
-        //commitReferences
-        //commitBinaries
-        
-        this->statepath = statepath;
-        this->componentHash = componentHash;
-        this->hashAlgo = hashAlgo;
-        this->hash = hash;
-        this->stateIdentifier = stateIdentifier;
-        this->enabled = enabled;
-        this->shareType = shareType;
-        this->synchronization = synchronization;
-        this->createDirsBeforeInstall = createDirsBeforeInstall;
-        this->runtimeStateArgs = runtimeStateArgs;
-        this->username = username;
-        this->sharedState = sharedState;
-		this->externalState = externalState;        
-    }
-    
-    bool getEnabled(){
-    	return string2bool(enabled);
-    }
-    
-    bool getCreateDirsBeforeInstall(){
-    	return string2bool(createDirsBeforeInstall);
-    }
-    
-    /*
-     * This sets all the paramters to "" to ensure they're not taken into account for the hash calculation in primops.cc 
-     */
-    void clearAllRuntimeParamters(){
-    	this->statepath = "";
-    	this->componentHash = "";
-        //this->hashAlgo;						//Clear this one?
-        //this->hash;							//Clear this one?
-        //this->stateIdentifier;				//Changes the statepath directly
-        this->enabled = "";
-        this->shareType = "";
-        this->synchronization = "";
-        this->createDirsBeforeInstall = "";
-        this->runtimeStateArgs = "";
-        //this->username;						//Changes the statepath directly
-        this->sharedState = "";
-        this->externalState = "";
-         
-    }
-};
-
-struct DerivationStateOutputDir
-{
-    string path;
-    string type;					//none, manual, interval, full
-    string interval;				//type int
-    DerivationStateOutputDir()
-    {
-    }
-    DerivationStateOutputDir(string path, string type, string interval)
-    {
-        if(type != "none" && type != "manual" && type != "interval" && type != "full")
-        	throw Error(format("interval '%1%' is not a correct type") % type);
-        
-        this->path = path;
-        this->type = type;
-        this->interval = interval;
-    }
-    
-    int getInterval(){
-    	if(interval == "") 
-          return 0;
-    	else{
-	    	int i;
-	    	if (!string2Int(interval, i)) throw Error("interval is not a number");
-	    	return i;
-    	}
-    }
-    
-    //sort function
-	bool operator<(const DerivationStateOutputDir& a) const { return path < a.path; }      
-};
-
-
 typedef std::map<string, DerivationOutput> DerivationOutputs;
-typedef std::map<string, DerivationStateOutput> DerivationStateOutputs;
-typedef std::map<string, DerivationStateOutputDir> DerivationStateOutputDirs;
-
 
 /* For inputs that are sub-derivations, we specify exactly which
    output IDs we are interested in. */
 typedef std::map<Path, StringSet> DerivationInputs;
+
 typedef std::map<string, string> StringPairs;
 
 struct Derivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */
-    DerivationStateOutputs stateOutputs; /* TODO */
-    DerivationStateOutputDirs stateOutputDirs; /* TODO */
     DerivationInputs inputDrvs; /* inputs that are sub-derivations */
     PathSet inputSrcs; /* inputs that are sources */
     string platform;

src/libstore/gc.cc

@@ -2,11 +2,12 @@
 #include "misc.hh"
 #include "pathlocks.hh"
 #include "local-store.hh"
-#include "db.hh"
-#include "util.hh"
 
 #include <boost/shared_ptr.hpp>
 
+#include <functional>
+#include <queue>
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <errno.h>
@@ -125,7 +126,11 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
         createSymlink(gcRoot, storePath, false);
     }
 
-    /* Check that the root can be found by the garbage collector. */
+    /* Check that the root can be found by the garbage collector.
+       !!! This can be very slow on machines that have many roots.
+       Instead of reading all the roots, it would be more efficient to
+       check if the root is in a directory in or linked from the
+       gcroots directory. */
     if (queryBoolSetting("gc-check-reachability", true)) {
         Roots roots = store->findRoots();
         if (roots.find(gcRoot) == roots.end())
@@ -242,9 +247,7 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
     Strings tempRootFiles = readDirectory(
         (format("%1%/%2%") % nixStateDir % tempRootsDir).str());
 
-    for (Strings::iterator i = tempRootFiles.begin();
-         i != tempRootFiles.end(); ++i)
-    {
+    foreach (Strings::iterator, i, tempRootFiles) {
         Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
 
         debug(format("reading temporary root file `%1%'") % path);
@@ -323,7 +326,7 @@ static void findRoots(const Path & path, bool recurseSymlinks,
 
         if (S_ISDIR(st.st_mode)) {
             Strings names = readDirectory(path);
-            for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+            foreach (Strings::iterator, i, names)
                 findRoots(path + "/" + *i, recurseSymlinks, deleteStale, roots);
         }
 
@@ -394,7 +397,7 @@ static void addAdditionalRoots(PathSet & roots)
 
     Strings paths = tokenizeString(result, "\n");
     
-    for (Strings::iterator i = paths.begin(); i != paths.end(); ++i) {
+    foreach (Strings::iterator, i, paths) {
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
             if (roots.find(path) == roots.end() && store->isValidPath(path)) {
@@ -409,16 +412,14 @@ static void addAdditionalRoots(PathSet & roots)
 static void dfsVisit(const PathSet & paths, const Path & path,
     PathSet & visited, Paths & sorted)
 {
-    if (visited.find(path) != visited.end()) 
-    	return;
+    if (visited.find(path) != visited.end()) return;
     visited.insert(path);
     
     PathSet references;
     if (store->isValidPath(path))
-        store->queryStoreReferences(path, references, 0);				
+        store->queryReferences(path, references);
     
-    for (PathSet::iterator i = references.begin();
-         i != references.end(); ++i)
+    foreach (PathSet::iterator, i, references)
         /* Don't traverse into paths that don't exist.  That can
            happen due to substitutes for non-existent paths. */
         if (*i != path && paths.find(*i) != paths.end())
@@ -427,25 +428,151 @@ static void dfsVisit(const PathSet & paths, const Path & path,
     sorted.push_front(path);
 }
 
-//TODO maybe? stateTopoSortPaths(const PathSet & paths)
-
 
 Paths topoSortPaths(const PathSet & paths)
 {
     Paths sorted;
     PathSet visited;
-    for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i)
+    foreach (PathSet::const_iterator, i, paths)
         dfsVisit(paths, *i, visited, sorted);
     return sorted;
 }
 
 
-void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,				//TODO ? statePathsToDelete
-    bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
+static time_t lastFileAccessTime(const Path & path)
 {
-    result.clear();
-    bytesFreed = 0;
+    checkInterrupt();
+    
+    struct stat st;
+    if (lstat(path.c_str(), &st) == -1)
+        throw SysError(format("statting `%1%'") % path);
 
+    if (S_ISDIR(st.st_mode)) {
+        time_t last = 0;
+	Strings names = readDirectory(path);
+	foreach (Strings::iterator, i, names) {
+            time_t t = lastFileAccessTime(path + "/" + *i);
+            if (t > last) last = t;
+        }
+        return last;
+    }
+
+    else if (S_ISLNK(st.st_mode)) return 0;
+
+    else return st.st_atime;
+}
+
+
+struct GCLimitReached { };
+
+
+void LocalStore::gcPath(const GCOptions & options, GCResults & results, 
+    const Path & path)
+{
+    results.paths.insert(path);
+
+    if (!pathExists(path)) return;
+                
+    /* Okay, it's safe to delete. */
+    unsigned long long bytesFreed, blocksFreed;
+    deleteFromStore(path, bytesFreed, blocksFreed);
+    results.bytesFreed += bytesFreed;
+    results.blocksFreed += blocksFreed;
+
+    if (options.maxFreed && results.bytesFreed > options.maxFreed) {
+        printMsg(lvlInfo, format("deleted more than %1% bytes; stopping") % options.maxFreed);
+        throw GCLimitReached();
+    }
+
+    if (options.maxLinks) {
+        struct stat st;
+        if (stat(nixStore.c_str(), &st) == -1)
+            throw SysError(format("statting `%1%'") % nixStore);
+        if (st.st_nlink < options.maxLinks) {
+            printMsg(lvlInfo, format("link count on the store has dropped below %1%; stopping") % options.maxLinks);
+            throw GCLimitReached();
+        }
+    }
+}
+
+
+void LocalStore::gcPathRecursive(const GCOptions & options,
+    GCResults & results, PathSet & done, const Path & path)
+{
+    if (done.find(path) != done.end()) return;
+    done.insert(path);
+
+    startNest(nest, lvlDebug, format("looking at `%1%'") % path);
+        
+    /* Delete all the referrers first.  They must be garbage too,
+       since if they were live, then the current path would also be
+       live.  Note that deleteFromStore() below still makes sure that
+       the referrer set has become empty, just in case.  (However that
+       doesn't guard against deleting top-level paths that are only
+       reachable from GC roots.) */
+    PathSet referrers;
+    if (isValidPath(path))
+        queryReferrers(path, referrers);
+    foreach (PathSet::iterator, i, referrers)
+        if (*i != path) gcPathRecursive(options, results, done, *i);
+
+    printMsg(lvlInfo, format("deleting `%1%'") % path);
+            
+    gcPath(options, results, path);
+}
+
+
+struct CachingAtimeComparator : public std::binary_function<Path, Path, bool> 
+{
+    std::map<Path, time_t> cache;
+
+    time_t lookup(const Path & p)
+    {
+        std::map<Path, time_t>::iterator i = cache.find(p);
+        if (i != cache.end()) return i->second;
+        debug(format("computing atime of `%1%'") % p);
+        cache[p] = lastFileAccessTime(p);
+        assert(cache.find(p) != cache.end());
+        return cache[p];
+    }
+        
+    bool operator () (const Path & p1, const Path & p2)
+    {
+        return lookup(p2) < lookup(p1);
+    }
+};
+
+
+static string showTime(const string & format, time_t t)
+{
+    char s[128];
+    strftime(s, sizeof s, format.c_str(), localtime(&t));
+    return string(s);
+}
+
+
+static bool isLive(const Path & path, const PathSet & livePaths,
+    const PathSet & tempRoots, const PathSet & tempRootsClosed)
+{
+    if (livePaths.find(path) != livePaths.end() ||
+        tempRootsClosed.find(path) != tempRootsClosed.end()) return true;
+
+    /* A lock file belonging to a path that we're building right
+       now isn't garbage. */
+    if (hasSuffix(path, ".lock") && tempRoots.find(string(path, 0, path.size() - 5)) != tempRoots.end())
+        return true;
+
+    /* Don't delete .chroot directories for derivations that are
+       currently being built. */
+    if (hasSuffix(path, ".chroot") && tempRoots.find(string(path, 0, path.size() - 7)) != tempRoots.end())
+        return true;
+
+    return false;
+}
+
+
+void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
     bool gcKeepOutputs =
         queryBoolSetting("gc-keep-outputs", false);
     bool gcKeepDerivations =
@@ -453,121 +580,69 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
     int gcKeepOutputsThreshold = 
         queryIntSetting ("gc-keep-outputs-threshold", defaultGcLevel);
 
-	//printMsg(lvlError, format("gcKeepOutputs %1% gcKeepDerivations: %2%") % gcKeepOutputs % gcKeepDerivations);
-
     /* Acquire the global GC root.  This prevents
        a) New roots from being added.
        b) Processes from creating new temporary root files. */
-    AutoCloseFD fdGCLock = openGCLock(ltWrite);												//TODO !!!!!!!!!!!!!!!!!!! GET STATE LOCKS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    AutoCloseFD fdGCLock = openGCLock(ltWrite);
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);
+    printMsg(lvlError, format("finding garbage collector roots..."));
+    Roots rootMap = options.ignoreLiveness ? Roots() : nix::findRoots(true);
 
     PathSet roots;
-    for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i)
-        roots.insert(i->second);
-        
+    foreach (Roots::iterator, i, rootMap) roots.insert(i->second);
 
     /* Add additional roots returned by the program specified by the
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
-    if (!ignoreLiveness)
+    if (!options.ignoreLiveness)
         addAdditionalRoots(roots);
 
-    if (action == gcReturnRoots) {
-        result = roots;
+    if (options.action == GCOptions::gcReturnRoots) {
+        results.paths = roots;
         return;
     }
 
     /* Determine the live paths which is just the closure of the
        roots under the `references' relation. */
+    printMsg(lvlError, format("computing live paths..."));
     PathSet livePaths;
-    for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
-        computeFSClosure(canonPath(*i), livePaths, true, true, 0);
+    foreach (PathSet::const_iterator, i, roots)
+        computeFSClosure(canonPath(*i), livePaths);
 
-    /* Note that the deriver need not be valid (e.g., if we
-    previously ran the collector with `gcKeepDerivations'
-	turned off). */
     if (gcKeepDerivations) {
-        for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i)
-        {
-        	if (store->isStateComponent(*i)){
-        		//printMsg(lvlError, format("Live state store '%1%'") % *i);
-        		
-        		PathSet derivers = store->queryDerivers(*i);		//we select ALL state Derivations here
-        		for (PathSet::const_iterator j = derivers.begin(); j != derivers.end(); ++j)
-        			if (*j != "" && store->isValidPath(*j))
-                		computeFSClosure(*j, livePaths, true, true, 0);
-        	}
-			else if (store->isValidPath(*i)){
-				//printMsg(lvlError, format("Live Store '%1%'") % *i);
-				
-	           	Path deriver = store->queryDeriver(*i);
-				if (deriver != "" && store->isValidPath(deriver))
-                	computeFSClosure(deriver, livePaths, true, true, 0);
-            }
-            else if (store->isValidStatePath(*i)){
-				//printMsg(lvlError, format("Live State '%1%'") % *i);
-            	Path deriver = queryStatePathDrvTxn(noTxn, *i);
-            	
-            	if(!store->isValidPath(deriver))
-            		throw Error(format("deriver `%1%' of state-store component `%2%' in GC is not valid") % deriver % *i);
-            	
-            	computeFSClosure(deriver, livePaths, true, true, 0);
-            }
+        foreach (PathSet::iterator, i, livePaths) {
+            /* Note that the deriver need not be valid (e.g., if we
+               previously ran the collector with `gcKeepDerivations'
+               turned off). */
+            Path deriver = queryDeriver(*i);
+            if (deriver != "" && isValidPath(deriver))
+                computeFSClosure(deriver, livePaths);
         }
     }
-    
+
     if (gcKeepOutputs) {
         /* Hmz, identical to storePathRequisites in nix-store. */
-        for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i)
+        foreach (PathSet::iterator, i, livePaths)
             if (isDerivation(*i)) {
-                Derivation drv = derivationFromPathTxn(noTxn, *i);
+                Derivation drv = derivationFromPath(*i);
 
-/*
- * TODO REMOVE
-<<<<<<< .working
-                for (DerivationOutputs::iterator j = drv.outputs.begin();
-                     j != drv.outputs.end(); ++j)
-                    if (store->isValidPath(j->second.path))
-                        computeFSClosure(j->second.path, livePaths, true, true, 0);
-                    else if (store->isValidStatePath(j->second.path))
-                    	computeFSClosure(j->second.path, livePaths, true, true, 0);
-=======
-
-
-			string gcLevelStr = drv.env["__gcLevel"];
-			int gcLevel;
-			if (!string2Int(gcLevelStr,gcLevel)) {
-			    gcLevel = defaultGcLevel;
-			}
-			
-			if (gcLevel >= gcKeepOutputsThreshold)    
-			    for (DerivationOutputs::iterator j = drv.outputs.begin();
-			            j != drv.outputs.end(); ++j)
-				if (store->isValidPath(j->second.path) || store->isValidStatePath(j->second.path))
-				    computeFSClosure(j->second.path, livePaths, true, true, 0);
-           }
-=======
-*/
-
-			string gcLevelStr = drv.env["__gcLevel"];
-			int gcLevel;
-			if (!string2Int(gcLevelStr, gcLevel))
-			    gcLevel = defaultGcLevel;
-			
-			if (gcLevel >= gcKeepOutputsThreshold)    
-			    for (DerivationOutputs::iterator j = drv.outputs.begin();
-	                         j != drv.outputs.end(); ++j)
-				if (store->isValidPath(j->second.path) || store->isValidStatePath(j->second.path))
-				    computeFSClosure(j->second.path, livePaths, true, true, 0);
-	     }
+		string gcLevelStr = drv.env["__gcLevel"];
+		int gcLevel;
+		if (!string2Int(gcLevelStr, gcLevel))
+		    gcLevel = defaultGcLevel;
+		
+		if (gcLevel >= gcKeepOutputsThreshold)    
+		    foreach (DerivationOutputs::iterator, j, drv.outputs)
+			if (isValidPath(j->second.path))
+			    computeFSClosure(j->second.path, livePaths);
+            }
     }
 
-    if (action == gcReturnLive) {
-        result = livePaths;
+    if (options.action == GCOptions::gcReturnLive) {
+        results.paths = livePaths;
         return;
     }
 
@@ -577,7 +652,7 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
     PathSet tempRoots;
     FDs fds;
     readTempRoots(tempRoots, fds);
-    
+
     /* Close the temporary roots.  Note that we *cannot* do this in
        readTempRoots(), because there we may not have all locks yet,
        meaning that an invalid path can become valid (and thus add to
@@ -585,230 +660,129 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
        (and computeFSClosure() assumes that the presence of a path
        means that it has already been closed). */
     PathSet tempRootsClosed;
-    for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
-        if (store->isValidPath(*i))
-            computeFSClosure(*i, tempRootsClosed, true, true, 0);
-        else if(store->isValidStatePath(*i))
-        	computeFSClosure(*i, tempRootsClosed, true, true, 0);
+    foreach (PathSet::iterator, i, tempRoots)
+        if (isValidPath(*i))
+            computeFSClosure(*i, tempRootsClosed);
         else
             tempRootsClosed.insert(*i);
 
-    /* For testing - see tests/gc-concurrent.sh. */
-    if (getenv("NIX_DEBUG_GC_WAIT"))
-        sleep(2);
-    
     /* After this point the set of roots or temporary roots cannot
        increase, since we hold locks on everything.  So everything
        that is not currently in in `livePaths' or `tempRootsClosed'
        can be deleted. */
     
-    /*
-     * We lookup all state (also shared) paths for: livePaths, tempRootsClosed
-     */
-    PathSet allLiveStatePaths;
-    for (PathSet::iterator i = livePaths.begin(); i != livePaths.end(); ++i)
-    	if(store->isValidStatePath(*i)){
-			allLiveStatePaths.insert(*i);
-			allLiveStatePaths = pathSets_union(getSharedWithPathSetRecTxn(noTxn, *i), allLiveStatePaths);
-    	}    		
-    for (PathSet::iterator i = tempRootsClosed.begin(); i != tempRootsClosed.end(); ++i)
-    	if(store->isValidStatePath(*i)){
-    		allLiveStatePaths.insert(*i);
-			allLiveStatePaths = pathSets_union(getSharedWithPathSetRecTxn(noTxn, *i), allLiveStatePaths);
-    	}
-
+    /* Read the Nix store directory to find all currently existing
+       paths and filter out all live paths. */
+    printMsg(lvlError, format("reading the Nix store..."));
+    PathSet storePaths;
     
-    /* Read the Nix store and state directory's to find all currently existing
-       paths. */
-    PathSet storePathSet;
-    PathSet statePathSet;
-    
-    if (action != gcDeleteSpecific) {
-    	
+    if (options.action != GCOptions::gcDeleteSpecific) {
         Paths entries = readDirectory(nixStore);
-        Paths stateEntries = readDirectory(nixStoreState);
-        
-        for (Paths::iterator i = entries.begin(); i != entries.end(); ++i)
-            storePathSet.insert(canonPath(nixStore + "/" + *i));
-        
-        for (Paths::iterator i = stateEntries.begin(); i != stateEntries.end(); ++i)
-            statePathSet.insert(canonPath(nixStoreState + "/" + *i));
-        
-    } else {
-        for (PathSet::iterator i = pathsToDelete.begin();
-             i != pathsToDelete.end(); ++i)
-        {
-            assertStorePath(*i);											//TODO ASSERTSTATEPATH, but for now we have no arg statePathsToDelete
-            storePathSet.insert(*i);
+        foreach (Paths::iterator, i, entries) {
+            Path path = canonPath(nixStore + "/" + *i);
+            if (!isLive(path, livePaths, tempRoots, tempRootsClosed)) storePaths.insert(path);
         }
     }
 
-    /* Topologically sort them under the `referrers' relation.  That
-       is, a < b if a is in referrers(b).  This gives us the order in
-       which things can be deleted safely. */
-    /* !!! when we have multiple output paths per derivation, this
-       will not work anymore because we get cycles. */
-    Paths storePaths = topoSortPaths(storePathSet);
-
-	for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
-		
-		if(*i != "/nix/store/zg8x9wdhcs4j0hvf33vg34c7m65adrpa-env-manifest")
-			continue;
-		
-		PathSet references;
-		store->queryStoreReferences(*i, references, 0);
-		for (PathSet::iterator j = references.begin(); j != references.end(); ++j) {
-			printMsg(lvlError, format("REF `%1%'") % *j);
-		}
-		
-	}
-
-    /* Try to delete store paths in the topologically sorted order. */
-    for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
-
-        debug(format("considering deletion of `%1%'") % *i);
-        
-        if (livePaths.find(*i) != livePaths.end()) {
-            if (action == gcDeleteSpecific)
+    else {
+        foreach (PathSet::iterator, i, options.pathsToDelete) {
+            assertStorePath(*i);
+            storePaths.insert(*i);
+            if (isLive(*i, livePaths, tempRoots, tempRootsClosed))
                 throw Error(format("cannot delete path `%1%' since it is still alive") % *i);
-            debug(format("live path `%1%'") % *i);
-            continue;
-        }
-
-        if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
-            debug(format("temporary root `%1%'") % *i);
-            continue;
-        }
-        
-        debug(format("dead path `%1%'") % *i);
-        result.insert(*i);
-
-        /* If just returning the set of dead paths, we also return the
-           space that would be freed if we deleted them. */
-        if (action == gcReturnDead)
-            bytesFreed += computePathSize(*i);
-
-        if (action == gcDeleteDead || action == gcDeleteSpecific) {
-
-#ifndef __CYGWIN__
-            AutoCloseFD fdLock;
-
-            /* Only delete a lock file if we can acquire a write lock
-               on it.  That means that it's either stale, or the
-               process that created it hasn't locked it yet.  In the
-               latter case the other process will detect that we
-               deleted the lock, and retry (see pathlocks.cc). */
-            if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
-                fdLock = openLockFile(*i, false);
-                if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
-                    debug(format("skipping active lock `%1%'") % *i);
-                    continue;
-                }
-            }
-#endif
-
-            if (!pathExists(*i)) continue;
-                
-            printMsg(lvlInfo, format("deleting store path `%1%'") % *i);
-            
-            /* Okay, it's safe to delete. */
-            try {
-                unsigned long long freed;
-                deleteFromStore(*i, freed);
-                bytesFreed += freed;
-            } catch (PathInUse & e) {
-                printMsg(lvlError, format("warning: %1%") % e.msg());
-            } 
-
-#ifndef __CYGWIN__
-            if (fdLock != -1)
-                /* Write token to stale (deleted) lock file. */
-                writeFull(fdLock, (const unsigned char *) "d", 1);
-#endif
         }
     }
 
-    /* Try to delete state paths. */
-    /* Topologically sort them under the `referrers' relation.  That
-       is, a < b if a is in referrers(b).  This gives us the order in
-       which things can be deleted safely. */
-    /* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! when we have multiple output paths per derivation, this
-       will not work anymore because we get cycles. */
-    Paths statePaths;
-    for (PathSet::iterator i = statePathSet.begin(); i != statePathSet.end(); ++i)
-    	statePaths.push_back(*i);
-    
-    //
-    for (Paths::iterator i = statePaths.begin(); i != statePaths.end(); ++i) {
-    	
-    	debug(format("considering deletion of state path `%1%'") % *i);
+    if (options.action == GCOptions::gcReturnDead) {
+        results.paths.insert(storePaths.begin(), storePaths.end());
+        return;
+    }
 
-        if (allLiveStatePaths.find(*i) != allLiveStatePaths.end()) {
-            debug(format("State path `%1%' is alive (possibyly shared with another live path)") % *i);
-            continue;
+    /* Delete all dead store paths (or until one of the stop
+       conditions is reached). */
+
+    PathSet done;
+    try {
+
+        if (!options.useAtime) {
+            /* Delete the paths, respecting the partial ordering
+               determined by the references graph. */
+            printMsg(lvlError, format("deleting garbage..."));
+            foreach (PathSet::iterator, i, storePaths)
+                gcPathRecursive(options, results, done, *i);
         }
-        
-        string lostAndFound = nixStoreState + "/lost+found";
-        if(*i == lostAndFound){
-        	debug(format("Keeping `%1%' because its a special path") % lostAndFound);
-            continue;
-        }        
-        
-        debug(format("dead path `%1%'") % *i);
-        result.insert(*i);
 
-        /* If just returning the set of dead paths, we also return the
-           space that would be freed if we deleted them. */
-        if (action == gcReturnDead)
-            bytesFreed += computePathSize(*i);
+        else {
 
-        if (action == gcDeleteDead || action == gcDeleteSpecific) {
+            /* Delete in order of ascending last access time, still
+               maintaining the partial ordering of the reference
+               graph.  Note that we can't use a topological sort for
+               this because that takes time O(V+E), and in this case
+               E=O(V^2) (i.e. the graph is dense because of the edges
+               due to the atime ordering).  So instead we put all
+               deletable paths in a priority queue (ordered by atime),
+               and after deleting a path, add additional paths that
+               have become deletable to the priority queue. */
 
-//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! state locks
-/*
-#ifndef __CYGWIN__
-            AutoCloseFD fdLock;
+            CachingAtimeComparator atimeComp;
 
-            /* Only delete a lock file if we can acquire a write lock
-               on it.  That means that it's either stale, or the
-               process that created it hasn't locked it yet.  In the
-               latter case the other process will detect that we
-               deleted the lock, and retry (see pathlocks.cc). * /
-            if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
-                fdLock = openLockFile(*i, false);
-                if (fdLock != -1 && !lockFile(fdLock, ltWrite, false)) {
-                    debug(format("skipping active lock `%1%'") % *i);
+            /* Create a priority queue that orders paths by ascending
+               atime.  This is why C++ needs type inferencing... */
+            std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> > prioQueue =
+                std::priority_queue<Path, vector<Path>, binary_function_ref_adapter<CachingAtimeComparator> >(binary_function_ref_adapter<CachingAtimeComparator>(&atimeComp));
+
+           /* Initially put the paths that are invalid or have no
+              referrers into the priority queue. */
+            printMsg(lvlError, format("finding deletable paths..."));
+            foreach (PathSet::iterator, i, storePaths) {
+                checkInterrupt();
+                /* We can safely delete a path if it's invalid or
+                   it has no referrers.  Note that all the invalid
+                   paths will be deleted in the first round. */
+                if (isValidPath(*i)) {
+                    if (queryReferrersNoSelf(*i).empty()) prioQueue.push(*i);
+                } else prioQueue.push(*i);
+            }
+
+            debug(format("%1% initially deletable paths") % prioQueue.size());
+
+            /* Now delete everything in the order of the priority
+               queue until nothing is left. */
+            printMsg(lvlError, format("deleting garbage..."));
+            while (!prioQueue.empty()) {
+                checkInterrupt();
+                Path path = prioQueue.top(); prioQueue.pop();
+
+                if (options.maxAtime != (time_t) -1 &&
+                    atimeComp.lookup(path) > options.maxAtime)
                     continue;
+                
+                printMsg(lvlInfo, format("deleting `%1%' (last accessed %2%)") % path % showTime("%F %H:%M:%S", atimeComp.lookup(path)));
+
+                PathSet references;
+                if (isValidPath(path)) references = queryReferencesNoSelf(path);
+
+                gcPath(options, results, path);
+
+                /* For each reference of the current path, see if the
+                   reference has now become deletable (i.e. is in the
+                   set of dead paths and has no referrers left).  If
+                   so add it to the priority queue. */
+                foreach (PathSet::iterator, i, references) {
+                    if (storePaths.find(*i) != storePaths.end() &&
+                        queryReferrersNoSelf(*i).empty())
+                    {
+                        debug(format("path `%1%' has become deletable") % *i);
+                        prioQueue.push(*i);
+                    }
                 }
             }
-#endif
-*/
-
-            if (!pathExists(*i)) continue;
-                
-            printMsg(lvlInfo, format("deleting state path `%1%'") % *i);
             
-            /* Okay, it's safe to delete. */
-            try {
-                unsigned long long freed;
-                deleteFromState(*i, freed);
-                bytesFreed += freed;
-            } catch (PathInUse & e) {
-                printMsg(lvlError, format("warning: %1%") % e.msg());
-            }
-
-/*
-#ifndef __CYGWIN__
-            if (fdLock != -1)
-                /* Write token to stale (deleted) lock file. * /
-                writeFull(fdLock, (const unsigned char *) "d", 1);
-#endif
-*/
-
         }
         
+    } catch (GCLimitReached & e) {
     }
 }
 
- 
+
 }

src/libstore/globals.cc

@@ -3,19 +3,17 @@
 
 #include <map>
 #include <algorithm>
-#include <pwd.h>
+
 
 namespace nix {
 
 
 string nixStore = "/UNINIT";
-string nixStoreState = "/UNINIT";
 string nixDataDir = "/UNINIT";
 string nixLogDir = "/UNINIT";
 string nixStateDir = "/UNINIT";
 string nixDBPath = "/UNINIT";
-string nixExt3CowHeader = "/UNINIT";
-string nixRsync = "/UNINIT";
+string nixCacheFile = "/UNINIT";
 string nixConfDir = "/UNINIT";
 string nixLibexecDir = "/UNINIT";
 string nixBinDir = "/UNINIT";
@@ -27,20 +25,19 @@ Verbosity buildVerbosity = lvlInfo;
 unsigned int maxBuildJobs = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
-unsigned int maxSilentTime = 0;
+time_t maxSilentTime = 0;
 Paths substituters;
 bool useBuildHook = true;
+bool printBuildTrace = false;
 
 
 static bool settingsRead = false;
 
-uid_t callingUID = 0;			//A root user will not set this value, so the default uid is 0
-bool singleThreaded = false;	//TODO Gives an error: cannot start worker (environment already open) / waiting for process X: No child processes
-bool sendOutput = true;
-bool sleepForGDB = false;
-
 static std::map<string, Strings> settings;
 
+/* Overriden settings. */
+std::map<string, Strings> settingsCmdline;
+
 
 string & at(Strings & ss, unsigned int n)
 {
@@ -80,6 +77,8 @@ static void readSettings()
         advance(i, 2);
         settings[name] = Strings(i, tokens.end());
     };
+
+    settings.insert(settingsCmdline.begin(), settingsCmdline.end());
     
     settingsRead = true;
 }
@@ -124,39 +123,19 @@ unsigned int queryIntSetting(const string & name, unsigned int def)
     return n;
 }
 
-uid_t queryCallingUID()
+
+void overrideSetting(const string & name, const Strings & value)
 {
-	/* A root user will not even bother calling the daemon, so there is no way to check
-	 * If the uid is not yet set...
-	 */	 
-	return callingUID;	
+    if (settingsRead) settings[name] = value;
+    settingsCmdline[name] = value;
 }
 
-void setCallingUID(uid_t uid, bool reset)
+
+void reloadSettings()
 {
-	if(callingUID != 0 && !reset)
-		throw Error(format("The UID of the caller aleady set! at this point"));
-	
-	callingUID = uid;
+    settingsRead = false;
+    settings.clear();
 }
 
-string uidToUsername(uid_t uid)
-{
-	passwd *pwd = getpwuid(uid);
-	char *pw_name = pwd->pw_name;
-	return (string)pw_name;
-}
-
-string queryCallingUsername()
-{
-	uid_t uid = queryCallingUID();
-	return uidToUsername(uid);
-}
-
-string queryCurrentUsername()
-{
-	return uidToUsername(geteuid());	
-}
-
-     
+ 
 }

src/libstore/globals.hh

@@ -18,20 +18,14 @@ extern string nixDataDir; /* !!! fix */
 /* nixLogDir is the directory where we log various operations. */ 
 extern string nixLogDir;
 
-/* nixStoreState is the directory where the state dirs of the components are stored.  */
-extern string nixStoreState;
-
 /* nixStateDir is the directory where state is stored. */
 extern string nixStateDir;
 
 /* nixDBPath is the path name of our Berkeley DB environment. */
 extern string nixDBPath;
 
-/* nixExt3CowHeader is the header file used to communicate with ext3cow. */
-extern string nixExt3CowHeader;
-
-/* nixRsync is used to copy from one statedir to the other. */
-extern string nixRsync;
+/* nixCacheFile is the path name of the normal form cache. */
+extern string nixCacheFile;
 
 /* nixConfDir is the directory where configuration files are
    stored. */
@@ -44,6 +38,7 @@ extern string nixLibexecDir;
 /* nixBinDir is the directory where the main programs are stored. */
 extern string nixBinDir;
 
+
 /* Misc. global flags. */
 
 /* Whether to keep temporary directories of failed builds. */
@@ -73,7 +68,7 @@ extern string thisSystem;
 /* The maximum time in seconds that a builer can go without producing
    any output on stdout/stderr before it is killed.  0 means
    infinity. */
-extern unsigned int maxSilentTime;
+extern time_t maxSilentTime;
 
 /* The substituters.  There are programs that can somehow realise a
    store path without building, e.g., by downloading it or copying it
@@ -84,6 +79,23 @@ extern Paths substituters;
    users want to disable this from the command-line. */
 extern bool useBuildHook;
 
+/* Whether buildDerivations() should print out lines on stderr in a
+   fixed format to allow its progress to be monitored.  Each line
+   starts with a "@".  The following are defined:
+
+   @ build-started <drvpath> <outpath> <system> <logfile>
+   @ build-failed <drvpath> <outpath> <exitcode> <error text>
+   @ build-succeeded <drvpath> <outpath>
+   @ substituter-started <outpath> <substituter>
+   @ substituter-failed <outpath> <exitcode> <error text>
+   @ substituter-succeeded <outpath>
+
+   Best combined with --no-build-output, otherwise stderr might
+   conceivably contain lines in this format printed by the builders.
+*/
+extern bool printBuildTrace;
+
+
 Strings querySetting(const string & name, const Strings & def);
 
 string querySetting(const string & name, const string & def);
@@ -92,27 +104,11 @@ bool queryBoolSetting(const string & name, bool def);
 
 unsigned int queryIntSetting(const string & name, unsigned int def);
 
-/* TODO PRIVATE: UID of the user that calls the nix-worker daemon */
-extern uid_t callingUID;
+void overrideSetting(const string & name, const Strings & value);
 
-/* get/set the UID of the user that calls the nix-worker daemon */
-uid_t queryCallingUID();
-void setCallingUID(uid_t uid, bool reset = false);
+void reloadSettings();
 
-/* Convert a uid to a username: Watch it! this segfaults when given a wrong uid !! */
-string uidToUsername(uid_t uid);
-
-/* get the username based on the UID of the user that calls the nix-worker daemon */
-string queryCallingUsername();
-
-/* get the username based on the UID of the user currently runs the process */
-string queryCurrentUsername();
-
-/* Debugging variables */
-extern bool singleThreaded;
-extern bool sendOutput;
-extern bool sleepForGDB;    
-  
+    
 }