* Sync with thesis: do not include store location in the hash

computation (in the intensional model).

NEWS

@@ -1,3 +1,9 @@
+Version 0.9
+
+* Unpacking of patch sequences is much faster now by not doing
+  redundant unpacking and repacking of intermediate paths.
+
+
 Version 0.8 (April 11, 2005)
 
 NOTE: the hashing scheme in Nix 0.8 changed (as detailed below).  As a

configure.ac

@@ -1,11 +1,11 @@
-AC_INIT(nix, "0.8")
+AC_INIT(nix, "0.9")
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
-AM_INIT_AUTOMAKE
+AM_INIT_AUTOMAKE([dist-bzip2])
 
 # Change to `1' to produce a `stable' release (i.e., the `preREVISION'
 # suffix is not added).
-STABLE=1
+STABLE=0
 
 # Put the revision number in the version.
 if test "$STABLE" != "1"; then

corepkgs/channels/unpack.sh.in

@@ -10,7 +10,7 @@ echo '[' > $expr
 nr=0
 for i in $inputs; do
     echo "unpacking $i"
-    @bunzip2@ < $i | @tar@ xvf -
+    @bunzip2@ < $i | @tar@ xf -
     @coreutils@/mv * ../$nr # !!! hacky
     echo "(import ./$nr)" >> $expr
     nr=$(($nr + 1))

doc/manual/nix-env.xml

@@ -311,7 +311,7 @@ $ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv</screen>
 <para>To install a specific output path:
 
 <screen>
-$ nix-env -i /nix/store/x3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3</screen>
+$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3</screen>
 
 </para>
 

doc/manual/nix-store.xml

@@ -500,8 +500,8 @@ query is applied to the target of the symlink.</para>
     
 <screen>
 $ nix-store -qR $(which svn)
-/nix/store/4mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
-/nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
+/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
+/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
 <replaceable>...</replaceable></screen>
 
 </para>
@@ -538,7 +538,7 @@ $ nix-store -q --tree $(nix-store -qd $(which svn))
 <screen>
 $ nix-store -q --referers $(nix-store -q --binding openssl $(nix-store -qd $(which svn)))
 /nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0
-/nix/store/4mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
+/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4
 /nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3
 /nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5</screen>
 
@@ -550,7 +550,7 @@ $ nix-store -q --referers $(nix-store -q --binding openssl $(nix-store -qd $(whi
 <screen>
 $ nix-store -q --referers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}')
 /nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2
-/nix/store/05l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
+/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4
 <replaceable>...</replaceable></screen>
 
 Note that <command>ldd</command> is a command that prints out the

externals/Makefile.am

@@ -1,11 +1,11 @@
 # Berkeley DB
 
-DB = db-4.2.52
+DB = db-4.3.28.NC
 
 $(DB).tar.gz:
 	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.2.52 from"
-	@echo "  http://www.sleepycat.com/update/snapshot/db-4.2.52.tar.gz"
+	@echo "Please download version 4.3.28 from"
+	@echo "  http://downloads.sleepycat.com/db-4.3.28.NC.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -26,8 +26,8 @@ build-db: have-db
 	 ../dist/configure --prefix=$$pfx/inst-bdb \
 	 --enable-cxx --disable-shared --disable-cryptography \
          --disable-replication --disable-verify && \
-	make && \
-	make install)
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-db
 endif
 
@@ -57,8 +57,8 @@ build-aterm: have-aterm
 	(pfx=`pwd` && \
 	cd $(ATERM) && \
 	CC="$(CC)" ./configure --prefix=$$pfx/inst-aterm && \
-	make && \
-	make install)
+	$(MAKE) && \
+	$(MAKE) install)
 	touch build-aterm
 endif
 

make/examples/aterm/aterm/default.nix

@@ -0,0 +1,34 @@
+{sharedLib ? true}:
+
+rec {
+
+  inherit (import ../../../lib) compileC makeLibrary;
+
+  sources = [
+    ./afun.c
+    ./aterm.c
+    ./bafio.c
+    ./byteio.c
+    ./gc.c
+    ./hash.c
+    ./list.c
+    ./make.c
+    ./md5c.c
+    ./memory.c
+    ./tafio.c
+    ./version.c
+  ];
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    forSharedLib = sharedLib;
+  };
+
+  libATerm = makeLibrary {
+    libraryName = "ATerm";
+    objects = map compile sources;
+    inherit sharedLib;
+  };
+
+}

make/examples/aterm/default.nix

@@ -0,0 +1 @@
+import test/default.nix

make/examples/aterm/test/default.nix

@@ -0,0 +1,18 @@
+let {
+
+  inherit (import ../../../lib) compileC link;
+
+  inherit (import ../aterm {}) libATerm;
+
+  compile = fn: compileC {
+    main = fn;
+    localIncludes = "auto";
+    cFlags = "-I../aterm";
+  };
+
+  fib = link {objects = compile ./fib.c; libraries = libATerm;};
+
+  primes = link {objects = compile ./primes.c; libraries = libATerm;};
+  
+  body = [fib primes];
+}

make/examples/default.nix

@@ -0,0 +1,6 @@
+[ (import ./trivial)
+  (import ./simple-header)
+  (import ./not-so-simple-header)
+  (import ./not-so-simple-header-auto)
+  (import ./aterm)
+]

make/examples/not-so-simple-header-auto/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header-auto/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC findIncludes link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = "auto";
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header-auto/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header-auto/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/not-so-simple-header/bar/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/not-so-simple-header/default.nix

@@ -0,0 +1,14 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {programName = "hello"; objects = compileC {
+    main = ./foo/hello.c;
+    localIncludes = [
+      [./foo/fnord/indirect.h "fnord/indirect.h"]
+      [./bar/hello.h "fnord/../../bar/hello.h"]
+    ];
+  };};
+
+  body = [hello];
+}

make/examples/not-so-simple-header/foo/fnord/indirect.h

@@ -0,0 +1,3 @@
+#define HELLO "Hello"
+
+#include "../../bar/hello.h"

make/examples/not-so-simple-header/foo/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "fnord/indirect.h"
+
+int main(int argc, char * * argv)
+{
+    printf(HELLO " " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/default.nix

@@ -0,0 +1,11 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {
+    main = ./hello.c;
+    localIncludes = [ [./hello.h "hello.h"] ];
+  };};
+
+  body = [hello];
+}

make/examples/simple-header/hello.c

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#include "hello.h"
+
+int main(int argc, char * * argv)
+{
+    printf("Hello " WHAT "\n");
+    return 0;
+}

make/examples/simple-header/hello.h

@@ -0,0 +1 @@
+#define WHAT "World"

make/examples/trivial/default.nix

@@ -0,0 +1,8 @@
+let {
+
+  inherit (import ../../lib) compileC link;
+
+  hello = link {objects = compileC {main = ./hello.c;};};
+
+  body = [hello];
+}

make/examples/trivial/hello.c

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+int main(int argc, char * * argv)
+{
+    printf("Hello World\n");
+    return 0;
+}

make/lib/compile-c.sh

@@ -0,0 +1,73 @@
+. $stdenv/setup
+
+mainName=$(basename $main | cut -c34-)
+
+echo "compiling \`$mainName'..."
+
+# Turn $localIncludes into an array.
+localIncludes=($localIncludes)
+
+# Determine how many `..' levels appear in the header file references.
+# E.g., if there is some reference `../../foo.h', then we have to
+# insert two extra levels in the directory structure, so that `a.c' is
+# stored at `dotdot/dotdot/a.c', and a reference from it to
+# `../../foo.h' resolves to `dotdot/dotdot/../../foo.h' == `foo.h'.
+n=0
+maxDepth=0
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    target=${localIncludes[$((n + 1))]}
+
+    # Split the target name into path components using some IFS magic.
+    savedIFS="$IFS"
+    IFS=/
+    components=($target)
+    depth=0
+    for ((m = 0; m < ${#components[*]}; m++)); do
+        c=${components[m]}
+        if test "$c" = ".."; then
+            depth=$((depth + 1))
+        fi
+    done
+    IFS="$savedIFS"
+
+    if test $depth -gt $maxDepth; then
+        maxDepth=$depth;
+    fi
+done
+
+# Create the extra levels in the directory hierarchy.
+prefix=
+for ((n = 0; n < maxDepth; n++)); do
+    prefix="dotdot/$prefix"
+done
+
+# Create symlinks to the header files.
+for ((n = 0; n < ${#localIncludes[*]}; n += 2)); do
+    source=${localIncludes[n]}
+    target=${localIncludes[$((n + 1))]}
+
+    # Create missing directories.  We use IFS magic to split the path
+    # into path components.
+    savedIFS="$IFS"
+    IFS=/
+    components=($prefix$target)
+    fullPath=(.)
+    for ((m = 0; m < ${#components[*]} - 1; m++)); do
+        fullPath=("${fullPath[@]}" ${components[m]})
+        if ! test -d "${fullPath[*]}"; then
+            mkdir "${fullPath[*]}"
+        fi
+    done
+    IFS="$savedIFS"
+    
+    ln -sf $source $prefix$target
+done
+
+# Create a symlink to the main file.
+if ! test "$(readlink $prefix$mainName)" = $main; then
+    ln -s $main $prefix$mainName
+fi
+
+mkdir $out
+test "$prefix" && cd $prefix
+gcc -Wall $cFlags -c $mainName -o $out/$mainName.o

make/lib/default.nix

@@ -0,0 +1,59 @@
+rec {
+
+  # Should point at your Nixpkgs installation.
+  pkgPath = ./pkgs;
+
+  pkgs = import (pkgPath + /system/all-packages.nix) {};
+
+  stdenv = pkgs.stdenv;
+  
+
+  compileC = {main, localIncludes ? [], cFlags ? "", forSharedLib ? false}:
+  stdenv.mkDerivation {
+    name = "compile-c";
+    builder = ./compile-c.sh;
+    localIncludes =
+      if localIncludes == "auto" then
+        import (findIncludes {
+          main = toString main;
+          hack = __currentTime;
+          inherit cFlags;
+        })
+      else
+        localIncludes;
+    inherit main;
+    cFlags = [
+      cFlags
+      (if forSharedLib then ["-fpic"] else [])
+    ];
+  };
+
+  /*
+  runCommand = {command}: {
+    name = "run-command";
+    builder = ./run-command.sh;
+    inherit command;
+  };
+  */
+
+  findIncludes = {main, hack, cFlags ? ""}: stdenv.mkDerivation {
+    name = "find-includes";
+    builder = ./find-includes.sh;
+    inherit main hack cFlags;
+  };
+  
+  link = {objects, programName ? "program", libraries ? []}: stdenv.mkDerivation {
+    name = "link";
+    builder = ./link.sh;
+    inherit objects programName libraries;
+  };
+
+  makeLibrary = {objects, libraryName ? [], sharedLib ? false}:
+  # assert sharedLib -> fold (obj: x: assert obj.sharedLib && x) false objects
+  stdenv.mkDerivation {
+    name = "library";
+    builder = ./make-library.sh;
+    inherit objects libraryName sharedLib;
+  };
+
+}

make/lib/find-includes.sh

@@ -0,0 +1,20 @@
+. $stdenv/setup
+
+echo "finding includes of \`$(basename $main)'..."
+
+makefile=$NIX_BUILD_TOP/makefile
+
+mainDir=$(dirname $main)
+(cd $mainDir && gcc $cFlags -MM $(basename $main) -MF $makefile) || false
+
+echo "[" >$out
+
+while read line; do
+    line=$(echo "$line" | sed 's/.*://')
+    for i in $line; do
+        fullPath=$(readlink -f $mainDir/$i)
+        echo "  [ $fullPath \"$i\" ]" >>$out
+    done
+done < $makefile
+
+echo "]" >>$out

make/lib/link.sh

@@ -0,0 +1,21 @@
+. $stdenv/setup
+
+shopt -s nullglob
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+libs=
+for i in $libraries; do
+    lib=$(echo $i/*.a; echo $i/*.so)
+    name=$(echo $(basename $lib) | sed -e 's/^lib//' -e 's/.a$//' -e 's/.so$//')
+    libs="$libs -L$(dirname $lib) -l$name" 
+done
+
+echo "linking object files into \`$programName'..."
+
+mkdir $out
+gcc -o $out/$programName $objs $libs

make/lib/make-library.sh

@@ -0,0 +1,28 @@
+. $stdenv/setup
+
+objs=
+for i in $objects; do
+    obj=$(echo $i/*.o)
+    objs="$objs $obj"
+done
+
+echo "archiving object files into library \`$libraryName'..."
+
+ensureDir $out
+
+if test -z "$sharedLib"; then
+
+    outPath=$out/lib${libraryName}.a
+
+    ar crs $outPath $objs
+    ranlib $outPath
+
+else
+
+    outPath=$out/lib${libraryName}.so
+
+    gcc -shared -o $outPath $objs
+
+fi    
+
+    

nix.spec.in

@@ -14,7 +14,7 @@ Release: 1
 License: GPL
 Group: Software Deployment
 URL: http://www.cs.uu.nl/groups/ST/Trace/Nix
-Source0: %{name}-@version@.tar.gz
+Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
 %define _prefix /nix
 Prefix: %{_prefix}

scripts/download-using-manifests.pl.in

@@ -76,7 +76,7 @@ addToQueue $targetPath;
 
 sub isValidPath {
     my $p = shift;
-    system "@bindir@/nix-store --isvalid '$p' 2> /dev/null";
+    system "@bindir@/nix-store --check-validity '$p' 2> /dev/null";
     return $? == 0;
 }
 
@@ -207,11 +207,19 @@ while (scalar @path > 0) {
     my $v = $edge->{end};
 
     print "\n*** Step $curStep/$maxStep: ";
-    $curStep++;
 
     if ($edge->{type} eq "present") {
         print "using already present path `$v'\n";
         print LOGFILE "$$ present $v\n";
+
+        if ($curStep < $maxStep) {
+            # Since this is not the last step, the path will be used
+            # as a base to one or more patches.  So turn the base path
+            # into a NAR archive, to which we can apply the patch.
+            print "  packing base path...\n";
+            system "@bindir@/nix-store --dump $v > /tmp/nar";
+            die "cannot dump `$v'" if ($? != 0);
+        }
     }
 
     elsif ($edge->{type} eq "patch") {
@@ -224,21 +232,22 @@ while (scalar @path > 0) {
         print "  downloading patch...\n";
         my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
 
-        # Turn the base path into a NAR archive, to which we can
-        # actually apply the patch.
-        print "  packing base path...\n";
-        system "@bindir@/nix-store --dump $patch->{basePath} > /tmp/nar";
-        die "cannot dump `$patch->{basePath}'" if ($? != 0);
-
-        # Apply the patch.
+        # Apply the patch to the NAR archive produced in step 1 (for
+        # the already present path) or a later step (for patch sequences).
         print "  applying patch...\n";
         system "@libexecdir@/bspatch /tmp/nar /tmp/nar2 $patchPath";
         die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
 
-        # Unpack the resulting NAR archive into the target path.
-        print "  unpacking patched archive...\n";
-        system "@bindir@/nix-store --restore $v < /tmp/nar2";
-        die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
+        if ($curStep < $maxStep) {
+            # The archive will be used as the base of the next patch.
+            rename "/tmp/nar2", "/tmp/nar" or die "cannot rename NAR archive: $!";
+        } else {
+            # This was the last patch.  Unpack the final NAR archive
+            # into the target path.
+            print "  unpacking patched archive...\n";
+            system "@bindir@/nix-store --restore $v < /tmp/nar2";
+            die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
+        }
     }
 
     elsif ($edge->{type} eq "narfile") {
@@ -251,11 +260,18 @@ while (scalar @path > 0) {
         print "  downloading archive...\n";
         my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
 
-        # Unpack the archive into the target path.
-        print "  unpacking archive...\n";
-        system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
-        die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
+        if ($curStep < $maxStep) {
+            # The archive will be used a base to a patch.
+            system "@bunzip2@ < '$narFilePath' > /tmp/nar";
+        } else {
+            # Unpack the archive into the target path.
+            print "  unpacking archive...\n";
+            system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
+            die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
+        }
     }
+
+    $curStep++;
 }
 
 

scripts/generate-patches.pl.in

@@ -237,10 +237,10 @@ foreach my $p (keys %dstOutPaths) {
             $ratio = 1 / $ratio if $ratio < 1;
             print "  USE $srcUses $dstUses $ratio $q\n";
             
-            if ($ratio >= 2) {
-                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
-                next;
-            }
+#            if ($ratio >= 2) {
+#                print "  SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
+#                next;
+#            }
 
             # If there are multiple matching names, include the ones
             # with the closest version numbers.

src/boost/shared_ptr.hpp

@@ -324,7 +324,7 @@ public:
 // Tasteless as this may seem, making all members public allows member templates
 // to work in the absence of member template friends. (Matthew Langston)
 
-#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+#if 0
 
 private:
 

src/boost/weak_ptr.hpp

@@ -154,7 +154,7 @@ public:
 // Tasteless as this may seem, making all members public allows member templates
 // to work in the absence of member template friends. (Matthew Langston)
 
-#ifndef BOOST_NO_MEMBER_TEMPLATE_FRIENDS
+#if 0
 
 private:
 

src/libexpr/nixexpr.cc

@@ -7,6 +7,7 @@
 
 
 ATermMap::ATermMap(unsigned int initialSize, unsigned int maxLoadPct)
+    : table(0)
 {
     this->maxLoadPct = maxLoadPct;
     table = ATtableCreate(initialSize, maxLoadPct);
@@ -16,6 +17,36 @@ ATermMap::ATermMap(unsigned int initialSize, unsigned int maxLoadPct)
 
 ATermMap::ATermMap(const ATermMap & map)
     : table(0)
+{
+    copy(map);
+}
+
+
+ATermMap::~ATermMap()
+{
+    free();
+}
+
+
+ATermMap & ATermMap::operator = (const ATermMap & map)
+{
+    if (this == &map) return *this;
+    free();
+    copy(map);
+    return *this;
+}
+
+
+void ATermMap::free()
+{
+    if (table) {
+        ATtableDestroy(table);
+        table = 0;
+    }
+}
+
+
+void ATermMap::copy(const ATermMap & map)
 {
     ATermList keys = map.keys();
 
@@ -28,12 +59,6 @@ ATermMap::ATermMap(const ATermMap & map)
 }
 
 
-ATermMap::~ATermMap()
-{
-    if (table) ATtableDestroy(table);
-}
-
-
 void ATermMap::set(ATerm key, ATerm value)
 {
     return ATtablePut(table, key, value);

src/libexpr/nixexpr.hh

@@ -29,6 +29,8 @@ public:
     ATermMap(const ATermMap & map);
     ~ATermMap();
 
+    ATermMap & ATermMap::operator = (const ATermMap & map);
+        
     void set(ATerm key, ATerm value);
     void set(const string & key, ATerm value);
 
@@ -46,6 +48,9 @@ public:
 
 private:
     void add(const ATermMap & map, ATermList & keys);
+
+    void free();
+    void copy(const ATermMap & map);
 };
 
 

src/libexpr/primops.cc

@@ -1,6 +1,7 @@
 #include "build.hh"
 #include "eval.hh"
 #include "globals.hh"
+#include "misc.hh"
 #include "nixexpr-ast.hh"
 
 
@@ -8,11 +9,36 @@
    argument. */ 
 static Expr primImport(EvalState & state, const ATermVector & args)
 {
-    ATerm path;
-    Expr fn = evalExpr(state, args[0]);
-    if (!matchPath(fn, path))
-        throw Error("path expected");
-    return evalFile(state, aterm2String(path));
+    ATermList es;
+    Path path;
+
+    Expr arg = evalExpr(state, args[0]), arg2;
+    
+    if (matchPath(arg, arg2))
+        path = aterm2String(arg2);
+
+    else if (matchAttrs(arg, es)) {
+        Expr a = queryAttr(arg, "type");
+
+        /* If it is a derivation, we have to realise it and load the
+           Nix expression created at the derivation's output path. */
+        if (a && evalString(state, a) == "derivation") {
+            a = queryAttr(arg, "drvPath");
+            if (!a) throw Error("bad derivation in import");
+            Path drvPath = evalPath(state, a);
+
+            buildDerivations(singleton<PathSet>(drvPath));
+ 
+            a = queryAttr(arg, "outPath");
+            if (!a) throw Error("bad derivation in import");
+            path = evalPath(state, a);
+        }
+    }
+
+    if (path == "")
+        throw Error("path or derivation expected in import");
+    
+    return evalFile(state, path);
 }
 
 
@@ -46,8 +72,8 @@ static Hash hashDerivationModulo(EvalState & state, Derivation drv)
         {
             return hashString(htSHA256, "fixed:out:"
                 + i->second.hashAlgo + ":"
-                + i->second.hash + ":"
-                + i->second.path);
+                + i->second.hash /* !!! + ":"
+                                    + i->second.path */);
         }
     }
 
@@ -188,14 +214,12 @@ static string concatStrings(const Strings & ss)
    derivation; `drvPath' containing the path of the Nix expression;
    and `type' set to `derivation' to indicate that this is a
    derivation. */
-static Expr primDerivation(EvalState & state, const ATermVector & _args)
+static Expr primDerivationStrict(EvalState & state, const ATermVector & args)
 {
     startNest(nest, lvlVomit, "evaluating derivation");
 
     ATermMap attrs;
-    Expr args = _args[0];
-    args = evalExpr(state, args);
-    queryAllAttrs(args, attrs, true);
+    queryAllAttrs(evalExpr(state, args[0]), attrs, true);
 
     /* Build the derivation expression by processing the attributes. */
     Derivation drv;
@@ -296,8 +320,10 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
         
     /* Use the masked derivation expression to compute the output
        path. */
-    Path outPath = makeStorePath("output:out",
-        hashDerivationModulo(state, drv), drvName);
+    /* XXX */
+    Path outPath;
+    PathHash outPathHash;
+    makeStorePath(hashDerivationModulo(state, drv), drvName, outPath, outPathHash);
 
     /* Construct the final derivation store expression. */
     drv.env["out"] = outPath;
@@ -316,10 +342,27 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
     state.drvHashes[drvPath] = hashDerivationModulo(state, drv);
 
     /* !!! assumes a single output */
-    attrs.set("outPath", makeAttrRHS(makePath(toATerm(outPath)), makeNoPos()));
-    attrs.set("drvPath", makeAttrRHS(makePath(toATerm(drvPath)), makeNoPos()));
+    ATermMap outAttrs;
+    outAttrs.set("outPath", makeAttrRHS(makePath(toATerm(outPath)), makeNoPos()));
+    outAttrs.set("drvPath", makeAttrRHS(makePath(toATerm(drvPath)), makeNoPos()));
+
+    return makeAttrs(outAttrs);
+}
+
+
+static Expr primDerivationLazy(EvalState & state, const ATermVector & args)
+{
+    Expr eAttrs = evalExpr(state, args[0]);
+    ATermMap attrs;
+    queryAllAttrs(eAttrs, attrs, true);
+
     attrs.set("type", makeAttrRHS(makeStr(toATerm("derivation")), makeNoPos()));
 
+    Expr drvStrict = makeCall(makeVar(toATerm("derivation!")), eAttrs);
+
+    attrs.set("outPath", makeAttrRHS(makeSelect(drvStrict, toATerm("outPath")), makeNoPos()));
+    attrs.set("drvPath", makeAttrRHS(makeSelect(drvStrict, toATerm("drvPath")), makeNoPos()));
+    
     return makeAttrs(attrs);
 }
 
@@ -339,7 +382,7 @@ static Expr primToString(EvalState & state, const ATermVector & args)
     ATerm s;
     if (matchStr(arg, s) || matchPath(arg, s) || matchUri(arg, s))
         return makeStr(s);
-    else throw Error("cannot coerce value to string");
+    throw Error("cannot coerce value to string");
 }
 
 
@@ -398,18 +441,46 @@ static Expr primCurrentSystem(EvalState & state, const ATermVector & args)
 }
 
 
+static Expr primCurrentTime(EvalState & state, const ATermVector & args)
+{
+    return ATmake("Int(<int>)", time(0));
+}
+
+
+static Expr primRemoveAttrs(EvalState & state, const ATermVector & args)
+{
+    ATermMap attrs;
+    queryAllAttrs(evalExpr(state, args[0]), attrs, true);
+    
+    ATermList list;
+    if (!matchList(evalExpr(state, args[1]), list))
+        throw Error("`removeAttrs' expects a list as its second argument");
+
+    for (ATermIterator i(list); i; ++i)
+        /* It's not an error for *i not to exist. */
+        attrs.remove(evalString(state, *i));
+
+    return makeAttrs(attrs);
+}
+
+
 void EvalState::addPrimOps()
 {
     addPrimOp("true", 0, primTrue);
     addPrimOp("false", 0, primFalse);
     addPrimOp("null", 0, primNull);
     addPrimOp("__currentSystem", 0, primCurrentSystem);
+    addPrimOp("__currentTime", 0, primCurrentTime);
 
     addPrimOp("import", 1, primImport);
-    addPrimOp("derivation", 1, primDerivation);
+    addPrimOp("derivation!", 1, primDerivationStrict);
+    addPrimOp("derivation", 1, primDerivationLazy);
     addPrimOp("baseNameOf", 1, primBaseNameOf);
     addPrimOp("toString", 1, primToString);
     addPrimOp("isNull", 1, primIsNull);
 
     addPrimOp("map", 2, primMap);
+    addPrimOp("removeAttrs", 2, primRemoveAttrs);
 }
+
+

src/libmain/shared.cc

@@ -139,6 +139,21 @@ static void initAndRun(int argc, char * * argv)
 
     /* ATerm stuff.  !!! find a better place to put this */
     initDerivationsHelpers();
+
+    /* Random number generator needed by makeRandomStorePath(); !!!
+       improve. */
+    srand(time(0));
+
+    /* Set the trust ID to the user name. */
+    currentTrustId = getEnv("NIX_USER_ID"); /* !!! dangerous? */
+    if (currentTrustId == "") {
+        SwitchToOriginalUser sw;
+        uid_t uid = geteuid();
+        struct passwd * pw = getpwuid(uid);
+        if (!pw) throw Error(format("unknown user ID %1%, go away") % uid);
+        currentTrustId = pw->pw_name;
+    }
+    printMsg(lvlError, format("trust ID is `%1%'") % currentTrustId);
     
     /* Put the arguments in a vector. */
     Strings args, remaining;

src/libstore/Makefile.am

@@ -2,7 +2,7 @@ noinst_LIBRARIES = libstore.a
 
 libstore_a_SOURCES = \
  store.cc store.hh derivations.cc derivations.hh \
- build.cc misc.cc build.hh \
+ build.cc build.hh misc.cc misc.hh \
  globals.cc globals.hh db.cc db.hh \
  references.cc references.hh pathlocks.cc pathlocks.hh \
  gc.cc gc.hh derivations-ast.hh
@@ -15,4 +15,4 @@ AM_CXXFLAGS = -Wall \
 derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
 	$(perl) ../aterm-helper.pl derivations-ast.hh derivations-ast.cc < derivations-ast.def
 
-derivations.cc store.cc: derivations-ast.hh
+derivations.cc store.cc: derivations-ast.hh

src/libstore/build.cc

@@ -14,6 +14,7 @@
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "gc.hh"
+#include "misc.hh"
 
 
 /* !!! TODO derivationFromPath shouldn't be used here */
@@ -314,6 +315,16 @@ private:
     
     /* The remainder is state held during the build. */
 
+    /* The map of output equivalence classes to temporary output
+       paths. */
+    typedef map<OutputEqClass, Path> OutputMap;
+    OutputMap tmpOutputs;
+
+    /* The hash rewrite map that rewrites output equivalences occuring
+       in the command-line arguments and environment variables to the
+       actual paths to be used. */
+    HashRewrites rewrites;
+
     /* Locks on the output paths. */
     PathLocks outputLocks;
 
@@ -399,7 +410,8 @@ private:
     void writeLog(int fd, const unsigned char * buf, size_t count);
 
     /* Return the set of (in)valid paths. */
-    PathSet checkPathValidity(bool returnValid);
+    typedef set<OutputEqClass> OutputEqClasses;
+    OutputEqClasses checkOutputValidity(bool returnValid);
 };
 
 
@@ -463,12 +475,15 @@ void DerivationGoal::haveStoreExpr()
     /* Get the derivation. */
     drv = derivationFromPath(drvPath);
 
+#if 0    
     for (DerivationOutputs::iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
         addTempRoot(i->second.path);
+#endif    
 
-    /* Check what outputs paths are not already valid. */
-    PathSet invalidOutputs = checkPathValidity(false);
+    /* Check for what output path equivalence classes we do not
+       already have valid, trusted output paths. */
+    OutputEqClasses invalidOutputs = checkOutputValidity(false);
 
     /* If they are all valid, then we're done. */
     if (invalidOutputs.size() == 0) {
@@ -476,6 +491,7 @@ void DerivationGoal::haveStoreExpr()
         return;
     }
 
+#if 0    
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -485,6 +501,7 @@ void DerivationGoal::haveStoreExpr()
            substitutes. */
         if (querySubstitutes(noTxn, *i).size() > 0)
             addWaitee(worker.makeSubstitutionGoal(*i));
+#endif    
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         outputsSubstituted();
@@ -502,7 +519,7 @@ void DerivationGoal::outputsSubstituted()
 
     nrFailed = 0;
 
-    if (checkPathValidity(false).size() == 0) {
+    if (checkOutputValidity(false).size() == 0) {
         amDone(true);
         return;
     }
@@ -647,6 +664,7 @@ void DerivationGoal::buildDone()
 }
 
 
+#if 0
 static string readLine(int fd)
 {
     string s;
@@ -686,33 +704,27 @@ static void drain(int fd)
         else writeFull(STDERR_FILENO, buffer, rd);
     }
 }
+#endif
 
 
+#if 0
 PathSet outputPaths(const DerivationOutputs & outputs)
 {
     PathSet paths;
+    /* XXX */
     for (DerivationOutputs::const_iterator i = outputs.begin();
          i != outputs.end(); ++i)
         paths.insert(i->second.path);
     return paths;
 }
-
-
-string showPaths(const PathSet & paths)
-{
-    string s;
-    for (PathSet::const_iterator i = paths.begin();
-         i != paths.end(); ++i)
-    {
-        if (s.size() != 0) s += ", ";
-        s += "`" + *i + "'";
-    }
-    return s;
-}
+#endif
 
 
 DerivationGoal::HookReply DerivationGoal::tryBuildHook()
 {
+    return rpDecline;
+    
+#if 0    
     Path buildHook = getEnv("NIX_BUILD_HOOK");
     if (buildHook == "") return rpDecline;
     buildHook = absPath(buildHook);
@@ -861,6 +873,7 @@ DerivationGoal::HookReply DerivationGoal::tryBuildHook()
     }
 
     else throw Error(format("bad hook reply `%1%'") % reply);
+#endif
 }
 
 
@@ -881,11 +894,42 @@ void DerivationGoal::terminateBuildHook()
 
 bool DerivationGoal::prepareBuild()
 {
+    /* We direct each output of the derivation to a temporary location
+       in the Nix store.  Afterwards, we move the outputs to their
+       final, content-addressed location. */
+    for (DerivationOutputs::iterator i = drv.outputs.begin();
+         i != drv.outputs.end(); ++i)
+    {
+        Path tmpPath = makeRandomStorePath(namePartOf(i->second.eqClass));
+        printMsg(lvlError, format("mapping output id `%1%', class `%2%' to `%3%'")
+            % i->first % i->second.eqClass % tmpPath);
+        assert(i->second.eqClass.size() == tmpPath.size());
+
+        debug(format("building path `%1%'") % tmpPath);
+        
+        tmpOutputs[i->second.eqClass] = tmpPath;
+
+        /* This is a referenceable path.  Make a note of that for when
+           we are scanning for references in the output. */
+        allPaths.insert(tmpPath);
+        
+        /* The environment variables and command-line arguments of the
+           builder refer to the output path equivalence class.  Cause
+           those references to be rewritten to the temporary
+           locations. */
+        rewrites[hashPartOf(i->second.eqClass)] = hashPartOf(tmpPath);
+    }
+
     /* Obtain locks on all output paths.  The locks are automatically
        released when we exit this function or Nix crashes. */
     /* !!! BUG: this could block, which is not allowed. */
+#if 0
+    /* !!! acquire lock on the derivation or something? or on a
+       pseudo-path representing the output equivalence class? */
     outputLocks.lockPaths(outputPaths(drv.outputs));
+#endif
 
+#if 0    
     /* Now check again whether the outputs are valid.  This is because
        another process may have started building in parallel.  After
        it has finished and released the locks, we can (and should)
@@ -907,17 +951,7 @@ bool DerivationGoal::prepareBuild()
             format("derivation `%1%' is blocked by its output paths")
             % drvPath);
     }
-
-    /* Gather information necessary for computing the closure and/or
-       running the build hook. */
-    
-    /* The outputs are referenceable paths. */
-    for (DerivationOutputs::iterator i = drv.outputs.begin();
-         i != drv.outputs.end(); ++i)
-    {
-        debug(format("building path `%1%'") % i->second.path);
-        allPaths.insert(i->second.path);
-    }
+#endif    
 
     /* Determine the full set of input paths. */
 
@@ -930,14 +964,22 @@ bool DerivationGoal::prepareBuild()
            that are specified as inputs. */
         assert(isValidPath(i->first));
         Derivation inDrv = derivationFromPath(i->first);
+
         for (StringSet::iterator j = i->second.begin();
              j != i->second.end(); ++j)
-            if (inDrv.outputs.find(*j) != inDrv.outputs.end())
-                computeFSClosure(inDrv.outputs[*j].path, inputPaths);
-            else
-                throw Error(
-                    format("derivation `%1%' requires non-existent output `%2%' from input derivation `%3%'")
-                    % drvPath % *j % i->first);
+        {
+            OutputEqClass eqClass = findOutputEqClass(inDrv, *j);
+            PathSet inputs = findTrustedEqClassMembers(eqClass, currentTrustId);
+            if (inputs.size() == 0)
+                /* !!! shouldn't happen, except for garbage
+                   collection? */
+                throw Error(format("output `%1%' of derivation `%2%' is missing!")
+                    % *j % i->first);
+            for (PathSet::iterator k = inputs.begin(); k != inputs.end(); ++k) {
+                rewrites[hashPartOf(eqClass)] = hashPartOf(*k);
+                computeFSClosure(*k, inputPaths);
+            }
+        }
     }
 
     /* Second, the input sources. */
@@ -945,7 +987,31 @@ bool DerivationGoal::prepareBuild()
          i != drv.inputSrcs.end(); ++i)
         computeFSClosure(*i, inputPaths);
 
-    debug(format("added input paths %1%") % showPaths(inputPaths));
+    /* There might be equivalence class collisions now.  That is,
+       different input closures might contain different paths from the
+       *same* output path equivalence class.  We should pick one from
+       each, and rewrite dependent paths. */
+    Replacements replacements;
+    inputPaths = consolidatePaths(inputPaths, false, replacements);
+
+    HashRewrites rewrites2;
+    for (Replacements::iterator i = replacements.begin();
+         i != replacements.end(); ++i)
+    {
+        printMsg(lvlError, format("HASH REWRITE %1% %2%")
+            % hashPartOf(i->first).toString() % hashPartOf(i->second).toString());
+        rewrites2[hashPartOf(i->first)] = hashPartOf(i->second);
+    }
+
+    for (HashRewrites::iterator i = rewrites.begin();
+         i != rewrites.end(); ++i)
+        rewrites[i->first] = PathHash(rewriteHashes(i->second.toString(), rewrites2));
+    
+    /* !!! remove, debug only */
+    Replacements dummy;
+    consolidatePaths(inputPaths, true, dummy);
+    
+    printMsg(lvlError, format("added input paths %1%") % showPaths(inputPaths)); /* !!! */
 
     allPaths.insert(inputPaths.begin(), inputPaths.end());
 
@@ -956,7 +1022,7 @@ bool DerivationGoal::prepareBuild()
 void DerivationGoal::startBuilder()
 {
     startNest(nest, lvlInfo,
-        format("building path(s) %1%") % showPaths(outputPaths(drv.outputs)))
+        format("building derivation `%1%'") % drvPath)
     
     /* Right platform? */
     if (drv.platform != thisSystem)
@@ -966,6 +1032,7 @@ void DerivationGoal::startBuilder()
 
     /* If any of the outputs already exist but are not registered,
        delete them. */
+#if 0    
     for (DerivationOutputs::iterator i = drv.outputs.begin(); 
          i != drv.outputs.end(); ++i)
     {
@@ -977,6 +1044,7 @@ void DerivationGoal::startBuilder()
             deletePath(path);
         }
     }
+#endif
 
     /* Construct the environment passed to the builder. */
     typedef map<string, string> Environment;
@@ -1004,7 +1072,7 @@ void DerivationGoal::startBuilder()
     /* Add all bindings specified in the derivation. */
     for (StringPairs::iterator i = drv.env.begin();
          i != drv.env.end(); ++i)
-        env[i->first] = i->second;
+        env[i->first] = rewriteHashes(i->second, rewrites);
 
     /* Create a temporary directory where the build will take
        place. */
@@ -1027,8 +1095,8 @@ void DerivationGoal::startBuilder()
             env["NIX_OUTPUT_CHECKED"] = "1";
 
     /* Run the builder. */
-    printMsg(lvlChatty, format("executing builder `%1%'") %
-        drv.builder);
+    string builder = rewriteHashes(drv.builder, rewrites);
+    printMsg(lvlChatty, format("executing builder `%1%'") % builder);
 
     /* Create the log file and pipe. */
     openLogFile();
@@ -1054,22 +1122,22 @@ void DerivationGoal::startBuilder()
 
             /* Fill in the arguments. */
             Strings args(drv.args);
-            args.push_front(baseNameOf(drv.builder));
+            args.push_front(baseNameOf(builder));
             const char * * argArr = strings2CharPtrs(args);
 
             /* Fill in the environment. */
             Strings envStrs;
             for (Environment::const_iterator i = env.begin();
                  i != env.end(); ++i)
-                envStrs.push_back(i->first + "=" + i->second);
+                envStrs.push_back(i->first + "=" +
+                    rewriteHashes(i->second, rewrites));
             const char * * envArr = strings2CharPtrs(envStrs);
 
             /* Execute the program.  This should not return. */
-            execve(drv.builder.c_str(),
+            execve(builder.c_str(),
                 (char * *) argArr, (char * *) envArr);
 
-            throw SysError(format("executing `%1%'")
-                % drv.builder);
+            throw SysError(format("executing `%1%'") % builder);
             
         } catch (exception & e) {
             cerr << format("build error: %1%\n") % e.what();
@@ -1089,23 +1157,17 @@ void DerivationGoal::computeClosure()
 {
     map<Path, PathSet> allReferences;
     map<Path, Hash> contentHashes;
-    
-    /* Check whether the output paths were created, and grep each
-       output path to determine what other paths it references.  Also make all
-       output paths read-only. */
+
     for (DerivationOutputs::iterator i = drv.outputs.begin(); 
          i != drv.outputs.end(); ++i)
     {
-        Path path = i->second.path;
+        Path path = tmpOutputs[i->second.eqClass];
         if (!pathExists(path)) {
             throw BuildError(
                 format("builder for `%1%' failed to produce output path `%2%'")
                 % drvPath % path);
         }
-
-        startNest(nest, lvlTalkative,
-            format("scanning for references inside `%1%'") % path);
-
+        
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */ 
@@ -1139,70 +1201,55 @@ void DerivationGoal::computeClosure()
             Hash h2 = recursive ? hashPath(ht, path) : hashFile(ht, path);
             if (h != h2)
                 throw Error(
-                    format("output path `%1% should have %2% hash `%3%', instead has `%4%'")
+                    format("output path `%1%' should have %2% hash `%3%', instead has `%4%'")
                     % path % algo % printHash(h) % printHash(h2));
         }
 
-	canonicalisePathMetaData(path);
-
-	/* For this output path, find the references to other paths contained
-	   in it. */
-        PathSet references;
+	/* For this output path, find the references to other paths
+	   contained in it. */
+        PathSet referenced;
         if (!pathExists(path + "/nix-support/no-scan")) {
-            startNest(nest2, lvlChatty,
-                format("scanning for store references in `%1%'") % path);
-            Paths references2;
-            references2 = filterReferences(path, 
+            Paths referenced2 = filterReferences(path, 
                 Paths(allPaths.begin(), allPaths.end()));
-            references = PathSet(references2.begin(), references2.end());
+            referenced = PathSet(referenced2.begin(), referenced2.end());
 
             /* For debugging, print out the referenced and
                unreferenced paths. */
-            for (PathSet::iterator i = inputPaths.begin();
-                 i != inputPaths.end(); ++i)
-            {
-                PathSet::iterator j = references.find(*i);
-                if (j == references.end())
-                    debug(format("unreferenced input: `%1%'") % *i);
-                else
-                    debug(format("referenced input: `%1%'") % *i);
-            }
-            
-            nest2.close();
+            PathSet unreferenced;
+            insert_iterator<PathSet> ins(unreferenced, unreferenced.begin());
+            set_difference(
+                inputPaths.begin(), inputPaths.end(),
+                referenced.begin(), referenced.end(), ins);
+            printMsg(lvlError, format("unreferenced inputs: %1%") % showPaths(unreferenced));
+            printMsg(lvlError, format("referenced inputs: %1%") % showPaths(referenced));
         }
 
-        allReferences[path] = references;
+        /* Rewrite each output to a name matching its content hash.
+           I.e., enforce the hash invariant: the hash part of a store
+           path matches the contents at that path.
 
-        /* Hash the contents of the path.  The hash is stored in the
-           database so that we can verify later on whether nobody has
-           messed with the store.  !!! inefficient: it would be nice
-           if we could combine this with filterReferences(). */
-        contentHashes[path] = hashPath(htSHA256, path);
+           This also registers the final output path as valid, and
+           sets it references. */
+        Path finalPath = addToStore(path,
+            hashPartOf(path), namePartOf(path),
+            referenced);
+        printMsg(lvlError, format("produced final path `%1%'") % finalPath);
+
+        /* Register the fact that this output path is a member of some
+           output path equivalence class (for a certain user, at
+           least).  This is how subsequent derivations will be able to
+           find it. */
+        Transaction txn;
+        createStoreTransaction(txn);
+        addOutputEqMember(txn, i->second.eqClass, currentTrustId, finalPath);
+        txn.commit();
+
+        /* Get rid of the temporary output.  !!! optimise all this by
+           *moving* the temporary output to the new location and
+           applying rewrites in situ. */
+        deletePath(path);
     }
-
-    /* Register each output path as valid, and register the sets of
-       paths referenced by each of them.  This is wrapped in one
-       database transaction to ensure that if we crash, either
-       everything is registered or nothing is.  This is for
-       recoverability: unregistered paths in the store can be deleted
-       arbitrarily, while registered paths can only be deleted by
-       running the garbage collector.
-
-       The reason that we do the transaction here and not on the fly
-       while we are scanning (above) is so that we don't hold database
-       locks for too long. */
-    Transaction txn;
-    createStoreTransaction(txn);
-    for (DerivationOutputs::iterator i = drv.outputs.begin(); 
-         i != drv.outputs.end(); ++i)
-    {
-        registerValidPath(txn, i->second.path,
-            contentHashes[i->second.path],
-            allReferences[i->second.path],
-            drvPath);
-    }
-    txn.commit();
-
+    
     /* It is now safe to delete the lock files, since all future
        lockers will see that the output paths are valid; they will not
        create new lock files with the same names as the old (unlinked)
@@ -1282,16 +1329,19 @@ void DerivationGoal::writeLog(int fd,
 }
 
 
-PathSet DerivationGoal::checkPathValidity(bool returnValid)
+DerivationGoal::OutputEqClasses DerivationGoal::checkOutputValidity(bool returnValid)
 {
-    PathSet result;
+    OutputEqClasses result;
     for (DerivationOutputs::iterator i = drv.outputs.begin();
          i != drv.outputs.end(); ++i)
-        if (isValidPath(i->second.path)) {
-            if (returnValid) result.insert(i->second.path);
+    {
+        PathSet paths = findTrustedEqClassMembers(i->second.eqClass, currentTrustId);
+        if (paths.size() > 0) {
+            if (returnValid) result.insert(i->second.eqClass);
         } else {
-            if (!returnValid) result.insert(i->second.path);
+            if (!returnValid) result.insert(i->second.eqClass);
         }
+    }
     return result;
 }
 
@@ -1306,11 +1356,13 @@ private:
     /* The store path that should be realised through a substitute. */
     Path storePath;
 
+#if 0    
     /* The remaining substitutes for this path. */
     Substitutes subs;
 
     /* The current substitute. */
     Substitute sub;
+#endif    
 
     /* Outgoing references for this path. */
     PathSet references;
@@ -1379,10 +1431,12 @@ void SubstitutionGoal::init()
         return;
     }
 
+#if 0
     /* Read the substitutes. */
     subs = querySubstitutes(noTxn, storePath);
+#endif
 
-    /* To maintain the closure invairant, we first have to realise the
+    /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
     queryReferences(noTxn, storePath, references);
 
@@ -1418,7 +1472,7 @@ void SubstitutionGoal::tryNext()
 {
     trace("trying next substitute");
 
-    if (subs.size() == 0) {
+    if (true /* !!! subs.size() == 0 */) {
         /* None left.  Terminate this goal and let someone else deal
            with it. */
         printMsg(lvlError,
@@ -1427,8 +1481,10 @@ void SubstitutionGoal::tryNext()
         amDone(false);
         return;
     }
+#if 0    
     sub = subs.front();
     subs.pop_front();
+#endif
 
     /* Wait until we can run the substitute program. */
     state = &SubstitutionGoal::tryToRun;
@@ -1438,6 +1494,7 @@ void SubstitutionGoal::tryNext()
 
 void SubstitutionGoal::tryToRun()
 {
+#if 0
     trace("trying to run");
 
     /* Make sure that we are allowed to start a build. */
@@ -1509,11 +1566,13 @@ void SubstitutionGoal::tryToRun()
         pid, logPipe.readSide, true);
 
     state = &SubstitutionGoal::finished;
+#endif    
 }
 
 
 void SubstitutionGoal::finished()
 {
+#if 0    
     trace("substitute finished");
 
     /* Since we got an EOF on the logger pipe, the substitute is
@@ -1570,6 +1629,7 @@ void SubstitutionGoal::finished()
         format("substitution of path `%1%' succeeded") % storePath);
 
     amDone();
+#endif
 }
 
 

src/libstore/build.hh

@@ -3,6 +3,7 @@
 
 #include "derivations.hh"
 
+
 /* Ensure that the output paths of the derivation are valid.  If they
    are already valid, this is a no-op.  Otherwise, validity can
    be reached in two ways.  First, if the output paths have
@@ -11,26 +12,10 @@
    sub-derivations. */
 void buildDerivations(const PathSet & drvPaths);
 
+
 /* Ensure that a path is valid.  If it is not currently valid, it may
    be made valid by running a substitute (if defined for the path). */
 void ensurePath(const Path & storePath);
 
-/* Read a derivation, after ensuring its existence through
-   ensurePath(). */
-Derivation derivationFromPath(const Path & drvPath);
-
-/* Place in `paths' the set of all store paths in the file system
-   closure of `storePath'; that is, all paths than can be directly or
-   indirectly reached from it.  `paths' is not cleared.  If
-   `flipDirection' is true, the set of paths that can reach
-   `storePath' is returned; that is, the closures under the `referers'
-   relation instead of the `references' relation is returned. */
-void computeFSClosure(const Path & storePath,
-    PathSet & paths, bool flipDirection = false);
-
-/* Return the path corresponding to the output identifier `id' in the
-   given derivation. */
-Path findOutput(const Derivation & drv, string id);
-    
 
 #endif /* !__BUILD_H */

src/libstore/db.cc

@@ -165,127 +165,152 @@ static int my_fsync(int fd)
 }
 
 
-void Database::open(const string & path)
+void Database::open2(const string & path, bool removeOldEnv)
 {
     if (env) throw Error(format("environment already open"));
 
-    try {
-
-        debug(format("opening database environment"));
+    debug(format("opening database environment"));
 
 
-        /* Create the database environment object. */
-        DbEnv * env = 0; /* !!! close on error */
-        env = new DbEnv(0);
-
-        /* Smaller log files. */
-        env->set_lg_bsize(32 * 1024); /* default */
-        env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */
-
-        /* Write the log, but don't sync.  This protects transactions
-           against application crashes, but if the system crashes,
-           some transactions may be undone.  An acceptable risk, I
-           think. */
-        env->set_flags(DB_TXN_WRITE_NOSYNC | DB_LOG_AUTOREMOVE, 1);
-
-        /* Increase the locking limits.  If you ever get `Dbc::get:
-           Cannot allocate memory' or similar, especially while
-           running `nix-store --verify', just increase the following
-           number, then run db_recover on the database to remove the
-           existing DB environment (since changes only take effect on
-           new environments). */
-        env->set_lk_max_locks(10000);
-        env->set_lk_max_lockers(10000);
-        env->set_lk_max_objects(10000);
-        env->set_lk_detect(DB_LOCK_DEFAULT);
-
-        /* Dangerous, probably, but from the docs it *seems* that BDB
-           shouldn't sync when DB_TXN_WRITE_NOSYNC is used, but it
-           still fsync()s sometimes. */
-        db_env_set_func_fsync(my_fsync);
+    /* Create the database environment object. */
+    DbEnv * env = 0; /* !!! close on error */
+    env = new DbEnv(0);
+    
+    /* Smaller log files. */
+    env->set_lg_bsize(32 * 1024); /* default */
+    env->set_lg_max(256 * 1024); /* must be > 4 * lg_bsize */
+    
+    /* Write the log, but don't sync.  This protects transactions
+       against application crashes, but if the system crashes, some
+       transactions may be undone.  An acceptable risk, I think. */
+    env->set_flags(DB_TXN_WRITE_NOSYNC | DB_LOG_AUTOREMOVE, 1);
+    
+    /* Increase the locking limits.  If you ever get `Dbc::get: Cannot
+       allocate memory' or similar, especially while running
+       `nix-store --verify', just increase the following number, then
+       run db_recover on the database to remove the existing DB
+       environment (since changes only take effect on new
+       environments). */
+    env->set_lk_max_locks(10000);
+    env->set_lk_max_lockers(10000);
+    env->set_lk_max_objects(10000);
+    env->set_lk_detect(DB_LOCK_DEFAULT);
+    
+    /* Dangerous, probably, but from the docs it *seems* that BDB
+       shouldn't sync when DB_TXN_WRITE_NOSYNC is used, but it still
+       fsync()s sometimes. */
+    db_env_set_func_fsync(my_fsync);
         
 
-        /* The following code provides automatic recovery of the
-           database environment.  Recovery is necessary when a process
-           dies while it has the database open.  To detect this,
-           processes atomically increment a counter when they open the
-           database, and decrement it when they close it.  If we see
-           that counter is > 0 but no processes are accessing the
-           database---determined by attempting to obtain a write lock
-           on a lock file on which all accessors have a read lock---we
-           must run recovery.  Note that this also ensures that we
-           only run recovery when there are no other accessors (which
-           could cause database corruption). */
+    /* The following code provides automatic recovery of the database
+       environment.  Recovery is necessary when a process dies while
+       it has the database open.  To detect this, processes atomically
+       increment a counter when they open the database, and decrement
+       it when they close it.  If we see that counter is > 0 but no
+       processes are accessing the database---determined by attempting
+       to obtain a write lock on a lock file on which all accessors
+       have a read lock---we must run recovery.  Note that this also
+       ensures that we only run recovery when there are no other
+       accessors (which could cause database corruption). */
 
-        /* !!! close fdAccessors / fdLock on exception */
+    /* !!! close fdAccessors / fdLock on exception */
 
-        /* Open the accessor count file. */
-        string accessorsPath = path + "/accessor_count";
-        fdAccessors = ::open(accessorsPath.c_str(), O_RDWR | O_CREAT, 0666);
-        if (fdAccessors == -1)
-            if (errno == EACCES)
-                throw DbNoPermission(
-                    format("permission denied to database in `%1%'") % accessorsPath);
-            else
-                throw SysError(format("opening file `%1%'") % accessorsPath);
-
-        /* Open the lock file. */
-        string lockPath = path + "/access_lock";
-        fdLock = ::open(lockPath.c_str(), O_RDWR | O_CREAT, 0666);
-        if (fdLock == -1)
-            throw SysError(format("opening lock file `%1%'") % lockPath);
-
-        /* Try to acquire a write lock. */
-        debug(format("attempting write lock on `%1%'") % lockPath);
-        if (lockFile(fdLock, ltWrite, false)) { /* don't wait */
-
-            debug(format("write lock granted"));
-
-            /* We have a write lock, which means that there are no
-               other readers or writers. */
-
-            int n = getAccessorCount(fdAccessors);
-
-            if (n != 0) {
-                printMsg(lvlTalkative,
-                    format("accessor count is %1%, running recovery") % n);
-
-                /* Open the environment after running recovery. */
-                openEnv(env, path, DB_RECOVER);
-            }
+    /* Open the accessor count file. */
+    string accessorsPath = path + "/accessor_count";
+    fdAccessors = ::open(accessorsPath.c_str(), O_RDWR | O_CREAT, 0666);
+    if (fdAccessors == -1)
+        if (errno == EACCES)
+            throw DbNoPermission(
+                format("permission denied to database in `%1%'") % accessorsPath);
+        else    
+            throw SysError(format("opening file `%1%'") % accessorsPath);
+    
+    /* Open the lock file. */
+    string lockPath = path + "/access_lock";
+    fdLock = ::open(lockPath.c_str(), O_RDWR | O_CREAT, 0666);
+    if (fdLock == -1)
+        throw SysError(format("opening lock file `%1%'") % lockPath);
+    
+    /* Try to acquire a write lock. */
+    debug(format("attempting write lock on `%1%'") % lockPath);
+    if (lockFile(fdLock, ltWrite, false)) { /* don't wait */
+        
+        debug(format("write lock granted"));
+        
+        /* We have a write lock, which means that there are no other
+           readers or writers. */
+        
+        if (removeOldEnv) {
+            printMsg(lvlError, "removing old Berkeley DB database environment...");
+            env->remove(path.c_str(), DB_FORCE);
+            return;
+        }
+        
+        int n = getAccessorCount(fdAccessors);
+        
+        if (n != 0) {
+            printMsg(lvlTalkative,
+                format("accessor count is %1%, running recovery") % n);
             
-            else 
-                /* Open the environment normally. */
-                openEnv(env, path, 0);
-
-            setAccessorCount(fdAccessors, 1);
-
-            /* Downgrade to a read lock. */
-            debug(format("downgrading to read lock on `%1%'") % lockPath);
-            lockFile(fdLock, ltRead, true);
-
-        } else {
-            /* There are other accessors. */ 
-            debug(format("write lock refused"));
-
-            /* Acquire a read lock. */
-            debug(format("acquiring read lock on `%1%'") % lockPath);
-            lockFile(fdLock, ltRead, true); /* wait indefinitely */
-
-            /* Increment the accessor count. */
-            lockFile(fdAccessors, ltWrite, true);
-            int n = getAccessorCount(fdAccessors) + 1;
-            setAccessorCount(fdAccessors, n);
-            debug(format("incremented accessor count to %1%") % n);
-            lockFile(fdAccessors, ltNone, true);
-
+            /* Open the environment after running recovery. */
+            openEnv(env, path, DB_RECOVER);
+        }
+            
+        else 
             /* Open the environment normally. */
             openEnv(env, path, 0);
-        }
 
-        this->env = env;
+        setAccessorCount(fdAccessors, 1);
+
+        /* Downgrade to a read lock. */
+        debug(format("downgrading to read lock on `%1%'") % lockPath);
+        lockFile(fdLock, ltRead, true);
+
+    } else {
+        /* There are other accessors. */ 
+        debug(format("write lock refused"));
+
+        /* Acquire a read lock. */
+        debug(format("acquiring read lock on `%1%'") % lockPath);
+        lockFile(fdLock, ltRead, true); /* wait indefinitely */
         
-    } catch (DbException e) { rethrow(e); }
+        /* Increment the accessor count. */
+        lockFile(fdAccessors, ltWrite, true);
+        int n = getAccessorCount(fdAccessors) + 1;
+        setAccessorCount(fdAccessors, n);
+        debug(format("incremented accessor count to %1%") % n);
+        lockFile(fdAccessors, ltNone, true);
+        
+        /* Open the environment normally. */
+        openEnv(env, path, 0);
+    }
+    
+    this->env = env;
+}       
+
+
+void Database::open(const string & path)
+{
+    try {
+
+        open2(path, false);
+        
+    } catch (DbException e) {
+        
+        if (e.get_errno() == DB_VERSION_MISMATCH) {
+            /* Remove the environment while we are holding the global
+               lock.  If things go wrong there, we bail out.  !!!
+               there is some leakage here op DbEnv and lock
+               handles. */
+            open2(path, true);
+
+            /* Try again. */
+            open2(path, false);
+        }
+        else
+            rethrow(e);
+    }
+    
 }
 
 

src/libstore/db.hh

@@ -58,6 +58,8 @@ private:
 
     Db * getDb(TableId table);
 
+    void open2(const string & path, bool removeOldEnv);
+    
 public:
     Database();
     ~Database();

src/libstore/derivations.cc

@@ -62,12 +62,12 @@ Derivation parseDerivation(ATerm t)
         throwBadDrv(t);
 
     for (ATermIterator i(outs); i; ++i) {
-        ATerm id, path, hashAlgo, hash;
-        if (!matchDerivationOutput(*i, id, path, hashAlgo, hash))
+        ATerm id, eqClass, hashAlgo, hash;
+        if (!matchDerivationOutput(*i, id, eqClass, hashAlgo, hash))
             throwBadDrv(t);
         DerivationOutput out;
-        out.path = aterm2String(path);
-        checkPath(out.path);
+        out.eqClass = aterm2String(eqClass);
+        // !!!        checkPath(out.path);
         out.hashAlgo = aterm2String(hashAlgo);
         out.hash = aterm2String(hash);
         drv.outputs[aterm2String(id)] = out;
@@ -125,7 +125,7 @@ ATerm unparseDerivation(const Derivation & drv)
         outputs = ATinsert(outputs,
             makeDerivationOutput(
                 toATerm(i->first),
-                toATerm(i->second.path),
+                toATerm(i->second.eqClass),
                 toATerm(i->second.hashAlgo),
                 toATerm(i->second.hash)));
 

src/libstore/derivations.hh

@@ -13,15 +13,15 @@ const string drvExtension = ".drv";
 
 struct DerivationOutput
 {
-    Path path;
+    OutputEqClass eqClass;
     string hashAlgo; /* hash used for expected hash computation */
     string hash; /* expected hash, may be null */
     DerivationOutput()
     {
     }
-    DerivationOutput(Path path, string hashAlgo, string hash)
+    DerivationOutput(OutputEqClass eqClass, string hashAlgo, string hash)
     {
-        this->path = path;
+        this->eqClass = eqClass;
         this->hashAlgo = hashAlgo;
         this->hash = hash;
     }

src/libstore/gc.cc

@@ -2,6 +2,7 @@
 #include "gc.hh"
 #include "build.hh"
 #include "pathlocks.hh"
+#include "misc.hh"
 
 #include <boost/shared_ptr.hpp>
 
@@ -249,7 +250,12 @@ static void findRoots(const Path & path, bool recurseSymlinks,
         if (isInStore(target2)) {
             debug(format("found root `%1%' in `%2%'")
                 % target2 % path);
-            roots.insert(toStorePath(target2));
+            Path target3 = toStorePath(target2);
+            if (isValidPath(target3)) 
+                roots.insert(target3);
+            else
+                printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'")
+                    % path % target3);
         }
 
         else if (recurseSymlinks) {
@@ -333,12 +339,14 @@ void collectGarbage(GCAction action, PathSet & result)
         for (PathSet::iterator i = livePaths.begin();
              i != livePaths.end(); ++i)
         {
+#if 0            
             /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
             Path deriver = queryDeriver(noTxn, *i);
             if (deriver != "" && isValidPath(deriver))
                 computeFSClosure(deriver, livePaths);
+#endif            
         }
     }
 
@@ -348,10 +356,13 @@ void collectGarbage(GCAction action, PathSet & result)
              i != livePaths.end(); ++i)
             if (isDerivation(*i)) {
                 Derivation drv = derivationFromPath(*i);
+                assert(0);
+#if 0                
                 for (DerivationOutputs::iterator j = drv.outputs.begin();
                      j != drv.outputs.end(); ++j)
                     if (isValidPath(j->second.path))
                         computeFSClosure(j->second.path, livePaths);
+#endif
             }
     }
 

src/libstore/globals.cc

@@ -22,6 +22,8 @@ unsigned int maxBuildJobs = 1;
 
 bool readOnlyMode = false;
 
+string currentTrustId;
+
 
 static bool settingsRead = false;
 

src/libstore/globals.hh

@@ -52,6 +52,10 @@ extern unsigned int maxBuildJobs;
    database. */
 extern bool readOnlyMode;
 
+/* Current trust ID.  !!! Of course, this shouldn't be a global
+   variable. */
+extern string currentTrustId;
+
 
 string querySetting(const string & name, const string & def);
 

src/libstore/misc.cc

@@ -1,4 +1,6 @@
 #include "build.hh"
+#include "misc.hh"
+#include "globals.hh"
 
 
 Derivation derivationFromPath(const Path & drvPath)
@@ -29,10 +31,224 @@ void computeFSClosure(const Path & storePath,
 }
 
 
- Path findOutput(const Derivation & drv, string id)
+OutputEqClass findOutputEqClass(const Derivation & drv, const string & id)
 {
-    for (DerivationOutputs::const_iterator i = drv.outputs.begin();
-         i != drv.outputs.end(); ++i)
-        if (i->first == id) return i->second.path;
-    throw Error(format("derivation has no output `%1%'") % id);
+    DerivationOutputs::const_iterator i = drv.outputs.find(id);
+    if (i == drv.outputs.end())
+        throw Error(format("derivation has no output `%1%'") % id);
+    return i->second.eqClass;
+}
+
+
+PathSet findTrustedEqClassMembers(const OutputEqClass & eqClass,
+    const TrustId & trustId)
+{
+    OutputEqMembers members;
+    queryOutputEqMembers(noTxn, eqClass, members);
+
+    PathSet result;
+    for (OutputEqMembers::iterator j = members.begin(); j != members.end(); ++j)
+        if (j->trustId == trustId || j->trustId == "root") result.insert(j->path);
+
+    return result;
+}
+
+
+Path findTrustedEqClassMember(const OutputEqClass & eqClass,
+    const TrustId & trustId)
+{
+    PathSet paths = findTrustedEqClassMembers(eqClass, trustId);
+    if (paths.size() == 0)
+        throw Error(format("no output path in equivalence class `%1%' is known") % eqClass);
+    return *(paths.begin());
+}
+
+
+typedef map<OutputEqClass, PathSet> ClassMap;
+typedef map<OutputEqClass, Path> FinalClassMap;
+
+
+static void findBestRewrite(const ClassMap::const_iterator & pos,
+    const ClassMap::const_iterator & end,
+    const PathSet & selection, const PathSet & unselection,
+    unsigned int & bestCost, PathSet & bestSelection)
+{
+    if (pos != end) {
+        for (PathSet::iterator i = pos->second.begin();
+             i != pos->second.end(); ++i)
+        {
+            PathSet selection2(selection);
+            selection2.insert(*i);
+            
+            PathSet unselection2(unselection);
+            for (PathSet::iterator j = pos->second.begin();
+                 j != pos->second.end(); ++j)
+                if (i != j) unselection2.insert(*j);
+            
+            ClassMap::const_iterator j = pos; ++j;
+            findBestRewrite(j, end, selection2, unselection2,
+                bestCost, bestSelection);
+        }
+        return;
+    }
+
+    PathSet badPaths;
+    for (PathSet::iterator i = selection.begin();
+         i != selection.end(); ++i)
+    {
+        PathSet closure;
+        computeFSClosure(*i, closure); 
+        for (PathSet::iterator j = closure.begin();
+             j != closure.end(); ++j)
+            if (unselection.find(*j) != unselection.end())
+                badPaths.insert(*i);
+    }
+    
+    //    printMsg(lvlError, format("cost %1% %2%") % badPaths.size() % showPaths(badPaths));
+
+    if (badPaths.size() < bestCost) {
+        bestCost = badPaths.size();
+        bestSelection = selection;
+    }
+}
+
+
+static Path maybeRewrite(const Path & path, const PathSet & selection,
+    const FinalClassMap & finalClassMap, const PathSet & sources,
+    Replacements & replacements,
+    unsigned int & nrRewrites)
+{
+    startNest(nest, lvlError, format("considering rewriting `%1%'") % path);
+
+    assert(selection.find(path) != selection.end());
+
+    if (replacements.find(path) != replacements.end()) return replacements[path];
+    
+    PathSet references;
+    queryReferences(noTxn, path, references);
+
+    HashRewrites rewrites;
+    PathSet newReferences;
+    
+    for (PathSet::iterator i = references.begin(); i != references.end(); ++i) {
+
+        if (*i == path || sources.find(*i) != sources.end()) {
+            newReferences.insert(*i);
+            continue; /* ignore self-references */
+        }
+
+        OutputEqClasses classes;
+        queryOutputEqClasses(noTxn, *i, classes);
+        assert(classes.size() > 0);
+
+        FinalClassMap::const_iterator j = finalClassMap.find(*(classes.begin()));
+        assert(j != finalClassMap.end());
+        
+        Path newPath = maybeRewrite(j->second, selection,
+            finalClassMap, sources, replacements, nrRewrites);
+
+        if (*i != newPath)
+            rewrites[hashPartOf(*i)] = hashPartOf(newPath);
+
+        newReferences.insert(newPath);
+    }
+
+    if (rewrites.size() == 0) {
+        replacements[path] = path;
+        return path;
+    }
+
+    printMsg(lvlError, format("rewriting `%1%'") % path);
+
+    Path newPath = addToStore(path,
+        hashPartOf(path), namePartOf(path),
+        references, rewrites);
+
+    /* !!! we don't know which eqClass `path' is in!  That is to say,
+       we don't know which one is intended here. */
+    OutputEqClasses classes;
+    queryOutputEqClasses(noTxn, path, classes);
+    for (PathSet::iterator i = classes.begin(); i != classes.end(); ++i) {
+        Transaction txn;
+        createStoreTransaction(txn);
+        addOutputEqMember(txn, *i, currentTrustId, newPath);
+        txn.commit();
+    }
+
+    nrRewrites++;
+
+    printMsg(lvlError, format("rewrote `%1%' to `%2%'") % path % newPath);
+
+    replacements[path] = newPath;
+        
+    return newPath;
+}
+
+
+PathSet consolidatePaths(const PathSet & paths, bool checkOnly,
+    Replacements & replacements)
+{
+    printMsg(lvlError, format("consolidating"));
+    
+    ClassMap classMap;
+    PathSet sources;
+    
+    for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i) {
+        OutputEqClasses classes;
+        queryOutputEqClasses(noTxn, *i, classes);
+
+        if (classes.size() == 0)
+            sources.insert(*i);
+        else
+            for (OutputEqClasses::iterator j = classes.begin(); j != classes.end(); ++j)
+                classMap[*j].insert(*i);
+    }
+
+    printMsg(lvlError, format("found %1% sources %2%") % sources.size() % showPaths(sources));
+
+    bool conflict = false;
+    for (ClassMap::iterator i = classMap.begin(); i != classMap.end(); ++i)
+        if (i->second.size() >= 2) {
+            printMsg(lvlError, format("conflict in eq class `%1%'") % i->first);
+            conflict = true;
+        }
+
+    if (!conflict) return paths;
+    
+    assert(!checkOnly);
+
+    
+    /* !!! exponential-time algorithm! */
+    const unsigned int infinity = 1000000;
+    unsigned int bestCost = infinity;
+    PathSet bestSelection;
+    findBestRewrite(classMap.begin(), classMap.end(),
+        PathSet(), PathSet(), bestCost, bestSelection);
+
+    assert(bestCost != infinity);
+
+    printMsg(lvlError, format("cheapest selection %1% %2%")
+        % bestCost % showPaths(bestSelection));
+
+    FinalClassMap finalClassMap;
+    for (ClassMap::iterator i = classMap.begin(); i != classMap.end(); ++i)
+        for (PathSet::const_iterator j = i->second.begin(); j != i->second.end(); ++j)
+            if (bestSelection.find(*j) != bestSelection.end())
+                finalClassMap[i->first] = *j;
+
+    PathSet newPaths;
+    unsigned int nrRewrites = 0;
+    replacements.clear();
+    for (PathSet::iterator i = bestSelection.begin();
+         i != bestSelection.end(); ++i)
+        newPaths.insert(maybeRewrite(*i, bestSelection, finalClassMap,
+                            sources, replacements, nrRewrites));
+
+    newPaths.insert(sources.begin(), sources.end());
+
+    assert(nrRewrites == bestCost);
+
+    assert(newPaths.size() < paths.size());
+
+    return newPaths;
 }

src/libstore/misc.hh

@@ -0,0 +1,47 @@
+#ifndef __MISC_H
+#define __MISC_H
+
+#include "derivations.hh"
+#include "store.hh"
+
+
+/* Read a derivation, after ensuring its existence through
+   ensurePath(). */
+Derivation derivationFromPath(const Path & drvPath);
+
+
+/* Place in `paths' the set of all store paths in the file system
+   closure of `storePath'; that is, all paths than can be directly or
+   indirectly reached from it.  `paths' is not cleared.  If
+   `flipDirection' is true, the set of paths that can reach
+   `storePath' is returned; that is, the closures under the `referers'
+   relation instead of the `references' relation is returned. */
+void computeFSClosure(const Path & storePath,
+    PathSet & paths, bool flipDirection = false);
+
+
+/* Return the output equivalence class denoted by `id' in the
+   derivation `drv'. */
+OutputEqClass findOutputEqClass(const Derivation & drv,
+    const string & id);
+
+
+/* Return anll trusted path (wrt to the given trust ID) in the given
+   output path equivalence class, or an empty set if no such paths
+   currently exist. */
+PathSet findTrustedEqClassMembers(const OutputEqClass & eqClass,
+    const TrustId & trustId);
+
+/* Like `findTrustedEqClassMembers', but returns an arbitrary trusted
+   path, or throws an exception if no such path currently exists. */
+Path findTrustedEqClassMember(const OutputEqClass & eqClass,
+    const TrustId & trustId);
+
+
+typedef map<Path, Path> Replacements;
+
+PathSet consolidatePaths(const PathSet & paths, bool checkOnly,
+    Replacements & replacements);
+
+
+#endif /* !__MISC_H */

src/libstore/store.cc

@@ -39,6 +39,21 @@ static TableId dbReferences = 0;
    This table is just the reverse mapping of dbReferences. */
 static TableId dbReferers = 0;
 
+/* dbEquivalences :: OutputEqClass -> [(TrustId, Path)]
+
+   Lists the output paths that have been produced for each extension
+   class; i.e., the extension of an extension class. */
+static TableId dbEquivalences = 0;
+
+/* dbEquivalenceClasses :: Path -> [OutputEqClass]
+
+   !!! should be [(TrustId, OutputEqClass)] ?
+
+   Lists for each output path the extension classes that it is in. */
+static TableId dbEquivalenceClasses = 0;
+
+
+#if 0
 /* dbSubstitutes :: Path -> [[Path]]
 
    Each pair $(p, subs)$ tells Nix that it can use any of the
@@ -61,13 +76,16 @@ static TableId dbSubstitutes = 0;
    only be multiple such paths for fixed-output derivations (i.e.,
    derivations specifying an expected hash). */
 static TableId dbDerivers = 0;
+#endif
 
 
-bool Substitute::operator == (const Substitute & sub)
+#if 0
+bool Substitute::operator == (const Substitute & sub) const
 {
     return program == sub.program
         && args == sub.args;
 }
+#endif
 
 
 static void upgradeStore();
@@ -87,8 +105,12 @@ void openDB()
     dbValidPaths = nixDB.openTable("validpaths");
     dbReferences = nixDB.openTable("references");
     dbReferers = nixDB.openTable("referers");
+#if 0    
     dbSubstitutes = nixDB.openTable("substitutes");
     dbDerivers = nixDB.openTable("derivers");
+#endif
+    dbEquivalences = nixDB.openTable("equivalences");
+    dbEquivalenceClasses = nixDB.openTable("equivalence-classes");
 
     int curSchema = 0;
     Path schemaFN = nixDBPath + "/schema";
@@ -121,6 +143,60 @@ void createStoreTransaction(Transaction & txn)
 }
 
 
+
+/* Path hashes. */
+
+const unsigned int pathHashLen = 32; /* characters */
+const string nullPathHashRef(pathHashLen, 0);
+
+
+PathHash::PathHash()
+{
+    rep = nullPathHashRef;
+}
+
+
+PathHash::PathHash(const Hash & h)
+{
+    assert(h.type == htSHA256);
+    rep = printHash32(compressHash(h, 20));
+}
+
+
+PathHash::PathHash(const string & h)
+{
+    /* !!! hacky; check whether this is a valid 160 bit hash */
+    assert(h.size() == pathHashLen);
+    parseHash32(htSHA1, h); 
+    rep = h;
+}
+
+
+string PathHash::toString() const
+{
+    return rep;
+}
+
+
+bool PathHash::isNull() const
+{
+    return rep == nullPathHashRef;
+}
+
+
+bool PathHash::operator ==(const PathHash & hash2) const
+{
+    return rep == hash2.rep;
+}
+
+
+bool PathHash::operator <(const PathHash & hash2) const
+{
+    return rep < hash2.rep;
+}
+
+
+
 /* Path copying. */
 
 struct CopySink : DumpSink
@@ -167,12 +243,14 @@ void copyPath(const Path & src, const Path & dst)
 }
 
 
+
 bool isInStore(const Path & path)
 {
     return path[0] == '/'
-        && path.compare(0, nixStore.size(), nixStore) == 0
+        && string(path, 0, nixStore.size()) == nixStore
         && path.size() >= nixStore.size() + 2
-        && path[nixStore.size()] == '/';
+        && path[nixStore.size()] == '/'
+        && path[nixStore.size() + 1 + pathHashLen] == '-';
 }
 
 
@@ -202,6 +280,20 @@ Path toStorePath(const Path & path)
 }
 
 
+PathHash hashPartOf(const Path & path)
+{
+    assertStorePath(path);
+    return PathHash(string(path, nixStore.size() + 1, pathHashLen));
+}
+
+
+string namePartOf(const Path & path)
+{
+    assertStorePath(path);
+    return string(path, nixStore.size() + 1 + pathHashLen + 1);
+}
+
+
 void checkStoreName(const string & name)
 {
     string validChars = "+-._?=";
@@ -275,14 +367,16 @@ bool isValidPath(const Path & path)
 }
 
 
+#if 0
 static Substitutes readSubstitutes(const Transaction & txn,
     const Path & srcPath);
+#endif
 
 
 static bool isRealisablePath(const Transaction & txn, const Path & path)
 {
     return isValidPathTxn(txn, path)
-        || readSubstitutes(txn, path).size() > 0;
+        /* !!! || readSubstitutes(txn, path).size() > 0 */;
 }
 
 
@@ -356,6 +450,73 @@ void queryReferers(const Transaction & txn,
 }
 
 
+void addOutputEqMember(const Transaction & txn,
+    const OutputEqClass & eqClass, const TrustId & trustId,
+    const Path & path)
+{
+    OutputEqMembers members;
+    queryOutputEqMembers(txn, eqClass, members);
+
+    for (OutputEqMembers::iterator i = members.begin();
+         i != members.end(); ++i)
+        if (i->trustId == trustId && i->path == path) return;
+    
+    OutputEqMember member;
+    member.trustId = trustId;
+    member.path = path;
+    members.push_back(member);
+
+    Strings ss;
+    
+    for (OutputEqMembers::iterator i = members.begin();
+         i != members.end(); ++i)
+    {
+        Strings ss2;
+        ss2.push_back(i->trustId);
+        ss2.push_back(i->path);
+        ss.push_back(packStrings(ss2));
+    }
+
+    nixDB.setStrings(txn, dbEquivalences, eqClass, ss);
+
+    OutputEqClasses classes;
+    queryOutputEqClasses(txn, path, classes);
+
+    classes.insert(eqClass);
+
+    nixDB.setStrings(txn, dbEquivalenceClasses, path,
+        Strings(classes.begin(), classes.end()));
+}
+
+
+void queryOutputEqMembers(const Transaction & txn,
+    const OutputEqClass & eqClass, OutputEqMembers & members)
+{
+    Strings ss;
+    nixDB.queryStrings(txn, dbEquivalences, eqClass, ss);
+
+    for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
+        Strings ss2 = unpackStrings(*i);
+        if (ss2.size() != 2) continue;
+        Strings::iterator j = ss2.begin();
+        OutputEqMember member;
+        member.trustId = *j++;
+        member.path = *j++;
+        members.push_back(member);
+    }
+}
+
+
+void queryOutputEqClasses(const Transaction & txn,
+    const Path & path, OutputEqClasses & classes)
+{
+    Strings ss;
+    nixDB.queryStrings(txn, dbEquivalenceClasses, path, ss);
+    classes.insert(ss.begin(), ss.end());
+}
+
+
+#if 0
 void setDeriver(const Transaction & txn, const Path & storePath,
     const Path & deriver)
 {
@@ -378,8 +539,10 @@ Path queryDeriver(const Transaction & txn, const Path & storePath)
     else
         return "";
 }
+#endif
 
 
+#if 0
 const int substituteVersion = 2;
 
 
@@ -488,6 +651,7 @@ void clearSubstitutes()
     
     txn.commit();
 }
+#endif
 
 
 static void setHash(const Transaction & txn, const Path & storePath,
@@ -563,7 +727,9 @@ void registerValidPaths(const Transaction & txn,
                 throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid")
                     % i->path % *j);
 
+#if 0
         setDeriver(txn, i->path, i->deriver);
+#endif
     }
 }
 
@@ -578,30 +744,39 @@ static void invalidatePath(Transaction & txn, const Path & path)
        inverse `referers' entries, and the `derivers' entry; but only
        if there are no substitutes for this path.  This maintains the
        cleanup invariant. */
-    if (querySubstitutes(txn, path).size() == 0) {
+    if (1 /*querySubstitutes(txn, path).size() == 0 !!! */) {
         setReferences(txn, path, PathSet());
-        nixDB.delPair(txn, dbDerivers, path);
+        // !!!        nixDB.delPair(txn, dbDerivers, path);
     }
     
     nixDB.delPair(txn, dbValidPaths, path);
 }
 
 
-Path makeStorePath(const string & type,
-    const Hash & hash, const string & suffix)
+void makeStorePath(const Hash & contentHash, const string & suffix,
+    Path & path, PathHash & pathHash)
 {
-    /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
-    string s = type + ":sha256:" + printHash(hash) + ":"
-        + nixStore + ":" + suffix;
-
     checkStoreName(suffix);
 
-    return nixStore + "/"
-        + printHash32(compressHash(hashString(htSHA256, s), 20))
-        + "-" + suffix;
+    /* e.g., "sha256:1abc...:foo.tar.gz" */
+    string s = "sha256:" + printHash(contentHash) + ":" + suffix;
+
+    pathHash = PathHash(hashString(htSHA256, s));
+    
+    path = nixStore + "/" + pathHash.toString() + "-" + suffix;
 }
 
 
+Path makeRandomStorePath(const string & suffix)
+{
+    Hash hash(htSHA256);
+    for (unsigned int i = 0; i < hash.hashSize; ++i)
+        hash.hash[i] = rand() % 256; // !!! improve
+    return nixStore + "/" + PathHash(hash).toString() + "-" + suffix;
+}
+
+
+#if 0
 Path makeFixedOutputPath(bool recursive,
     string hashAlgo, Hash hash, string name)
 {
@@ -612,39 +787,101 @@ Path makeFixedOutputPath(bool recursive,
         + "");
     return makeStorePath("output:out", h, name);
 }
+#endif
 
 
-static Path _addToStore(bool fixed, bool recursive,
-    string hashAlgo, const Path & _srcPath)
+typedef map<PathHash, PathHash> HashRewrites;
+
+string rewriteHashes(string s, const HashRewrites & rewrites,
+    vector<int> & positions)
 {
-    Path srcPath(absPath(_srcPath));
-    debug(format("adding `%1%' to the store") % srcPath);
-
-    Hash h(htSHA256);
+    for (HashRewrites::const_iterator i = rewrites.begin();
+         i != rewrites.end(); ++i)
     {
-        SwitchToOriginalUser sw;
-        h = hashPath(htSHA256, srcPath);
-    }
+        string from = i->first.toString(), to = i->second.toString();
 
-    string baseName = baseNameOf(srcPath);
+        assert(from.size() == to.size());
 
-    Path dstPath;
-    
-    if (fixed) {
-
-        HashType ht(parseHashType(hashAlgo));
-        Hash h2(ht);
-        {
-            SwitchToOriginalUser sw;
-            h2 = recursive ? hashPath(ht, srcPath) : hashFile(ht, srcPath);
+        unsigned int j = 0;
+        while ((j = s.find(from, j)) != string::npos) {
+            debug(format("rewriting @ %1%") % j);
+            positions.push_back(j);
+            s.replace(j, to.size(), to);
+            j += to.size();
         }
-        
-        dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
     }
-        
-    else dstPath = makeStorePath("source", h, baseName);
 
-    addTempRoot(dstPath);
+    return s;
+}
+
+
+string rewriteHashes(const string & s, const HashRewrites & rewrites)
+{
+    vector<int> dummy;
+    return rewriteHashes(s, rewrites, dummy);
+}
+
+
+static Hash hashModulo(string s, const PathHash & modulus)
+{
+    vector<int> positions;
+    
+    if (!modulus.isNull()) {
+        /* Zero out occurences of `modulus'. */
+        HashRewrites rewrites;
+        rewrites[modulus] = PathHash(); /* = null hash */
+        s = rewriteHashes(s, rewrites, positions);
+    }
+
+    string positionPrefix;
+    
+    for (vector<int>::iterator i = positions.begin();
+         i != positions.end(); ++i)
+        positionPrefix += (format("|%1%") % *i).str();
+
+    positionPrefix += "||";
+
+    debug(format("positions %1%") % positionPrefix);
+    
+    return hashString(htSHA256, positionPrefix + s);
+}
+
+
+static PathSet rewriteReferences(const PathSet & references,
+    const HashRewrites & rewrites)
+{
+    PathSet result;
+    for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i)
+        result.insert(rewriteHashes(*i, rewrites));
+    return result;
+}
+
+
+static Path _addToStore(const string & suffix, string dump,
+    const PathHash & selfHash, const PathSet & references)
+{
+    /* Hash the contents, modulo the previous hash reference (if it
+       had one). */
+    Hash contentHash = hashModulo(dump, selfHash);
+
+    /* Construct the new store path. */ 
+    Path dstPath;
+    PathHash pathHash;
+    makeStorePath(contentHash, suffix, dstPath, pathHash);
+
+    /* If the contents had a previous hash reference, rewrite those
+       references to the new hash. */
+    HashRewrites rewrites;
+    if (!selfHash.isNull()) {
+        rewrites[selfHash] = pathHash;
+        vector<int> positions;
+        dump = rewriteHashes(dump, rewrites, positions);
+        /* !!! debug code, remove */
+        PathHash contentHash2 = hashModulo(dump, pathHash);
+        assert(contentHash2 == contentHash);
+    }
+
+    if (!readOnlyMode) addTempRoot(dstPath);
 
     if (!readOnlyMode && !isValidPath(dstPath)) { 
 
@@ -659,17 +896,18 @@ static Path _addToStore(bool fixed, bool recursive,
 
             if (pathExists(dstPath)) deletePath(dstPath);
 
-            copyPath(srcPath, dstPath);
-
-            Hash h2 = hashPath(htSHA256, dstPath);
-            if (h != h2)
-                throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
-                    % srcPath % dstPath % printHash(h) % printHash(h2));
+            CopySource source(dump);
+            restorePath(dstPath, source);
 
             canonicalisePathMetaData(dstPath);
+
+            /* Set the references for the new path.  Of course, any
+               hash rewrites have to be applied to the references,
+               too. */
+            PathSet references2 = rewriteReferences(references, rewrites);
             
             Transaction txn(nixDB);
-            registerValidPath(txn, dstPath, h, PathSet(), "");
+            registerValidPath(txn, dstPath, contentHash, references2, "");
             txn.commit();
         }
 
@@ -680,51 +918,41 @@ static Path _addToStore(bool fixed, bool recursive,
 }
 
 
-Path addToStore(const Path & srcPath)
+Path addToStore(const Path & _srcPath, const PathHash & selfHash,
+    const string & suffix, const PathSet & references, const HashRewrites & rewrites)
 {
-    return _addToStore(false, false, "", srcPath);
+    Path srcPath(absPath(_srcPath));
+    debug(format("adding `%1%' to the store") % srcPath);
+
+    CopySink sink;
+    {
+        SwitchToOriginalUser sw;
+        dumpPath(srcPath, sink);
+    }
+
+    if (rewrites.size() != 0) sink.s = rewriteHashes(sink.s, rewrites);
+
+    return _addToStore(suffix == "" ? baseNameOf(srcPath) : suffix,
+        sink.s, selfHash,
+        rewriteReferences(references, rewrites));
 }
 
 
+#if 0
 Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath)
 {
     return _addToStore(true, recursive, hashAlgo, srcPath);
 }
+#endif
 
 
 Path addTextToStore(const string & suffix, const string & s,
     const PathSet & references)
 {
-    Hash hash = hashString(htSHA256, s);
-
-    Path dstPath = makeStorePath("text", hash, suffix);
+    CopySink sink;
+    makeSingletonArchive(s, sink);
     
-    addTempRoot(dstPath);
-
-    if (!readOnlyMode && !isValidPath(dstPath)) {
-
-        PathSet lockPaths;
-        lockPaths.insert(dstPath);
-        PathLocks outputLock(lockPaths);
-
-        if (!isValidPath(dstPath)) {
-
-            if (pathExists(dstPath)) deletePath(dstPath);
-
-            writeStringToFile(dstPath, s);
-
-            canonicalisePathMetaData(dstPath);
-            
-            Transaction txn(nixDB);
-            registerValidPath(txn, dstPath,
-                hashPath(htSHA256, dstPath), references, "");
-            txn.commit();
-        }
-
-        outputLock.setDeletion(true);
-    }
-
-    return dstPath;
+    return _addToStore(suffix, sink.s, PathHash(), references);
 }
 
 
@@ -737,10 +965,10 @@ void deleteFromStore(const Path & _path)
     Transaction txn(nixDB);
     if (isValidPathTxn(txn, path)) {
         PathSet referers = getReferers(txn, path);
-        if (referers.size() > 1 ||
-            (referers.size() == 1 &&
-                *referers.begin() != path))
-            throw Error(format("cannot delete path `%1%' because it is in use") % path);
+        for (PathSet::iterator i = referers.begin();
+             i != referers.end(); ++i)
+            if (*i != path && isValidPathTxn(txn, *i))
+                throw Error(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i);
         invalidatePath(txn, path);
     }
     txn.commit();
@@ -751,6 +979,7 @@ void deleteFromStore(const Path & _path)
 
 void verifyStore(bool checkContents)
 {
+#if 0    
     Transaction txn(nixDB);
 
     Paths paths;
@@ -891,6 +1120,7 @@ void verifyStore(bool checkContents)
     }
 
     txn.commit();
+#endif    
 }
 
 
@@ -902,6 +1132,7 @@ void verifyStore(bool checkContents)
 static void upgradeStore()
 {
     printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
+#if 0    
 
     Transaction txn(nixDB);
 
@@ -982,4 +1213,5 @@ static void upgradeStore()
 
     /* !!! maybe this transaction is way too big */
     txn.commit();
+#endif    
 }

src/libstore/store.hh

@@ -12,6 +12,27 @@ using namespace std;
 const int nixSchemaVersion = 2;
 
 
+/* Path hashes are the hash components of store paths, e.g., the
+   `zvhgns772jpj68l40mq1jb74wpfsf0ma' in
+   `/nix/store/zvhgns772jpj68l40mq1jb74wpfsf0ma-glibc'.  These are
+   truncated SHA-256 hashes of the path contents,  */
+struct PathHash
+{
+private:
+    string rep;
+public:
+    PathHash();
+    PathHash(const Hash & h);
+    PathHash(const string & h);
+    string toString() const;
+    bool PathHash::isNull() const;
+    bool operator ==(const PathHash & hash2) const;
+    bool operator <(const PathHash & hash2) const;
+};
+
+
+
+#if 0
 /* A substitute is a program invocation that constructs some store
    path (typically by fetching it from somewhere, e.g., from the
    network). */
@@ -28,10 +49,43 @@ struct Substitute
        is the store path to be substituted). */
     Strings args;
 
-    bool operator == (const Substitute & sub);
+    bool operator == (const Substitute & sub) const;
 };
 
 typedef list<Substitute> Substitutes;
+#endif
+
+
+/* A trust identifier, which is a name of an entity involved in a
+   trust relation.  Right now this is just a user ID (e.g.,
+   `root'). */
+typedef string TrustId;
+
+
+/* An output path equivalence class.  They represent outputs of
+   derivations.  That is, a derivation can have several outputs (e.g.,
+   `out', `lib', `man', etc.), each of which maps to a output path
+   equivalence class.  They can map to a number of concrete paths,
+   depending on what users built the derivation.
+
+   Equivalence classes are actually "placeholder" store paths that
+   never get built.  They do occur in derivations however in
+   command-line arguments and environment variables, but get
+   substituted with concrete paths when we actually build. */
+typedef Path OutputEqClass;
+
+typedef set<OutputEqClass> OutputEqClasses;
+
+
+/* A member of an output path equivalence class, i.e., a store path
+   that has been produced by a certain derivation. */
+struct OutputEqMember
+{
+    TrustId trustId;
+    Path path;
+};
+
+typedef list<OutputEqMember> OutputEqMembers;
 
 
 /* Open the database environment. */
@@ -43,9 +97,12 @@ void initDB();
 /* Get a transaction object. */
 void createStoreTransaction(Transaction & txn);
 
+
 /* Copy a path recursively. */
 void copyPath(const Path & src, const Path & dst);
 
+
+#if 0
 /* Register a substitute. */
 void registerSubstitute(const Transaction & txn,
     const Path & srcPath, const Substitute & sub);
@@ -55,6 +112,8 @@ Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath);
 
 /* Deregister all substitutes. */
 void clearSubstitutes();
+#endif
+
 
 /* Register the validity of a path, i.e., that `path' exists, that the
    paths referenced by it exists, and in the case of an output path of
@@ -79,6 +138,7 @@ typedef list<ValidPathInfo> ValidPathInfos;
 void registerValidPaths(const Transaction & txn,
     const ValidPathInfos & infos);
 
+
 /* Throw an exception if `path' is not directly in the Nix store. */
 void assertStorePath(const Path & path);
 
@@ -91,6 +151,11 @@ void checkStoreName(const string & name);
    /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
 Path toStorePath(const Path & path);
 
+PathHash hashPartOf(const Path & path);
+
+string namePartOf(const Path & path);
+
+
 /* "Fix", or canonicalise, the meta-data of the files in a store path
    after it has been built.  In particular:
    - the last modification date on each file is set to 0 (i.e.,
@@ -105,6 +170,7 @@ void canonicalisePathMetaData(const Path & path);
 bool isValidPathTxn(const Transaction & txn, const Path & path);
 bool isValidPath(const Path & path);
 
+
 /* Queries the hash of a valid path. */ 
 Hash queryPathHash(const Path & path);
 
@@ -123,6 +189,17 @@ void queryReferences(const Transaction & txn,
 void queryReferers(const Transaction & txn,
     const Path & storePath, PathSet & referers);
 
+void addOutputEqMember(const Transaction & txn,
+    const OutputEqClass & eqClass, const TrustId & trustId,
+    const Path & path);
+
+void queryOutputEqMembers(const Transaction & txn,
+    const OutputEqClass & eqClass, OutputEqMembers & members);
+
+void queryOutputEqClasses(const Transaction & txn,
+    const Path & path, OutputEqClasses & classes);
+
+#if 0
 /* Sets the deriver of a store path.  Use with care! */
 void setDeriver(const Transaction & txn, const Path & storePath,
     const Path & deriver);
@@ -130,27 +207,48 @@ void setDeriver(const Transaction & txn, const Path & storePath,
 /* Query the deriver of a store path.  Return the empty string if no
    deriver has been set. */
 Path queryDeriver(const Transaction & txn, const Path & storePath);
+#endif
+
 
 /* Constructs a unique store path name. */
-Path makeStorePath(const string & type,
-    const Hash & hash, const string & suffix);
-    
+void makeStorePath(const Hash & contentHash, const string & suffix,
+    Path & path, PathHash & pathHash);
+
+/* Constructs a random store path name.  Only to be used for temporary
+   build outputs, since these will violate the hash invariant. */
+Path makeRandomStorePath(const string & suffix);
+
+
+/* Hash rewriting. */
+typedef map<PathHash, PathHash> HashRewrites;
+
+string rewriteHashes(string s, const HashRewrites & rewrites,
+    vector<int> & positions);
+
+string rewriteHashes(const string & s, const HashRewrites & rewrites);
+
+
 /* Copy the contents of a path to the store and register the validity
    the resulting path.  The resulting path is returned. */
-Path addToStore(const Path & srcPath);
+Path addToStore(const Path & srcPath, const PathHash & selfHash = PathHash(),
+    const string & suffix = "", const PathSet & references = PathSet(),
+    const HashRewrites & rewrites = HashRewrites());
 
+#if 0
 /* Like addToStore(), but for pre-adding the outputs of fixed-output
    derivations. */
 Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath);
 
 Path makeFixedOutputPath(bool recursive,
     string hashAlgo, Hash hash, string name);
+#endif
 
 /* Like addToStore, but the contents written to the output path is a
    regular file containing the given string. */
 Path addTextToStore(const string & suffix, const string & s,
     const PathSet & references);
 
+
 /* Delete a value from the nixStore directory. */
 void deleteFromStore(const Path & path);
 

src/libstore/test-builder-1.sh

@@ -1,3 +0,0 @@
-#! /bin/sh
-
-echo "Hello World" > $out

src/libutil/archive.cc

@@ -140,6 +140,26 @@ void dumpPath(const Path & path, DumpSink & sink)
 }
 
 
+void makeSingletonArchive(const string & contents, DumpSink & sink)
+{
+    /* !!! hacky; have to keep this synchronised with dumpPath().  It
+       would be better to parameterise dumpPath() with a file system
+       "traverser". */
+    writeString(archiveVersion1, sink);
+    writeString("(", sink);
+    writeString("type", sink);
+    writeString("regular", sink);
+
+    unsigned int size = contents.size();
+    writeString("contents", sink);
+    writeInt(size, sink);
+    sink((const unsigned char *) contents.c_str(), size);
+    writePadding(size, sink);
+
+    writeString(")", sink);
+}
+
+
 static Error badArchive(string s)
 {
     return Error("bad archive: " + s);

src/libutil/archive.hh

@@ -48,6 +48,10 @@ struct DumpSink
 
 void dumpPath(const Path & path, DumpSink & sink);
 
+/* Make an archive consisting of a single non-executable regular
+   file, with specified string contents. */
+void makeSingletonArchive(const string & contents, DumpSink & sink);
+
 
 struct RestoreSource
 {

src/libutil/md5.h

@@ -21,7 +21,7 @@
 #ifndef _MD5_H
 #define _MD5_H 1
 
-#include <stdint.h>
+#include <inttypes.h>
 typedef uint32_t md5_uint32;
 typedef uintptr_t md5_uintptr;
 

src/libutil/sha1.h

@@ -1,7 +1,7 @@
 #ifndef _SHA_H
 #define _SHA_H
 
-#include <stdint.h>
+#include <inttypes.h>
 
 /* The SHA block size and message digest sizes, in bytes */
 

src/libutil/sha256.h

@@ -1,7 +1,7 @@
 #ifndef _SHA256_H
 #define _SHA256_H 1
 
-#include <stdint.h>
+#include <inttypes.h>
 
 #define SHA_LBLOCK	16
 #define SHA_CBLOCK	(SHA_LBLOCK*4)	/* SHA treats input data as a

src/libutil/util.cc

@@ -357,6 +357,19 @@ void printMsg_(Verbosity level, const format & f)
 }
 
 
+string showPaths(const PathSet & paths)
+{
+    string s;
+    for (PathSet::const_iterator i = paths.begin();
+         i != paths.end(); ++i)
+    {
+        if (s.size() != 0) s += ", ";
+        s += "`" + *i + "'";
+    }
+    return s;
+}
+
+
 void readFull(int fd, unsigned char * buf, size_t count)
 {
     while (count) {
@@ -688,5 +701,5 @@ bool string2Int(const string & s, int & n)
 {
     istringstream str(s);
     str >> n;
-    return str && str.eof();
+    return str && str.get() == EOF;
 }

src/libutil/util.hh

@@ -173,6 +173,8 @@ void printMsg_(Verbosity level, const format & f);
 
 #define debug(f) printMsg(lvlDebug, f)
 
+string showPaths(const PathSet & paths);
+
 
 /* Wrappers arount read()/write() that read/write exactly the
    requested number of bytes. */

src/nix-env/main.cc

@@ -3,6 +3,7 @@
 #include "globals.hh"
 #include "build.hh"
 #include "gc.hh"
+#include "misc.hh"
 #include "shared.hh"
 #include "parser.hh"
 #include "eval.hh"
@@ -11,6 +12,7 @@
 
 #include <cerrno>
 #include <ctime>
+#include <algorithm>
 
 
 typedef enum {
@@ -48,13 +50,47 @@ typedef void (* Operation) (Globals & globals,
 
 struct UserEnvElem
 {
+private:
+    string drvPath;
+    string outPath;
+    
+public:
     string name;
     string system;
-    Path drvPath;
-    Path outPath;
+
+    ATermMap attrs;
+
+    string queryDrvPath(EvalState & state) const
+    {
+        if (drvPath == "") {
+            Expr a = attrs.get("drvPath");
+            (string &) drvPath = a ? evalPath(state, a) : "";
+        }
+        return drvPath;
+    }
+    
+    string queryOutPath(EvalState & state) const
+    {
+        if (outPath == "") {
+            Expr a = attrs.get("outPath");
+            if (!a) throw Error("output path missing");
+            (string &) outPath = evalPath(state, a);
+        }
+        return outPath;
+    }
+
+    void setDrvPath(const string & s)
+    {
+        drvPath = s;
+    }
+    
+    void setOutPath(const string & s)
+    {
+        outPath = s;
+    }
 };
 
-typedef map<Path, UserEnvElem> UserEnvElems;
+typedef map<unsigned int, UserEnvElem> UserEnvElems;
 
 
 void printHelp()
@@ -68,30 +104,32 @@ static bool parseDerivation(EvalState & state, Expr e, UserEnvElem & elem)
     ATermList es;
     e = evalExpr(state, e);
     if (!matchAttrs(e, es)) return false;
-    Expr a = queryAttr(e, "type");
+
+    ATermMap attrs;
+    queryAllAttrs(e, attrs, false);
+    
+    Expr a = attrs.get("type");
     if (!a || evalString(state, a) != "derivation") return false;
 
-    a = queryAttr(e, "name");
+    a = attrs.get("name");
     if (!a) throw badTerm("derivation name missing", e);
     elem.name = evalString(state, a);
 
-    a = queryAttr(e, "system");
+    a = attrs.get("system");
     if (!a)
         elem.system = "unknown";
     else
         elem.system = evalString(state, a);
 
-    a = queryAttr(e, "drvPath");
-    if (a) elem.drvPath = evalPath(state, a);
-
-    a = queryAttr(e, "outPath");
-    if (!a) throw badTerm("output path missing", e);
-    elem.outPath = evalPath(state, a);
+    elem.attrs = attrs;
 
     return true;
 }
 
 
+static unsigned int elemCounter = 0;
+
+
 static void parseDerivations(EvalState & state, Expr e, UserEnvElems & elems)
 {
     ATermList es;
@@ -100,15 +138,15 @@ static void parseDerivations(EvalState & state, Expr e, UserEnvElems & elems)
     e = evalExpr(state, e);
 
     if (parseDerivation(state, e, elem)) 
-        elems[elem.outPath] = elem;
+        elems[elemCounter++] = elem;
 
     else if (matchAttrs(e, es)) {
         ATermMap drvMap;
         queryAllAttrs(e, drvMap);
         for (ATermIterator i(drvMap.keys()); i; ++i) {
-            debug(format("evaluating attribute `%1%'") % *i);
+            debug(format("evaluating attribute `%1%'") % aterm2String(*i));
             if (parseDerivation(state, drvMap.get(*i), elem))
-                elems[elem.outPath] = elem;
+                elems[elemCounter++] = elem;
             else
                 parseDerivations(state, drvMap.get(*i), elems);
         }
@@ -118,7 +156,7 @@ static void parseDerivations(EvalState & state, Expr e, UserEnvElems & elems)
         for (ATermIterator i(es); i; ++i) {
             debug(format("evaluating list element"));
             if (parseDerivation(state, *i, elem))
-                elems[elem.outPath] = elem;
+                elems[elemCounter++] = elem;
             else
                 parseDerivations(state, *i, elems);
         }
@@ -199,8 +237,9 @@ static void createUserEnv(EvalState & state, const UserEnvElems & elems,
          i != elems.end(); ++i)
         /* Call to `isDerivation' is for compatibility with Nix <= 0.7
            user environments. */
-        if (i->second.drvPath != "" && isDerivation(i->second.drvPath))
-            drvsToBuild.insert(i->second.drvPath);
+        if (i->second.queryDrvPath(state) != "" &&
+            isDerivation(i->second.queryDrvPath(state)))
+            drvsToBuild.insert(i->second.queryDrvPath(state));
 
     debug(format("building user environment dependencies"));
     buildDerivations(drvsToBuild);
@@ -216,7 +255,7 @@ static void createUserEnv(EvalState & state, const UserEnvElems & elems,
     for (UserEnvElems::const_iterator i = elems.begin(); 
          i != elems.end(); ++i)
     {
-        Path drvPath = keepDerivations ? i->second.drvPath : "";
+        Path drvPath = keepDerivations ? i->second.queryDrvPath(state) : "";
         ATerm t = makeAttrs(ATmakeList5(
             makeBind(toATerm("type"),
                 makeStr(toATerm("derivation")), makeNoPos()),
@@ -227,17 +266,17 @@ static void createUserEnv(EvalState & state, const UserEnvElems & elems,
             makeBind(toATerm("drvPath"),
                 makePath(toATerm(drvPath)), makeNoPos()),
             makeBind(toATerm("outPath"),
-                makePath(toATerm(i->second.outPath)), makeNoPos())
+                makePath(toATerm(i->second.queryOutPath(state))), makeNoPos())
             ));
         manifest = ATinsert(manifest, t);
-        inputs = ATinsert(inputs, makeStr(toATerm(i->second.outPath)));
+        inputs = ATinsert(inputs, makeStr(toATerm(i->second.queryOutPath(state))));
 
         /* This is only necessary when installing store paths, e.g.,
            `nix-env -i /nix/store/abcd...-foo'. */
-        addTempRoot(i->second.outPath);
-        ensurePath(i->second.outPath);
+        addTempRoot(i->second.queryOutPath(state));
+        ensurePath(i->second.queryOutPath(state));
         
-        references.insert(i->second.outPath);
+        references.insert(i->second.queryOutPath(state));
         if (drvPath != "") references.insert(drvPath);
     }
 
@@ -268,11 +307,11 @@ static void createUserEnv(EvalState & state, const UserEnvElems & elems,
     
     /* Realise the resulting store expression. */
     debug(format("building user environment"));
-    buildDerivations(singleton<PathSet>(topLevelDrv.drvPath));
+    buildDerivations(singleton<PathSet>(topLevelDrv.queryDrvPath(state)));
 
     /* Switch the current user environment to the output path. */
     debug(format("switching to new user environment"));
-    Path generation = createGeneration(profile, topLevelDrv.outPath);
+    Path generation = createGeneration(profile, topLevelDrv.queryOutPath(state));
     switchLink(profile, generation);
 }
 
@@ -345,7 +384,6 @@ static void queryInstSources(EvalState & state,
            (import ./foo.nix)' = `(import ./foo.nix).bar'. */
         case srcNixExprs: {
                 
-
             Expr e1 = parseExprFromFile(state,
                 absPath(instSource.nixExprPath));
 
@@ -377,17 +415,20 @@ static void queryInstSources(EvalState & state,
                     name = string(name, dash + 1);
 
                 if (isDerivation(*i)) {
-                    elem.drvPath = *i;
-                    elem.outPath = findOutput(derivationFromPath(*i), "out");
+                    elem.setDrvPath(*i);
+                    elem.setOutPath(
+                        /* XXX check this; may not give a result */
+                        findTrustedEqClassMember(
+                            findOutputEqClass(derivationFromPath(*i), "out"), currentTrustId));
                     if (name.size() >= drvExtension.size() &&
                         string(name, name.size() - drvExtension.size()) == drvExtension)
                         name = string(name, 0, name.size() - drvExtension.size());
                 }
-                else elem.outPath = *i;
+                else elem.setOutPath(*i);
 
                 elem.name = name;
 
-                elems[elem.outPath] = elem;
+                elems[elemCounter++] = elem;
             }
             
             break;
@@ -510,7 +551,8 @@ static void upgradeDerivations(Globals & globals,
         }
 
         if (bestElem != availElems.end() &&
-            i->second.outPath != bestElem->second.outPath)
+            i->second.queryOutPath(globals.state) !=
+                bestElem->second.queryOutPath(globals.state))
         {
             printMsg(lvlInfo,
                 format("upgrading `%1%' to `%2%'")
@@ -677,10 +719,15 @@ static void opQuery(Globals & globals,
 
     /* We only need to know the installed paths when we are querying
        the status of the derivation. */
-    UserEnvElems installed; /* installed paths */
+    PathSet installed; /* installed paths */
     
-    if (printStatus)
-        installed = queryInstalled(globals.state, globals.profile);
+    if (printStatus) {
+        UserEnvElems installedElems; 
+        installedElems = queryInstalled(globals.state, globals.profile);
+        for (UserEnvElems::iterator i = installedElems.begin();
+             i != installedElems.end(); ++i)
+            installed.insert(i->second.queryOutPath(globals.state));
+    }
             
     /* Print the desired columns. */
     Table table;
@@ -691,22 +738,25 @@ static void opQuery(Globals & globals,
         Strings columns;
         
         if (printStatus) {
-            Substitutes subs = querySubstitutes(noTxn, i->drvPath);
+#if 0            
+            Substitutes subs = querySubstitutes(noTxn, i->queryDrvPath(globals.state));
+#endif            
             columns.push_back(
-                (string) (installed.find(i->outPath)
+                (string) (installed.find(i->queryOutPath(globals.state))
                     != installed.end() ? "I" : "-")
-                + (isValidPath(i->outPath) ? "P" : "-")
-                + (subs.size() > 0 ? "S" : "-"));
+                + (isValidPath(i->queryOutPath(globals.state)) ? "P" : "-")
+                + (/* XXX subs.size() > 0 */ false ? "S" : "-"));
         }
 
         if (printName) columns.push_back(i->name);
 
         if (printSystem) columns.push_back(i->system);
 
-        if (printDrvPath) columns.push_back(i->drvPath == "" ? "-" : i->drvPath);
+        if (printDrvPath) columns.push_back(
+            i->queryDrvPath(globals.state) == ""
+            ? "-" : i->queryDrvPath(globals.state));
         
-        if (printOutPath) columns.push_back(i->outPath);
-
+        if (printOutPath) columns.push_back(i->queryOutPath(globals.state));
         table.push_back(columns);
     }
 
@@ -871,7 +921,7 @@ static void opDefaultExpr(Globals & globals,
 
 
 static string needArg(Strings::iterator & i,
-    const Strings & args, const string & arg)
+    Strings & args, const string & arg)
 {
     ++i;
     if (i == args.end()) throw UsageError(

src/nix-env/profiles.cc

@@ -4,6 +4,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <errno.h>
+#include <stdio.h>
 
 
 static bool cmpGensByNumber(const Generation & a, const Generation & b)

src/nix-store/main.cc

@@ -1,8 +1,10 @@
 #include <iostream>
+#include <algorithm>
 
 #include "globals.hh"
 #include "build.hh"
 #include "gc.hh"
+#include "misc.hh"
 #include "archive.hh"
 #include "shared.hh"
 #include "dotgraph.hh"
@@ -44,7 +46,8 @@ static Path realisePath(const Path & path)
         PathSet paths;
         paths.insert(path);
         buildDerivations(paths);
-        Path outPath = findOutput(derivationFromPath(path), "out");
+        Path outPath = findTrustedEqClassMember(
+            findOutputEqClass(derivationFromPath(path), "out"), currentTrustId);
         
         if (gcRoot == "")
             printGCWarning();
@@ -95,6 +98,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
 }
 
 
+#if 0
 /* Preload the output of a fixed-output derivation into the Nix
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
@@ -140,6 +144,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs)
     cout << format("%1%\n") %
         makeFixedOutputPath(recursive, hashAlgo, h, name);
 }
+#endif
 
 
 /* Place in `paths' the set of paths that are required to `realise'
@@ -164,6 +169,8 @@ static void storePathRequisites(const Path & storePath,
     computeFSClosure(storePath, paths);
 
     if (includeOutputs) {
+        assert(0);
+#if 0        
         for (PathSet::iterator i = paths.begin();
              i != paths.end(); ++i)
             if (isDerivation(*i)) {
@@ -173,6 +180,7 @@ static void storePathRequisites(const Path & storePath,
                     if (isValidPath(j->second.path))
                         computeFSClosure(j->second.path, paths);
             }
+#endif        
     }
 }
 
@@ -182,7 +190,8 @@ static Path maybeUseOutput(const Path & storePath, bool useOutput, bool forceRea
     if (forceRealise) realisePath(storePath);
     if (useOutput && isDerivation(storePath)) {
         Derivation drv = derivationFromPath(storePath);
-        return findOutput(drv, "out");
+        return findTrustedEqClassMember(
+            findOutputEqClass(drv, "out"), currentTrustId);
     }
     else return storePath;
 }
@@ -312,7 +321,9 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 *i = fixPath(*i);
                 if (forceRealise) realisePath(*i);
                 Derivation drv = derivationFromPath(*i);
-                cout << format("%1%\n") % findOutput(drv, "out");
+                cout << format("%1%\n") % findTrustedEqClassMember(
+                    findOutputEqClass(drv, "out"),
+                    currentTrustId);
             }
             break;
         }
@@ -337,6 +348,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
         }
 
         case qDeriver:
+#if 0            
             for (Strings::iterator i = opArgs.begin();
                  i != opArgs.end(); ++i)
             {
@@ -344,6 +356,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 cout << format("%1%\n") %
                     (deriver == "" ? "unknown-deriver" : deriver);
             }
+#endif
+            assert(0);
             break;
 
         case qBinding:
@@ -394,6 +408,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
 }
 
 
+#if 0
 static void opRegisterSubstitutes(Strings opFlags, Strings opArgs)
 {
     if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -440,6 +455,7 @@ static void opClearSubstitutes(Strings opFlags, Strings opArgs)
 
     clearSubstitutes();
 }
+#endif
 
 
 static void opRegisterValidity(Strings opFlags, Strings opArgs)
@@ -601,16 +617,20 @@ void run(Strings args)
             op = opRealise;
         else if (arg == "--add" || arg == "-A")
             op = opAdd;
+#if 0        
         else if (arg == "--add-fixed")
             op = opAddFixed;
         else if (arg == "--print-fixed-path")
             op = opPrintFixedPath;
+#endif
         else if (arg == "--query" || arg == "-q")
             op = opQuery;
+#if 0        
         else if (arg == "--register-substitutes")
             op = opRegisterSubstitutes;
         else if (arg == "--clear-substitutes")
             op = opClearSubstitutes;
+#endif            
         else if (arg == "--register-validity")
             op = opRegisterValidity;
         else if (arg == "--check-validity")

tests/lang/eval-fail-remove.nix

@@ -0,0 +1,5 @@
+let {
+  attrs = {x = 123; y = 456;};
+
+  body = (removeAttrs attrs ["x"]).x;
+}

tests/lang/eval-okay-remove.exp

@@ -0,0 +1 @@
+Int(456)

tests/lang/eval-okay-remove.nix

@@ -0,0 +1,5 @@
+let {
+  attrs = {x = 123; y = 456;};
+
+  body = (removeAttrs attrs ["x"]).y;
+}

tests/nix-pull.sh

@@ -34,4 +34,4 @@ cat $outPath/input-2/bar
 
 # Check that the derivers are set properly.
 test $($TOP/src/nix-store/nix-store -q --deriver "$outPath") = "$drvPath"
-$TOP/src/nix-store/nix-store -q --deriver $(readlink $outPath/input-2) | grep -q -- "-input-2.drv" 
+$TOP/src/nix-store/nix-store -q --deriver $(/bin/ls -l $outPath/input-2 | sed 's/.*->\ //') | grep -q -- "-input-2.drv"