* Option to turn off position information to test the impact on

maximal sharing.

Makefile.am

@@ -20,10 +20,11 @@ install-data-local: init-state
 	fi
 
 if INIT_STATE
-if SETUID_HACK
-INIT_FLAGS = -g @NIX_GROUP@ -o @NIX_USER@
-GROUP_WRITABLE = -m 775
-endif
+
+# For setuid operation, you can enable the following:
+# INIT_FLAGS = -g @NIX_GROUP@ -o @NIX_USER@
+# GROUP_WRITABLE = -m 775
+
 init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/db
@@ -34,12 +35,13 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
-	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
-	ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
+	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
 	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(prefix)/store
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
+	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
 #	$(bindir)/nix-store --init
+
 else
 init-state:
 endif

configure.ac

@@ -1,22 +1,22 @@
-AC_INIT(nix, 0.10)
+AC_INIT(nix, 0.11)
 AC_CONFIG_SRCDIR(README)
 AC_CONFIG_AUX_DIR(config)
 AM_INIT_AUTOMAKE([dist-bzip2 foreign])
         
 # Change to `1' to produce a `stable' release (i.e., the `preREVISION'
 # suffix is not added).
-STABLE=1
+STABLE=0
 
 # Put the revision number in the version.
 if test "$STABLE" != "1"; then
-    if REVISION=`test -d $srcdir/.svn && svnversion $srcdir 2> /dev/null`; then
+    if REVISION=`test -d $srcdir/.svn && svnversion -n $srcdir 2> /dev/null`; then
         VERSION=${VERSION}pre${REVISION}
     elif REVISION=`cat $srcdir/svn-revision 2> /dev/null`; then
         VERSION=${VERSION}pre${REVISION}
     fi
 fi
 
-AC_DEFINE_UNQUOTED(NIX_VERSION, ["$(echo $VERSION)"], [version])
+AC_DEFINE_UNQUOTED(NIX_VERSION, ["$VERSION"], [version])
 
 AC_PREFIX_DEFAULT(/nix)
 
@@ -32,6 +32,9 @@ case $machine_name in
     i*86)
         machine_name=i686
         ;;
+    x86_64)
+        machine_name=x86_64
+        ;;
     ppc)
         machine_name=powerpc
         ;;
@@ -122,8 +125,21 @@ AC_PATH_PROG(bison, bison, false)
 NEED_PROG(perl, perl)
 NEED_PROG(tar, tar)
 AC_PATH_PROG(dot, dot)
-                
+
+AC_PATH_PROG(openssl_prog, openssl, openssl) # if not found, call openssl in $PATH
+AC_SUBST(openssl_prog)
+AC_DEFINE_UNQUOTED(OPENSSL_PATH, ["$openssl_prog"], [Path of the OpenSSL binary])
+
+# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond).
+AC_MSG_CHECKING([whether Perl is recent enough])
+if ! $perl -e 'open(FOO, "-|", "true"); while (<FOO>) { print; }; close FOO or die;'; then
+    AC_MSG_RESULT(no)
+    AC_MSG_ERROR([Your Perl version is too old.  Nix requires Perl 5.8.0 or newer.])
+fi
+AC_MSG_RESULT(yes)
+
 NEED_PROG(cat, cat)
+NEED_PROG(tr, tr)
 AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
   [path of cat, mkdir, etc.]),
   coreutils=$withval, coreutils=$(dirname $cat))
@@ -226,33 +242,9 @@ AM_CONDITIONAL(INIT_STATE, test "$init_state" = "yes")
 
 
 # Setuid installations.
-AC_ARG_ENABLE(setuid, AC_HELP_STRING([--enable-setuid],
-  [install Nix setuid]),
-  setuid_hack=$enableval, setuid_hack=no)
-AM_CONDITIONAL(SETUID_HACK, test "$setuid_hack" = "yes")
-if test "$setuid_hack" = "yes"; then
-    AC_DEFINE(SETUID_HACK, 1, [whether to install Nix setuid])
-fi
+AC_CHECK_FUNCS([setresuid setreuid lchown])
 
-AC_CHECK_FUNC(setresuid, [HAVE_SETRESUID=1], [HAVE_SETRESUID=])
-AM_CONDITIONAL(HAVE_SETRESUID, test "$HAVE_SETRESUID" = "1")
-if test "$HAVE_SETRESUID" = "1"; then
-    AC_DEFINE(HAVE_SETRESUID, 1, [whether we have setresuid()])
-fi
 
-AC_ARG_WITH(nix-user, AC_HELP_STRING([--with-nix-user=USER],
-  [user for Nix setuid binaries]),
-  NIX_USER=$withval, NIX_USER=nix)
-AC_SUBST(NIX_USER)
-AC_DEFINE_UNQUOTED(NIX_USER, ["$NIX_USER"], [Nix user])
-                                                      
-AC_ARG_WITH(nix-group, AC_HELP_STRING([--with-nix-group=USER],
-  [group for Nix setuid binaries]),
-  NIX_GROUP=$withval, NIX_GROUP=nix)
-AC_SUBST(NIX_GROUP)
-AC_DEFINE_UNQUOTED(NIX_GROUP, ["$NIX_GROUP"], [Nix group])
-
-                                                                                                            
 # This is needed if ATerm, Berkeley DB or bzip2 are static libraries,
 # and the Nix libraries are dynamic.
 if test "$(uname)" = "Darwin"; then
@@ -275,6 +267,8 @@ AC_CONFIG_FILES([Makefile
    src/libexpr/Makefile
    src/nix-instantiate/Makefile
    src/nix-env/Makefile
+   src/nix-worker/Makefile
+   src/nix-setuid-helper/Makefile
    src/nix-log2xml/Makefile
    src/bsdiff-4.3/Makefile
    scripts/Makefile

corepkgs/buildenv/builder.pl.in

@@ -12,13 +12,15 @@ mkdir "$out", 0755 || die "error creating $out";
 
 my $symlinks = 0;
 
+my %priorities;
+
 
 # For each activated package, create symlinks.
 
 sub createLinks {
     my $srcDir = shift;
     my $dstDir = shift;
-    my $ignoreCollisions = shift;
+    my $priority = shift;
 
     my @srcFiles = glob("$srcDir/*");
 
@@ -42,7 +44,7 @@ sub createLinks {
             lstat $dstFile;
             
             if (-d _) {
-                createLinks($srcFile, $dstFile, $ignoreCollisions);
+                createLinks($srcFile, $dstFile, $priority);
             }
 
             elsif (-l _) {
@@ -53,27 +55,35 @@ sub createLinks {
                 unlink $dstFile or die "error unlinking `$dstFile': $!";
                 mkdir $dstFile, 0755 || 
                     die "error creating directory `$dstFile': $!";
-                createLinks($target, $dstFile, $ignoreCollisions);
-                createLinks($srcFile, $dstFile, $ignoreCollisions);
+                createLinks($target, $dstFile, $priorities{$dstFile});
+                createLinks($srcFile, $dstFile, $priority);
             }
 
             else {
                 symlink($srcFile, $dstFile) ||
                     die "error creating link `$dstFile': $!";
+                $priorities{$dstFile} = $priority;
                 $symlinks++;
             }
         }
 
-        elsif (-l $dstFile) {
-            if (!$ignoreCollisions) {
-                my $target = readlink $dstFile;
-                die "collission between `$srcFile' and `$target'";
-            }
-        }
-
         else {
+
+            if (-l $dstFile) {
+                my $target = readlink $dstFile;
+                my $prevPriority = $priorities{$dstFile};
+                die ( "Collission between `$srcFile' and `$target'. "
+                    . "Suggested solution: use `nix-env --set-flag "
+                    . "priority NUMBER PKGNAME' to change the priority of "
+                    . "one of the conflicting packages.\n" )
+                    if $prevPriority == $priority;
+                next if $prevPriority < $priority;
+                unlink $dstFile or die;
+            }
+            
             symlink($srcFile, $dstFile) ||
                 die "error creating link `$dstFile': $!";
+            $priorities{$dstFile} = $priority;
             $symlinks++;
         }
     }
@@ -86,13 +96,13 @@ my %postponed;
 sub addPkg;
 sub addPkg {
     my $pkgDir = shift;
-    my $ignoreCollisions = shift;
+    my $priority = shift;
 
     return if (defined $done{$pkgDir});
     $done{$pkgDir} = 1;
 
 #    print "symlinking $pkgDir\n";
-    createLinks("$pkgDir", "$out", $ignoreCollisions);
+    createLinks("$pkgDir", "$out", $priority);
 
     my $propagatedFN = "$pkgDir/nix-support/propagated-user-env-packages";
     if (-e $propagatedFN) {
@@ -107,11 +117,29 @@ sub addPkg {
 }
 
 
-# Symlink to the packages that have been installed explicitly by the user.
-my @args = split ' ', $ENV{"derivations"};
+# Convert the stuff we get from the environment back into a coherent
+# data type.
+my @paths = split ' ', $ENV{"paths"};
+my @active = split ' ', $ENV{"active"};
+my @priority = split ' ', $ENV{"priority"};
 
-foreach my $pkgDir (sort @args) {
-    addPkg($pkgDir, 0);
+die if scalar @paths != scalar @active;
+die if scalar @paths != scalar @priority;
+
+my %pkgs;
+
+for (my $n = 0; $n < scalar @paths; $n++) {
+    $pkgs{$paths[$n]} =
+        { active => $active[$n]
+        , priority => $priority[$n] };
+}
+
+
+# Symlink to the packages that have been installed explicitly by the
+# user.
+foreach my $pkg (sort (keys %pkgs)) {
+    #print $pkg, " ", $pkgs{$pkg}->{priority}, "\n";
+    addPkg($pkg, $pkgs{$pkg}->{priority}) if $pkgs{$pkg}->{active} ne "false";
 }
 
 
@@ -119,11 +147,12 @@ foreach my $pkgDir (sort @args) {
 # installed by the user (i.e., package X declares that it want Y
 # installed as well).  We do these later because they have a lower
 # priority in case of collisions.
+my $priorityCounter = 1000; # don't care about collisions
 while (scalar(keys %postponed) > 0) {
     my @pkgDirs = keys %postponed;
     %postponed = ();
     foreach my $pkgDir (sort @pkgDirs) {
-        addPkg($pkgDir, 1);
+        addPkg($pkgDir, $priorityCounter++);
     }
 }
 

corepkgs/buildenv/default.nix

@@ -4,6 +4,11 @@ derivation {
   name = "user-environment";
   system = system;
   builder = ./builder.pl;
-  derivations = derivations;
+  
   manifest = manifest;
+
+  # !!! grmbl, need structured data for passing this in a clean way.
+  paths = derivations;
+  active = map (x: if x ? meta && x.meta ? active then x.meta.active else "true") derivations;
+  priority = map (x: if x ? meta && x.meta ? priority then x.meta.priority else "5") derivations;
 }

corepkgs/channels/unpack.sh.in

@@ -4,19 +4,23 @@
 @coreutils@/mkdir $out/tmp
 cd $out/tmp
 
-expr=$out/default.nix
-echo '[' > $expr
+inputs=($inputs)
+for ((n = 0; n < ${#inputs[*]}; n += 2)); do
+    channelName=${inputs[n]}
+    channelTarball=${inputs[n+1]}
+    echo "unpacking channel $channelName"
+    @bunzip2@ < $channelTarball | @tar@ xf -
 
-nr=0
-for i in $inputs; do
-    echo "unpacking $i"
-    @bunzip2@ < $i | @tar@ xf -
-    @coreutils@/mv * ../$nr # !!! hacky
-    echo "(import ./$nr)" >> $expr
-    nr=$(($nr + 1))
+    nr=1
+    attrName=$(echo $channelName | @tr@ -- '- ' '__')
+    dirName=$attrName
+    while test -e ../$dirName; do
+        nr=$((nr+1))
+        dirName=$attrName-$nr
+    done
+
+    @coreutils@/mv * ../$dirName # !!! hacky
 done
 
-echo ']' >> $expr
-
 cd ..
 @coreutils@/rmdir tmp

doc/dev/release-procedures.txt

@@ -1,5 +1,8 @@
 To produce a `stable' release from the trunk:
 
+-1. Update the release notes; make sure that the release date is
+    correct.
+
 0. Make sure that the trunk builds in the release supervisor.
 
 1. Branch the trunk, e.g., `svn cp .../trunk
@@ -22,8 +25,8 @@ To produce a `stable' release from the trunk:
    branch (e.g., `.../branches/0.5') should be created from the
    original revision of the trunk (since maintenance releases should
    also be tested first; hence, we cannot have `STABLE=1').  The same
-   procedure can then be followed to produce maintenance release; just
-   substitute `.../branches/VERSION' for the trunk.
+   procedure can then be followed to produce maintenance releases;
+   just substitute `.../branches/VERSION' for the trunk.
 
 7. Switch back to the trunk.
 

doc/manual/Makefile.am

@@ -15,7 +15,7 @@ man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
  nix-collect-garbage.1 nix-push.1 nix-pull.1 \
  nix-prefetch-url.1 nix-channel.1 \
  nix-pack-closure.1 nix-unpack-closure.1 \
- nix-install-package.1 nix-hash.1
+ nix-install-package.1 nix-hash.1 nix-copy-closure.1
 
 FIGURES = figures/user-environments.png
 
@@ -31,7 +31,7 @@ MANUAL_SRCS = manual.xml introduction.xml installation.xml \
 manual.is-valid: $(MANUAL_SRCS) version.txt
 #	$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --relaxng $(docbookrng)/docbook.rng -
 	if test "$(jing)" != "false"; then \
-		$(XMLLINT) --xinclude $< | $(jing) $(docbookrng)/docbook.rng /dev/stdin; \
+		$(XMLLINT) --xinclude $< | $(jing) $(docbookrng)/docbook.rng /dev/fd/0; \
 	else \
 		echo "Not validating."; \
 	fi

doc/manual/bugs.xml

@@ -15,18 +15,6 @@ generation 43 is created which is a descendant of 39, not 42.  So a
 rollback from 43 ought to go back to 39.  This is not currently
 implemented; generations form a linear sequence.</para></listitem>
 
-<listitem><para><emphasis>Build management.</emphasis> In principle it
-is already possible to do build management using Nix (by writing
-builders that perform appropriate build steps), but the Nix expression
-language is not yet powerful enough to make this pleasant (?).  The
-language should be extended with features from the <link
-xlink:href='http://www.cs.uu.nl/~eelco/maak/'>Maak build
-manager</link>.  Another interesting idea is to write a
-<command>make</command> implementation that uses Nix as a back-end to
-support <link
-xlink:href='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</link>
-build files.</para></listitem>
-
 <listitem><para>For security, <command>nix-push</command> manifests
 should be digitally signed, and <command>nix-pull</command> should
 verify the signatures.  The actual NAR archives in the cache do not

doc/manual/env-common.xml

@@ -10,17 +10,6 @@
 <variablelist>
 
   
-<varlistentry><term><envar>NIX_ROOT</envar></term>
-
-  <listitem><para>If <envar>NIX_ROOT</envar> is set, the Nix command
-  will on startup perform a <function>chroot()</function> to the
-  specified directory.  This is useful in certain bootstrapping
-  situations (e.g., when installing a Nix installation onto a hard
-  disk from CD-ROM).</para></listitem>
-
-</varlistentry>
-
-  
 <varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term>
 
   <listitem>

doc/manual/installation.xml

@@ -6,6 +6,39 @@
 <title>Installation</title>
 
 
+<section><title>Supported platforms</title>
+
+<para>Nix is currently supported on the following platforms:
+
+<itemizedlist>
+
+  <listitem><para>Linux (particularly on x86, x86_64, and
+  PowerPC).</para></listitem>
+
+  <listitem><para>Mac OS X, both on Intel and
+  PowerPC.</para></listitem>
+
+  <listitem><para>FreeBSD (only tested on Intel).</para></listitem>
+
+  <listitem><para>Windows through <link
+  xlink:href="http://www.cygwin.com/">Cygwin</link>.</para>
+
+  <warning><para>On Cygwin, Nix <emphasis>must</emphasis> be installed
+  on an NTFS partition.  It will not work correctly on a FAT
+  partition.</para></warning>
+
+  </listitem>
+
+</itemizedlist>
+
+</para>
+
+<para>Nix is pretty portable, so it should work on most other Unix
+platforms as well.</para>
+
+</section>
+
+
 <section><title>Obtaining Nix</title>
 
 <para>The easiest way to obtain Nix is to download a <link
@@ -57,14 +90,15 @@ repository.</para>
 <para>To build the parser, very <emphasis>recent</emphasis> versions
 of Bison and Flex are required.  (This is because Nix needs GLR
 support in Bison and reentrancy support in Flex.)  For Bison, you need
-version 1.875c or higher (1.875 does <emphasis>not</emphasis> work),
-which can be obtained from the <link
-xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP server</link>.
-For Flex, you need version 2.5.31, which is available on <link
-xlink:href="http://lex.sourceforge.net/">SourceForge</link>.  Slightly
-older versions may also work, but ancient versions like the ubiquitous
-2.5.4a won't.  Note that these are only required if you modify the
-parser or when you are building from the Subversion repository.</para>
+version 2.3 or higher (1.875 does <emphasis>not</emphasis> work),
+which can be obtained from
+the <link xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP
+server</link>.  For Flex, you need version 2.5.33, which is available
+on <link xlink:href="http://lex.sourceforge.net/">SourceForge</link>.
+Slightly older versions may also work, but ancient versions like the
+ubiquitous 2.5.4a won't.  Note that these are only required if you
+modify the parser or when you are building from the Subversion
+repository.</para>
 
 <para>Nix uses Sleepycat's Berkeley DB and CWI's ATerm library.  These
 are included in the Nix source distribution.  If you build from the
@@ -75,7 +109,7 @@ these packages.  Alternatively, if you already have them installed,
 you can use <command>configure</command>'s <option>--with-bdb</option>
 and <option>--with-aterm</option> options to point to their respective
 locations.  Note that Berkeley DB <emphasis>must</emphasis> be version
-4.4; other versions may not have compatible database formats.</para>
+4.5; other versions may not have compatible database formats.</para>
 
 </section>
 
@@ -96,7 +130,7 @@ preceded by the command:
     </para>
 
 <screen>
-$ autoreconf -i</screen>
+$ ./boostrap</screen>
 
 <para>The installation path can be specified by passing the
 <option>--prefix=<replaceable>prefix</replaceable></option> to
@@ -152,7 +186,7 @@ $ rm -rf /nix/var</screen>
 
 <para>You can install the latest stable version of Nix through Nix
 itself by subscribing to the channel <link
-xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-stable" /> ,
+xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-stable" />,
 or the latest unstable version by subscribing to the channel<link
 xlink:href="http://nix.cs.uu.nl/dist/nix/channels-v3/nix-unstable" />.
 You can also do a <link linkend="sec-one-click">one-click
@@ -162,54 +196,70 @@ xlink:href="http://nix.cs.uu.nl/dist/nix/" />.</para>
 </section>
 
 
-<section><title>Permissions</title>
+<section><title>Security</title>
 
-<para>All Nix operations must be performed under the user ID that owns
-the Nix store and database
-(<filename><replaceable>prefix</replaceable>/store</filename> and
-<filename><replaceable>prefix</replaceable>/var/nix/db</filename>,
-respectively).  When installed from the RPM packages, these
-directories are owned by <systemitem class="username">root</systemitem>.</para>
+<para>Nix has two basic security models.  First, it can be used in
+“single-user mode”, which is similar to what most other package
+management tools do: there is a single user (typically <systemitem
+class="username">root</systemitem>) who performs all package
+management operations.  All other users can then use the installed
+packages, but they cannot perform package management operations
+themselves.</para>
 
-<section><title>Setuid installation</title>
+<para>Alternatively, you can configure Nix in “multi-user mode”.  In
+this model, all users can perform package management operations — for
+instance, every user can install software without requiring root
+privileges.  Nix ensures that this is secure.  For instance, it’s not
+possible for one user to overwrite a package used by another user with
+a Trojan horse.</para>
 
-<para>As a somewhat <emphasis>ad hoc</emphasis> hack, you can also
-install the Nix binaries <quote>setuid</quote> so that a Nix store can
-be shared among several users.  To do this, configure Nix with the
-<emphasis>--enable-setuid</emphasis> option.  Nix will be installed as
-owned by a user and group specified by the
-<option>--with-nix-user=</option><parameter>user</parameter> and
-<option>--with-nix-group=</option><parameter>group</parameter>
-options.  E.g.,
 
-<screen>
-$ ./configure --enable-setuid --with-nix-user=my_nix_user --with-nix-group=my_nix_group</screen>
-
-The user and group default to <literal>nix</literal>.  You should make
-sure that both the user and the group exist.  Any <quote>real</quote>
-users that you want to allow access should be added to the Nix
-group.</para>
-
-<warning><para>A setuid installation should only by used if the users
-in the Nix group are mutually trusted, since any user in that group
-has the ability to change anything in the Nix store or database.  For
-instance, they could install a trojan horse in executables used by
-other users.</para></warning>
-
-<warning><para>On some platforms, the Nix binaries will be installed
-as setuid <literal>root</literal>.  They drop root privileges
-immediately after startup and switch to the Nix user.  The reason for
-this is that both the real and effective user must be set to the Nix
-user, and POSIX has no system call to do this.  This is not the case
-on systems that have the <function>setresuid()</function> system call
-(such as Linux and FreeBSD), so on those systems the binaries are
-simply owned by the Nix user.</para></warning>
+<section><title>Single-user mode</title>
+  
+<para>In single-user mode, all Nix operations that access the database
+in <filename><replaceable>prefix</replaceable>/var/nix/db</filename>
+or modify the Nix store in
+<filename><replaceable>prefix</replaceable>/store</filename> must be
+performed under the user ID that owns those directories.  This is
+typically <systemitem class="username">root</systemitem>.  (If you
+install from RPM packages, that’s in fact the default ownership.)
+However, on single-user machines, it is often convenient to
+<command>chown</command> those directories to your normal user account
+so that you don’t have to <command>su</command> to <systemitem
+class="username">root</systemitem> all the time.</para>
 
 </section>
 
+
+<section><title>Multi-user mode</title>
+
+<para></para>
+
+
+<!--
+
+warning: the nix-builders group should contain *only* the Nix
+builders, and nothing else.  If the Nix account is compromised, you
+can execute programs under the accounts in the nix-builders group, so
+it obviously shouldn’t contain any “real” user accounts.  So don’t use
+an existing group like <literal>users</literal> — just create a new
+one.
+
+-->
+
+<note><para>Multi-user mode has one important limitation: only
+<systemitem class="username">root</systemitem> can run <command
+linkend="sec-nix-pull">nix-pull</command> to register the availability
+of pre-built binaries.  However, those registrations
+<emphasis>are</emphasis> used by all users to speed up
+builds.</para></note>
+
 </section>
 
 
+</section> <!-- end of permissions section -->
+
+
 <section><title>Using Nix</title>
 
 <para>To use Nix, some environment variables should be set.  In

doc/manual/manual.xml

@@ -13,14 +13,21 @@
         <firstname>Eelco</firstname>
         <surname>Dolstra</surname>
       </personname>
+      <affiliation>
+        <orgname>Utrecht University</orgname>
+        <orgdiv>Faculty of Science, Department of Information and Computing Sciences</orgdiv>
+      </affiliation>
     </author>
 
     <copyright>
       <year>2004</year>
       <year>2005</year>
       <year>2006</year>
+      <year>2007</year>
       <holder>Eelco Dolstra</holder>
     </copyright>
+
+    <date>September 2007</date>
     
   </info>
 
@@ -49,7 +56,7 @@
         <title>nix-instantiate</title>
         <xi:include href="nix-instantiate.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-store">
         <title>nix-store</title>
         <xi:include href="nix-store.xml" />
       </section>
@@ -65,10 +72,14 @@
         <title>nix-channel</title>
         <xi:include href="nix-channel.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-collect-garbage">
         <title>nix-collect-garbage</title>
         <xi:include href="nix-collect-garbage.xml" />
       </section>
+      <section xml:id="sec-nix-copy-closure">
+        <title>nix-copy-closure</title>
+        <xi:include href="nix-copy-closure.xml" />
+      </section>
       <section xml:id="sec-nix-hash">
         <title>nix-hash</title>
         <xi:include href="nix-hash.xml" />
@@ -77,15 +88,15 @@
         <title>nix-install-package</title>
         <xi:include href="nix-install-package.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-pack-closure">
         <title>nix-pack-closure</title>
         <xi:include href="nix-pack-closure.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-prefetch-url">
         <title>nix-prefetch-url</title>
         <xi:include href="nix-prefetch-url.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-pull">
         <title>nix-pull</title>
         <xi:include href="nix-pull.xml" />
       </section>
@@ -93,7 +104,7 @@
         <title>nix-push</title>
         <xi:include href="nix-push.xml" />
       </section>
-      <section>
+      <section xml:id="sec-nix-unpack-closure">
         <title>nix-unpack-closure</title>
         <xi:include href="nix-unpack-closure.xml" />
       </section>

doc/manual/nix-build.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-build</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-build</refname>
@@ -9,7 +17,7 @@
 <refsynopsisdiv>
   <cmdsynopsis>
     <command>nix-build</command>
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
     <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
     <arg>
       <group choice='req'>

doc/manual/nix-channel.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-channel</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-channel</refname>
   <refpurpose>manage Nix channels</refpurpose>

doc/manual/nix-collect-garbage.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-collect-garbage</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-collect-garbage</refname>
   <refpurpose>delete unreachable store paths</refpurpose>

doc/manual/nix-copy-closure.xml

@@ -0,0 +1,151 @@
+<refentry xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-copy-closure</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
+<refnamediv>
+  <refname>nix-copy-closure</refname>
+  <refpurpose>copy a closure to or from a remote machine via SSH</refpurpose>
+</refnamediv>
+
+<refsynopsisdiv>
+  <cmdsynopsis>
+    <command>nix-copy-closure</command>
+    <group>
+      <arg choice='plain'><option>--to</option></arg>
+      <arg choice='plain'><option>--from</option></arg>
+    </group>
+    <arg><option>--sign</option></arg>
+    <arg><option>--gzip</option></arg>
+    <arg choice='plain'>
+      <arg><replaceable>user@</replaceable></arg><replaceable>machine</replaceable>
+    </arg>
+    <arg choice='plain'><replaceable>paths</replaceable></arg>
+  </cmdsynopsis>
+</refsynopsisdiv>
+
+
+<refsection><title>Description</title>
+
+<para><command>nix-copy-closure</command> gives you an easy and
+efficient way to exchange software between machines.  Given one or
+more Nix store paths <replaceable>paths</replaceable> on the local
+machine, <command>nix-copy-closure</command> computes the closure of
+those paths (i.e. all their dependencies in the Nix store), and copies
+all paths in the closure to the remote machine via the
+<command>ssh</command> (Secure Shell) command.  With the
+<option>--from</option>, the direction is reversed:
+the closure of <replaceable>paths</replaceable> on a remote machine is
+copied to the Nix store on the local machine.</para>
+
+<para>This command is efficient because it only sends the store paths
+that are missing on the target machine.</para>
+
+<para>Since <command>nix-copy-closure</command> calls
+<command>ssh</command>, you may be asked to type in the appropriate
+password or passphrase.  In fact, you may be asked
+<emphasis>twice</emphasis> because <command>nix-copy-closure</command>
+currently connects twice to the remote machine, first to get the set
+of paths missing on the target machine, and second to send the dump of
+those paths.  If this bothers you, use
+<command>ssh-agent</command>.</para>
+
+
+<refsection><title>Options</title>
+
+<variablelist>
+  
+  <varlistentry><term><option>--to</option></term>
+
+    <listitem><para>Copy the closure of
+    <replaceable>paths</replaceable> from the local Nix store to the
+    Nix store on <replaceable>machine</replaceable>.  This is the
+    default.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--from</option></term>
+
+    <listitem><para>Copy the closure of
+    <replaceable>paths</replaceable> from the Nix store on
+    <replaceable>machine</replaceable> to the local Nix
+    store.</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--sign</option></term>
+
+    <listitem><para>Let the sending machine cryptographically sign the
+    dump of each path with the key in
+    <filename>/nix/etc/nix/signing-key.sec</filename>.  If the user on
+    the target machine does not have direct access to the Nix store
+    (i.e., if the target machine has a multi-user Nix installation),
+    then the target machine will check the dump against
+    <filename>/nix/etc/nix/signing-key.pub</filename> before unpacking
+    it in its Nix store.  This allows secure sharing of store paths
+    between untrusted users on two machines, provided that there is a
+    trust relation between the Nix installations on both machines
+    (namely, they have matching public/secret keys).</para></listitem>
+
+  </varlistentry>
+
+  <varlistentry><term><option>--gzip</option></term>
+
+    <listitem><para>Compress the dump of each path with
+    <command>gzip</command> before sending it.</para></listitem>
+
+  </varlistentry>
+
+</variablelist>
+
+</refsection>
+
+
+<refsection><title>Environment variables</title>
+
+<variablelist>
+
+  <varlistentry><term><envar>NIX_SSHOPTS</envar></term>
+
+    <listitem><para>Additional options to be passed to
+    <command>ssh</command> on the command line.</para></listitem>
+
+  </varlistentry>
+  
+</variablelist>
+
+</refsection>
+
+
+<refsection><title>Examples</title>
+
+<para>Copy Firefox with all its dependencies to a remote machine:
+
+<screen>
+$ nix-copy-closure alice@itchy.labs $(type -tP firefox)</screen>
+
+</para>
+
+<para>Copy Subversion from a remote machine and then install it into a
+user environment:
+
+<screen>
+$ nix-copy-closure --from alice@itchy.labs \
+    /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4
+</screen>
+
+</para>
+
+</refsection>
+
+
+</refsection>
+
+</refentry>

doc/manual/nix-env.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-env</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-env</refname>
   <refpurpose>manipulate or query Nix user environments</refpurpose>
@@ -9,7 +17,7 @@
 <refsynopsisdiv>
   <cmdsynopsis>
     <command>nix-env</command>
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
     <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
     <arg>
       <group choice='req'>
@@ -142,14 +150,14 @@ linkend="sec-common-options" />.</para>
 <variablelist>
 
   <varlistentry><term><filename>~/.nix-defexpr</filename></term>
+
+  <!-- !!! .nix-defexpr can be a directory now -->
   
     <listitem><para>The default Nix expression used by the
     <option>--install</option>, <option>--upgrade</option>, and
     <option>--query --available</option> operations to obtain
-    derivations.  It is generally a symbolic link to some other
-    location set using the <option>--import</option> operation.  The
-    <option>--file</option> option may be used to override this
-    default.</para></listitem>
+    derivations.  The <option>--file</option> option may be used to
+    override this default.</para></listitem>
     
   </varlistentry>
 
@@ -1061,43 +1069,4 @@ error: no generation older than the current (91) exists</screen>
 
 
   
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--import</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-env</command>
-  <group choice='req'>
-    <arg choice='plain'><option>--import</option></arg>
-    <arg choice='plain'><option>-I</option></arg>
-  </group>
-  <arg choice='req'><replaceable>path</replaceable></arg>
-</cmdsynopsis>
-
-</refsection>
-
-
-<refsection><title>Description</title>
-            
-<para>This operation makes <replaceable>path</replaceable> the default
-active Nix expression for the user.  That is, the symlink
-<filename>~/.nix-userenv</filename> is made to point to
-<replaceable>path</replaceable>.</para>
-
-</refsection>
-
-
-<refsection><title>Examples</title>
-
-<screen>
-$ nix-env -I ~/nixpkgs-0.5/</screen>
-
-</refsection>
-    
-</refsection>
-
-
-  
 </refentry>

doc/manual/nix-hash.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-hash</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-hash</refname>
   <refpurpose>compute the cryptographic hash of a path</refpurpose>

doc/manual/nix-install-package.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-install-package</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-install-package</refname>
   <refpurpose>install a Nix Package file</refpurpose>

doc/manual/nix-instantiate.xml

@@ -1,6 +1,14 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-instantiate</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-instantiate</refname>
   <refpurpose>instantiate store derivations from Nix expressions</refpurpose>
@@ -9,7 +17,7 @@
 <refsynopsisdiv>
   <cmdsynopsis>
     <command>nix-instantiate</command>
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
     <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg>
     <arg>
       <group choice='req'>

doc/manual/nix-pack-closure.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-pack-closure</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-pack-closure</refname>
@@ -58,6 +66,16 @@ $ nix-pack-closure $(which azureus) | ssh scratchy nix-unpack-closure</screen>
     
 </para>
 
+<para>As a variation on the previous example, copy
+<command>azureus</command>, and also install it in the user’s profile
+on the target machine:
+
+<screen>
+$ nix-pack-closure $(which azureus) | ssh scratchy 'nix-env -i $(nix-unpack-closure)'</screen>
+
+</para>
+ 
+
 </refsection>
 
 

doc/manual/nix-prefetch-url.xml

@@ -1,5 +1,14 @@
-<refentry>
+<refentry xmlns="http://docbook.org/ns/docbook"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
   
+<refmeta>
+  <refentrytitle>nix-prefetch-url</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
+
 <refnamediv>
   <refname>nix-prefetch-url</refname>
   <refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
@@ -38,7 +47,7 @@ avoided.</para>
 <para>The environment variable <envar>NIX_HASH_ALGO</envar> specifies
 which hash algorithm to use.  It can be either <literal>md5</literal>,
 <literal>sha1</literal>, or <literal>sha256</literal>.  The default is
-<literal>md5</literal>.</para>
+<literal>sha256</literal>.</para>
 
 <para>If <replaceable>hash</replaceable> is specified, then a download
 is not performed if the Nix store already contains a file with the

doc/manual/nix-pull.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-pull</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-pull</refname>

doc/manual/nix-push.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-push</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-push</refname>

doc/manual/nix-store.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-store</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-store</refname>
@@ -9,7 +17,7 @@
 <refsynopsisdiv>
   <cmdsynopsis>
     <command>nix-store</command>
-    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
+    <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" />
     <arg><option>--add-root</option> <replaceable>path</replaceable></arg>
     <arg><option>--indirect</option></arg>
     <arg choice='plain'><replaceable>operation</replaceable></arg>
@@ -651,37 +659,7 @@ $ gv graph.ps</screen>
 
 <!--######################################################################-->
 
-<!--
-<refsection><title>Operation <option>-XXX-substitute</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>-XXX-substitute</option></arg>
-  <arg choice='plain'
-       rep='repeat'><replaceable>srcpath</replaceable> <replaceable>subpath</replaceable></arg>
-</cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-            
-<para>The operation <option>-XXX-substitute</option> registers that the
-store path <replaceable>srcpath</replaceable> can be built by
-realising the derivation expression in
-<replaceable>subpath</replaceable>.  This is used to implement binary
-deployment.</para>
-
-</refsection>
-            
-</refsection>
--->
-
-
-
-<!--######################################################################-->
-
-<refsection><title>Operation <option>--verify</option></title>
+<refsection xml:id='refsec-nix-store-verify'><title>Operation <option>--verify</option></title>
 
 <refsection>
   <title>Synopsis</title>

doc/manual/nix-unpack-closure.xml

@@ -1,5 +1,13 @@
 <refentry xmlns="http://docbook.org/ns/docbook"
-          xmlns:xlink="http://www.w3.org/1999/xlink">
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          xmlns:xi="http://www.w3.org/2001/XInclude">
+
+<refmeta>
+  <refentrytitle>nix-unpack-closure</refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo class="source">Nix</refmiscinfo>
+  <refmiscinfo class="version"><xi:include href="version.txt" parse="text"/></refmiscinfo>
+</refmeta>
 
 <refnamediv>
   <refname>nix-unpack-closure</refname>
@@ -22,6 +30,12 @@ closure is a single file read from standard input.  See the
 description of <command>nix-pack-closure</command> for details and
 examples.</para>
 
+<para>The top-level paths in the closure (i.e., the paths passed to
+the original <command>nix-pack-closure</command> call that created the
+closure) are printed on standard output.  These paths can be passed,
+for instance, to <literal>nix-env -i</literal> to install them into a
+user environment on the target machine.</para>
+
 </refsection>
 
 

doc/manual/opt-common-syn.xml

@@ -1,4 +1,4 @@
-<nop>
+<nop xmlns="http://docbook.org/ns/docbook">
   
 <arg><option>--help</option></arg>
 <arg><option>--version</option></arg>

doc/manual/package-management.xml

@@ -92,7 +92,7 @@ component is installed in your current user environment.  The second
 very quick operation).  The last one (<literal>S</literal>) indicates
 whether there is a so-called <emphasis>substitute</emphasis> for the
 component, which is Nix’s mechanism for doing binary deployment.  It
-just means that Nix know that it can fetch a pre-built component from
+just means that Nix knows that it can fetch a pre-built component from
 somewhere (typically a network server) instead of building it
 locally.</para>
 

doc/manual/release-notes.xml

@@ -1,10 +1,141 @@
 <article xmlns="http://docbook.org/ns/docbook"
-         xmlns:xlink="http://www.w3.org/1999/xlink">
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xml:id="sec-relnotes">
 
 <title>Nix Release Notes</title>
 
 
 
+<!--==================================================================-->
+
+<section xml:id="ssec-relnotes-0.11"><title>Release 0.11 (TBA)</title>
+
+<itemizedlist>
+
+
+  <listitem><para>TODO: multi-user support.  The old setuid method for
+  sharing a store between multiple users has been
+  removed.</para></listitem>
+
+
+  <listitem><para>The new command <command>nix-copy-closure</command>
+  gives you an easy and efficient way to exchange software between
+  machines.  It copies the missing parts of the closure of a set of
+  store path to or from a remote machine.</para></listitem>
+
+
+  <listitem><para><command>nix-prefetch-url</command> now by default
+  computes the SHA-256 hash of the file instead of the MD5 hash.  In
+  calls to <function>fetchurl</function> you should pass an
+  <literal>sha256</literal> attribute instead of
+  <literal>md5</literal>.  You can pass either a hexadecimal or a
+  base-32 encoding of the hash.</para></listitem>
+
+  
+  <listitem><para><command>nix-store</command> has a new operation
+  <option>--read-log</option> (<option>-l</option>)
+  <parameter>paths</parameter> that shows the build log of the given
+  paths.</para></listitem>
+
+  
+  <listitem><para>TODO: <varname>allowedReferences</varname> for
+  checking the set of references in the output of a
+  derivation.</para></listitem>
+
+
+  <listitem><para>TODO: semantic cleanups of string concatenation
+  etc. (mostly in r6740).</para></listitem>
+
+
+  <listitem><para>TODO: now using Berkeley DB 4.5.</para></listitem>
+
+
+  <listitem><para>TODO: option <option>--reregister</option> in
+  <command>nix-store --register-validity</command>.</para></listitem>
+
+
+  <listitem><para>TODO: magic <varname>exportReferencesGraph</varname>
+  attribute.</para></listitem>
+
+
+  <listitem><para>TODO: option <option>--max-silent-time</option>,
+  configuration setting
+  <literal>build-max-silent-time</literal>.</para></listitem>
+
+
+  <listitem><para>TODO: <command>nix-env</command>
+  <option>--set</option>.</para></listitem>
+  
+
+  <listitem><para>TODO: <option>--argstr</option>.</para></listitem>
+
+
+  <listitem><para>TODO: <command>nix-env</command> now maintains meta
+  info about installed packages in user environments.  <option>-q
+  --xml --meta</option> to show all meta info.</para></listitem>
+
+  
+  <listitem><para>TODO: <command>nix-env</command>
+  <option>--set-flag</option>.  Specific flags:
+  <literal>active</literal>, <literal>priority</literal>,
+  <literal>keep</literal>.</para></listitem>
+
+  
+  <listitem><para>TODO: <command>nix-env</command> <option>-i</option>
+  / <option>-u</option> take package priorities into
+  account.</para></listitem>
+  
+
+  <listitem><para><command>nix-env -q</command> now has a flag
+  <option>--prebuilt-only</option> (<option>-b</option>) that causes
+  <command>nix-env</command> to show only those derivations whose
+  output is already in the Nix store or that can be substituted (i.e.,
+  downloaded from somewhere).  In other words, it shows the packages
+  that can be installed “quickly”, i.e., don’t need to be built from
+  source.</para></listitem>
+  
+
+  <listitem><para>TODO: new built-ins
+  <function>builtins.attrNames</function>,
+  <function>builtins.filterSource</function>,
+  <function>builtins.sub</function>,
+  <function>builtins.stringLength</function>,
+  <function>builtins.substring</function>.</para></listitem>
+
+
+  <listitem><para>TODO: each subscribed channel is its own attribute
+  in the top-level expression generated for the channel, this allows
+  disambiguation (<command>nix-env -qaA</command>).</para></listitem>
+
+
+  <listitem><para>TODO: substitutes table is gone, registering
+  substitutes is now much faster.</para></listitem>
+
+
+  <listitem><para><command>nix-prefetch-url</command> now has a
+  limited form of caching.  This is used by
+  <command>nix-channel</command> to prevent unnecessary downloads when
+  the channel hasn’t changed.</para></listitem>
+  
+
+</itemizedlist>
+
+</section>
+
+
+
+<!--==================================================================-->
+
+<section><title>Release 0.10.1 (October 11, 2006)</title>
+
+<para>This release fixes two somewhat obscure bugs that occur when
+evaluating Nix expressions that are stored inside the Nix store
+(<literal>NIX-67</literal>).  These do not affect most users.</para>
+
+</section>
+
+
+
 <!--==================================================================-->
 
 <section><title>Release 0.10 (October 6, 2006)</title>

doc/manual/troubleshooting.xml

@@ -33,6 +33,51 @@ $ rm __db.00*</screen>
 </section>
 
 
+<section><title>Berkeley DB gives weird error messages</title>
+
+<para>Symptom: you get error messages such as
+
+<screen>
+Berkeley DB message: Finding last valid log LSN: file: 1 offset 28
+Berkeley DB error: file validpaths (meta pgno = 0) has LSN [483][34721].
+Berkeley DB error: end of log is [1][28]
+Berkeley DB error: /nix/var/nix/db/validpaths: unexpected file type or format</screen>
+
+or other weird Berkeley DB errors, and they don’t go away (i.e.,
+automatic recovery doesn’t work).  This may be the case after a system
+crash.</para>
+
+<para>Solution: first try to run <command>db_recover</command> and
+then <link linkend='refsec-nix-store-verify'><command>nix-store
+--verify</command></link>:
+
+<screen>
+$ db_recover -h /nix/var/nix/db
+$ nix-store --verify</screen>
+
+(Make sure that you have the right version of
+<command>db_recover</command>, namely, Berkeley DB 4.4 for Nix 0.10,
+and 4.5 for Nix 0.11.)</para>
+
+<para>If that doesn’t work, it’s time to bring out the big guns:
+
+<screen>
+$ cd /nix/var/nix
+$ cp -pr db db-backup  <lineannotation>(making a backup just in case)</lineannotation>
+$ cd db
+$ rm __db.* log*  <lineannotation>(removing the Berkeley DB environment)</lineannotation>
+$ mkdir tmp
+$ for i in *; do db_dump $i | (cd tmp &amp;&amp; db_load $i); done 
+<lineannotation>(ignore error messages about non-database files like “reserved”)</lineannotation>
+$ mv tmp/* .
+$ nix-store --verify</screen>
+
+</para>
+
+</section>
+
+
+
 <section><title>Collisions in <command>nix-env</command></title>
 
 <para>Symptom: when installing or upgrading, you get an error message such as

doc/manual/writing-nix-expressions.xml

@@ -635,13 +635,14 @@ language.</para>
     complicated example (from the Nix expression for <link
     xlink:href='http://www.trolltech.com/products/qt'>Qt</link>):
 
-<programlisting>    
+<programlisting>
 configureFlags = "
   -system-zlib -system-libpng -system-libjpeg
   ${if openglSupport then "-dlopen-opengl
     -L${mesa}/lib -I${mesa}/include
     -L${libXmu}/lib -I${libXmu}/include" else ""}
-  ${if threadSupport then "-thread" else "-no-thread"}</programlisting>
+  ${if threadSupport then "-thread" else "-no-thread"}
+";</programlisting>
 
     Note that Nix expressions and strings can be arbitrarily nested;
     in this case the outer string contains various antiquotations that
@@ -649,16 +650,20 @@ configureFlags = "
     some of which in turn contain expressions (e.g.,
     <literal>${mesa}</literal>).</para>
 
+    <para>As a convenience, <emphasis>URIs</emphasis> as defined in
+    appendix B of <link
+    xlink:href='http://www.ietf.org/rfc/rfc2396.txt'>RFC 2396</link>
+    can be written <emphasis>as is</emphasis>, without quotes.  For
+    instance, the string
+    <literal>"https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2"</literal>
+    can also be written as
+    <literal>https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2</literal>.</para>
+
   </listitem>
 
   <listitem><para><emphasis>Integers</emphasis>, e.g.,
   <literal>123</literal>.</para></listitem>
 
-  <listitem><para><emphasis>URIs</emphasis> as defined in appendix B
-  of <link xlink:href='http://www.ietf.org/rfc/rfc2396.txt'>RFC
-  2396</link>, e.g.,
-  <literal>https://svn.cs.uu.nl:12443/dist/trace/trace-nix-trunk.tar.bz2</literal>.</para></listitem>
-
   <listitem><para><emphasis>Paths</emphasis>, e.g.,
   <filename>/bin/sh</filename> or <filename>./builder.sh</filename>.
   A path must contain at least one slash to be recognised as such; for
@@ -1212,6 +1217,10 @@ set, the attributes of which specify the inputs of the build.</para>
       They are simply concatenated, separated by
       spaces.</para></listitem>
 
+      <listitem><para><emphasis>true</emphasis> is passed as 
+      <emphasis>1</emphasis>, <emphasis>false</emphasis>
+      and <emphasis>null</emphasis> are passed as empty string.
+      </para></listitem>
     </itemizedlist>
 
   </para></listitem>
@@ -1357,6 +1366,24 @@ is also available as <function>builtins.derivation</function>.</para>
   </varlistentry>
 
   
+  <varlistentry><term><function>builtins.attrNames</function>
+  <replaceable>attrs</replaceable></term>
+
+    <listitem><para>Return the names of the attributes in the
+    attribute set <replaceable>attrs</replaceable> in a sorted list.
+    For instance, <literal>builtins.attrNames {y = 1; x =
+    "foo";}</literal> evaluates to <literal>["x" "y"]</literal>.
+    There is no built-in function <function>attrValues</function>, but
+    you can easily define it yourself:
+
+<programlisting>
+attrValues = attrs: map (name: builtins.getAttr name attrs) (builtins.attrNames attrs);</programlisting>
+
+    </para></listitem>
+
+  </varlistentry>
+
+  
   <varlistentry><term><function>baseNameOf</function> <replaceable>s</replaceable></term>
 
     <listitem><para>Return the <emphasis>base name</emphasis> of the

doc/signing.txt

@@ -0,0 +1,24 @@
+Generate a private key:
+
+$ (umask 277 && openssl genrsa -out /nix/etc/nix/signing-key.sec 2048)
+
+The private key should be kept secret (only readable to the Nix daemon
+user).
+
+
+Generate the corresponding public key:
+
+$ openssl rsa -in /nix/etc/nix/signing-key.sec -pubout > /nix/etc/nix/signing-key.pub
+
+The public key should be copied to all machines to which you want to
+export store paths.
+
+
+Signing:
+
+$ nix-hash --type sha256 --flat svn.nar | openssl rsautl -sign -inkey mykey.sec > svn.nar.sign
+
+
+Verifying a signature:
+
+$ test "$(nix-hash --type sha256 --flat svn.nar)" = "$(openssl rsautl -verify -inkey mykey.pub -pubin -in svn.nar.sign)"

externals/Makefile.am

@@ -1,11 +1,11 @@
 # Berkeley DB
 
-DB = db-4.4.20.NC
+DB = db-4.5.20
 
 $(DB).tar.gz:
 	@echo "Nix requires Berkeley DB to build."
-	@echo "Please download version 4.4.20 from"
-	@echo "  http://downloads.sleepycat.com/db-4.4.20.NC.tar.gz"
+	@echo "Please download version 4.5.20 from"
+	@echo "  http://download-east.oracle.com/berkeley-db/db-4.5.20.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -35,19 +35,17 @@ endif
 
 # CWI ATerm
 
-ATERM = aterm-2.4.2
+ATERM = aterm-2.4.2-fixes-r2
 
-$(ATERM).tar.gz:
+$(ATERM).tar.bz2:
 	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.4.2 from"
-	@echo "  http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.4.2.tar.gz"
+	@echo "Please download version 2.4.2-fixes-r2 from"
+	@echo "  http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes-r2.tar.bz2"
 	@echo "and place it in the externals/ directory."
 	false
 
-$(ATERM): $(ATERM).tar.gz
-	gunzip < $(srcdir)/$(ATERM).tar.gz | tar xvf -
-	(cd $(ATERM) && $(patch) -p1) < $(srcdir)/aterm-aliasing.patch
-#	(cd $(ATERM) && $(patch) -p1) < $(srcdir)/aterm-64-bit.patch
+$(ATERM): $(ATERM).tar.bz2
+	bunzip2 < $(srcdir)/$(ATERM).tar.bz2 | tar xvf -
 
 have-aterm:
 	$(MAKE) $(ATERM)
@@ -69,12 +67,12 @@ endif
 
 # bzip2
 
-BZIP2 = bzip2-1.0.3
+BZIP2 = bzip2-1.0.4
 
 $(BZIP2).tar.gz:
 	@echo "Nix requires bzip2 to build."
-	@echo "Please download version 1.0.3 from"
-	@echo "  http://www.bzip.org/1.0.3/bzip2-1.0.3.tar.gz"
+	@echo "Please download version 1.0.4 from"
+	@echo "  http://www.bzip.org/1.0.4/bzip2-1.0.4.tar.gz"
 	@echo "and place it in the externals/ directory."
 	false
 
@@ -103,8 +101,8 @@ endif
 
 all: build-db build-aterm build-bzip2
 
-EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.gz $(BZIP2).tar.gz \
-  bdb-cygwin.patch aterm-aliasing.patch aterm-64-bit.patch
+EXTRA_DIST = $(DB).tar.gz $(ATERM).tar.bz2 $(BZIP2).tar.gz \
+  bdb-cygwin.patch
 
 ext-clean:
 	$(RM) -f have-db build-db have-aterm build-aterm

externals/aterm-64-bit.patch

@@ -1,661 +0,0 @@
-diff -rc aterm-2.4.2-orig/aterm/bafio.c aterm-2.4.2/aterm/bafio.c
-*** aterm-2.4.2-orig/aterm/bafio.c	2004-02-02 12:24:34.000000000 +0100
---- aterm-2.4.2/aterm/bafio.c	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 222,227 ****
---- 222,229 ----
-      }
-    }
-  
-+   if (val) return -1;
-+ 
-    /* Ok */
-    return 0;
-  }
-***************
-*** 544,551 ****
-    * terms have been sorted by symbol.
-  	*/
-  
-! void gather_top_symbols(sym_entry *cur_entry, int cur_arg, 
-! 			int total_top_symbols)
-  {
-    int index;
-    unsigned int hnr;
---- 546,553 ----
-    * terms have been sorted by symbol.
-  	*/
-  
-! static void gather_top_symbols(sym_entry *cur_entry, int cur_arg, 
-! 			       int total_top_symbols)
-  {
-    int index;
-    unsigned int hnr;
-***************
-*** 899,905 ****
-    } else {
-      switch(ATgetType(t)) {
-      case AT_INT:
-!       if(writeBits(ATgetInt((ATermInt)t), HEADER_BITS, writer) < 0) {
-  	return ATfalse;
-        }
-  #if 0
---- 901,908 ----
-    } else {
-      switch(ATgetType(t)) {
-      case AT_INT:
-!       /* If ATerm integers are > 32 bits, then this can fail. */
-!       if(writeBits(ATgetInt((ATermInt)t), INT_SIZE_IN_BAF, writer) < 0) {
-  	return ATfalse;
-        }
-  #if 0
-***************
-*** 1033,1039 ****
-  /*}}}  */
-  /*{{{  ATbool write_baf(ATerm t, byte_writer *writer) */
-  
-! ATbool
-  write_baf(ATerm t, byte_writer *writer)
-  {
-    int nr_unique_terms = 0;
---- 1036,1042 ----
-  /*}}}  */
-  /*{{{  ATbool write_baf(ATerm t, byte_writer *writer) */
-  
-! static ATbool
-  write_baf(ATerm t, byte_writer *writer)
-  {
-    int nr_unique_terms = 0;
-***************
-*** 1233,1239 ****
-  	* Read a single symbol from file.
-  	*/
-  
-! Symbol read_symbol(byte_reader *reader)
-  {
-    unsigned int arity, quoted;
-    int len;
---- 1236,1242 ----
-  	* Read a single symbol from file.
-  	*/
-  
-! static Symbol read_symbol(byte_reader *reader)
-  {
-    unsigned int arity, quoted;
-    int len;
-***************
-*** 1260,1266 ****
-   * Read all symbols from file.
-   */
-  
-! ATbool read_all_symbols(byte_reader *reader)
-  {
-    unsigned int val;
-    int i, j, k, arity;
---- 1263,1269 ----
-   * Read all symbols from file.
-   */
-  
-! static ATbool read_all_symbols(byte_reader *reader)
-  {
-    unsigned int val;
-    int i, j, k, arity;
-***************
-*** 1280,1293 ****
-      /*}}}  */
-      /*{{{  Read term count and allocate space */
-  
-!     if(readInt(&val, reader) < 0)
-        return ATfalse;
-      read_symbols[i].nr_terms = val;
-      read_symbols[i].term_width = bit_width(val);
-!     if(val == 0)
-!       read_symbols[i].terms = NULL;
-!     else
-!       read_symbols[i].terms = (ATerm *)calloc(val, sizeof(ATerm));
-      if(!read_symbols[i].terms)
-        ATerror("read_symbols: could not allocate space for %d terms.\n", val);
-      ATprotectArray(read_symbols[i].terms, val);
---- 1283,1293 ----
-      /*}}}  */
-      /*{{{  Read term count and allocate space */
-  
-!     if(readInt(&val, reader) < 0 || val == 0)
-        return ATfalse;
-      read_symbols[i].nr_terms = val;
-      read_symbols[i].term_width = bit_width(val);
-!     read_symbols[i].terms = (ATerm *)calloc(val, sizeof(ATerm));
-      if(!read_symbols[i].terms)
-        ATerror("read_symbols: could not allocate space for %d terms.\n", val);
-      ATprotectArray(read_symbols[i].terms, val);
-***************
-*** 1351,1357 ****
-  /*}}}  */
-  /*{{{  ATerm read_term(sym_read_entry *sym, byte_reader *reader) */
-  
-! ATerm read_term(sym_read_entry *sym, byte_reader *reader)
-  {
-    unsigned int val;
-    int i, arity = sym->arity;
---- 1351,1357 ----
-  /*}}}  */
-  /*{{{  ATerm read_term(sym_read_entry *sym, byte_reader *reader) */
-  
-! static ATerm read_term(sym_read_entry *sym, byte_reader *reader)
-  {
-    unsigned int val;
-    int i, arity = sym->arity;
-***************
-*** 1365,1370 ****
---- 1365,1371 ----
-      ATprotectArray(args, arity);
-      if(!args)
-        ATerror("could not allocate space for %d arguments.\n", arity);
-+     /* !!! leaks memory on the "return NULL" paths */
-    }
-  
-    /*ATfprintf(stderr, "reading term over symbol %y\n", sym->sym);*/
-***************
-*** 1372,1377 ****
---- 1373,1380 ----
-      /*ATfprintf(stderr, "  reading argument %d (%d)", i, sym->sym_width[i]);*/
-      if(readBits(&val, sym->sym_width[i], reader) < 0)
-        return NULL;
-+     if(val >= sym->nr_topsyms[i])
-+       return NULL;
-      arg_sym = &read_symbols[sym->topsyms[i][val]];
-      /*		ATfprintf(stderr, "argument %d, symbol index = %d, symbol = %y\n", 
-  		i, val, arg_sym->sym);*/
-***************
-*** 1381,1386 ****
---- 1384,1391 ----
-      if(readBits(&val, arg_sym->term_width, reader) < 0)
-        return NULL;
-      /*		ATfprintf(stderr, "arg term index = %d\n", val);*/
-+     if(val >= arg_sym->nr_terms)
-+       return NULL;
-      if(!arg_sym->terms[val]) {
-        arg_sym->terms[val] = read_term(arg_sym, reader);
-        if(!arg_sym->terms[val])
-***************
-*** 1396,1402 ****
-    case AS_INT:
-      /*{{{  Read an integer */
-  
-!     if(readBits(&val, HEADER_BITS, reader) < 0)
-        return NULL;
-  
-      result = (ATerm)ATmakeInt((int)val);
---- 1401,1407 ----
-    case AS_INT:
-      /*{{{  Read an integer */
-  
-!     if(readBits(&val, INT_SIZE_IN_BAF, reader) < 0)
-        return NULL;
-  
-      result = (ATerm)ATmakeInt((int)val);
-***************
-*** 1494,1502 ****
-    for(i=0; i<nr_unique_symbols; i++) {
-      sym_read_entry *entry = &read_symbols[i];
-  
-!     ATunprotectArray(entry->terms);
-!     if(entry->terms)
-        free(entry->terms);
-      if(entry->nr_topsyms)
-        free(entry->nr_topsyms);
-      if(entry->sym_width)
---- 1499,1508 ----
-    for(i=0; i<nr_unique_symbols; i++) {
-      sym_read_entry *entry = &read_symbols[i];
-  
-!     if(entry->terms) {
-!       ATunprotectArray(entry->terms);
-        free(entry->terms);
-+     }
-      if(entry->nr_topsyms)
-        free(entry->nr_topsyms);
-      if(entry->sym_width)
-Only in aterm-2.4.2/aterm: config.h.in
-diff -rc aterm-2.4.2-orig/aterm/encoding.h aterm-2.4.2/aterm/encoding.h
-*** aterm-2.4.2-orig/aterm/encoding.h	2004-06-01 10:29:02.000000000 +0200
---- aterm-2.4.2/aterm/encoding.h	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 10,15 ****
---- 10,17 ----
-  {
-  #endif/* __cplusplus */
-  
-+ #include "config.h"
-+ 
-  /*
-   |--------------------------------|
-   |info|type |arity|quoted|mark|age|
-***************
-*** 31,37 ****
-  #define SHIFT_REMOVE_MARK_AGE 3
-  #define MASK_AGE_MARK   (MASK_AGE|MASK_MARK)
-  
-! #if AT_64BIT
-  #define SHIFT_LENGTH  34
-  #define HEADER_BITS 64
-  typedef unsigned long header_type;
---- 33,39 ----
-  #define SHIFT_REMOVE_MARK_AGE 3
-  #define MASK_AGE_MARK   (MASK_AGE|MASK_MARK)
-  
-! #ifdef AT_64BIT
-  #define SHIFT_LENGTH  34
-  #define HEADER_BITS 64
-  typedef unsigned long header_type;
-***************
-*** 137,142 ****
---- 139,150 ----
-  #define AT_TABLE_SIZE(table_class)  (1<<(table_class))
-  #define AT_TABLE_MASK(table_class)  (AT_TABLE_SIZE(table_class)-1)
-  
-+ 
-+ /* Integers in BAF are always exactly 32 bits.  The size must be fixed so that
-+  * BAF terms can be exchanged between platforms. */
-+ #define INT_SIZE_IN_BAF 32
-+ 
-+ 
-  #ifdef __cplusplus
-  }
-  #endif/* __cplusplus */ 
-diff -rc aterm-2.4.2-orig/aterm/gc.c aterm-2.4.2/aterm/gc.c
-*** aterm-2.4.2-orig/aterm/gc.c	2004-06-01 10:29:02.000000000 +0200
---- aterm-2.4.2/aterm/gc.c	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 154,166 ****
-      }
-  
-  #ifdef AT_64BIT
-!     odd_term = *((ATerm *)((MachineWord)cur)+4);
-      real_term = AT_isInsideValidTerm(odd_term);
-      if (real_term != NULL) {
-        AT_markTerm(odd_term);
-      }
-  
-!     odd_sym = *((AFun *)((MachineWord)cur)+4);
-      if (AT_isValidSymbol(odd_sym)) {
-          /*fprintf(stderr,"mark_memory: AT_markSymbol(%d)\n",odd_sym);*/
-        AT_markSymbol(odd_sym);
---- 154,166 ----
-      }
-  
-  #ifdef AT_64BIT
-!     odd_term = *((ATerm *)(((MachineWord)cur)+4));
-      real_term = AT_isInsideValidTerm(odd_term);
-      if (real_term != NULL) {
-        AT_markTerm(odd_term);
-      }
-  
-!     odd_sym = *((AFun *)(((MachineWord)cur)+4));
-      if (AT_isValidSymbol(odd_sym)) {
-          /*fprintf(stderr,"mark_memory: AT_markSymbol(%d)\n",odd_sym);*/
-        AT_markSymbol(odd_sym);
-***************
-*** 198,210 ****
-      }
-  
-  #ifdef AT_64BIT
-!     odd_term = *((ATerm *)((MachineWord)cur)+4);
-      real_term = AT_isInsideValidTerm(odd_term);
-      if (real_term != NULL) {
-        AT_markTerm_young(odd_term);
-      }
-  
-!     odd_sym = *((AFun *)((MachineWord)cur)+4);
-      if (AT_isValidSymbol(odd_sym)) {
-          /*fprintf(stderr,"mark_memory_young: AT_markSymbol_young(%d)\n",odd_sym);*/
-        AT_markSymbol_young(odd_sym);
---- 198,210 ----
-      }
-  
-  #ifdef AT_64BIT
-!     odd_term = *((ATerm *)(((MachineWord)cur)+4));
-      real_term = AT_isInsideValidTerm(odd_term);
-      if (real_term != NULL) {
-        AT_markTerm_young(odd_term);
-      }
-  
-!     odd_sym = *((AFun *)(((MachineWord)cur)+4));
-      if (AT_isValidSymbol(odd_sym)) {
-          /*fprintf(stderr,"mark_memory_young: AT_markSymbol_young(%d)\n",odd_sym);*/
-        AT_markSymbol_young(odd_sym);
-***************
-*** 225,235 ****
-    ATerm *start, *stop;
-    ProtEntry *prot;
-  
-- #ifdef AT_64BIT
--   ATerm oddTerm;
--   AFun oddSym;
-- #endif
-- 
-  #ifdef WIN32
-  
-    unsigned int r_eax, r_ebx, r_ecx, r_edx, \
---- 225,230 ----
-***************
-*** 287,293 ****
-    /* Traverse possible register variables */
-    sigsetjmp(env,0);
-  
-!   start = (ATerm *)env;
-    stop  = ((ATerm *)(((char *)env) + sizeof(sigjmp_buf)));
-    mark_memory(start, stop);
-  #endif
---- 282,288 ----
-    /* Traverse possible register variables */
-    sigsetjmp(env,0);
-  
-!   start = (ATerm *)env; /* !!! illegal aliasing  */
-    stop  = ((ATerm *)(((char *)env) + sizeof(sigjmp_buf)));
-    mark_memory(start, stop);
-  #endif
-***************
-*** 338,348 ****
-    ATerm *start, *stop;
-    ProtEntry *prot;
-  
-- #ifdef AT_64BIT
--   ATerm oddTerm;
--   AFun oddSym;
-- #endif
-- 
-  #ifdef WIN32
-  
-    unsigned int r_eax, r_ebx, r_ecx, r_edx, \
---- 333,338 ----
-***************
-*** 400,406 ****
-      /* Traverse possible register variables */
-    sigsetjmp(env,0);
-  
-!   start = (ATerm *)env;
-    stop  = ((ATerm *)(((char *)env) + sizeof(sigjmp_buf)));
-    mark_memory_young(start, stop);
-  #endif
---- 390,396 ----
-      /* Traverse possible register variables */
-    sigsetjmp(env,0);
-  
-!   start = (ATerm *)env; /* !!! illegal aliasing  */
-    stop  = ((ATerm *)(((char *)env) + sizeof(sigjmp_buf)));
-    mark_memory_young(start, stop);
-  #endif
-***************
-*** 1047,1053 ****
-        /*fprintf(stderr,"minor_sweep_phase_young: ensure empty freelist[%d]\n",size);*/
-        for(data = at_freelist[size] ; data ; data=data->next) {
-          if(!EQUAL_HEADER(data->header,FREE_HEADER)) {
-!           fprintf(stderr,"data = %x header = %x\n",(unsigned int)data,data->header);
-          }
-          assert(EQUAL_HEADER(data->header,FREE_HEADER)); 
-          assert(ATgetType(data) == AT_FREE);   
---- 1037,1043 ----
-        /*fprintf(stderr,"minor_sweep_phase_young: ensure empty freelist[%d]\n",size);*/
-        for(data = at_freelist[size] ; data ; data=data->next) {
-          if(!EQUAL_HEADER(data->header,FREE_HEADER)) {
-!           fprintf(stderr,"data = %p header = %x\n",data,(unsigned int) data->header);
-          }
-          assert(EQUAL_HEADER(data->header,FREE_HEADER)); 
-          assert(ATgetType(data) == AT_FREE);   
-diff -rc aterm-2.4.2-orig/aterm/Makefile.am aterm-2.4.2/aterm/Makefile.am
-*** aterm-2.4.2-orig/aterm/Makefile.am	2005-08-03 11:45:19.000000000 +0200
---- aterm-2.4.2/aterm/Makefile.am	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 37,43 ****
-    aterm2.h \
-    atypes.h \
-    deprecated.h \
-!   encoding.h
-  
-  PRIVATE_INCL = \
-    _afun.h \
---- 37,44 ----
-    aterm2.h \
-    atypes.h \
-    deprecated.h \
-!   encoding.h \
-!   config.h
-  
-  PRIVATE_INCL = \
-    _afun.h \
-diff -rc aterm-2.4.2-orig/aterm/md5.h aterm-2.4.2/aterm/md5.h
-*** aterm-2.4.2-orig/aterm/md5.h	2003-09-02 15:32:46.000000000 +0200
---- aterm-2.4.2/aterm/md5.h	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 24,29 ****
---- 24,31 ----
-  documentation and/or software.
-   */
-  
-+ #include <stdint.h>
-+ 
-  /* GLOBAL.H - RSAREF types and constants
-   */
-  
-***************
-*** 46,55 ****
-  typedef unsigned char *POINTER;
-  
-  /* UINT2 defines a two byte word */
-! typedef unsigned short int UINT2;
-  
-  /* UINT4 defines a four byte word */
-! typedef unsigned long int UINT4;
-  
-  /* PROTO_LIST is defined depending on how PROTOTYPES is defined above.
-  If using PROTOTYPES, then PROTO_LIST returns the list, otherwise it
---- 48,57 ----
-  typedef unsigned char *POINTER;
-  
-  /* UINT2 defines a two byte word */
-! typedef uint16_t UINT2;
-  
-  /* UINT4 defines a four byte word */
-! typedef uint32_t UINT4;
-  
-  /* PROTO_LIST is defined depending on how PROTOTYPES is defined above.
-  If using PROTOTYPES, then PROTO_LIST returns the list, otherwise it
-diff -rc aterm-2.4.2-orig/aterm/memory.c aterm-2.4.2/aterm/memory.c
-*** aterm-2.4.2-orig/aterm/memory.c	2005-10-11 13:19:25.000000000 +0200
---- aterm-2.4.2/aterm/memory.c	2006-09-22 13:39:16.000000000 +0200
-***************
-*** 96,102 ****
-  #define HASHNUMBER4(t) hash_number(t,4)
-  #define HASHINT(val)   (tmp[0]=(MachineWord)(AT_INT<<SHIFT_TYPE),\
-                          tmp[1]=(MachineWord)0,\
-!                         tmp[2]=(MachineWord)val,\
-                          hash_number(tmp,3)) 
-  
-  #else
---- 96,102 ----
-  #define HASHNUMBER4(t) hash_number(t,4)
-  #define HASHINT(val)   (tmp[0]=(MachineWord)(AT_INT<<SHIFT_TYPE),\
-                          tmp[1]=(MachineWord)0,\
-!                         tmp[2]=(MachineWord)((unsigned int) val),\
-                          hash_number(tmp,3)) 
-  
-  #else
-***************
-*** 127,133 ****
-  		       ((MachineWord*)t)[2]),((MachineWord*)t)[3]))
-  
-  #define HASHINT(val) \
-! FINISH(COMBINE(START( (AT_INT<<SHIFT_TYPE) ), val))
-  
-  
-  #endif /* HASHPEM */
---- 127,133 ----
-  		       ((MachineWord*)t)[2]),((MachineWord*)t)[3]))
-  
-  #define HASHINT(val) \
-! FINISH(COMBINE(START( (AT_INT<<SHIFT_TYPE) ), (unsigned int) val))
-  
-  
-  #endif /* HASHPEM */
-***************
-*** 708,714 ****
-    at_blocks[size] = newblock;
-    top_at_blocks[size] = newblock->data;
-    assert(at_blocks[size] != NULL);
-!   assert(((int)top_at_blocks[size] % MAX(sizeof(double), sizeof(void *))) == 0);
-    
-      /* [pem: Feb 14 02] TODO: fast allocation */
-    assert(at_freelist[size] == NULL);
---- 708,714 ----
-    at_blocks[size] = newblock;
-    top_at_blocks[size] = newblock->data;
-    assert(at_blocks[size] != NULL);
-!   assert(((long)top_at_blocks[size] % MAX(sizeof(double), sizeof(void *))) == 0);
-    
-      /* [pem: Feb 14 02] TODO: fast allocation */
-    assert(at_freelist[size] == NULL);
-***************
-*** 1009,1018 ****
-    do {
-      if(!cur) {
-          /*printf("freeterm = %d\n",t);*/
-!       fprintf(stderr,"### cannot find term %x in hashtable at pos %d header = %x\n", (unsigned int)t, hnr, t->header);
-! 
-!       ATabort("### cannot find term %n at %p in hashtable at pos %d"
-! 	      ", header = %d\n", t, t, hnr, t->header);
-      }
-      if (cur == t) {
-        if(prev)
---- 1009,1016 ----
-    do {
-      if(!cur) {
-          /*printf("freeterm = %d\n",t);*/
-!       ATabort("### cannot find term %p in hashtable at pos %d"
-! 	      ", header = %x\n", t, hnr, t->header);
-      }
-      if (cur == t) {
-        if(prev)
-***************
-*** 1728,1733 ****
---- 1726,1733 ----
-      hashtable[hnr] = cur;
-    }
-  
-+   assert((hnr & table_mask) == (hash_number(cur, TERM_SIZE_INT) & table_mask));
-+ 
-    return (ATermInt)cur;  
-  }
-  
-diff -rc aterm-2.4.2-orig/aterm.m4 aterm-2.4.2/aterm.m4
-*** aterm-2.4.2-orig/aterm.m4	2005-08-03 11:45:19.000000000 +0200
---- aterm-2.4.2/aterm.m4	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 8,15 ****
-      [AS_HELP_STRING([--with-sharing],[create libraries that do term sharing @<:@yes@:>@])],
-      [if test "$withval" = "no"; then
-         AC_MSG_RESULT([no])
-!        AC_DEFINE([NO_SHARING])
-!        AC_DEFINE([WITH_STATS])
-       else
-         if test "$withval" != "yes"; then
-           AC_MSG_RESULT([unknown value specified for --with-sharing.])
---- 8,15 ----
-      [AS_HELP_STRING([--with-sharing],[create libraries that do term sharing @<:@yes@:>@])],
-      [if test "$withval" = "no"; then
-         AC_MSG_RESULT([no])
-!        AC_DEFINE([NO_SHARING], [], [Whether terms are shared.])
-!        AC_DEFINE([WITH_STATS], [], [Whether to keep statistics.])
-       else
-         if test "$withval" != "yes"; then
-           AC_MSG_RESULT([unknown value specified for --with-sharing.])
-***************
-*** 73,78 ****
---- 73,102 ----
-    AC_SUBST([OPTIMIZECFLAGS])
-  ])
-  
-+ # ATERM_64_BIT
-+ # ------------
-+ # Enable 64-bit mode if pointers are 8 bytes large.
-+ AC_DEFUN([ATERM_64_BIT], [
-+   AC_CHECK_SIZEOF(void *)
-+   AC_CHECK_SIZEOF(int)
-+   AC_CHECK_SIZEOF(long)
-+ 
-+   AC_MSG_CHECKING([what kind of platform this is])
-+ 
-+   AC_SUBST([AT_64BIT], [0])
-+   if test "$ac_cv_sizeof_void_p" = "8" -a "$ac_cv_sizeof_int" = "4" -a "$ac_cv_sizeof_long" = "8"; then
-+     AC_MSG_RESULT([LP64])
-+     AC_SUBST([AT_64BIT], [1])
-+   elif test "$ac_cv_sizeof_void_p" = "8" -a "$ac_cv_sizeof_int" = "8" -a "$ac_cv_sizeof_long" = "8"; then
-+     AC_MSG_RESULT([ILP64 - warning, untested])
-+     AC_SUBST([AT_64BIT], [1])
-+   elif test "$ac_cv_sizeof_void_p" = "4" -a "$ac_cv_sizeof_int" = "4" -a "$ac_cv_sizeof_long" = "4"; then
-+     AC_MSG_RESULT([32 bits])
-+   else
-+     AC_MSG_RESULT([something weird - warning, untested])
-+   fi
-+ ])
-+ 
-  # XT_SVN_REVISION
-  # ---------------
-  AC_DEFUN([XT_SVN_REVISION],
-diff -rc aterm-2.4.2-orig/configure.ac aterm-2.4.2/configure.ac
-*** aterm-2.4.2-orig/configure.ac	2005-08-03 11:45:19.000000000 +0200
---- aterm-2.4.2/configure.ac	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 30,35 ****
---- 30,38 ----
-  # Add a configuration option to allow users to control sharing.
-  ATERM_WITH_SHARING
-  
-+ # Enable 64-bit mode if pointers are 8 bytes large.
-+ ATERM_64_BIT
-+ 
-  CURDATE=`date`
-  AC_SUBST([CURDATE])
-  
-***************
-*** 45,49 ****
---- 48,53 ----
-    doc/spec/Makefile
-    aterm.spec
-    aterm.pc
-+   aterm/config.h
-  ])
-  AC_OUTPUT
-diff -rc aterm-2.4.2-orig/utils/dicttoc.c aterm-2.4.2/utils/dicttoc.c
-*** aterm-2.4.2-orig/utils/dicttoc.c	2003-10-07 13:57:40.000000000 +0200
---- aterm-2.4.2/utils/dicttoc.c	2006-09-22 13:39:07.000000000 +0200
-***************
-*** 69,74 ****
---- 69,75 ----
-    fprintf(file, "#ifndef __%s_H\n", code_prefix);
-    fprintf(file, "#define __%s_H\n\n", code_prefix);
-    fprintf(file, "#include <aterm2.h>\n\n");
-+   fprintf(file, "#include <assert.h>\n\n");
-  
-    while (!ATisEmpty(afuns)) {
-      ATerm afun, alias, pair = ATgetFirst(afuns);
-***************
-*** 244,251 ****
-    ATfprintf(file, "{\n");
-    ATfprintf(file, "  ATermList afuns, terms;\n\n");
-  
-!   ATfprintf(file, "  _%s = ATreadFromBinaryString(_%s_baf, _%s_LEN);\n\n", 
-  	    code_prefix, code_prefix, code_prefix);
-    ATfprintf(file, "  ATprotect(&_%s);\n\n", code_prefix);
-  
-    ATfprintf(file, "  afuns = (ATermList)ATelementAt((ATermList)_%s, 0);\n\n", code_prefix);
---- 245,253 ----
-    ATfprintf(file, "{\n");
-    ATfprintf(file, "  ATermList afuns, terms;\n\n");
-  
-!   ATfprintf(file, "  _%s = ATreadFromBinaryString(_%s_baf, _%s_LEN);\n", 
-  	    code_prefix, code_prefix, code_prefix);
-+   ATfprintf(file, "  assert(_%s);\n\n", code_prefix);
-    ATfprintf(file, "  ATprotect(&_%s);\n\n", code_prefix);
-  
-    ATfprintf(file, "  afuns = (ATermList)ATelementAt((ATermList)_%s, 0);\n\n", code_prefix);

externals/aterm-aliasing.patch

@@ -1,224 +0,0 @@
-diff -rc aterm-1142707243.10633/aterm/aterm.c aterm/aterm/aterm.c
-*** aterm-1142707243.10633/aterm/aterm.c	2006-02-08 11:35:28.000000000 +0100
---- aterm/aterm/aterm.c	2006-04-25 17:10:52.000000000 +0200
-***************
-*** 193,198 ****
---- 193,199 ----
-    /* that have char == 2 bytes, and sizeof(header_type) == 2 */
-    assert(sizeof(header_type) == sizeof(ATerm *));
-    assert(sizeof(header_type) >= 4);
-+   assert(sizeof(ATerm) == sizeof(MachineWord));
-  
-    /*}}}  */
-    /*{{{  Initialize buffer */
-diff -rc aterm-1142707243.10633/aterm/memory.c aterm/aterm/memory.c
-*** aterm-1142707243.10633/aterm/memory.c	2006-03-09 15:02:56.000000000 +0100
---- aterm/aterm/memory.c	2006-04-25 18:22:00.000000000 +0200
-***************
-*** 119,130 ****
-                          hash_number(tmp,3)) 
-  */
-  
-  #define HASHNUMBER3(t)\
-! FINISH(COMBINE(START(((MachineWord*)t)[0]), ((MachineWord*)t)[2]))
-  
-  #define HASHNUMBER4(t)\
-! FINISH(COMBINE(COMBINE(START(((MachineWord*)t)[0]), \
-! 		       ((MachineWord*)t)[2]),((MachineWord*)t)[3]))
-  
-  #define HASHINT(val) \
-  FINISH(COMBINE(START( (AT_INT<<SHIFT_TYPE) ), val))
---- 119,171 ----
-                          hash_number(tmp,3)) 
-  */
-  
-+ /* The ATerm library use some heavy aliasing.  For instance, the
-+    various ATermXXX structures are referenced through MachineWord
-+    arrays.  This is not generally allowed by the C standard --- see
-+    C99, section 6.5, clause 7.  In particular, this means that you
-+    cannot assign something through an ATermXXX pointer, e.g.,
-+ 
-+      protoAppl->header = header;
-+ 
-+    and then read it through a MachineWord*, e.g.,
-+ 
-+      hnr = hash_number((ATerm) protoAppl, 2);
-+ 
-+    (hash_number walks over the term by casting it to a MachineWord*).
-+ 
-+    However, the same clause of the C standard also specifies that you
-+    *can* read the memory location through a union type that contains
-+    both the original type (e.g. ATermAppl) and the type used to read
-+    the memory location (e.g. MachineWord).  That's what we do
-+    below: we have a union of all the types that occur in the various
-+    ATerm types.  We then read the "w" element of the union.  The
-+    compiler is not allowed to assume absence of aliasing with the
-+    other types in the union.
-+ 
-+    A better solution would be to hash the term through a character
-+    pointer (since *any* memory location can be legally read as a
-+    character), but I'm too lazy right now.  Performance might also
-+    suffer if we do that. */
-+ 
-+ typedef union 
-+ {
-+     MachineWord w;
-+     header_type header;
-+     ATerm term;
-+     ATermList list;
-+     int i;
-+     double d;
-+     void* p;
-+ } Aliaser;
-+ 
-+ #define GET_WORD(t, n) (((Aliaser*) (((MachineWord*) t) + n))->w)
-+ 
-  #define HASHNUMBER3(t)\
-! FINISH(COMBINE(START(GET_WORD(t, 0)), GET_WORD(t, 2)))
-  
-  #define HASHNUMBER4(t)\
-! FINISH(COMBINE(COMBINE(START(GET_WORD(t, 0)), \
-! 		       GET_WORD(t, 2)), GET_WORD(t, 3)))
-  
-  #define HASHINT(val) \
-  FINISH(COMBINE(START( (AT_INT<<SHIFT_TYPE) ), val))
-***************
-*** 132,144 ****
-  
-  #endif /* HASHPEM */
-  
-! #define PROTO_APPL_ARGS ((ATerm *) (protoTerm + ARG_OFFSET))
-  
-  #define SET_PROTO_APPL_ARG(i, a) \
-!   (PROTO_APPL_ARGS[(i)] = (a))
-  
-  #define GET_PROTO_APPL_ARG(i) \
-!   (PROTO_APPL_ARGS[(i)])
-  
-  #define CHECK_TERM(t) \
-    assert((t) != NULL \
---- 173,185 ----
-  
-  #endif /* HASHPEM */
-  
-! #define PROTO_APPL_ARGS (protoTerm + ARG_OFFSET)
-  
-  #define SET_PROTO_APPL_ARG(i, a) \
-!   (PROTO_APPL_ARGS[(i)] = (MachineWord) (a))
-  
-  #define GET_PROTO_APPL_ARG(i) \
-!   ((ATerm) PROTO_APPL_ARGS[(i)])
-  
-  #define CHECK_TERM(t) \
-    assert((t) != NULL \
-***************
-*** 323,336 ****
-  #else
-  static HashNumber hash_number(ATerm t, int size)
-  {
--   MachineWord *words = (MachineWord *) t;
-    int i;
-    HashNumber hnr;
-  
-!   hnr = START(HIDE_AGE_MARK(words[0]));
-    
-    for (i=2; i<size; i++) {
-!     hnr = COMBINE(hnr, words[i]);
-    }
-  
-    return FINISH(hnr);
---- 364,376 ----
-  #else
-  static HashNumber hash_number(ATerm t, int size)
-  {
-    int i;
-    HashNumber hnr;
-  
-!   hnr = START(HIDE_AGE_MARK(GET_WORD(t, 0)));
-    
-    for (i=2; i<size; i++) {
-!     hnr = COMBINE(hnr, GET_WORD(t, i));
-    }
-  
-    return FINISH(hnr);
-***************
-*** 338,351 ****
-  
-  static HashNumber hash_number_anno(ATerm t, int size, ATerm anno)
-  {
--   MachineWord *words = (MachineWord *) t;
-    int i;
-    HashNumber hnr;
-  
-!   hnr = START(HIDE_AGE_MARK(words[0]));
-    
-    for (i=2; i<size; i++) {
-!     hnr = COMBINE(hnr, words[i]);
-    }
-    hnr = COMBINE(hnr, (MachineWord)anno);
-  
---- 378,390 ----
-  
-  static HashNumber hash_number_anno(ATerm t, int size, ATerm anno)
-  {
-    int i;
-    HashNumber hnr;
-  
-!   hnr = START(HIDE_AGE_MARK(GET_WORD(t, 0)));
-    
-    for (i=2; i<size; i++) {
-!     hnr = COMBINE(hnr, GET_WORD(t, i));
-    }
-    hnr = COMBINE(hnr, (MachineWord)anno);
-  
-***************
-*** 1639,1645 ****
-    protoAppl->header = header;
-    CHECK_HEADER(protoAppl->header);
-  
-!   if (args != PROTO_APPL_ARGS) {
-      for (i=0; i<arity; i++) {
-        CHECK_TERM(args[i]);
-        SET_PROTO_APPL_ARG(i, args[i]);
---- 1678,1684 ----
-    protoAppl->header = header;
-    CHECK_HEADER(protoAppl->header);
-  
-!   if (args != (ATerm *) PROTO_APPL_ARGS) {
-      for (i=0; i<arity; i++) {
-        CHECK_TERM(args[i]);
-        SET_PROTO_APPL_ARG(i, args[i]);
-***************
-*** 1680,1686 ****
-      hashtable[hnr] = cur;
-    }
-  
-!   if (args != PROTO_APPL_ARGS) {
-      for (i=0; i<arity; i++) {
-        protected_buffer[i] = NULL;
-      }
---- 1719,1725 ----
-      hashtable[hnr] = cur;
-    }
-  
-!   if (args != (ATerm *) PROTO_APPL_ARGS) {
-      for (i=0; i<arity; i++) {
-        protected_buffer[i] = NULL;
-      }
-***************
-*** 2144,2150 ****
-    }
-    SET_PROTO_APPL_ARG(n, arg);
-  
-!   result = ATmakeApplArray(sym, PROTO_APPL_ARGS);
-    annos = AT_getAnnotations((ATerm)appl);
-    if (annos != NULL) {
-      result = (ATermAppl)AT_setAnnotations((ATerm)result, annos);
---- 2183,2189 ----
-    }
-    SET_PROTO_APPL_ARG(n, arg);
-  
-!   result = ATmakeApplArray(sym, (ATerm *) PROTO_APPL_ARGS);
-    annos = AT_getAnnotations((ATerm)appl);
-    if (annos != NULL) {
-      result = (ATermAppl)AT_setAnnotations((ATerm)result, annos);

externals/bdb-cygwin.patch

@@ -1,21 +1,22 @@
-diff -rc db-4.4.20.NC-old/os/os_flock.c db-4.4.20.NC/os/os_flock.c
-*** db-4.4.20.NC-old/os/os_flock.c	Mon Jun 20 16:59:01 2005
---- db-4.4.20.NC/os/os_flock.c	Wed Jun  7 17:01:49 2006
+diff -rc db-4.5.20-orig/os/os_flock.c db-4.5.20/os/os_flock.c
+*** db-4.5.20-orig/os/os_flock.c	2006-10-13 12:36:12.000000000 +0200
+--- db-4.5.20/os/os_flock.c	2006-10-13 12:40:11.000000000 +0200
 ***************
-*** 36,41 ****
---- 36,50 ----
+*** 30,35 ****
+--- 30,44 ----
   
-  	DB_ASSERT(F_ISSET(fhp, DB_FH_OPENED) && fhp->fd != -1);
+  	DB_ASSERT(dbenv, F_ISSET(fhp, DB_FH_OPENED) && fhp->fd != -1);
   
 + #ifdef __CYGWIN__
 + 	/*
 + 	 * Windows file locking interferes with read/write operations, so we
 + 	 * map the ranges to an area past the end of the file.
 + 	 */
-+ 	DB_ASSERT(offset < (off_t) 1 << 62);
++ 	DB_ASSERT(dbenv, offset < (off_t) 1 << 62);
 + 	offset += (off_t) 1 << 62;
 + #endif
 + 
-  #ifdef HAVE_FCNTL
   	fl.l_start = offset;
   	fl.l_len = 1;
+  	fl.l_type = acquire ? F_WRLCK : F_UNLCK;
+Only in db-4.5.20/os: os_flock.c~

nix.conf.example

@@ -78,35 +78,61 @@
 #build-max-jobs = 1
 
 
-### Option `build-allow-root'
+### Option `build-max-silent-time'
 #
-# This option controls Nix's behaviour when it is invoked under the
-# `root' user (or setuid-root).  If `true' (default), builds are
-# performed under the `root' user.  If `false', builds are performed
-# under one of the users listed in the `build-users' option (see
-# below).
-#build-allow-root = true
-
-
-### Option `build-users'
+# This option defines the maximum number of seconds that builder can
+# go without producing any data on standard output or standard error.
+# This is useful (for instance in a automated build system) to catch
+# builds that are stuck in an infinite loop, or to catch remote builds
+# that are hanging due to network problems.  It can be overriden using
+# the `--max-silent-time' command line switch.
 #
-# This option is only applicable if `build-allow-root' is `false' and
-# Nix is invoked under the `root' user (or setuid-root).  It contains
-# a list of user names under which Nix can execute builds.  Builds
-# cannot be performed by root since that would allow users to take
-# over the system by supplying specially crafted builders; and they
-# cannot be performed by the calling user since that would allow
-# him/her to influence the build result.
-#
-# Thus this list should contain a number of `special' user accounts
-# created specifically for Nix, e.g., `nix-builder-1',
-# `nix-builder-2', and so on.  The more users the better, since at
-# most a number of builds equal to the number of build users can be
-# started.
+# The value 0 means that there is no timeout.  This is also the
+# default.
 #
 # Example:
-#   build-users = nix-builder-1 nix-builder-2 nix-builder-3
-#build-users =
+#   build-max-silent-time = 600 # = 10 minutes
+#build-max-silent-time = 0
+
+
+### Option `build-users-group'
+#
+# This options specifies the Unix group containing the Nix build user
+# accounts.  In multi-user Nix installations, builds should not
+# be performed by the Nix account since that would allow users to
+# arbitrarily modify the Nix store and database by supplying specially
+# crafted builders; and they cannot be performed by the calling user
+# since that would allow him/her to influence the build result.
+#
+# Therefore, if this option is non-empty and specifies a valid group,
+# builds will be performed under the user accounts that are a member
+# of the group specified here (as listed in /etc/group).  Those user
+# accounts should not be used for any other purpose!
+#
+# Nix will never run two builds under the same user account at the
+# same time.  This is to prevent an obvious security hole: a malicious
+# user writing a Nix expression that modifies the build result of a
+# legitimate Nix expression being built by another user.  Therefore it
+# is good to have as many Nix build user accounts as you can spare.
+# (Remember: uids are cheap.)
+#
+# The build users should have permission to create files in the Nix
+# store, but not delete them.  Therefore, /nix/store should be owned
+# by the Nix account, its group should be the group specified here,
+# and its mode should be 1775.
+#
+# If the build users group is empty, builds will be performed under
+# the uid of the Nix process (that is, the uid of the caller if
+# $NIX_REMOTE is empty, the uid under which the Nix daemon runs if
+# $NIX_REMOTE is `daemon', or the uid that owns the setuid nix-worker
+# program if $NIX_REMOTE is `slave').  Obviously, this should not be
+# used in multi-user settings with untrusted users.
+#
+# The default is empty.
+#
+# Example:
+#   build-users-group = nix-builders
+#build-users-group =
 
 
 ### Option `system'

nix.spec.in

@@ -13,7 +13,7 @@ Version: @version@
 Release: 1
 License: GPL
 Group: Software Deployment
-URL: http://www.cs.uu.nl/groups/ST/Trace/Nix
+URL: http://nix.cs.uu.nl/
 Source0: %{name}-@version@.tar.bz2
 BuildRoot: %{_tmppath}/%{name}-%{version}-buildroot
 %define _prefix /nix
@@ -26,7 +26,12 @@ Provides: perl(readmanifest)
                                                                
 %description
 
-Nix is a system for software deployment.
+Nix is a purely functional package manager. It allows multiple
+versions of a package to be installed side-by-side, ensures that
+dependency specifications are complete, supports atomic upgrades and
+rollbacks, allows non-root users to install software, and has many
+other features. It is the basis of the NixOS Linux distribution, but
+it can be used equally well under other Unix systems.
 
 %prep
 %setup -q
@@ -73,7 +78,6 @@ fi
 %{_prefix}/include
 %{_prefix}/var
 %{_prefix}/share
-%{_prefix}/man
 %{_prefix}/store
 %config
 %{_prefix}/etc

scripts/Makefile.am

@@ -1,7 +1,8 @@
 bin_SCRIPTS = nix-collect-garbage \
   nix-pull nix-push nix-prefetch-url \
   nix-install-package nix-channel nix-build \
-  nix-pack-closure nix-unpack-closure
+  nix-pack-closure nix-unpack-closure \
+  nix-copy-closure
 
 noinst_SCRIPTS = nix-profile.sh generate-patches.pl find-runtime-roots.pl
 
@@ -15,6 +16,7 @@ install-exec-local: readmanifest.pm download-using-manifests.pl find-runtime-roo
 	$(INSTALL_DATA) readconfig.pm $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL_PROGRAM) find-runtime-roots.pl $(DESTDIR)$(libexecdir)/nix 
+	$(INSTALL_PROGRAM) generate-patches.pl $(DESTDIR)$(libexecdir)/nix 
 	$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
 
 include ../substitute.mk
@@ -29,4 +31,5 @@ EXTRA_DIST = nix-collect-garbage.in \
   download-using-manifests.pl.in \
   generate-patches.pl.in \
   nix-pack-closure.in nix-unpack-closure.in \
+  nix-copy-closure.in \
   find-runtime-roots.pl.in

scripts/download-using-manifests.pl.in

@@ -10,8 +10,6 @@ my $logFile = "@localstatedir@/log/nix/downloads";
 
 open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
 
-delete $ENV{"NIX_ROOT"};
-
 # Create a temporary directory.
 my $tmpDir = tempdir("nix-download.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
@@ -24,9 +22,61 @@ my $tmpNar2 = "$tmpDir/nar2";
 END { unlink $tmpNar; unlink $tmpNar2; rmdir $tmpDir; }
 
 
-# Check the arguments.
-die unless scalar @ARGV == 1;
-my $targetPath = $ARGV[0];
+# Load all manifests.
+my %narFiles;
+my %localPaths;
+my %patches;
+
+for my $manifest (glob "$manifestDir/*.nixmanifest") {
+#    print STDERR "reading $manifest\n";
+    if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
+        print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
+        exit 1;
+    }
+}
+
+
+# Parse the arguments.
+
+if ($ARGV[0] eq "--query-paths") {
+    foreach my $storePath (keys %narFiles) { print "$storePath\n"; }
+    foreach my $storePath (keys %localPaths) { print "$storePath\n"; }
+    exit 0;
+}
+
+elsif ($ARGV[0] eq "--query-info") {
+    shift @ARGV;
+    foreach my $storePath (@ARGV) {
+        my $info;
+        if (defined $narFiles{$storePath}) {
+            $info = @{$narFiles{$storePath}}[0];
+        }
+        elsif (defined $localPaths{$storePath}) {
+            $info = @{$localPaths{$storePath}}[0];
+        }
+        else {
+            next; # not an error
+        }
+        print "$storePath\n";
+        print "$info->{deriver}\n";
+        my @references = split " ", $info->{references};
+        my $count = scalar @references;
+        print "$count\n";
+        foreach my $reference (@references) {
+            print "$reference\n";
+        }
+    }
+    exit 0;
+}
+
+elsif ($ARGV[0] ne "--substitute") {
+    die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n";
+}
+
+
+die unless scalar @ARGV == 2;
+my $targetPath = $ARGV[1];
+
 
 my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
 print LOGFILE "$$ get $targetPath $date\n";
@@ -34,16 +84,15 @@ print LOGFILE "$$ get $targetPath $date\n";
 print "\n*** Trying to download/patch `$targetPath'\n";
 
 
-# Load all manifests.
-my %narFiles;
-my %patches;
-my %successors;
-
-for my $manifest (glob "$manifestDir/*.nixmanifest") {
-#    print STDERR "reading $manifest\n";
-    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
-        print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
-        exit 1;
+# If we can copy from a local path, do that.
+my $localPathList = $localPaths{$targetPath};
+foreach my $localPath (@{$localPathList}) {
+    my $sourcePath = $localPath->{copyFrom};
+    if (-e $sourcePath) {
+        print "\n*** Step 1/1: copying from $sourcePath\n";
+        system("@bindir@/nix-store --dump $sourcePath | @bindir@/nix-store --restore $targetPath") == 0
+            or die "cannot copy `$sourcePath' to `$targetPath'";
+        exit 0;
     }
 }
 

scripts/find-runtime-roots.pl.in

@@ -56,3 +56,20 @@ sub lsof {
 
 readProc;
 lsof;
+
+
+sub readFile {
+    my $path = shift;
+    if (-e $path) {
+        if (open FILE, "$path") {
+            while (<FILE>) {
+                print;
+            }
+            close FILE;
+        }
+    }
+}
+
+# This is rather NixOS-specific, so it probably shouldn't be here.
+readFile "/proc/sys/kernel/modprobe";
+readFile "/proc/sys/kernel/fbsplash";

scripts/generate-patches.pl.in

@@ -4,6 +4,18 @@ use strict;
 use File::Temp qw(tempdir);
 use readmanifest;
 
+
+# Some patch generations options.
+
+# Max size of NAR archives to generate patches for.
+my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
+$maxNarSize = 100 * 1024 * 1024 if !defined $maxNarSize;
+
+# If patch is bigger than this fraction of full archive, reject.
+my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
+$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
+
+
 die unless scalar @ARGV == 5;
 
 my $hashAlgo = "sha256";
@@ -22,23 +34,22 @@ print "TEMP = $tmpDir\n";
 #END { rmdir $tmpDir; }
 
 my %srcNarFiles;
+my %srcLocalPaths;
 my %srcPatches;
-my %srcSuccessors;
 
 my %dstNarFiles;
+my %dstLocalPaths;
 my %dstPatches;
-my %dstSuccessors;
 
 readManifest "$srcDir/MANIFEST",
-    \%srcNarFiles, \%srcPatches, \%srcSuccessors;
+    \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
 
 readManifest "$dstDir/MANIFEST",
-    \%dstNarFiles, \%dstPatches, \%dstSuccessors;
+    \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
 
 
 sub findOutputPaths {
     my $narFiles = shift;
-    my $successors = shift;
 
     my %outPaths;
     
@@ -63,10 +74,10 @@ sub findOutputPaths {
 }
 
 print "finding src output paths...\n";
-my %srcOutPaths = findOutputPaths \%srcNarFiles, \%srcSuccessors;
+my %srcOutPaths = findOutputPaths \%srcNarFiles;
 
 print "finding dst output paths...\n";
-my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
+my %dstOutPaths = findOutputPaths \%dstNarFiles;
 
 
 sub getNameVersion {
@@ -277,8 +288,6 @@ foreach my $p (keys %dstOutPaths) {
         my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
         my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
 
-        my $maxNarSize = 150 * 1024 * 1024;
-
         system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
             or die "cannot unpack $srcNarBz2";
 
@@ -310,16 +319,21 @@ foreach my $p (keys %dstOutPaths) {
         my $narDiffSize = (stat "$tmpDir/DIFF")[7];
         my $dstNarBz2Size = (stat $dstNarBz2)[7];
 
+        print "    size $narDiffSize; full size $dstNarBz2Size\n";
+        
         if ($narDiffSize >= $dstNarBz2Size) {
             print "    rejecting; patch bigger than full archive\n";
             next;
         }
     
+        if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
+            print "    rejecting; patch too large relative to full archive\n";
+            next;
+        }
+    
         my $finalName =
             "$narDiffHash.nar-bsdiff";
 
-        print "    size $narDiffSize; full size $dstNarBz2Size\n";
-        
         if (-e "$patchesDir/$finalName") {
             print "    not copying, already exists\n";
         }
@@ -348,11 +362,22 @@ foreach my $p (keys %dstOutPaths) {
 # patches that produce either paths in the destination or paths that
 # can be used as the base for other useful patches).
 
+print "propagating patches...\n";
+
 my $changed;
 do {
     # !!! we repeat this to reach the transitive closure; inefficient
     $changed = 0;
 
+    print "loop\n";
+
+    my %dstBasePaths;
+    foreach my $q (keys %dstPatches) {
+	foreach my $patch (@{$dstPatches{$q}}) {
+	    $dstBasePaths{$patch->{basePath}} = 1;
+	}
+    }
+
     foreach my $p (keys %srcPatches) {
         my $patchList = $srcPatches{$p};
 
@@ -360,22 +385,18 @@ do {
 
         # Is path $p included in the destination?  If so, include
         # patches that produce it.
-        $include = 1 if (defined $dstNarFiles{$p});
+        $include = 1 if defined $dstNarFiles{$p};
 
         # Is path $p a path that serves as a base for paths in the
         # destination?  If so, include patches that produce it.
-        foreach my $q (keys %dstPatches) {
-            foreach my $patch (@{$dstPatches{$q}}) {
-                # !!! check baseHash
-                $include = 1 if ($p eq $patch->{basePath});
-            }
-        }
+	# !!! check baseHash
+        $include = 1 if defined $dstBasePaths{$p};
 
         if ($include) {
             foreach my $patch (@{$patchList}) {
                 $changed = 1 if addPatch \%dstPatches, $p, $patch;
             }
-        }            
+        }
         
     }
     
@@ -384,4 +405,4 @@ do {
 
 # Rewrite the manifest of the destination (with the new patches).
 writeManifest "$dstDir/MANIFEST",
-    \%dstNarFiles, \%dstPatches, \%dstSuccessors;
+    \%dstNarFiles, \%dstPatches;

scripts/maintenance/gc-releases.pl

@@ -7,12 +7,12 @@ use readcache;
 
 # Read the manifests.
 my %narFiles;
+my %localPaths;
 my %patches;
-my %successors;
 
 foreach my $manifest (@ARGV) {
     print STDERR "loading $manifest\n";
-    if (readManifest($manifest, \%narFiles, \%patches, \%successors, 1) < 3) {
+    if (readManifest($manifest, \%narFiles, \%localPaths, \%patches, 1) < 3) {
 #        die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n";
     }
 }

scripts/maintenance/readcache.pm

@@ -15,7 +15,7 @@ sub readDir {
 }
 
 readDir "/data/webserver/dist/nix-cache";
-readDir "/data/webserver/dist/test";
+readDir "/data/webserver/dist/test-cache";
 readDir "/data/webserver/dist/patches";
 
 print STDERR scalar (keys %archives), "\n";

scripts/maintenance/shrink-manifest.pl

@@ -6,12 +6,12 @@ use readcache;
 
 
 my %allNarFiles;
+my %allLocalPaths;
 my %allPatches;
-my %allSuccessors;
 
 foreach my $manifest (glob "/data/webserver/dist/*/*/MANIFEST") {
     print STDERR "loading $manifest\n";
-    readManifest($manifest, \%allNarFiles, \%allPatches, \%allSuccessors, 1);
+    readManifest($manifest, \%allNarFiles, \%allLocalPaths, \%allPatches, 1);
 }
 
 
@@ -22,9 +22,8 @@ foreach my $manifest (@ARGV) {
 
     my %narFiles;
     my %patches;
-    my %successors;
 
-    if (readManifest($manifest, \%narFiles, \%patches, \%successors, 1) < 3) {
+    if (readManifest($manifest, \%narFiles, \%patches, 1) < 3) {
         print STDERR "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n";
 	next;
     }

scripts/nix-build.in

@@ -56,7 +56,7 @@ EOF
     }
 
     elsif ($arg eq "--no-out-link" or $arg eq "--no-link") {
-        $addOutLink = 1;
+        $addOutLink = 0;
     }
 
     elsif ($arg eq "--drv-link") {
@@ -83,6 +83,12 @@ EOF
         $n += 2;
     }
 
+    elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time") {
+        $n++;
+        die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV;
+        push @buildArgs, ($arg, $ARGV[$n]);
+    }
+
     elsif (substr($arg, 0, 1) eq "-") {
         push @buildArgs, $arg;
     }
@@ -116,17 +122,19 @@ foreach my $expr (@exprs) {
     close DRVPATHS or exit 1;
 
     foreach my $drvPath (@drvPaths) {
-        my $target = readlink $drvPath;
+        my $target = readlink $drvPath or die "cannot read symlink `$drvPath'";
         print STDERR "store derivation is $target\n";
     }
 
     # Build.
-    my $outPaths = `@bindir@/nix-store --add-root "$outLink" --indirect -rv @buildArgs @drvPaths`;
-    my @outPaths = split ' ', $outPaths;
-    
+    my @outPaths;
+    $pid = open(OUTPATHS, "-|") || exec "@bindir@/nix-store", "--add-root", $outLink, "--indirect", "-rv",
+        @buildArgs, @drvPaths;
+    while (<OUTPATHS>) {chomp; push @outPaths, $_;}
+    close OUTPATHS or exit 1;
+
     foreach my $outPath (@outPaths) {
-        my $target = readlink $outPath;
+        my $target = readlink $outPath or die "cannot read symlink `$outPath'";
         print "$target\n";
     }
-
 }

scripts/nix-channel.in

@@ -2,16 +2,24 @@
 
 use strict;
 
-my $rootsDir = "@localstatedir@/nix/gcroots/channels";
+my $rootsDir = "@localstatedir@/nix/gcroots";
 
 my $stateDir = $ENV{"NIX_STATE_DIR"};
 $stateDir = "@localstatedir@/nix" unless defined $stateDir;
 
 
+# Turn on caching in nix-prefetch-url.
+my $channelCache = "$stateDir/channel-cache";
+mkdir $channelCache, 0755 unless -e $channelCache;
+$ENV{'NIX_DOWNLOAD_CACHE'} = $channelCache if -W $channelCache;
+
+
 # Figure out the name of the `.nix-channels' file to use.
 my $home = $ENV{"HOME"};
 die '$HOME not set' unless defined $home;
 my $channelsList = "$home/.nix-channels";
+
+my $nixDefExpr = "$home/.nix-defexpr";
     
 
 my @channels;
@@ -70,48 +78,56 @@ sub removeChannel {
 sub update {
     readChannels;
 
-    # Get rid of all the old substitutes.
-    system("@bindir@/nix-store", "--clear-substitutes") == 0
-        or die "cannot clear substitutes";
+    # Do we have write permission to the manifests directory?  If not,
+    # then just skip pulling the manifest and just download the Nix
+    # expressions.  If the user is a non-privileged user in a
+    # multi-user Nix installation, he at least gets installation from
+    # source.
+    if (-W "$stateDir/manifests") {
 
-    # Remove all the old manifests.
-    for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
-        unlink $manifest or die "cannot remove `$manifest': $!";
-    }
+        # Remove all the old manifests.
+        for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
+            unlink $manifest or die "cannot remove `$manifest': $!";
+        }
+
+        # Pull cache manifests.
+        foreach my $url (@channels) {
+            #print "pulling cache manifest from `$url'\n";
+            system("@bindir@/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
+                or die "cannot pull cache manifest from `$url'";
+        }
 
-    # Pull cache manifests.
-    foreach my $url (@channels) {
-        print "pulling cache manifest from `$url'\n";
-        system("@bindir@/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
-            or die "cannot pull cache manifest from `$url'";
     }
 
     # Create a Nix expression that fetches and unpacks the channel Nix
     # expressions.
 
-    my $nixExpr = "[";
+    my $inputs = "[";
     foreach my $url (@channels) {
+        $url =~ /\/([^\/]+)\/?$/;
+        my $channelName = $1;
+        $channelName = "unnamed" unless defined $channelName;
+
         my $fullURL = "$url/nixexprs.tar.bz2";
+        print "downloading Nix expressions from `$fullURL'...\n";
         $ENV{"PRINT_PATH"} = 1;
-        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
+        $ENV{"QUIET"} = 1;
+        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL'`;
         die "cannot fetch `$fullURL'" if $? != 0;
         chomp $path;
-        $nixExpr .= $path . " ";
+        $inputs .= '"' . $channelName . '"' . " " . $path . " ";
     }
-    $nixExpr .= "]";
-
-    $nixExpr =
-        "(import @datadir@/nix/corepkgs/channels/unpack.nix) " .
-        "{inputs = $nixExpr; system = \"@system@\";}";
+    $inputs .= "]";
 
     # Figure out a name for the GC root.
     my $userName = getpwuid($<);
     die "who ARE you? go away" unless defined $userName;
 
-    my $rootFile = "$rootsDir/$userName";
+    my $rootFile = "$rootsDir/per-user/$userName/channels";
     
     # Instantiate the Nix expression.
-    my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate --add-root '$rootFile'.tmp -`
+    print "unpacking channel Nix expressions...\n";
+    my $storeExpr = `@bindir@/nix-instantiate --add-root '$rootFile'.tmp @datadir@/nix/corepkgs/channels/unpack.nix --argstr system @system@ --arg inputs '$inputs'`
         or die "cannot instantiate Nix expression";
     chomp $storeExpr;
 
@@ -122,9 +138,12 @@ sub update {
 
     unlink "$rootFile.tmp";
 
-    # Make it the default Nix expression for `nix-env'.
-    system("@bindir@/nix-env", "--import", "$outPath") == 0
-         or die "cannot pull set default Nix expression to `$outPath'";
+    # Make the channels appear in nix-env.
+    unlink $nixDefExpr if -l $nixDefExpr; # old-skool ~/.nix-defexpr
+    mkdir $nixDefExpr or die "cannot create directory `$nixDefExpr'" if !-e $nixDefExpr;
+    my $channelLink = "$nixDefExpr/channels";
+    unlink $channelLink; # !!! not atomic
+    symlink($outPath, $channelLink) or die "cannot symlink `$channelLink' to `$outPath'";
 }
 
 

scripts/nix-collect-garbage.in

@@ -21,22 +21,31 @@ for my $arg (@ARGV) {
 # If `-d' was specified, remove all old generations of all profiles.
 # Of course, this makes rollbacks to before this point in time
 # impossible.
-if ($removeOld) {
 
-    opendir DIR, $profilesDir or die;
+sub removeOldGenerations;
+sub removeOldGenerations {
+    my $dir = shift;
 
-    foreach my $name (sort (readdir DIR)) {
-        $name = $profilesDir . "/" . $name;
+    my $dh;
+    opendir $dh, $dir or die;
+
+    foreach my $name (sort (readdir $dh)) {
+        next if $name eq "." || $name eq "..";
+        $name = $dir . "/" . $name;
         if (-l $name && (readlink($name) =~ /link/)) {
             print STDERR "removing old generations of profile $name\n";
             system("@bindir@/nix-env", "-p", $name, "--delete-generations", "old");
         }
+        elsif (! -l $name && -d $name) {
+            removeOldGenerations $name;
+        }
     }
     
-    closedir DIR or die;
-    
+    closedir $dh or die;
 }
 
+removeOldGenerations $profilesDir if $removeOld;
+
 
 # Run the actual garbage collector.
 exec "@bindir@/nix-store", "--gc", @args;

scripts/nix-copy-closure.in

@@ -0,0 +1,148 @@
+#! @perl@ -w
+
+my $binDir = $ENV{"NIX_BIN_DIR"};
+$binDir = "@bindir@" unless defined $binDir;
+
+
+if (scalar @ARGV < 1) {
+    print STDERR <<EOF
+Usage: nix-copy-closure [--from | --to] HOSTNAME [--sign] PATHS...
+EOF
+    ;
+    exit 1;
+}
+
+
+# Get the target host.
+my $sshHost;
+my @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or "");
+
+my $sign = 0;
+
+my $compressor = "cat";
+my $decompressor = "cat";
+
+my $toMode = 1;
+
+
+# !!! Copied from nix-pack-closure, should put this in a module.
+my @storePaths = ();
+
+while (@ARGV) {
+    my $arg = shift @ARGV;
+    
+    if ($arg eq "--sign") {
+        $sign = 1;
+    }
+    elsif ($arg eq "--gzip") {
+        $compressor = "gzip";
+        $decompressor = "gunzip";
+    }
+    elsif ($arg eq "--from") {
+        $toMode = 0;
+    }
+    elsif ($arg eq "--to") {
+        $toMode = 1;
+    }
+    elsif (!defined $sshHost) {
+        $sshHost = $arg;
+    }
+    else {
+        push @storePaths, $arg;
+    }
+}
+
+
+if ($toMode) { # Copy TO the remote machine.
+
+    my @allStorePaths;
+    my %storePathsSeen;
+
+    foreach my $storePath (@storePaths) {
+        # $arg might be a symlink to the store, so resolve it.
+        my $storePath2 = (`$binDir/nix-store --query --resolve '$storePath'`
+            or die "cannot resolve `$storePath'");
+        chomp $storePath2;
+
+        # Get the closure of this path.
+        my $pid = open(READ,
+            "$binDir/nix-store --query --requisites '$storePath2'|") or die;
+    
+        while (<READ>) {
+            chomp;
+            die "bad: $_" unless /^\//;
+            if (!defined $storePathsSeen{$_}) {
+                push @allStorePaths, $_;
+                $storePathsSeen{$_} = 1;
+            }
+        }
+
+        close READ or die "nix-store failed: $?";
+    }
+
+
+    # Ask the remote host which paths are invalid.
+    open(READ, "ssh @sshOpts $sshHost nix-store --check-validity --print-invalid @allStorePaths|");
+    my @missing = ();
+    while (<READ>) {
+        chomp;
+        print STDERR "target machine needs $_\n";
+        push @missing, $_;
+    }
+    close READ or die;
+
+
+    # Export the store paths and import them on the remote machine.
+    if (scalar @missing > 0) {
+        my $extraOpts = "";
+        $extraOpts .= "--sign" if $sign == 1;
+        system("nix-store --export $extraOpts @missing | $compressor | ssh @sshOpts $sshHost '$decompressor | nix-store --import'") == 0
+            or die "copying store paths to remote machine `$sshHost' failed: $?";
+    }
+
+}
+
+
+else { # Copy FROM the remote machine.
+
+    # Query the closure of the given store paths on the remote
+    # machine.  Paths are assumed to be store paths; there is no
+    # resolution (following of symlinks).
+    my $pid = open(READ,
+        "ssh @sshOpts $sshHost nix-store --query --requisites @storePaths|") or die;
+    
+    my @allStorePaths;
+    my %storePathsSeen;
+
+    while (<READ>) {
+        chomp;
+        die "bad: $_" unless /^\//;
+        if (!defined $storePathsSeen{$_}) {
+            push @allStorePaths, $_;
+            $storePathsSeen{$_} = 1;
+        }
+    }
+
+    close READ or die "nix-store on remote machine `$sshHost' failed: $?";
+
+
+    # What paths are already valid locally?
+    open(READ, "@bindir@/nix-store --check-validity --print-invalid @allStorePaths|");
+    my @missing = ();
+    while (<READ>) {
+        chomp;
+        print STDERR "local machine needs $_\n";
+        push @missing, $_;
+    }
+    close READ or die;
+    
+
+    # Export the store paths on the remote machine and import them on locally.
+    if (scalar @missing > 0) {
+        my $extraOpts = "";
+        $extraOpts .= "--sign" if $sign == 1;
+        system("ssh @sshOpts $sshHost 'nix-store --export $extraOpts @missing | $compressor' | $decompressor | @bindir@/nix-store --import") == 0
+            or die "copying store paths to remote machine `$sshHost' failed: $?";
+    }
+
+}

scripts/nix-install-package.in

@@ -13,7 +13,7 @@ downloading it from URL.
 
 Flags:
   --profile / -p LINK: install into the specified profile
-  --non-interactive: don't run inside a new terminal XXX
+  --non-interactive: don't run inside a new terminal
 EOF
     ; # '
     exit 1;

scripts/nix-pack-closure.in

@@ -17,10 +17,11 @@ $binDir = "@bindir@" unless defined $binDir;
 my $tmpDir = tempdir("nix-pack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-mkdir "$tmpDir/contents", 0777 or die;
-mkdir "$tmpDir/references", 0777 or die;
-mkdir "$tmpDir/derivers", 0777 or die;
+mkdir "$tmpDir/contents", 0755 or die;
+mkdir "$tmpDir/references", 0755 or die;
+mkdir "$tmpDir/derivers", 0755 or die;
 
+open TOPLEVEL, ">$tmpDir/top-level" or die;
 
 
 my %storePaths;
@@ -29,6 +30,12 @@ my %storePaths;
 while (@ARGV) {
     my $storePath = shift @ARGV;
 
+    # $storePath might be a symlink to the store, so resolve it.
+    $storePath = (`$binDir/nix-store --query --resolve '$storePath'`
+        or die "cannot resolve `$storePath'");
+    chomp $storePath;
+    print TOPLEVEL $storePath, "\n";
+
     # Get the closure of this path.
     my $pid = open(READ,
         "$binDir/nix-store --query --requisites '$storePath'|") or die;
@@ -43,6 +50,9 @@ while (@ARGV) {
 }
 
 
+close TOPLEVEL or die;
+
+
 foreach my $storePath (sort(keys %storePaths)) {
     print STDERR "packing `$storePath'...\n";
 

scripts/nix-prefetch-url.in

@@ -3,15 +3,12 @@
 url=$1
 expHash=$2
 
-# to prevent doing more than 1 chroot
-unset NIX_ROOT
-
 # needed to make it work on NixOS
 export PATH=$PATH:@coreutils@
 
 hashType=$NIX_HASH_ALGO
 if test -z "$hashType"; then
-    hashType=md5
+    hashType=sha256
 fi
 
 hashFormat=
@@ -39,29 +36,75 @@ if test -n "$expHash"; then
 fi
 
 
+doDownload() {
+    @curl@ $cacheFlags --fail -# --location --max-redirs 20 --disable-epsv \
+        --cookie-jar $tmpPath/cookies "$url" -o $tmpFile
+}
+
+
 # If we don't know the hash or a file with that hash doesn't exist,
 # download the file and add it to the store.
 if test -z "$finalPath"; then
 
     tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
     tmpFile=$tmpPath/$name
-    mkdir $tmpPath
+    mkdir $tmpPath # !!! retry if tmpPath already exists
+
+    # Optionally do timestamp-based caching of the download.
+    # Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is
+    # the hash and the timestamp of the file at $url.  The caching of
+    # the file *contents* is done in Nix store, where it can be
+    # garbage-collected independently.
+    if test -n "$NIX_DOWNLOAD_CACHE"; then
+        echo -n "$url" > $tmpPath/url
+        urlHash=$(nix-hash --type sha256 --base32 --flat $tmpPath/url)
+        echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url"
+        cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType"
+        cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp"
+        cacheFlags="--remote-time"
+        if test -e "$cachedTimestampFN" -a -e "$cachedHashFN"; then
+            # Only download the file if it is newer than the cached version.
+            cacheFlags="$cacheFlags --time-cond $cachedTimestampFN"
+        fi
+    fi
 
     # Perform the download.
-    @curl@ --fail --location --max-redirs 20 "$url" > $tmpFile
+    doDownload
 
-    # Compute the hash.
-    hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
-    if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
+    if test -n "$NIX_DOWNLOAD_CACHE" -a ! -e $tmpFile; then
+        # Curl didn't create $tmpFile, so apparently there's no newer
+        # file on the server.
+        hash=$(cat $cachedHashFN)
+        finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$hash" "$name") 
+        if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
+            echo "cached contents of \`$url' disappeared, redownloading..." >&2
+            finalPath=
+            cacheFlags="--remote-time"
+            doDownload
+        fi
+    fi
 
-    # Add the downloaded file to the Nix store.
-    finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+    if test -z "$finalPath"; then
 
-    if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+        # Compute the hash.
+        hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
+        if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
 
-    if test -n "$expHash" -a "$expHash" != "$hash"; then
-        echo "hash mismatch for URL \`$url'"
-        exit 1
+        if test -n "$NIX_DOWNLOAD_CACHE"; then
+            echo $hash > $cachedHashFN
+            touch -r $tmpFile $cachedTimestampFN
+        fi
+
+        # Add the downloaded file to the Nix store.
+        finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+
+        if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+
+        if test -n "$expHash" -a "$expHash" != "$hash"; then
+            echo "hash mismatch for URL \`$url'" >&2
+            exit 1
+        fi
+        
     fi
 fi
 

scripts/nix-pull.in

@@ -7,8 +7,6 @@ use readmanifest;
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-my $manifest = "$tmpDir/manifest";
-
 my $binDir = $ENV{"NIX_BIN_DIR"};
 $binDir = "@bindir@" unless defined $binDir;
 
@@ -28,23 +26,52 @@ umask 0022;
 
 # Process the URLs specified on the command line.
 my %narFiles;
+my %localPaths;
 my %patches;
-my %successors;
 
 my $skipWrongStore = 0;
 
+sub downloadFile {
+    my $url = shift;
+    $ENV{"PRINT_PATH"} = 1;
+    $ENV{"QUIET"} = 1;
+    my ($dummy, $path) = `$binDir/nix-prefetch-url '$url'`;
+    die "cannot fetch `$url'" if $? != 0;
+    die "nix-prefetch-url did not return a path" unless defined $path;
+    chomp $path;
+    return $path;
+}
+
 sub processURL {
     my $url = shift;
 
     $url =~ s/\/$//;
-    print "obtaining list of Nix archives at $url...\n";
 
-    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
-           "'$url' > '$manifest'") == 0
-           or die "curl failed: $?";
+    my $manifest;
 
-    if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
-        die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
+    # First see if a bzipped manifest is available.
+    if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) {
+        print "obtaining list of Nix archives at `$url.bz2'...\n";
+        my $bzipped = downloadFile "$url.bz2";
+
+        $manifest = "$tmpDir/MANIFEST";
+
+        system("@bunzip2@ < $bzipped > $manifest") == 0
+            or die "cannot decompress manifest";
+
+        $manifest = (`$binDir/nix-store --add $manifest`
+                     or die "cannot copy $manifest to the store");
+        chomp $manifest;
+    }
+
+    # Otherwise, just get the uncompressed manifest.
+    else {
+        print "obtaining list of Nix archives at `$url'...\n";
+        $manifest = downloadFile $url;
+    }
+    
+    if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
+        die "`$url' is not manifest or it is too old (i.e., for Nix <= 0.7)\n";
     }
 
     if ($skipWrongStore) {
@@ -67,8 +94,8 @@ sub processURL {
     
     my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
     
-    system ("@coreutils@/mv", "-f", "$manifest", "$finalPath") == 0
-        or die "cannot move `$manifest' to `$finalPath";
+    system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
+        or die "cannot link `$finalPath to `$manifest'";
 }
 
 while (@ARGV) {
@@ -81,30 +108,5 @@ while (@ARGV) {
 }
 
 
-my $size = scalar (keys %narFiles);
+my $size = scalar (keys %narFiles) + scalar (keys %localPaths);
 print "$size store paths in manifest\n";
-
-
-# Register all substitutes.
-print STDERR "registering substitutes...\n";
-
-my $pid = open(WRITE, "|$binDir/nix-store --register-substitutes")
-    or die "cannot run nix-store";
-
-foreach my $storePath (keys %narFiles) {
-    my $narFileList = $narFiles{$storePath};
-    foreach my $narFile (@{$narFileList}) {
-        print WRITE "$storePath\n";
-        print WRITE "$narFile->{deriver}\n";
-        print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
-        print WRITE "0\n";
-        my @references = split " ", $narFile->{references};
-        my $count = scalar @references;
-        print WRITE "$count\n";
-        foreach my $reference (@references) {
-            print WRITE "$reference\n";
-        }
-    }
-}
-
-close WRITE or die "nix-store failed: $?";

scripts/nix-push.in

@@ -264,8 +264,12 @@ foreach my $narArchive (@narArchives) {
 print STDERR "uploading manifest...\n";
 if ($localCopy) {
     copyFile $manifest, $localManifestFile;
+    copyFile "$manifest.bz2", "$localManifestFile.bz2";
 } else {
-    system("$curl  --show-error --upload-file " .
+    system("$curl --show-error --upload-file " .
            "'$manifest' '$manifestPutURL' > /dev/null") == 0 or
            die "curl failed on $manifest: $?";
+    system("$curl --show-error --upload-file " .
+           "'$manifest'.bz2 '$manifestPutURL'.bz2 > /dev/null") == 0 or
+           die "curl failed on $manifest: $?";
 }

scripts/nix-unpack-closure.in

@@ -77,3 +77,12 @@ closedir(DIR) or die;
 # Register the invalid paths as valid.
 system("nix-store --register-validity <'$tmpDir/validity'") == 0
     or die "nix-store --register-validity failed";
+
+
+# Show the top-level paths so that something useful can be done with
+# them, e.g., passing them to `nix-env -i'.
+if (-e "$tmpDir/unpacked/top-level") {
+    open TOPLEVEL, "<$tmpDir/unpacked/top-level" or die;
+    while (<TOPLEVEL>) { print "$_"; }
+    close TOPLEVEL;
+}

scripts/optimise-store.pl

@@ -1,91 +0,0 @@
-#! /usr/bin/perl -w
-
-use strict;
-use File::Basename;
-
-
-my @paths = ("/nix/store");
-
-
-print "hashing...\n";
-
-my $hashList = "/tmp/nix-optimise-hash-list";
-
-system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0
-    or die "cannot hash store files";
-
-
-print "sorting by hash...\n";
-
-system("sort $hashList > $hashList.sorted") == 0
-    or die "cannot sort list";
-
-
-sub atomicLink {
-    my $target = shift;
-    my $new = shift;
-    my $tmpNew = "${new}_optimise.$$";
-
-    # Make the directory writable temporarily.
-    my $dir = dirname $new;
-    my @st = stat $dir or die;
-
-    chmod ($st[2] | 0200, $dir) or die "cannot make `$dir' writable: $!";
-    
-    link $target, $tmpNew or die "cannot create hard link `$tmpNew': $!";
-
-    rename $tmpNew, $new or die "cannot rename `$tmpNew' to `$new': $!";
-
-    chmod ($st[2], $dir) or die "cannot restore permission on `$dir': $!";
-    utime ($st[8], $st[9], $dir) or die "cannot restore timestamp on `$dir': $!";
-}
-
-
-print "hard-linking...\n";
-
-open LIST, "<$hashList.sorted" or die;
-
-my $prevFile;
-my $prevHash;
-my $prevInode;
-my $prevExec;
-
-my $totalSpace = 0;
-my $savedSpace = 0;
-
-while (<LIST>) {
-    /^([0-9a-f]*)\s+(.*)$/ or die;
-    my $curFile = $2;
-    my $curHash = $1;
-
-    my @st = stat $curFile or die;
-    next if ($st[2] & 0222) != 0; # skip writable files
-
-    my $fileSize = $st[7];
-    $totalSpace += $fileSize;
-    my $isExec = ($st[2] & 0111) == 0111;
-
-    if (defined $prevHash && $curHash eq $prevHash
-        && $prevExec == $isExec)
-    {
-        
-        if ($st[1] != $prevInode) {
-            print "$curFile = $prevFile\n";
-            atomicLink $prevFile, $curFile;
-            $savedSpace += $fileSize;
-        }
-        
-    } else {
-        $prevFile = $curFile;
-        $prevHash = $curHash;
-        $prevInode = $st[1];
-        $prevExec = ($st[2] & 0111) == 0111;
-    }
-}
-
-print "total space = $totalSpace\n";
-print "saved space = $savedSpace\n";
-my $savings = ($savedSpace / $totalSpace) * 100.0;
-print "savings = $savings %\n";
-
-close LIST;

scripts/readmanifest.pm.in

@@ -34,8 +34,8 @@ sub addPatch {
 sub readManifest {
     my $manifest = shift;
     my $narFiles = shift;
+    my $localPaths = shift;
     my $patches = shift;
-    my $successors = shift;
     my $allowConflicts = shift;
     $allowConflicts = 0 unless defined $allowConflicts;
 
@@ -51,7 +51,6 @@ sub readManifest {
     my $url;
     my $hash;
     my $size;
-    my @preds;
     my $basePath;
     my $baseHash;
     my $patchType;
@@ -59,6 +58,7 @@ sub readManifest {
     my $references;
     my $deriver;
     my $hashAlgo;
+    my $copyFrom;
 
     while (<MANIFEST>) {
         chomp;
@@ -75,7 +75,6 @@ sub readManifest {
                 undef $url;
                 undef $hash;
                 undef $size;
-                @preds = ();
                 undef $narHash;
                 undef $basePath;
                 undef $baseHash;
@@ -117,10 +116,6 @@ sub readManifest {
                             };
                     }
                 
-                    foreach my $p (@preds) {
-                        $$successors{$p} = $storePath;
-                    }
-
                 }
 
                 elsif ($type eq "patch") {
@@ -132,13 +127,29 @@ sub readManifest {
                         }, $allowConflicts;
                 }
 
+                elsif ($type eq "localPath") {
+
+                    $$localPaths{$storePath} = []
+                        unless defined $$localPaths{$storePath};
+
+                    my $localPathsList = $$localPaths{$storePath};
+
+                    # !!! remove duplicates
+                    
+                    push @{$localPathsList},
+                        { copyFrom => $copyFrom, references => $references
+                        , deriver => ""
+                        };
+                }
+
             }
             
             elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; }
+            elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; }
             elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; }
             elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; }
             elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; }
-            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { push @preds, $1; }
+            elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { } # obsolete
             elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
             elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
             elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
@@ -165,6 +176,7 @@ sub writeManifest
     my $manifest = shift;
     my $narFiles = shift;
     my $patches = shift;
+    my $copySources = shift;
 
     open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
 
@@ -210,6 +222,14 @@ sub writeManifest
 
     rename("$manifest.tmp", $manifest)
         or die "cannot rename $manifest.tmp: $!";
+
+
+    # Create a bzipped manifest.
+    system("@bzip2@ < $manifest > $manifest.bz2.tmp") == 0
+        or die "cannot compress manifest";
+
+    rename("$manifest.bz2.tmp", "$manifest.bz2")
+        or die "cannot rename $manifest.bz2.tmp: $!";
 }
 
 

scripts/remove-patches.pl

@@ -6,14 +6,12 @@ use readmanifest;
 for my $p (@ARGV) {
 
     my %narFiles;
+    my %localPaths;
     my %patches;
-    my %successors;
 
-    readManifest $p,
-        \%narFiles, \%patches, \%successors;
+    readManifest $p, \%narFiles, \%localPaths, \%patches;
 
     %patches = ();
     
-    writeManifest $p,
-        \%narFiles, \%patches, \%successors;
+    writeManifest $p, \%narFiles, \%patches;
 }

scripts/update-manifest.pl

@@ -8,10 +8,10 @@ die unless scalar @ARGV == 2;
 my $cache = $ARGV[0];
 my $manifest = $ARGV[1];
 my %narFiles;
+my %localPaths;
 my %patches;
-my %successors;
 
-readManifest $manifest, \%narFiles, \%patches, \%successors;
+readManifest $manifest, \%narFiles, \%localPaths, \%patches;
 
 foreach my $storePath (keys %narFiles) {
     my $narFileList = $narFiles{$storePath};
@@ -50,4 +50,4 @@ if (! -e "$manifest.backup") {
     system "mv --reply=no '$manifest' '$manifest.backup'";
 }
 
-writeManifest $manifest, \%narFiles, \%patches, \%successors;
+writeManifest $manifest, \%narFiles, \%patches;

src/Makefile.am

@@ -1,15 +1,5 @@
 SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
- libexpr nix-instantiate nix-env nix-log2xml bsdiff-4.3
+ libexpr nix-instantiate nix-env nix-worker nix-setuid-helper \
+ nix-log2xml bsdiff-4.3
 
 EXTRA_DIST = aterm-helper.pl
-
-SETUID_PROGS = nix-store nix-instantiate nix-env
-install-exec-hook:
-if SETUID_HACK
-if HAVE_SETRESUID
-	cd $(DESTDIR)$(bindir) && chown @NIX_USER@ $(SETUID_PROGS) \
-	  && chgrp @NIX_GROUP@ $(SETUID_PROGS) && chmod ug+s $(SETUID_PROGS)
-else
-	cd $(DESTDIR)$(bindir) && chown root $(SETUID_PROGS) && chmod u+s $(SETUID_PROGS)
-endif
-endif

src/aterm-helper.pl

@@ -47,6 +47,7 @@ print HEADER "#endif\n\n\n";
 print IMPL "namespace nix {\n";
 
 while (<STDIN>) {
+    s/\#.*//;
     next if (/^\s*$/);
     
     if (/^\s*(\w*)\s*\|([^\|]*)\|\s*(\w+)\s*\|\s*(\w+)?/) {

src/libexpr/Makefile.am

@@ -2,11 +2,11 @@ pkglib_LTLIBRARIES = libexpr.la
 
 libexpr_la_SOURCES = \
  nixexpr.cc eval.cc primops.cc lexer-tab.cc parser-tab.cc \
- get-drvs.cc attr-path.cc expr-to-xml.cc
+ get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc
 
 pkginclude_HEADERS = \
  nixexpr.hh eval.hh parser.hh lexer-tab.hh parser-tab.hh \
- get-drvs.hh attr-path.hh expr-to-xml.hh
+ get-drvs.hh attr-path.hh expr-to-xml.hh common-opts.hh
 
 libexpr_la_LIBADD = ../libutil/libutil.la ../libstore/libstore.la \
  ../boost/format/libformat.la

src/libexpr/attr-path.cc

@@ -46,7 +46,7 @@ Expr findAlongAttrPath(EvalState & state, const string & attrPath,
 
         if (apType == apAttr) {
 
-            ATermMap attrs(128);
+            ATermMap attrs;
 
             if (!isAttrs(state, e, attrs))
                 throw TypeError(

src/libexpr/common-opts.cc

@@ -0,0 +1,32 @@
+#include "common-opts.hh"
+#include "../libmain/shared.hh"
+#include "util.hh"
+#include "parser.hh"
+
+
+namespace nix {
+
+
+bool parseOptionArg(const string & arg, Strings::iterator & i,
+    const Strings::iterator & argsEnd, EvalState & state,
+    ATermMap & autoArgs)
+{
+    if (arg != "--arg" && arg != "--argstr") return false;
+
+    UsageError error(format("`%1%' requires two arguments") % arg);
+    
+    if (i == argsEnd) throw error;
+    string name = *i++;
+    if (i == argsEnd) throw error;
+    string value = *i++;
+    
+    Expr e = arg == "--arg"
+        ? parseExprFromString(state, value, absPath("."))
+        : makeStr(value);
+    autoArgs.set(toATerm(name), e);
+    
+    return true;
+}
+
+ 
+}

src/libexpr/common-opts.hh

@@ -0,0 +1,17 @@
+#ifndef __COMMON_OPTS_H
+#define __COMMON_OPTS_H
+
+#include "eval.hh"
+
+
+namespace nix {
+
+/* Some common option parsing between nix-env and nix-instantiate. */
+bool parseOptionArg(const string & arg, Strings::iterator & i,
+    const Strings::iterator & argsEnd, EvalState & state,
+    ATermMap & autoArgs);
+    
+}
+
+
+#endif /* !__COMMON_OPTS_H */

src/libexpr/eval.hh

@@ -27,6 +27,9 @@ struct EvalState;
 typedef Expr (* PrimOp) (EvalState &, const ATermVector & args);
 
 
+extern int cacheTerms; // 0 = don't, 1 = do, 2 = "cell" terms only
+
+
 struct EvalState 
 {
     ATermMap normalForms;
@@ -38,6 +41,10 @@ struct EvalState
     unsigned int nrEvaluated;
     unsigned int nrCached;
 
+    bool strictMode;
+
+    ATermMap parsings; /* path -> expr mapping */
+
     EvalState();
 
     void addPrimOps();
@@ -56,21 +63,29 @@ Expr evalFile(EvalState & state, const Path & path);
    attributes.  If `canonicalise' is true, we remove things like
    position information and make sure that attribute sets are in
    sorded order. */
-Expr strictEvalExpr(EvalState & state, Expr e,
-    bool canonicalise = false);
+Expr strictEvalExpr(EvalState & state, Expr e);
 
 /* Specific results. */
-string evalString(EvalState & state, Expr e);
-Path evalPath(EvalState & state, Expr e);
+string evalString(EvalState & state, Expr e, PathSet & context);
+string evalStringNoCtx(EvalState & state, Expr e);
 int evalInt(EvalState & state, Expr e);
 bool evalBool(EvalState & state, Expr e);
 ATermList evalList(EvalState & state, Expr e);
-ATerm coerceToString(Expr e);
 
-/* Contexts. */
-string coerceToStringWithContext(EvalState & state,
-    ATermList & context, Expr e, bool & isPath);
-Expr wrapInContext(ATermList context, Expr e);
+/* Flatten nested lists into a single list (or expand a singleton into
+   a list). */
+ATermList flattenList(EvalState & state, Expr e);
+
+/* String coercion.  Converts strings, paths and derivations to a
+   string.  If `coerceMore' is set, also converts nulls, integers,
+   booleans and lists to a string. */
+string coerceToString(EvalState & state, Expr e, PathSet & context,
+    bool coerceMore = false, bool copyToStore = true);
+
+/* Path coercion.  Converts strings, paths and derivations to a path.
+   The result is guaranteed to be an canonicalised, absolute path.
+   Nothing is copied to the store. */
+Path coerceToPath(EvalState & state, Expr e, PathSet & context);
 
 /* Automatically call a function for which each argument has a default
    value or has a binding in the `args' map.  Note: result is a call,

src/libexpr/expr-to-xml.cc

@@ -2,6 +2,7 @@
 #include "xml-writer.hh"
 #include "nixexpr-ast.hh"
 #include "aterm.hh"
+#include "util.hh"
 
 
 namespace nix {
@@ -15,29 +16,45 @@ static XMLAttrs singletonAttrs(const string & name, const string & value)
 }
 
 
-static void printTermAsXML(Expr e, XMLWriter & doc, ATermList & context)
+/* set<Expr> is safe because all the expressions are also reachable
+   from the stack, therefore can't be garbage-collected. */
+typedef set<Expr> ExprSet;
+
+
+static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
+    ExprSet & drvsSeen);
+
+
+static void showAttrs(const ATermMap & attrs, XMLWriter & doc,
+    PathSet & context, ExprSet & drvsSeen)
+{
+    StringSet names;
+    for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i)
+        names.insert(aterm2String(i->key));
+    for (StringSet::iterator i = names.begin(); i != names.end(); ++i) {
+        XMLOpenElement _(doc, "attr", singletonAttrs("name", *i));
+        printTermAsXML(attrs.get(toATerm(*i)), doc, context, drvsSeen);
+    }
+}
+
+
+static void printTermAsXML(Expr e, XMLWriter & doc, PathSet & context,
+    ExprSet & drvsSeen)
 {
     XMLAttrs attrs;
-    ATerm s;
+    string s;
+    ATerm s2;
     int i;
-    Expr e2;
     ATermList as, es, formals;
     ATerm body, pos;
 
-    while (matchContext(e, es, e2)) {
-        e = e2;
-        for (ATermIterator i(es); i; ++i)
-            context = ATinsert(context, *i);
-    }
+    checkInterrupt();
 
-    if (matchStr(e, s))
-        doc.writeEmptyElement("string", singletonAttrs("value", aterm2String(s)));
+    if (matchStr(e, s, context)) /* !!! show the context? */
+        doc.writeEmptyElement("string", singletonAttrs("value", s));
 
-    else if (matchPath(e, s))
-        doc.writeEmptyElement("path", singletonAttrs("value", aterm2String(s)));
-
-    else if (matchUri(e, s))
-        doc.writeEmptyElement("uri", singletonAttrs("value", aterm2String(s)));
+    else if (matchPath(e, s2))
+        doc.writeEmptyElement("path", singletonAttrs("value", aterm2String(s2)));
 
     else if (matchNull(e))
         doc.writeEmptyElement("null");
@@ -52,22 +69,42 @@ static void printTermAsXML(Expr e, XMLWriter & doc, ATermList & context)
         doc.writeEmptyElement("bool", singletonAttrs("value", "false"));
 
     else if (matchAttrs(e, as)) {
-        XMLOpenElement _(doc, "attrs");
-        ATermMap attrs(128);
+        ATermMap attrs;
         queryAllAttrs(e, attrs);
-        StringSet names;
-        for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i)
-            names.insert(aterm2String(i->key));
-        for (StringSet::iterator i = names.begin(); i != names.end(); ++i) {
-            XMLOpenElement _(doc, "attr", singletonAttrs("name", *i));
-            printTermAsXML(attrs.get(toATerm(*i)), doc, context);
+
+        Expr a = attrs.get(toATerm("type"));
+        if (a && matchStr(a, s, context) && s == "derivation") {
+
+            XMLAttrs xmlAttrs;
+            Path outPath, drvPath;
+            
+            a = attrs.get(toATerm("drvPath"));
+            if (matchStr(a, drvPath, context))
+                xmlAttrs["drvPath"] = drvPath;
+        
+            a = attrs.get(toATerm("outPath"));
+            if (matchStr(a, outPath, context))
+                xmlAttrs["outPath"] = outPath;
+        
+            XMLOpenElement _(doc, "derivation", xmlAttrs);
+
+            if (drvsSeen.find(e) == drvsSeen.end()) {
+                drvsSeen.insert(e);
+                showAttrs(attrs, doc, context, drvsSeen);
+            } else
+                doc.writeEmptyElement("repeated");
+        }
+
+        else {
+            XMLOpenElement _(doc, "attrs");
+            showAttrs(attrs, doc, context, drvsSeen);
         }
     }
 
     else if (matchList(e, es)) {
         XMLOpenElement _(doc, "list");
         for (ATermIterator i(es); i; ++i)
-            printTermAsXML(*i, doc, context);
+            printTermAsXML(*i, doc, context, drvsSeen);
     }
 
     else if (matchFunction(e, formals, body, pos)) {
@@ -82,7 +119,7 @@ static void printTermAsXML(Expr e, XMLWriter & doc, ATermList & context)
             if (matchValidValues(valids, valids2)) {
                 for (ATermIterator j(valids2); j; ++j) {
                     XMLOpenElement _(doc, "value");
-                    printTermAsXML(*j, doc, context);
+                    printTermAsXML(*j, doc, context, drvsSeen);
                 }
             }
         }
@@ -93,11 +130,12 @@ static void printTermAsXML(Expr e, XMLWriter & doc, ATermList & context)
 }
 
 
-void printTermAsXML(Expr e, std::ostream & out, ATermList & context)
+void printTermAsXML(Expr e, std::ostream & out, PathSet & context)
 {
     XMLWriter doc(true, out);
     XMLOpenElement root(doc, "expr");
-    printTermAsXML(e, doc, context);
+    ExprSet drvsSeen;    
+    printTermAsXML(e, doc, context, drvsSeen);
 }
 
  

src/libexpr/expr-to-xml.hh

@@ -9,7 +9,7 @@
 
 namespace nix {
 
-void printTermAsXML(Expr e, std::ostream & out, ATermList & context);
+void printTermAsXML(Expr e, std::ostream & out, PathSet & context);
     
 }
 

src/libexpr/get-drvs.cc

@@ -10,7 +10,15 @@ string DrvInfo::queryDrvPath(EvalState & state) const
 {
     if (drvPath == "") {
         Expr a = attrs->get(toATerm("drvPath"));
-        (string &) drvPath = a ? evalPath(state, a) : "";
+
+        /* Backwards compatibility hack with user environments made by
+           Nix <= 0.10: these contain illegal Path("") expressions. */
+        ATerm t;
+        if (a && matchPath(evalExpr(state, a), t))
+            return aterm2String(t);
+        
+        PathSet context;
+        (string &) drvPath = a ? coerceToPath(state, a, context) : "";
     }
     return drvPath;
 }
@@ -21,7 +29,8 @@ string DrvInfo::queryOutPath(EvalState & state) const
     if (outPath == "") {
         Expr a = attrs->get(toATerm("outPath"));
         if (!a) throw TypeError("output path missing");
-        (string &) outPath = evalPath(state, a);
+        PathSet context;
+        (string &) outPath = coerceToPath(state, a, context);
     }
     return outPath;
 }
@@ -34,13 +43,15 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
     Expr a = attrs->get(toATerm("meta"));
     if (!a) return meta; /* fine, empty meta information */
 
-    ATermMap attrs2(16); /* !!! */
+    ATermMap attrs2;
     queryAllAttrs(evalExpr(state, a), attrs2);
 
     for (ATermMap::const_iterator i = attrs2.begin(); i != attrs2.end(); ++i) {
-        ATerm s = coerceToString(evalExpr(state, i->value));
-        if (s)
-            meta[aterm2String(i->key)] = aterm2String(s);
+        Expr e = evalExpr(state, i->value);
+        string s;
+        PathSet context;
+        if (matchStr(e, s, context))
+            meta[aterm2String(i->key)] = s;
         /* For future compatibility, ignore attribute values that are
            not strings. */
     }
@@ -49,6 +60,25 @@ MetaInfo DrvInfo::queryMetaInfo(EvalState & state) const
 }
 
 
+string DrvInfo::queryMetaInfo(EvalState & state, const string & name) const
+{
+    /* !!! evaluates all meta attributes => inefficient */
+    MetaInfo meta = queryMetaInfo(state);
+    MetaInfo::iterator i = meta.find(name);
+    return i == meta.end() ? "" : i->second;
+}
+
+
+void DrvInfo::setMetaInfo(const MetaInfo & meta)
+{
+    ATermMap metaAttrs;
+    for (MetaInfo::const_iterator i = meta.begin(); i != meta.end(); ++i)
+        metaAttrs.set(toATerm(i->first),
+            makeAttrRHS(makeStr(i->second), makeNoPos()));
+    attrs->set(toATerm("meta"), makeAttrs(metaAttrs));
+}
+
+
 /* Cache for already evaluated derivations.  Usually putting ATerms in
    a STL container is unsafe (they're not scanning for GC roots), but
    here it doesn't matter; everything in this set is reachable from
@@ -70,11 +100,11 @@ static bool getDerivation(EvalState & state, Expr e,
         e = evalExpr(state, e);
         if (!matchAttrs(e, es)) return true;
 
-        boost::shared_ptr<ATermMap> attrs(new ATermMap(32)); /* !!! */
+        boost::shared_ptr<ATermMap> attrs(new ATermMap());
         queryAllAttrs(e, *attrs, false);
     
         Expr a = attrs->get(toATerm("type"));
-        if (!a || evalString(state, a) != "derivation") return true;
+        if (!a || evalStringNoCtx(state, a) != "derivation") return true;
 
         /* Remove spurious duplicates (e.g., an attribute set like
            `rec { x = derivation {...}; y = x;}'. */
@@ -86,13 +116,13 @@ static bool getDerivation(EvalState & state, Expr e,
         a = attrs->get(toATerm("name"));
         /* !!! We really would like to have a decent back trace here. */
         if (!a) throw TypeError("derivation name missing");
-        drv.name = evalString(state, a);
+        drv.name = evalStringNoCtx(state, a);
 
         a = attrs->get(toATerm("system"));
         if (!a)
             drv.system = "unknown";
         else
-            drv.system = evalString(state, a);
+            drv.system = evalStringNoCtx(state, a);
 
         drv.attrs = attrs;
 
@@ -140,12 +170,21 @@ static void getDerivations(EvalState & state, Expr e,
     if (matchAttrs(e, es)) {
         ATermMap drvMap(ATgetLength(es));
         queryAllAttrs(e, drvMap);
+
+        /* !!! undocumented hackery to support combining channels in
+           nix-env.cc. */
+        Expr e2 = drvMap.get(toATerm("_combineChannels"));
+        bool combineChannels = e2 && evalBool(state, e2);
         
         for (ATermMap::const_iterator i = drvMap.begin(); i != drvMap.end(); ++i) {
             startNest(nest, lvlDebug,
                 format("evaluating attribute `%1%'") % aterm2String(i->key));
             string pathPrefix2 = addToPath(pathPrefix, aterm2String(i->key));
-            if (getDerivation(state, i->value, pathPrefix2, drvs, doneExprs)) {
+            if (combineChannels) {
+                if (((string) aterm2String(i->key)) != "_combineChannels")
+                    getDerivations(state, i->value, pathPrefix2, autoArgs, drvs, doneExprs);
+            }
+            else if (getDerivation(state, i->value, pathPrefix2, drvs, doneExprs)) {
                 /* If the value of this attribute is itself an
                    attribute set, should we recurse into it?  => Only
                    if it has a `recurseForDerivations = true'
@@ -155,8 +194,8 @@ static void getDerivations(EvalState & state, Expr e,
                 if (matchAttrs(e, es)) {
                     ATermMap attrs(ATgetLength(es));
                     queryAllAttrs(e, attrs, false);
-                    Expr e2 = attrs.get(toATerm("recurseForDerivations"));
-                    if (e2 && evalBool(state, e2))
+                    if (((e2 = attrs.get(toATerm("recurseForDerivations")))
+                            && evalBool(state, e2)))
                         getDerivations(state, e, pathPrefix2, autoArgs, drvs, doneExprs);
                 }
             }

src/libexpr/get-drvs.hh

@@ -26,11 +26,15 @@ public:
     string attrPath; /* path towards the derivation */
     string system;
 
+    /* !!! these should really be hidden, and setMetaInfo() should
+       make a copy since the ATermMap can be shared between multiple
+       DrvInfos. */
     boost::shared_ptr<ATermMap> attrs;
 
     string queryDrvPath(EvalState & state) const;
     string queryOutPath(EvalState & state) const;
     MetaInfo queryMetaInfo(EvalState & state) const;
+    string queryMetaInfo(EvalState & state, const string & name) const;
 
     void setDrvPath(const string & s)
     {
@@ -41,6 +45,8 @@ public:
     {
         outPath = s;
     }
+
+    void setMetaInfo(const MetaInfo & meta);
 };
 
 

src/libexpr/lexer.l

@@ -64,7 +64,7 @@ static Expr unescapeStr(const char * s)
         }
         else t += c;
     }
-    return makeStr(toATerm(t));
+    return makeStr(toATerm(t), ATempty);
 }
 
  

src/libexpr/nixexpr-ast.def

@@ -24,9 +24,38 @@ Call | Expr Expr | Expr |
 Select | Expr string | Expr |
 Var | string | Expr |
 Int | int | Expr |
-Str | string | Expr |
+
+# Strings in the evaluator carry a so-called `context' (the ATermList)
+# which is a list of strings representing store paths.  This is to
+# allow users to write things like
+#
+#   "--with-freetype2-library=" + freetype + "/lib"
+#
+# where `freetype' is a derivation (or a source to be copied to the
+# store).  If we just concatenated the strings without keeping track
+# of the referenced store paths, then if the string is used as a
+# derivation attribute, the derivation will not have the correct
+# dependencies in its inputDrvs and inputSrcs.
+#
+# The semantics of the context is as follows: when a string with
+# context C is used as a derivation attribute, then the derivations in
+# C will be added to the inputDrvs of the derivation, and the other
+# store paths in C will be added to the inputSrcs of the derivations.
+#
+# For canonicity, the store paths should be in sorted order.
+Str | string ATermList | Expr |
+Str | string | Expr | ObsoleteStr
+
+# A path is a reference to a file system object that is to be copied
+# to the Nix store when used as a derivation attribute.  When it is
+# concatenated to a string (i.e., `str + path'), it is also copied and
+# the resulting store path is concatenated to the string (with the
+# store path in the context).  If a string or path is concatenated to
+# a path (i.e., `path + str' or `path + path'), the result is a new
+# path (if the right-hand side is a string, the context must be
+# empty).
 Path | string | Expr |
-Uri | string | Expr |
+
 List | ATermList | Expr |
 BlackHole | | Expr |
 Undefined | | Expr |
@@ -37,14 +66,15 @@ Closed | Expr | Expr |
 Rec | ATermList ATermList | Expr |
 Bool | ATerm | Expr |
 Null | | Expr |
-Context | ATermList Expr | Expr |
 
 Bind | string Expr Pos | ATerm |
-Bind | string Expr | ATerm | Bind2
+Bind | string Expr | ATerm | ObsoleteBind
 Inherit | Expr ATermList Pos | ATerm |
 
 Scope | | Expr |
 
+Cell | int Expr | Expr |
+
 Formal | string ValidValues DefaultValue | ATerm |
 
 ValidValues | ATermList | ValidValues |

src/libexpr/nixexpr.cc

@@ -2,6 +2,7 @@
 #include "derivations.hh"
 #include "util.hh"
 #include "aterm.hh"
+#include "eval.hh" // !!! urgh
 
 #include "nixexpr-ast.hh"
 #include "nixexpr-ast.cc"
@@ -54,7 +55,7 @@ void queryAllAttrs(Expr e, ATermMap & attrs, bool withPos)
 {
     ATermList bnds;
     if (!matchAttrs(e, bnds))
-        throw TypeError("attribute set expected");
+        throw TypeError(format("value is %1% while an attribute set was expected") % showType(e));
 
     for (ATermIterator i(bnds); i; ++i) {
         ATerm name;
@@ -77,7 +78,7 @@ Expr queryAttr(Expr e, const string & name, ATerm & pos)
 {
     ATermList bnds;
     if (!matchAttrs(e, bnds))
-        throw TypeError("attribute set expected");
+        throw TypeError(format("value is %1% while an attribute set was expected") % showType(e));
 
     for (ATermIterator i(bnds); i; ++i) {
         ATerm name2, pos2;
@@ -108,7 +109,16 @@ Expr makeAttrs(const ATermMap & attrs)
 }
 
 
-Expr substitute(const Substitution & subs, Expr e)
+extern unsigned int substs;
+extern unsigned int substsCached;
+extern bool closedTerms;
+extern bool substCache;
+
+
+static Expr substitute(ATermMap & done, const Substitution & subs, Expr e);
+
+
+static Expr substitute2(ATermMap & done, const Substitution & subs, Expr e)
 {
     checkInterrupt();
 
@@ -116,19 +126,20 @@ Expr substitute(const Substitution & subs, Expr e)
 
     ATerm name, pos, e2;
 
+    substs++;
+
     /* As an optimisation, don't substitute in subterms known to be
        closed. */
-    if (matchClosed(e, e2)) return e;
+    if (closedTerms && matchClosed(e, e2)) return e;
 
     if (matchVar(e, name)) {
         Expr sub = subs.lookup(name);
-        if (sub == makeRemoved()) sub = 0;
         Expr wrapped;
         /* Add a "closed" wrapper around terms that aren't already
            closed.  The check is necessary to prevent repeated
            wrapping, e.g., closed(closed(closed(...))), which kills
            caching. */
-        return sub ? (matchClosed(sub, wrapped) ? sub : makeClosed(sub)) : e;
+        return sub ? ((!closedTerms || matchClosed(sub, wrapped)) ? sub : makeClosed(sub)) : e;
     }
 
     /* In case of a function, filter out all variables bound by this
@@ -140,18 +151,30 @@ Expr substitute(const Substitution & subs, Expr e)
         for (ATermIterator i(formals); i; ++i) {
             ATerm d1, d2;
             if (!matchFormal(*i, name, d1, d2)) abort();
-            map.set(name, makeRemoved());
+            if (subs.lookup(name))
+                map.set(name, constRemoved);
+        }
+        if (map.size() == 0)
+            return makeFunction(
+                (ATermList) substitute(done, subs, (ATerm) formals),
+                substitute(done, subs, body), pos);
+        else {
+            Substitution subs2(&subs, &map);
+            ATermMap done2(128);
+            return makeFunction(
+                (ATermList) substitute(done2, subs2, (ATerm) formals),
+                substitute(done2, subs2, body), pos);
         }
-        Substitution subs2(&subs, &map);
-        return makeFunction(
-            (ATermList) substitute(subs2, (ATerm) formals),
-            substitute(subs2, body), pos);
     }
 
     if (matchFunction1(e, name, body, pos)) {
-        ATermMap map(1);
-        map.set(name, makeRemoved());
-        return makeFunction1(name, substitute(Substitution(&subs, &map), body), pos);
+        if (subs.lookup(name)) {
+            ATermMap map(1);
+            map.set(name, constRemoved);
+            ATermMap done2(128);
+            return makeFunction1(name, substitute(done2, Substitution(&subs, &map), body), pos);
+        } else
+            return makeFunction1(name, substitute(done, subs, body), pos);
     }
         
     /* Idem for a mutually recursive attribute set. */
@@ -159,14 +182,21 @@ Expr substitute(const Substitution & subs, Expr e)
     if (matchRec(e, rbnds, nrbnds)) {
         ATermMap map(ATgetLength(rbnds) + ATgetLength(nrbnds));
         for (ATermIterator i(rbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
+            if (matchBind(*i, name, e2, pos) && subs.lookup(name))
+                map.set(name, constRemoved);
         for (ATermIterator i(nrbnds); i; ++i)
-            if (matchBind(*i, name, e2, pos)) map.set(name, makeRemoved());
-            else abort(); /* can't happen */
-        return makeRec(
-            (ATermList) substitute(Substitution(&subs, &map), (ATerm) rbnds),
-            (ATermList) substitute(subs, (ATerm) nrbnds));
+            if (matchBind(*i, name, e2, pos) && subs.lookup(name))
+                map.set(name, constRemoved);
+        if (map.size() == 0)
+            return makeRec(
+                (ATermList) substitute(done, subs, (ATerm) rbnds),
+                (ATermList) substitute(done, subs, (ATerm) nrbnds));
+        else {
+            ATermMap done2(128);
+            return makeRec(
+                (ATermList) substitute(done2, Substitution(&subs, &map), (ATerm) rbnds),
+                (ATermList) substitute(done, subs, (ATerm) nrbnds));
+        }
     }
 
     if (ATgetType(e) == AT_APPL) {
@@ -177,7 +207,73 @@ Expr substitute(const Substitution & subs, Expr e)
 
         for (int i = 0; i < arity; ++i) {
             ATerm arg = ATgetArgument(e, i);
-            args[i] = substitute(subs, arg);
+            args[i] = substitute(done, subs, arg);
+            if (args[i] != arg) changed = true;
+        }
+        
+        return changed ? (ATerm) ATmakeApplArray(fun, args) : e;
+    }
+
+    if (ATgetType(e) == AT_LIST) {
+        unsigned int len = ATgetLength((ATermList) e);
+        ATerm es[len];
+        ATermIterator i((ATermList) e);
+        bool changed = false;
+        for (unsigned int j = 0; i; ++i, ++j) {
+            es[j] = substitute(done, subs, *i);
+            if (es[j] != *i) changed = true;
+        }
+        if (!changed) return e;
+        ATermList out = ATempty;
+        for (unsigned int j = len; j; --j)
+            out = ATinsert(out, es[j - 1]);
+        return (ATerm) out;
+    }
+
+    return e;
+}
+
+
+static Expr substitute(ATermMap & done, const Substitution & subs, Expr e)
+{
+    Expr res = done[e];
+    if (substCache && res) {
+        substsCached++;
+        return res;
+    }
+    res = substitute2(done, subs, e);
+    done.set(e, res);
+    return res;
+}
+
+
+Expr substitute(const Substitution & subs, Expr e)
+{
+    ATermMap done(256);
+    return substitute(done, subs, e);
+}
+
+
+Expr allocCells(Expr e)
+{
+    checkInterrupt();
+
+    ATerm e2;
+    if (matchClosed(e, e2)) return e;
+
+    int i;
+    if (matchCell(e, i, e2))
+        return allocCell(allocCells(e2));
+
+    if (ATgetType(e) == AT_APPL) {
+        AFun fun = ATgetAFun(e);
+        int arity = ATgetArity(fun);
+        ATerm args[arity];
+        bool changed = false;
+
+        for (int i = 0; i < arity; ++i) {
+            ATerm arg = ATgetArgument(e, i);
+            args[i] = allocCells(arg);
             if (args[i] != arg) changed = true;
         }
         
@@ -189,7 +285,7 @@ Expr substitute(const Substitution & subs, Expr e)
         ATerm es[len];
         ATermIterator i((ATermList) e);
         for (unsigned int j = 0; i; ++i, ++j)
-            es[j] = substitute(subs, *i);
+            es[j] = allocCells(*i);
         ATermList out = ATempty;
         for (unsigned int j = len; j; --j)
             out = ATinsert(out, es[j - 1]);
@@ -287,21 +383,81 @@ void checkVarDefs(const ATermMap & defs, Expr e)
 }
 
 
+struct Canonicalise : TermFun
+{
+    ATerm operator () (ATerm e)
+    {
+        /* Remove position info. */
+        ATerm path;
+        int line, column;
+        if (matchPos(e, path, line, column))
+            return makeNoPos();
+
+        /* Sort attribute sets. */
+        ATermList _;
+        if (matchAttrs(e, _)) {
+            ATermMap attrs;
+            queryAllAttrs(e, attrs);
+            StringSet names;
+            for (ATermMap::const_iterator i = attrs.begin(); i != attrs.end(); ++i)
+                names.insert(aterm2String(i->key));
+
+            ATermList attrs2 = ATempty;
+            for (StringSet::reverse_iterator i = names.rbegin(); i != names.rend(); ++i)
+                attrs2 = ATinsert(attrs2,
+                    makeBind(toATerm(*i), attrs.get(toATerm(*i)), makeNoPos()));
+
+            return makeAttrs(attrs2);
+        }
+        
+        return e;
+    }
+};
+
+
+Expr canonicaliseExpr(Expr e)
+{
+    Canonicalise canonicalise;
+    return bottomupRewrite(canonicalise, e);
+}
+
+
 Expr makeBool(bool b)
 {
     return b ? eTrue : eFalse;
 }
 
 
+bool matchStr(Expr e, string & s, PathSet & context)
+{
+    ATermList l;
+    ATerm s_;
+
+    if (!matchStr(e, s_, l)) return false;
+
+    s = aterm2String(s_);
+
+    for (ATermIterator i(l); i; ++i)
+        context.insert(aterm2String(*i));
+
+    return true;
+}
+
+
+Expr makeStr(const string & s, const PathSet & context)
+{
+    return makeStr(toATerm(s), toATermList(context));
+}
+
+
 string showType(Expr e)
 {
     ATerm t1, t2, t3;
     ATermList l1;
     ATermBlob b1;
     int i1;
-    if (matchStr(e, t1)) return "a string";
+    if (matchStr(e, t1, l1)) return "a string";
     if (matchPath(e, t1)) return "a path";
-    if (matchUri(e, t1)) return "a path";
     if (matchNull(e)) return "null";
     if (matchInt(e, i1)) return "an integer";
     if (matchBool(e, t1)) return "a boolean";
@@ -310,18 +466,19 @@ string showType(Expr e)
     if (matchAttrs(e, l1)) return "an attribute set";
     if (matchList(e, l1)) return "a list";
     if (matchPrimOp(e, i1, b1, l1)) return "a partially applied built-in function";
-    if (matchContext(e, l1, t1)) return "a context containing " + showType(t1);
     return "an unknown type";
 }
 
 
 string showValue(Expr e)
 {
-    ATerm s;
+    PathSet context;
+    string s;
+    ATerm s2;
     int i;
-    if (matchStr(e, s)) {
-        string t = aterm2String(s), u;
-        for (string::iterator i = t.begin(); i != t.end(); ++i)
+    if (matchStr(e, s, context)) {
+        string u;
+        for (string::iterator i = s.begin(); i != s.end(); ++i)
             if (*i == '\"' || *i == '\\') u += "\\" + *i;
             else if (*i == '\n') u += "\\n";
             else if (*i == '\r') u += "\\r";
@@ -329,8 +486,7 @@ string showValue(Expr e)
             else u += *i;
         return "\"" + u + "\"";
     }
-    if (matchPath(e, s)) return aterm2String(s);
-    if (matchUri(e, s)) return aterm2String(s);
+    if (matchPath(e, s2)) return aterm2String(s2);
     if (matchNull(e)) return "null";
     if (matchInt(e, i)) return (format("%1%") % i).str();
     if (e == eTrue) return "true";
@@ -339,5 +495,18 @@ string showValue(Expr e)
     return "<unknown>";
 }
 
+
+static unsigned int cellCount = 0;
+
+
+Expr allocCell(Expr e)
+{
+    if (cacheTerms != 2) return e;
+    int i;
+    Expr e2;
+    if (matchCell(e, i, e2)) return e;
+    return makeCell(cellCount++, e);
+}
+
  
 }

src/libexpr/nixexpr.hh

@@ -12,6 +12,7 @@ namespace nix {
 
 MakeError(EvalError, Error)
 MakeError(AssertionError, EvalError)
+MakeError(ThrownError, AssertionError)
 MakeError(Abort, EvalError)
 MakeError(TypeError, EvalError)
 
@@ -33,6 +34,9 @@ typedef ATerm Pos;
 typedef vector<ATerm> ATermVector;
 
 
+extern Expr constRemoved;
+
+
 /* A substitution is a linked list of ATermMaps that map names to
    identifiers.  We use a list of ATermMaps rather than a single to
    make it easy to grow or shrink a substitution when entering a
@@ -52,7 +56,8 @@ struct Substitution
     {
         Expr x;
         for (const Substitution * s(this); s; s = s->prev)
-            if ((x = s->map->get(name))) return x;
+            if ((x = s->map->get(name)))
+                return x == constRemoved ? 0 : x;
         return 0;
     }
 };
@@ -71,6 +76,7 @@ struct TermFun
 };
 ATerm bottomupRewrite(TermFun & f, ATerm e);
 
+
 /* Query all attributes in an attribute set expression.  The
    expression must be in normal form. */
 void queryAllAttrs(Expr e, ATermMap & attrs, bool withPos = false);
@@ -83,20 +89,42 @@ Expr queryAttr(Expr e, const string & name, ATerm & pos);
 /* Create an attribute set expression from an Attrs value. */
 Expr makeAttrs(const ATermMap & attrs);
 
+
 /* Perform a set of substitutions on an expression. */
 Expr substitute(const Substitution & subs, Expr e);
 
+
 /* Check whether all variables are defined in the given expression.
    Throw an exception if this isn't the case. */
 void checkVarDefs(const ATermMap & def, Expr e);
 
+
+/* Canonicalise a Nix expression by sorting attributes and removing
+   location information. */
+Expr canonicaliseExpr(Expr e);
+
+
 /* Create an expression representing a boolean. */
 Expr makeBool(bool b);
 
+
+/* Manipulation of Str() nodes.  Note: matchStr() does not clear
+   context!  */
+bool matchStr(Expr e, string & s, PathSet & context);
+
+Expr makeStr(const string & s, const PathSet & context = PathSet());
+
+
+/* Showing types, values. */
 string showType(Expr e);
 
 string showValue(Expr e);
 
+
+Expr allocCell(Expr e); // make an updateable cell (for simulating conventional laziness)
+
+Expr allocCells(Expr e); // re-allocate all cells in e
+
  
 }
 

src/libexpr/parser.y

@@ -3,7 +3,7 @@
 %locations
 %error-verbose
 %defines
-%no-lines
+/* %no-lines */
 %parse-param { yyscan_t scanner }
 %parse-param { ParseData * data }
 %lex-param { yyscan_t scanner }
@@ -57,7 +57,7 @@ static Expr fixAttrs(int recursive, ATermList as)
             bool fromScope = matchScope(src);
             for (ATermIterator j(names); j; ++j) {
                 Expr rhs = fromScope ? makeVar(*j) : makeSelect(src, *j);
-                *is = ATinsert(*is, makeBind(*j, rhs, pos));
+                *is = ATinsert(*is, makeBind(*j, allocCell(rhs), pos));
             }
         } else bs = ATinsert(bs, *i);
     }
@@ -70,11 +70,13 @@ static Expr fixAttrs(int recursive, ATermList as)
 
 void backToString(yyscan_t scanner);
 
+
+extern bool posInfo;
  
 static Pos makeCurPos(YYLTYPE * loc, ParseData * data)
 {
-    return makePos(toATerm(data->path),
-        loc->first_line, loc->first_column);
+    return posInfo ? makePos(toATerm(data->path),
+        loc->first_line, loc->first_column) : makeNoPos();
 }
 
 #define CUR_POS makeCurPos(yylocp, data)
@@ -195,12 +197,12 @@ expr_simple
   | INT { $$ = makeInt(ATgetInt((ATermInt) $1)); }
   | '"' string_parts '"' {
       /* For efficiency, and to simplify parse trees a bit. */
-      if ($2 == ATempty) $$ = makeStr(toATerm(""));
+      if ($2 == ATempty) $$ = makeStr(toATerm(""), ATempty);
       else if (ATgetNext($2) == ATempty) $$ = ATgetFirst($2);
       else $$ = makeConcatStrings(ATreverse($2));
   }
   | PATH { $$ = makePath(toATerm(absPath(aterm2String($1), data->basePath))); }
-  | URI { $$ = makeUri($1); }
+  | URI { $$ = makeStr($1, ATempty); }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
@@ -226,7 +228,7 @@ binds
 
 bind
   : ID '=' expr ';'
-    { $$ = makeBind($1, $3, CUR_POS); }
+    { $$ = makeBind($1, allocCell($3), CUR_POS); }
   | INHERIT inheritsrc ids ';'
     { $$ = makeInherit($2, $3, CUR_POS); }
   ;
@@ -249,6 +251,7 @@ expr_list
 formals
   : formal ',' formals { $$ = ATinsert($3, $1); } /* idem - right recursive */
   | formal { $$ = ATinsert(ATempty, $1); }
+  | { $$ = ATempty; }
   ;
 
 formal
@@ -358,8 +361,6 @@ static Expr parse(EvalState & state,
 
 Expr parseExprFromFile(EvalState & state, Path path)
 {
-    SwitchToOriginalUser sw;
-
     assert(path[0] == '/');
 
 #if 0
@@ -371,9 +372,12 @@ Expr parseExprFromFile(EvalState & state, Path path)
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
     struct stat st;
-    if (lstat(path.c_str(), &st))
-        throw SysError(format("getting status of `%1%'") % path);
-    if (S_ISLNK(st.st_mode)) path = absPath(readLink(path), dirOf(path));
+    while (true) {
+        if (lstat(path.c_str(), &st))
+            throw SysError(format("getting status of `%1%'") % path);
+        if (!S_ISLNK(st.st_mode)) break;
+        path = absPath(readLink(path), dirOf(path));
+    }
 
     /* If `path' refers to a directory, append `/default.nix'. */
     if (stat(path.c_str(), &st))
@@ -381,20 +385,13 @@ Expr parseExprFromFile(EvalState & state, Path path)
     if (S_ISDIR(st.st_mode))
         path = canonPath(path + "/default.nix");
 
-    /* Read the input file.  We can't use SGparseFile() because it's
-       broken, so we read the input ourselves and call
-       SGparseString(). */
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY);
-    if (fd == -1) throw SysError(format("opening `%1%'") % path);
+    Expr cached = state.parsings.get(toATerm(path));
+    if (cached) return cached;
 
-    if (fstat(fd, &st) == -1)
-        throw SysError(format("statting `%1%'") % path);
-
-    char text[st.st_size + 1];
-    readFull(fd, (unsigned char *) text, st.st_size);
-    text[st.st_size] = 0;
-
-    return parse(state, text, path, dirOf(path));
+    /* Read and parse the input file. */
+    cached = parse(state, readFile(path).c_str(), path, dirOf(path));
+    state.parsings.set(toATerm(path), cached);
+    return cached;
 }
 
 

src/libmain/Makefile.am

@@ -9,6 +9,7 @@ AM_CXXFLAGS = \
  -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \
  -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
  -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \
+ -DNIX_BIN_DIR=\"$(bindir)\" \
  -DNIX_VERSION=\"$(VERSION)\" \
  -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil \
  -I$(srcdir)/../libstore

src/libmain/shared.cc

@@ -1,13 +1,13 @@
+#include "config.h"
+
 #include "shared.hh"
 #include "globals.hh"
-#include "gc.hh"
-#include "store.hh"
+#include "store-api.hh"
 #include "util.hh"
 
-#include "config.h"
-
 #include <iostream>
 #include <cctype>
+#include <exception>
 
 #include <sys/stat.h>
 #include <unistd.h>
@@ -44,7 +44,7 @@ void printGCWarning()
 {
     static bool haveWarned = false;
     warnOnce(haveWarned, 
-        "warning: you did not specify `--add-root'; "
+        "you did not specify `--add-root'; "
         "the result might be removed by the garbage collector");
 }
 
@@ -58,6 +58,18 @@ static void setLogType(string lt)
 }
 
 
+static unsigned int getIntArg(const string & opt,
+    Strings::iterator & i, const Strings::iterator & end)
+{
+    ++i;
+    if (i == end) throw UsageError(format("`%1%' requires an argument") % opt);
+    int n;
+    if (!string2Int(*i, n) || n < 0)
+        throw UsageError(format("`%1%' requires a non-negative integer") % opt);
+    return n;
+}
+
+
 struct RemoveTempRoots 
 {
     ~RemoveTempRoots()
@@ -70,16 +82,27 @@ struct RemoveTempRoots
 void initDerivationsHelpers();
 
 
+static void closeStore()
+{
+    try {
+        throw;
+    } catch (std::exception & e) {
+        printMsg(lvlError,
+            format("FATAL: unexpected exception (closing store and aborting): %1%") % e.what());
+    }
+    try {
+        store.reset((StoreAPI *) 0);
+    } catch (...) {
+        ignoreException();
+    }
+    abort();
+}
+
+
 /* Initialize and reorder arguments, then call the actual argument
    processor. */
 static void initAndRun(int argc, char * * argv)
 {
-    string root = getEnv("NIX_ROOT");
-    if (root != "") {
-        if (chroot(root.c_str()) != 0)
-            throw SysError(format("changing root to `%1%'") % root);
-    }
-    
     /* Setup Nix paths. */
     nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
     nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
@@ -88,15 +111,18 @@ static void initAndRun(int argc, char * * argv)
     nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
     nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
     nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
+    nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
+
+    string subs = getEnv("NIX_SUBSTITUTERS", "default");
+    if (subs == "default")
+        substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl");
+    else
+        substituters = tokenizeString(subs, ":");
 
     /* Get some settings from the configuration file. */
     thisSystem = querySetting("system", SYSTEM);
-    {
-        int n;
-        if (!string2Int(querySetting("build-max-jobs", "1"), n) || n < 0)
-            throw Error("invalid value for configuration setting `build-max-jobs'");
-        maxBuildJobs = n;
-    }
+    maxBuildJobs = queryIntSetting("build-max-jobs", 1);
+    maxSilentTime = queryIntSetting("build-max-silent-time", 0);
 
     /* Catch SIGINT. */
     struct sigaction act, oact;
@@ -180,16 +206,12 @@ static void initAndRun(int argc, char * * argv)
             keepGoing = true;
         else if (arg == "--fallback")
             tryFallback = true;
-        else if (arg == "--max-jobs" || arg == "-j") {
-            ++i;
-            if (i == args.end()) throw UsageError("`--max-jobs' requires an argument");
-            int n;
-            if (!string2Int(*i, n) || n < 0)
-                throw UsageError(format("`--max-jobs' requires a non-negative integer"));
-            maxBuildJobs = n;
-        }
+        else if (arg == "--max-jobs" || arg == "-j")
+            maxBuildJobs = getIntArg(arg, i, args.end());
         else if (arg == "--readonly-mode")
             readOnlyMode = true;
+        else if (arg == "--max-silent-time")
+            maxSilentTime = getIntArg(arg, i, args.end());
         else remaining.push_back(arg);
     }
 
@@ -197,9 +219,66 @@ static void initAndRun(int argc, char * * argv)
        exit. */
     RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
 
+    /* Make sure that the database gets closed properly, even if
+       terminate() is called (which happens sometimes due to bugs in
+       destructor/exceptions interaction, but that needn't preclude a
+       clean shutdown of the database). */
+    std::set_terminate(closeStore);
+    
     run(remaining);
 
-    closeDB(); /* it's fine if the DB isn't actually open */
+    /* Close the Nix database. */
+    store.reset((StoreAPI *) 0);
+}
+
+
+bool setuidMode = false;
+
+
+static void setuidInit()
+{
+    /* Don't do anything if this is not a setuid binary. */
+    if (getuid() == geteuid() && getgid() == getegid()) return;
+
+    uid_t nixUid = geteuid();
+    gid_t nixGid = getegid();
+    
+    setuidCleanup();
+
+    /* Don't trust the current directory. */
+    if (chdir("/") == -1) abort();
+
+    /* Set the real (and preferably also the save) uid/gid to the
+       effective uid/gid.  This matters mostly when we're not using
+       build-users (bad!), since some builders (like Perl) complain
+       when real != effective.
+
+       On systems where setresuid is unavailable, we can't drop the
+       saved uid/gid.  This means that we could go back to the
+       original real uid (i.e., the uid of the caller).  That's not
+       really a problem, except maybe when we execute a builder and
+       we're not using build-users.  In that case, the builder may be
+       able to switch to the uid of the caller and possibly do bad
+       stuff.  But note that when not using build-users, the builder
+       could also modify the Nix executables (say, replace them by a
+       Trojan horse), so the problem is already there. */
+
+#if HAVE_SETRESUID
+    if (setresuid(nixUid, nixUid, nixUid)) abort();
+    if (setresgid(nixGid, nixGid, nixGid)) abort();
+#elif HAVE_SETREUID
+    /* Note: doesn't set saved uid/gid! */
+    fprintf(stderr, "warning: cannot set saved uid\n");
+    if (setreuid(nixUid, nixUid)) abort();
+    if (setregid(nixGid, nixGid)) abort();
+#else
+    /* Note: doesn't set real and saved uid/gid! */
+    fprintf(stderr, "warning: cannot set real and saved uids\n");
+    if (setuid(nixUid)) abort();
+    if (setgid(nixGid)) abort();
+#endif
+
+    setuidMode = true;
 }
 
 
@@ -211,10 +290,11 @@ static char buf[1024];
 int main(int argc, char * * argv)
 {
     using namespace nix;
-    
-    /* If we are setuid root, we have to get rid of the excess
-       privileges ASAP. */
-    switchToNixUser();
+
+    /* If we're setuid, then we need to take some security precautions
+       right away. */
+    if (argc == 0) abort();
+    setuidInit();
     
     /* ATerm setup. */
     ATerm bottomOfStack;
@@ -225,6 +305,8 @@ int main(int argc, char * * argv)
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
+    std::ios::sync_with_stdio(false);
+
     try {
         try {
             initAndRun(argc, argv);
@@ -244,7 +326,7 @@ int main(int argc, char * * argv)
                 "Try `%2% --help' for more information.")
             % e.what() % programId);
         return 1;
-    } catch (Error & e) {
+    } catch (BaseError & e) {
         printMsg(lvlError, format("error: %1%") % e.msg());
         return 1;
     } catch (std::exception & e) {

src/libmain/shared.hh

@@ -3,6 +3,8 @@
 
 #include "types.hh"
 
+#include <signal.h>
+
 
 /* These are not implemented here, but must be implemented by a
    program linking against libmain. */
@@ -24,6 +26,13 @@ namespace nix {
 Path makeRootName(const Path & gcRoot, int & counter);
 void printGCWarning();
 
+/* Whether we're running setuid. */
+extern bool setuidMode;
+
+extern volatile ::sig_atomic_t blockInt;
+
+MakeError(UsageError, nix::Error);
+
 }
 
 

src/libstore/Makefile.am

@@ -1,12 +1,13 @@
 pkglib_LTLIBRARIES = libstore.la
 
 libstore_la_SOURCES = \
- store.cc derivations.cc build.cc misc.cc globals.cc db.cc \
- references.cc pathlocks.cc gc.cc 
+ store-api.cc local-store.cc remote-store.cc derivations.cc build.cc misc.cc \
+ globals.cc db.cc references.cc pathlocks.cc gc.cc 
 
 pkginclude_HEADERS = \
- store.hh derivations.hh build.hh misc.hh globals.hh db.hh \
- references.hh pathlocks.hh gc.hh 
+ store-api.hh local-store.hh remote-store.hh derivations.hh misc.hh \
+ globals.hh db.hh references.hh pathlocks.hh \
+ worker-protocol.hh
 
 libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la
 

src/libstore/build.hh

@@ -1,27 +0,0 @@
-#ifndef __BUILD_H
-#define __BUILD_H
-
-
-#include "types.hh"
-
-
-namespace nix {
-
-    
-/* Ensure that the output paths of the derivation are valid.  If they
-   are already valid, this is a no-op.  Otherwise, validity can
-   be reached in two ways.  First, if the output paths have
-   substitutes, then those can be used.  Second, the output paths can
-   be created by running the builder, after recursively building any
-   sub-derivations. */
-void buildDerivations(const PathSet & drvPaths);
-
-/* Ensure that a path is valid.  If it is not currently valid, it may
-   be made valid by running a substitute (if defined for the path). */
-void ensurePath(const Path & storePath);
-
-    
-}
-
-
-#endif /* !__BUILD_H */

src/libstore/db.cc

@@ -30,7 +30,12 @@ class DestroyDbEnv
     DbEnv * dbenv;
 public:
     DestroyDbEnv(DbEnv * _dbenv) : dbenv(_dbenv) { }
-    ~DestroyDbEnv() { if (dbenv) { dbenv->close(0); delete dbenv; } }
+    ~DestroyDbEnv() {
+        if (dbenv) {
+            if (dbenv->get_DB_ENV()) dbenv->close(0);
+            delete dbenv;
+        }
+    }
     void release() { dbenv = 0; };
 };
 
@@ -137,6 +142,15 @@ Database::~Database()
 
 void openEnv(DbEnv * & env, const string & path, u_int32_t flags)
 {
+    try {
+        createDirs(path);
+    } catch (SysError & e) {
+        if (e.errNo == EPERM || e.errNo == EACCES)
+            throw DbNoPermission(format("cannot create the Nix database in `%1%'") % path);
+        else
+            throw;
+    }
+        
     try {
         env->open(path.c_str(),
             DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN |
@@ -180,7 +194,8 @@ void Database::open2(const string & path, bool removeOldEnv)
 
     env->set_errcall(errorPrinter);
     env->set_msgcall(messagePrinter);
-    //env->set_verbose(DB_VERB_REGISTER, 1);
+    if (getEnv("NIX_DEBUG_DB_REGISTER") == "1")
+        env->set_verbose(DB_VERB_REGISTER, 1);
     env->set_verbose(DB_VERB_RECOVERY, 1);
     
     /* Smaller log files. */
@@ -241,6 +256,8 @@ void Database::open(const string & path)
 
             /* Force a checkpoint, as per the BDB docs. */
             env->txn_checkpoint(DB_FORCE, 0, 0);
+
+            printMsg(lvlError, "database succesfully upgraded to new version");
         }
 
 #if 0        
@@ -438,4 +455,14 @@ void Database::enumTable(const Transaction & txn, TableId table,
 }
 
  
+void Database::clearTable(const Transaction & txn, TableId table)
+{
+    try {
+        Db * db = getDb(table);
+        u_int32_t count;
+        db->truncate(txn.txn, &count, 0);
+    } catch (DbException e) { rethrow(e); }
+}
+
+
 }

src/libstore/db.hh

@@ -89,6 +89,8 @@ public:
 
     void enumTable(const Transaction & txn, TableId table,
         Strings & keys, const string & keyPrefix = "");
+
+    void clearTable(const Transaction & txn, TableId table);
 };
 
 

src/libstore/derivations.cc

@@ -1,6 +1,7 @@
 #include "derivations.hh"
-#include "store.hh"
+#include "store-api.hh"
 #include "aterm.hh"
+#include "globals.hh"
 
 #include "derivations-ast.hh"
 #include "derivations-ast.cc"
@@ -25,8 +26,11 @@ Path writeDerivation(const Derivation & drv, const string & name)
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-    return addTextToStore(name + drvExtension,
-        atPrint(unparseDerivation(drv)), references);
+    string suffix = name + drvExtension;
+    string contents = atPrint(unparseDerivation(drv));
+    return readOnlyMode
+        ? computeStorePathForText(suffix, contents, references)
+        : store->addTextToStore(suffix, contents, references);
 }
 
 
@@ -113,16 +117,6 @@ Derivation parseDerivation(ATerm t)
 }
 
 
-static ATermList unparseStrings(const StringSet & paths)
-{
-    ATermList l = ATempty;
-    for (PathSet::const_reverse_iterator i = paths.rbegin();
-         i != paths.rend(); ++i)
-        l = ATinsert(l, toATerm(*i));
-    return l;
-}
-
-
 ATerm unparseDerivation(const Derivation & drv)
 {
     ATermList outputs = ATempty;
@@ -141,7 +135,7 @@ ATerm unparseDerivation(const Derivation & drv)
         inDrvs = ATinsert(inDrvs,
             makeDerivationInput(
                 toATerm(i->first),
-                unparseStrings(i->second)));
+                toATermList(i->second)));
     
     ATermList args = ATempty;
     for (Strings::const_reverse_iterator i = drv.args.rbegin();
@@ -159,7 +153,7 @@ ATerm unparseDerivation(const Derivation & drv)
     return makeDerive(
         outputs,
         inDrvs,
-        unparseStrings(drv.inputSrcs),
+        toATermList(drv.inputSrcs),
         toATerm(drv.platform),
         toATerm(drv.builder),
         args,

src/libstore/gc.cc

@@ -1,8 +1,7 @@
-#include "gc.hh"
 #include "globals.hh"
 #include "misc.hh"
 #include "pathlocks.hh"
-#include "store.hh"
+#include "local-store.hh"
 #include "db.hh"
 #include "util.hh"
 
@@ -62,6 +61,8 @@ void createSymlink(const Path & link, const Path & target, bool careful)
     /* Create directories up to `gcRoot'. */
     createDirs(dirOf(link));
 
+    /* !!! shouldn't removing and creating the symlink be atomic? */
+
     /* Remove the old symlink. */
     if (pathExists(link)) {
         if (careful && (!isLink(link) || !isInStore(readLink(link))))
@@ -69,13 +70,28 @@ void createSymlink(const Path & link, const Path & target, bool careful)
         unlink(link.c_str());
     }
 
-    /* And create the new own. */
+    /* And create the new one. */
     if (symlink(target.c_str(), link.c_str()) == -1)
         throw SysError(format("symlinking `%1%' to `%2%'")
             % link % target);
 }
 
 
+void LocalStore::syncWithGC()
+{
+    AutoCloseFD fdGCLock = openGCLock(ltRead);
+}
+
+
+void LocalStore::addIndirectRoot(const Path & path)
+{
+    string hash = printHash32(hashString(htSHA1, path));
+    Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
+        % nixStateDir % gcRootsDir % hash).str());
+    createSymlink(realRoot, path, false);
+}
+
+
 Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
     bool indirect, bool allowOutsideRootsDir)
 {
@@ -83,20 +99,14 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
     Path gcRoot(canonPath(_gcRoot));
     assertStorePath(storePath);
 
-    /* Grab the global GC root.  This prevents the set of permanent
-       roots from increasing while a GC is in progress. */
-    AutoCloseFD fdGCLock = openGCLock(ltRead);
+    if (isInStore(gcRoot))
+        throw Error(format(
+                "creating a garbage collector root (%1%) in the Nix store is forbidden "
+                "(are you running nix-build inside the store?)") % gcRoot);
 
     if (indirect) {
-        string hash = printHash32(hashString(htSHA1, gcRoot));
-        Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
-            % nixStateDir % gcRootsDir % hash).str());
-        
-        {
-            SwitchToOriginalUser sw;
-            createSymlink(gcRoot, storePath, true);
-        }
-        createSymlink(realRoot, gcRoot, false);
+        createSymlink(gcRoot, storePath, true);
+        store->addIndirectRoot(gcRoot);
     }
 
     else {
@@ -113,6 +123,22 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
         createSymlink(gcRoot, storePath, false);
     }
 
+    /* Check that the root can be found by the garbage collector. */
+    if (queryBoolSetting("gc-check-reachability", true)) {
+        Roots roots = store->findRoots();
+        if (roots.find(gcRoot) == roots.end())
+            printMsg(lvlError, 
+                format(
+                    "warning: `%1%' is not in a directory where the garbage collector looks for roots; "
+                    "therefore, `%2%' might be removed by the garbage collector")
+                % gcRoot % storePath);
+    }
+        
+    /* Grab the global GC root, causing us to block while a GC is in
+       progress.  This prevents the set of permanent roots from
+       increasing while a GC is in progress. */
+    store->syncWithGC();
+    
     return gcRoot;
 }
 
@@ -122,7 +148,7 @@ static Path fnTempRoots;
 static AutoCloseFD fdTempRoots;
 
 
-void addTempRoot(const Path & path)
+void LocalStore::addTempRoot(const Path & path)
 {
     /* Create the temporary roots file for this process. */
     if (fdTempRoots == -1) {
@@ -283,47 +309,73 @@ static void readTempRoots(PathSet & tempRoots, FDs & fds)
 
 
 static void findRoots(const Path & path, bool recurseSymlinks,
-    PathSet & roots)
+    bool deleteStale, Roots & roots)
 {
-    struct stat st;
-    if (lstat(path.c_str(), &st) == -1)
-        throw SysError(format("statting `%1%'") % path);
+    try {
+        
+        struct stat st;
+        if (lstat(path.c_str(), &st) == -1)
+            throw SysError(format("statting `%1%'") % path);
 
-    printMsg(lvlVomit, format("looking at `%1%'") % path);
+        printMsg(lvlVomit, format("looking at `%1%'") % path);
 
-    if (S_ISDIR(st.st_mode)) {
-	Strings names = readDirectory(path);
-	for (Strings::iterator i = names.begin(); i != names.end(); ++i)
-            findRoots(path + "/" + *i, recurseSymlinks, roots);
-    }
-
-    else if (S_ISLNK(st.st_mode)) {
-        string target = readLink(path);
-        Path target2 = absPath(target, dirOf(path));
-
-        if (isInStore(target2)) {
-            debug(format("found root `%1%' in `%2%'")
-                % target2 % path);
-            Path target3 = toStorePath(target2);
-            if (isValidPath(target3)) 
-                roots.insert(target3);
-            else
-                printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'")
-                    % path % target3);
+        if (S_ISDIR(st.st_mode)) {
+            Strings names = readDirectory(path);
+            for (Strings::iterator i = names.begin(); i != names.end(); ++i)
+                findRoots(path + "/" + *i, recurseSymlinks, deleteStale, roots);
         }
 
-        else if (recurseSymlinks) {
-            if (pathExists(target2))
-                findRoots(target2, false, roots);
-            else {
-                printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target2);
-                /* Note that we only delete when recursing, i.e., when
-                   we are still in the `gcroots' tree.  We never
-                   delete stuff outside that tree. */
-                unlink(path.c_str());
+        else if (S_ISLNK(st.st_mode)) {
+            Path target = absPath(readLink(path), dirOf(path));
+
+            if (isInStore(target)) {
+                debug(format("found root `%1%' in `%2%'")
+                    % target % path);
+                Path storePath = toStorePath(target);
+                if (store->isValidPath(storePath)) 
+                    roots[path] = storePath;
+                else
+                    printMsg(lvlInfo, format("skipping invalid root from `%1%' to `%2%'")
+                        % path % storePath);
+            }
+
+            else if (recurseSymlinks) {
+                if (pathExists(target))
+                    findRoots(target, false, deleteStale, roots);
+                else if (deleteStale) {
+                    printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target);
+                    /* Note that we only delete when recursing, i.e.,
+                       when we are still in the `gcroots' tree.  We
+                       never delete stuff outside that tree. */
+                    unlink(path.c_str());
+                }
             }
         }
+
     }
+
+    catch (SysError & e) {
+        /* We only ignore permanent failures. */
+        if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
+            printMsg(lvlInfo, format("cannot read potential root `%1%'") % path);
+        else
+            throw;
+    }
+}
+
+
+static Roots findRoots(bool deleteStale)
+{
+    Roots roots;
+    Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+    findRoots(rootsDir, true, deleteStale, roots);
+    return roots;
+}
+
+
+Roots LocalStore::findRoots()
+{
+    return nix::findRoots(false);
 }
 
 
@@ -343,7 +395,7 @@ static void addAdditionalRoots(PathSet & roots)
     for (Strings::iterator i = paths.begin(); i != paths.end(); ++i) {
         if (isInStore(*i)) {
             Path path = toStorePath(*i);
-            if (roots.find(path) == roots.end() && isValidPath(path)) {
+            if (roots.find(path) == roots.end() && store->isValidPath(path)) {
                 debug(format("found additional root `%1%'") % path);
                 roots.insert(path);
             }
@@ -359,8 +411,8 @@ static void dfsVisit(const PathSet & paths, const Path & path,
     visited.insert(path);
     
     PathSet references;
-    if (isValidPath(path))
-        queryReferences(noTxn, path, references);
+    if (store->isValidPath(path))
+        store->queryReferences(path, references);
     
     for (PathSet::iterator i = references.begin();
          i != references.end(); ++i)
@@ -373,7 +425,7 @@ static void dfsVisit(const PathSet & paths, const Path & path,
 }
 
 
-static Paths topoSort(const PathSet & paths)
+Paths topoSortPaths(const PathSet & paths)
 {
     Paths sorted;
     PathSet visited;
@@ -383,7 +435,7 @@ static Paths topoSort(const PathSet & paths)
 }
 
 
-void collectGarbage(GCAction action, const PathSet & pathsToDelete,
+void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
     bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
 {
     result.clear();
@@ -401,16 +453,18 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete,
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
-    Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
+    Roots rootMap = ignoreLiveness ? Roots() : nix::findRoots(true);
+
     PathSet roots;
-    if (!ignoreLiveness)
-        findRoots(rootsDir, true, roots);
+    for (Roots::iterator i = rootMap.begin(); i != rootMap.end(); ++i)
+        roots.insert(i->second);
 
     /* Add additional roots returned by the program specified by the
        NIX_ROOT_FINDER environment variable.  This is typically used
        to add running programs to the set of roots (to prevent them
        from being garbage collected). */
-    addAdditionalRoots(roots);
+    if (!ignoreLiveness)
+        addAdditionalRoots(roots);
 
     if (action == gcReturnRoots) {
         result = roots;
@@ -430,8 +484,8 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete,
             /* Note that the deriver need not be valid (e.g., if we
                previously ran the collector with `gcKeepDerivations'
                turned off). */
-            Path deriver = queryDeriver(noTxn, *i);
-            if (deriver != "" && isValidPath(deriver))
+            Path deriver = store->queryDeriver(*i);
+            if (deriver != "" && store->isValidPath(deriver))
                 computeFSClosure(deriver, livePaths);
         }
     }
@@ -444,7 +498,7 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete,
                 Derivation drv = derivationFromPath(*i);
                 for (DerivationOutputs::iterator j = drv.outputs.begin();
                      j != drv.outputs.end(); ++j)
-                    if (isValidPath(j->second.path))
+                    if (store->isValidPath(j->second.path))
                         computeFSClosure(j->second.path, livePaths);
             }
     }
@@ -469,7 +523,7 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete,
        means that it has already been closed). */
     PathSet tempRootsClosed;
     for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
-        if (isValidPath(*i))
+        if (store->isValidPath(*i))
             computeFSClosure(*i, tempRootsClosed);
         else
             tempRootsClosed.insert(*i);
@@ -504,7 +558,7 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete,
        which things can be deleted safely. */
     /* !!! when we have multiple output paths per derivation, this
        will not work anymore because we get cycles. */
-    Paths storePaths = topoSort(storePathSet);
+    Paths storePaths = topoSortPaths(storePathSet);
 
     /* Try to delete store paths in the topologically sorted order. */
     for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
@@ -528,12 +582,8 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete,
 
         /* If just returning the set of dead paths, we also return the
            space that would be freed if we deleted them. */
-        if (action == gcReturnDead) {
-            struct stat st;
-            if (lstat(i->c_str(), &st) == -1)
-                st.st_size = 0;
-            bytesFreed += st.st_size;
-        }
+        if (action == gcReturnDead)
+            bytesFreed += computePathSize(*i);
 
         if (action == gcDeleteDead || action == gcDeleteSpecific) {
 
@@ -554,12 +604,18 @@ void collectGarbage(GCAction action, const PathSet & pathsToDelete,
             }
 #endif
 
+            if (!pathExists(*i)) continue;
+                
             printMsg(lvlInfo, format("deleting `%1%'") % *i);
             
             /* Okay, it's safe to delete. */
-            unsigned long long freed;
-            deleteFromStore(*i, freed);
-            bytesFreed += freed;
+            try {
+                unsigned long long freed;
+                deleteFromStore(*i, freed);
+                bytesFreed += freed;
+            } catch (PathInUse & e) {
+                printMsg(lvlError, format("warning: %1%") % e.msg());
+            }
 
 #ifndef __CYGWIN__
             if (fdLock != -1)

src/libstore/gc.hh

@@ -1,48 +0,0 @@
-#ifndef __GC_H
-#define __GC_H
-
-#include "types.hh"
-
-
-namespace nix {
-
-
-/* Garbage collector operation. */
-typedef enum {
-    gcReturnRoots,
-    gcReturnLive,
-    gcReturnDead,
-    gcDeleteDead,
-    gcDeleteSpecific,
-} GCAction;
-
-/* If `action' is set to `gcReturnRoots', find and return the set of
-   roots for the garbage collector.  These are the store paths
-   symlinked to in the `gcroots' directory.  If `action' is
-   `gcReturnLive', return the set of paths reachable from (i.e. in the
-   closure of) the roots.  If `action' is `gcReturnDead', return the
-   set of paths not reachable from the roots.  If `action' is
-   `gcDeleteDead', actually delete the latter set. */
-void collectGarbage(GCAction action, const PathSet & pathsToDelete,
-    bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
-
-/* Register a temporary GC root.  This root will automatically
-   disappear when this process exits.  WARNING: this function should
-   not be called inside a BDB transaction, otherwise we can
-   deadlock. */
-void addTempRoot(const Path & path);
-
-/* Remove the temporary roots file for this process.  Any temporary
-   root becomes garbage after this point unless it has been registered
-   as a (permanent) root. */
-void removeTempRoots();
-
-/* Register a permanent GC root. */
-Path addPermRoot(const Path & storePath, const Path & gcRoot,
-    bool indirect, bool allowOutsideRootsDir = false);
-
-
-}
-
-
-#endif /* !__GC_H */

src/libstore/globals.cc

@@ -15,6 +15,7 @@ string nixStateDir = "/UNINIT";
 string nixDBPath = "/UNINIT";
 string nixConfDir = "/UNINIT";
 string nixLibexecDir = "/UNINIT";
+string nixBinDir = "/UNINIT";
 
 bool keepFailed = false;
 bool keepGoing = false;
@@ -23,6 +24,8 @@ Verbosity buildVerbosity = lvlInfo;
 unsigned int maxBuildJobs = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
+unsigned int maxSilentTime = 0;
+Paths substituters;
 
 
 static bool settingsRead = false;
@@ -103,5 +106,14 @@ bool queryBoolSetting(const string & name, bool def)
         % name % v);
 }
 
+
+unsigned int queryIntSetting(const string & name, unsigned int def)
+{
+    int n;
+    if (!string2Int(querySetting(name, int2String(def)), n) || n < 0)
+        throw Error(format("configuration setting `%1%' should have an integer value") % name);
+    return n;
+}
+
  
 }

src/libstore/globals.hh

@@ -32,6 +32,8 @@ extern string nixConfDir;
    stored. */
 extern string nixLibexecDir;
 
+/* nixBinDir is the directory where the main programs are stored. */
+extern string nixBinDir;
 
 
 /* Misc. global flags. */
@@ -60,6 +62,16 @@ extern bool readOnlyMode;
 /* The canonical system name, as returned by config.guess. */ 
 extern string thisSystem;
 
+/* The maximum time in seconds that a builer can go without producing
+   any output on stdout/stderr before it is killed.  0 means
+   infinity. */
+extern unsigned int maxSilentTime;
+
+/* The substituters.  There are programs that can somehow realise a
+   store path without building, e.g., by downloading it or copying it
+   from a CD. */
+extern Paths substituters;
+
 
 Strings querySetting(const string & name, const Strings & def);
 
@@ -67,6 +79,8 @@ string querySetting(const string & name, const string & def);
 
 bool queryBoolSetting(const string & name, bool def);
 
+unsigned int queryIntSetting(const string & name, unsigned int def);
+
     
 }
 

src/libstore/local-store.hh

@@ -0,0 +1,176 @@
+#ifndef __LOCAL_STORE_H
+#define __LOCAL_STORE_H
+
+#include <string>
+
+#include "store-api.hh"
+
+
+namespace nix {
+
+
+class Transaction;
+
+
+/* Nix store and database schema version.  Version 1 (or 0) was Nix <=
+   0.7.  Version 2 was Nix 0.8 and 0.9.  Version 3 is Nix 0.10.
+   Version 4 is Nix 0.11. */
+const int nixSchemaVersion = 4;
+
+
+extern string drvsLogDir;
+
+
+struct OptimiseStats
+{
+    unsigned long totalFiles;
+    unsigned long sameContents;
+    unsigned long filesLinked;
+    unsigned long long bytesFreed;
+    OptimiseStats()
+    {
+        totalFiles = sameContents = filesLinked = 0;
+        bytesFreed = 0;
+    }
+};
+
+
+class LocalStore : public StoreAPI
+{
+private:
+    bool substitutablePathsLoaded;
+    PathSet substitutablePaths;
+    
+public:
+
+    /* Open the database environment.  If `reserveSpace' is true, make
+       sure that a big empty file exists in /nix/var/nix/db/reserved.
+       If `reserveSpace' is false, delete this file if it exists.  The
+       idea is that on normal operation, the file exists; but when we
+       run the garbage collector, it is deleted.  This is to ensure
+       that the garbage collector has a small amount of disk space
+       available, which is required to open the Berkeley DB
+       environment. */
+    LocalStore(bool reserveSpace);
+
+    ~LocalStore();
+    
+    /* Implementations of abstract store API methods. */
+    
+    bool isValidPath(const Path & path);
+
+    Hash queryPathHash(const Path & path);
+
+    void queryReferences(const Path & path, PathSet & references);
+
+    void queryReferrers(const Path & path, PathSet & referrers);
+
+    Path queryDeriver(const Path & path);
+    
+    PathSet querySubstitutablePaths();
+    
+    bool hasSubstitutes(const Path & path);
+    
+    Path addToStore(const Path & srcPath, bool fixed = false,
+        bool recursive = false, string hashAlgo = "",
+        PathFilter & filter = defaultPathFilter);
+
+    Path addTextToStore(const string & suffix, const string & s,
+        const PathSet & references);
+
+    void exportPath(const Path & path, bool sign,
+        Sink & sink);
+
+    Path importPath(bool requireSignature, Source & source);
+    
+    void buildDerivations(const PathSet & drvPaths);
+
+    void ensurePath(const Path & path);
+
+    void addTempRoot(const Path & path);
+
+    void addIndirectRoot(const Path & path);
+    
+    void syncWithGC();
+
+    Roots findRoots();
+
+    void collectGarbage(GCAction action, const PathSet & pathsToDelete,
+        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
+
+    /* Optimise the disk space usage of the Nix store by hard-linking
+       files with the same contents. */
+    void optimiseStore(bool dryRun, OptimiseStats & stats);
+};
+
+
+/* Get a transaction object. */
+void createStoreTransaction(Transaction & txn);
+
+/* Copy a path recursively. */
+void copyPath(const Path & src, const Path & dst);
+
+/* Register the validity of a path, i.e., that `path' exists, that the
+   paths referenced by it exists, and in the case of an output path of
+   a derivation, that it has been produced by a succesful execution of
+   the derivation (or something equivalent).  Also register the hash
+   of the file system contents of the path.  The hash must be a
+   SHA-256 hash. */
+void registerValidPath(const Transaction & txn,
+    const Path & path, const Hash & hash, const PathSet & references,
+    const Path & deriver);
+
+typedef list<ValidPathInfo> ValidPathInfos;
+
+void registerValidPaths(const Transaction & txn,
+    const ValidPathInfos & infos);
+
+/* "Fix", or canonicalise, the meta-data of the files in a store path
+   after it has been built.  In particular:
+   - the last modification date on each file is set to 0 (i.e.,
+     00:00:00 1/1/1970 UTC)
+   - the permissions are set of 444 or 555 (i.e., read-only with or
+     without execute permission; setuid bits etc. are cleared)
+   - the owner and group are set to the Nix user and group, if we're
+     in a setuid Nix installation. */
+void canonicalisePathMetaData(const Path & path);
+
+/* Checks whether a path is valid. */ 
+bool isValidPathTxn(const Transaction & txn, const Path & path);
+
+/* Sets the set of outgoing FS references for a store path.  Use with
+   care! */
+void setReferences(const Transaction & txn, const Path & path,
+    const PathSet & references);
+
+/* Sets the deriver of a store path.  Use with care! */
+void setDeriver(const Transaction & txn, const Path & path,
+    const Path & deriver);
+
+/* Delete a value from the nixStore directory. */
+void deleteFromStore(const Path & path, unsigned long long & bytesFreed);
+
+MakeError(PathInUse, Error);
+
+void verifyStore(bool checkContents);
+
+/* Whether we are in build users mode. */
+bool haveBuildUsers();
+
+/* Whether we are root. */
+bool amPrivileged();
+
+/* Recursively change the ownership of `path' to the current uid. */
+void getOwnership(const Path & path);
+
+/* Like deletePath(), but changes the ownership of `path' using the
+   setuid wrapper if necessary (and possible). */
+void deletePathWrapped(const Path & path,
+    unsigned long long & bytesFreed);
+
+void deletePathWrapped(const Path & path);
+ 
+}
+
+
+#endif /* !__LOCAL_STORE_H */

src/libstore/misc.cc

@@ -1,6 +1,5 @@
 #include "misc.hh"
-#include "store.hh"
-#include "build.hh"
+#include "store-api.hh"
 #include "db.hh"
 
 #include <aterm2.h>
@@ -12,7 +11,7 @@ namespace nix {
 Derivation derivationFromPath(const Path & drvPath)
 {
     assertStorePath(drvPath);
-    ensurePath(drvPath);
+    store->ensurePath(drvPath);
     ATerm t = ATreadFromNamedFile(drvPath.c_str());
     if (!t) throw Error(format("cannot read aterm from `%1%'") % drvPath);
     return parseDerivation(t);
@@ -27,9 +26,9 @@ void computeFSClosure(const Path & storePath,
 
     PathSet references;
     if (flipDirection)
-        queryReferrers(noTxn, storePath, references);
+        store->queryReferrers(storePath, references);
     else
-        queryReferences(noTxn, storePath, references);
+        store->queryReferences(storePath, references);
 
     for (PathSet::iterator i = references.begin();
          i != references.end(); ++i)
@@ -58,14 +57,13 @@ void queryMissing(const PathSet & targets,
         done.insert(p);
 
         if (isDerivation(p)) {
-            if (!isValidPath(p)) continue;
+            if (!store->isValidPath(p)) continue;
             Derivation drv = derivationFromPath(p);
 
             bool mustBuild = false;
             for (DerivationOutputs::iterator i = drv.outputs.begin();
                  i != drv.outputs.end(); ++i)
-                if (!isValidPath(i->second.path) &&
-                    querySubstitutes(noTxn, i->second.path).size() == 0)
+                if (!store->isValidPath(i->second.path) && !store->hasSubstitutes(i->second.path))
                     mustBuild = true;
 
             if (mustBuild) {
@@ -81,11 +79,11 @@ void queryMissing(const PathSet & targets,
         }
 
         else {
-            if (isValidPath(p)) continue;
-            if (querySubstitutes(noTxn, p).size() > 0)
+            if (store->isValidPath(p)) continue;
+            if (store->hasSubstitutes(p))
                 willSubstitute.insert(p);
-            PathSet refs;
-            queryReferences(noTxn, p, todo);
+            // XXX call the substituters
+            // store->queryReferences(p, todo);
         }
     }
 }

src/libstore/pathlocks.cc

@@ -161,10 +161,8 @@ void PathLocks::lockPaths(const PathSet & _paths, const string & waitMsg)
 
         debug(format("locking path `%1%'") % path);
 
-        if (lockedPaths.find(lockPath) != lockedPaths.end()) {
-            debug(format("already holding lock on `%1%'") % lockPath);
-            continue;
-        }
+        if (lockedPaths.find(lockPath) != lockedPaths.end())
+            throw Error("deadlock: trying to re-acquire self-held lock");
 
         AutoCloseFD fd;
         
@@ -225,5 +223,12 @@ void PathLocks::setDeletion(bool deletePaths)
     this->deletePaths = deletePaths;
 }
 
+
+bool pathIsLockedByMe(const Path & path)
+{
+    Path lockPath = path + ".lock";
+    return lockedPaths.find(lockPath) != lockedPaths.end();
+}
+
  
 }

src/libstore/pathlocks.hh

@@ -40,6 +40,9 @@ public:
 };
 
 
+bool pathIsLockedByMe(const Path & path);
+
+
 }
 
 

src/libstore/remote-store.cc

@@ -0,0 +1,404 @@
+#include "serialise.hh"
+#include "util.hh"
+#include "remote-store.hh"
+#include "worker-protocol.hh"
+#include "archive.hh"
+#include "globals.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <fcntl.h>
+
+#include <iostream>
+#include <unistd.h>
+
+
+namespace nix {
+
+
+Path readStorePath(Source & from)
+{
+    Path path = readString(from);
+    assertStorePath(path);
+    return path;
+}
+
+
+PathSet readStorePaths(Source & from)
+{
+    PathSet paths = readStringSet(from);
+    for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
+        assertStorePath(*i);
+    return paths;
+}
+
+
+RemoteStore::RemoteStore()
+{
+    string remoteMode = getEnv("NIX_REMOTE");
+
+    if (remoteMode == "slave")
+        /* Fork off a setuid worker to do the privileged work. */
+        forkSlave();
+    else if (remoteMode == "daemon")
+        /* Connect to a daemon that does the privileged work for
+           us. */
+       connectToDaemon();
+    else
+         throw Error(format("invalid setting for NIX_REMOTE, `%1%'")
+             % remoteMode);
+            
+    from.fd = fdSocket;
+    to.fd = fdSocket;
+
+    /* Send the magic greeting, check for the reply. */
+    try {
+        writeInt(WORKER_MAGIC_1, to);
+        unsigned int magic = readInt(from);
+        if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
+
+        unsigned int daemonVersion = readInt(from);
+        if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
+            throw Error("Nix daemon protocol version not supported");
+        writeInt(PROTOCOL_VERSION, to);
+        processStderr();
+
+    } catch (Error & e) {
+        throw Error(format("cannot start worker (%1%)")
+            % e.msg());
+    }
+
+    setOptions();
+}
+
+
+void RemoteStore::forkSlave()
+{
+    int sockets[2];
+    if (socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) == -1)
+        throw SysError("cannot create sockets");
+
+    fdSocket = sockets[0];
+    AutoCloseFD fdChild = sockets[1];
+
+    /* Start the worker. */
+    Path worker = getEnv("NIX_WORKER");
+    if (worker == "")
+        worker = nixBinDir + "/nix-worker";
+
+    string verbosityArg = "-";
+    for (int i = 1; i < verbosity; ++i)
+        verbosityArg += "v";
+
+    child = fork();
+    
+    switch (child) {
+        
+    case -1:
+        throw SysError("unable to fork");
+
+    case 0:
+        try { /* child */
+            
+            if (dup2(fdChild, STDOUT_FILENO) == -1)
+                throw SysError("dupping write side");
+
+            if (dup2(fdChild, STDIN_FILENO) == -1)
+                throw SysError("dupping read side");
+
+            close(fdSocket);
+            close(fdChild);
+
+            execlp(worker.c_str(), worker.c_str(), "--slave",
+                /* hacky - must be at the end */
+                verbosityArg == "-" ? NULL : verbosityArg.c_str(),
+                NULL);
+
+            throw SysError(format("executing `%1%'") % worker);
+            
+        } catch (std::exception & e) {
+            std::cerr << format("child error: %1%\n") % e.what();
+        }
+        quickExit(1);
+    }
+
+    fdChild.close();
+
+}
+
+
+void RemoteStore::connectToDaemon()
+{
+    fdSocket = socket(PF_UNIX, SOCK_STREAM, 0);
+    if (fdSocket == -1)
+        throw SysError("cannot create Unix domain socket");
+
+    string socketPath = nixStateDir + DEFAULT_SOCKET_PATH;
+
+    struct sockaddr_un addr;
+    addr.sun_family = AF_UNIX;
+    if (socketPath.size() >= sizeof(addr.sun_path))
+        throw Error(format("socket path `%1%' is too long") % socketPath);
+    strcpy(addr.sun_path, socketPath.c_str());
+    
+    if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+        throw SysError(format("cannot connect to daemon at `%1%'") % socketPath);
+}
+
+
+RemoteStore::~RemoteStore()
+{
+    try {
+        fdSocket.close();
+        if (child != -1)
+            child.wait(true);
+    } catch (...) {
+        ignoreException();
+    }
+}
+
+
+void RemoteStore::setOptions()
+{
+    writeInt(wopSetOptions, to);
+    writeInt(keepFailed, to);
+    writeInt(keepGoing, to);
+    writeInt(tryFallback, to);
+    writeInt(verbosity, to);
+    writeInt(maxBuildJobs, to);
+    writeInt(maxSilentTime, to);
+    processStderr();
+}
+
+
+bool RemoteStore::isValidPath(const Path & path)
+{
+    writeInt(wopIsValidPath, to);
+    writeString(path, to);
+    processStderr();
+    unsigned int reply = readInt(from);
+    return reply != 0;
+}
+
+
+bool RemoteStore::hasSubstitutes(const Path & path)
+{
+    writeInt(wopHasSubstitutes, to);
+    writeString(path, to);
+    processStderr();
+    unsigned int reply = readInt(from);
+    return reply != 0;
+}
+
+
+Hash RemoteStore::queryPathHash(const Path & path)
+{
+    writeInt(wopQueryPathHash, to);
+    writeString(path, to);
+    processStderr();
+    string hash = readString(from);
+    return parseHash(htSHA256, hash);
+}
+
+
+void RemoteStore::queryReferences(const Path & path,
+    PathSet & references)
+{
+    writeInt(wopQueryReferences, to);
+    writeString(path, to);
+    processStderr();
+    PathSet references2 = readStorePaths(from);
+    references.insert(references2.begin(), references2.end());
+}
+
+
+void RemoteStore::queryReferrers(const Path & path,
+    PathSet & referrers)
+{
+    writeInt(wopQueryReferrers, to);
+    writeString(path, to);
+    processStderr();
+    PathSet referrers2 = readStorePaths(from);
+    referrers.insert(referrers2.begin(), referrers2.end());
+}
+
+
+Path RemoteStore::queryDeriver(const Path & path)
+{
+    writeInt(wopQueryDeriver, to);
+    writeString(path, to);
+    processStderr();
+    return readStorePath(from);
+}
+
+
+PathSet RemoteStore::querySubstitutablePaths()
+{
+    throw Error("not implemented");
+}
+
+
+Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
+    bool recursive, string hashAlgo, PathFilter & filter)
+{
+    Path srcPath(absPath(_srcPath));
+    
+    writeInt(wopAddToStore, to);
+    writeString(baseNameOf(srcPath), to);
+    writeInt(fixed ? 1 : 0, to);
+    writeInt(recursive ? 1 : 0, to);
+    writeString(hashAlgo, to);
+    dumpPath(srcPath, to, filter);
+    processStderr();
+    return readStorePath(from);
+}
+
+
+Path RemoteStore::addTextToStore(const string & suffix, const string & s,
+    const PathSet & references)
+{
+    writeInt(wopAddTextToStore, to);
+    writeString(suffix, to);
+    writeString(s, to);
+    writeStringSet(references, to);
+    
+    processStderr();
+    return readStorePath(from);
+}
+
+
+void RemoteStore::exportPath(const Path & path, bool sign,
+    Sink & sink)
+{
+    writeInt(wopExportPath, to);
+    writeString(path, to);
+    writeInt(sign ? 1 : 0, to);
+    processStderr(&sink); /* sink receives the actual data */
+    readInt(from);
+}
+
+
+Path RemoteStore::importPath(bool requireSignature, Source & source)
+{
+    writeInt(wopImportPath, to);
+    /* We ignore requireSignature, since the worker forces it to true
+       anyway. */
+    
+    processStderr(0, &source);
+    return readStorePath(from);
+}
+
+
+void RemoteStore::buildDerivations(const PathSet & drvPaths)
+{
+    writeInt(wopBuildDerivations, to);
+    writeStringSet(drvPaths, to);
+    processStderr();
+    readInt(from);
+}
+
+
+void RemoteStore::ensurePath(const Path & path)
+{
+    writeInt(wopEnsurePath, to);
+    writeString(path, to);
+    processStderr();
+    readInt(from);
+}
+
+
+void RemoteStore::addTempRoot(const Path & path)
+{
+    writeInt(wopAddTempRoot, to);
+    writeString(path, to);
+    processStderr();
+    readInt(from);
+}
+
+
+void RemoteStore::addIndirectRoot(const Path & path)
+{
+    writeInt(wopAddIndirectRoot, to);
+    writeString(path, to);
+    processStderr();
+    readInt(from);
+}
+
+
+void RemoteStore::syncWithGC()
+{
+    writeInt(wopSyncWithGC, to);
+    processStderr();
+    readInt(from);
+}
+
+
+Roots RemoteStore::findRoots()
+{
+    writeInt(wopFindRoots, to);
+    processStderr();
+    unsigned int count = readInt(from);
+    Roots result;
+    while (count--) {
+        Path link = readString(from);
+        Path target = readStorePath(from);
+        result[link] = target;
+    }
+    return result;
+}
+
+
+void RemoteStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
+    bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed)
+{
+    result.clear();
+    bytesFreed = 0;
+    writeInt(wopCollectGarbage, to);
+    writeInt(action, to);
+    writeStringSet(pathsToDelete, to);
+    writeInt(ignoreLiveness, to);
+    
+    processStderr();
+    
+    result = readStringSet(from);
+
+    /* Ugh - NAR integers are 64 bits, but read/writeInt() aren't. */
+    unsigned int lo = readInt(from);
+    unsigned int hi = readInt(from);
+    bytesFreed = (((unsigned long long) hi) << 32) | lo;
+}
+
+
+void RemoteStore::processStderr(Sink * sink, Source * source)
+{
+    unsigned int msg;
+    while ((msg = readInt(from)) == STDERR_NEXT
+        || msg == STDERR_READ || msg == STDERR_WRITE) {
+        if (msg == STDERR_WRITE) {
+            string s = readString(from);
+            if (!sink) throw Error("no sink");
+            (*sink)((const unsigned char *) s.c_str(), s.size());
+        }
+        else if (msg == STDERR_READ) {
+            if (!source) throw Error("no source");
+            unsigned int len = readInt(from);
+            unsigned char * buf = new unsigned char[len];
+            AutoDeleteArray<unsigned char> d(buf);
+            (*source)(buf, len);
+            writeString(string((const char *) buf, len), to);
+        }
+        else {
+            string s = readString(from);
+            writeToStderr((const unsigned char *) s.c_str(), s.size());
+        }
+    }
+    if (msg == STDERR_ERROR)
+        throw Error(readString(from));
+    else if (msg != STDERR_LAST)
+        throw Error("protocol error processing standard error");
+}
+
+
+}

src/libstore/remote-store.hh

@@ -0,0 +1,88 @@
+#ifndef __REMOTE_STORE_H
+#define __REMOTE_STORE_H
+
+#include <string>
+
+#include "store-api.hh"
+
+
+namespace nix {
+
+
+class Pipe;
+class Pid;
+struct FdSink;
+struct FdSource;
+
+
+class RemoteStore : public StoreAPI
+{
+public:
+
+    RemoteStore();
+
+    ~RemoteStore();
+    
+    /* Implementations of abstract store API methods. */
+    
+    bool isValidPath(const Path & path);
+
+    Hash queryPathHash(const Path & path);
+
+    void queryReferences(const Path & path, PathSet & references);
+
+    void queryReferrers(const Path & path, PathSet & referrers);
+
+    Path queryDeriver(const Path & path);
+    
+    PathSet querySubstitutablePaths();
+    
+    bool hasSubstitutes(const Path & path);
+    
+    Path addToStore(const Path & srcPath, bool fixed = false,
+        bool recursive = false, string hashAlgo = "",
+        PathFilter & filter = defaultPathFilter);
+
+    Path addTextToStore(const string & suffix, const string & s,
+        const PathSet & references);
+
+    void exportPath(const Path & path, bool sign,
+        Sink & sink);
+
+    Path importPath(bool requireSignature, Source & source);
+    
+    void buildDerivations(const PathSet & drvPaths);
+
+    void ensurePath(const Path & path);
+
+    void addTempRoot(const Path & path);
+
+    void addIndirectRoot(const Path & path);
+    
+    void syncWithGC();
+    
+    Roots findRoots();
+
+    void collectGarbage(GCAction action, const PathSet & pathsToDelete,
+        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed);
+    
+private:
+    AutoCloseFD fdSocket;
+    FdSink to;
+    FdSource from;
+    Pid child;
+
+    void processStderr(Sink * sink = 0, Source * source = 0);
+
+    void forkSlave();
+    
+    void connectToDaemon();
+
+    void setOptions();
+};
+
+
+}
+
+
+#endif /* !__REMOTE_STORE_H */

src/libstore/store-api.cc

@@ -0,0 +1,175 @@
+#include "store-api.hh"
+#include "globals.hh"
+#include "util.hh"
+
+
+namespace nix {
+
+
+bool StoreAPI::hasSubstitutes(const Path & path)
+{
+    PathSet paths = querySubstitutablePaths();
+    return paths.find(path) != paths.end();
+}
+
+
+bool isInStore(const Path & path)
+{
+    return path[0] == '/'
+        && string(path, 0, nixStore.size()) == nixStore
+        && path.size() >= nixStore.size() + 2
+        && path[nixStore.size()] == '/';
+}
+
+
+bool isStorePath(const Path & path)
+{
+    return isInStore(path)
+        && path.find('/', nixStore.size() + 1) == Path::npos;
+}
+
+
+void assertStorePath(const Path & path)
+{
+    if (!isStorePath(path))
+        throw Error(format("path `%1%' is not in the Nix store") % path);
+}
+
+
+Path toStorePath(const Path & path)
+{
+    if (!isInStore(path))
+        throw Error(format("path `%1%' is not in the Nix store") % path);
+    Path::size_type slash = path.find('/', nixStore.size() + 1);
+    if (slash == Path::npos)
+        return path;
+    else
+        return Path(path, 0, slash);
+}
+
+
+void checkStoreName(const string & name)
+{
+    string validChars = "+-._?=";
+    /* Disallow names starting with a dot for possible security
+       reasons (e.g., "." and ".."). */
+    if (string(name, 0, 1) == ".")
+        throw Error(format("illegal name: `%1%'") % name);
+    for (string::const_iterator i = name.begin(); i != name.end(); ++i)
+        if (!((*i >= 'A' && *i <= 'Z') ||
+              (*i >= 'a' && *i <= 'z') ||
+              (*i >= '0' && *i <= '9') ||
+              validChars.find(*i) != string::npos))
+        {
+            throw Error(format("invalid character `%1%' in name `%2%'")
+                % *i % name);
+        }
+}
+
+
+Path makeStorePath(const string & type,
+    const Hash & hash, const string & suffix)
+{
+    /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
+    string s = type + ":sha256:" + printHash(hash) + ":"
+        + nixStore + ":" + suffix;
+
+    checkStoreName(suffix);
+
+    return nixStore + "/"
+        + printHash32(compressHash(hashString(htSHA256, s), 20))
+        + "-" + suffix;
+}
+
+
+Path makeFixedOutputPath(bool recursive,
+    string hashAlgo, Hash hash, string name)
+{
+    /* !!! copy/paste from primops.cc */
+    Hash h = hashString(htSHA256, "fixed:out:"
+        + (recursive ? (string) "r:" : "") + hashAlgo + ":"
+        + printHash(hash) + ":"
+        + "");
+    return makeStorePath("output:out", h, name);
+}
+
+
+std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
+    bool fixed, bool recursive, string hashAlgo, PathFilter & filter)
+{
+    Hash h = hashPath(htSHA256, srcPath, filter);
+
+    string baseName = baseNameOf(srcPath);
+
+    Path dstPath;
+    
+    if (fixed) {
+        HashType ht(parseHashType(hashAlgo));
+        Hash h2 = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath);
+        dstPath = makeFixedOutputPath(recursive, hashAlgo, h2, baseName);
+    }
+        
+    else dstPath = makeStorePath("source", h, baseName);
+
+    return std::pair<Path, Hash>(dstPath, h);
+}
+
+
+Path computeStorePathForText(const string & suffix, const string & s,
+    const PathSet & references)
+{
+    Hash hash = hashString(htSHA256, s);
+    /* Stuff the references (if any) into the type.  This is a bit
+       hacky, but we can't put them in `s' since that would be
+       ambiguous. */
+    string type = "text";
+    for (PathSet::const_iterator i = references.begin(); i != references.end(); ++i) {
+        type += ":";
+        type += *i;
+    }
+    return makeStorePath(type, hash, suffix);
+}
+
+
+ValidPathInfo decodeValidPathInfo(std::istream & str)
+{
+    ValidPathInfo info;
+    getline(str, info.path);
+    if (str.eof()) { info.path = ""; return info; }
+    getline(str, info.deriver);
+    string s; int n;
+    getline(str, s);
+    if (!string2Int(s, n)) throw Error("number expected");
+    while (n--) {
+        getline(str, s);
+        info.references.insert(s);
+    }
+    if (!str || str.eof()) throw Error("missing input");
+    return info;
+}
+
+
+}
+
+
+#include "local-store.hh"
+#include "serialise.hh"
+#include "remote-store.hh"
+
+
+namespace nix {
+
+
+boost::shared_ptr<StoreAPI> store;
+
+
+boost::shared_ptr<StoreAPI> openStore(bool reserveSpace)
+{
+    if (getEnv("NIX_REMOTE") == "")
+        return boost::shared_ptr<StoreAPI>(new LocalStore(reserveSpace));
+    else
+        return boost::shared_ptr<StoreAPI>(new RemoteStore());
+}
+
+
+}

src/libstore/store-api.hh

@@ -0,0 +1,256 @@
+#ifndef __STOREAPI_H
+#define __STOREAPI_H
+
+#include <string>
+#include <map>
+
+#include <boost/shared_ptr.hpp>
+
+#include "hash.hh"
+#include "serialise.hh"
+
+
+namespace nix {
+
+
+typedef std::map<Path, Path> Roots;
+
+
+/* Garbage collector operation. */
+typedef enum {
+    gcReturnRoots,
+    gcReturnLive,
+    gcReturnDead,
+    gcDeleteDead,
+    gcDeleteSpecific,
+} GCAction;
+
+
+class StoreAPI 
+{
+public:
+
+    virtual ~StoreAPI() { }
+
+    /* Checks whether a path is valid. */ 
+    virtual bool isValidPath(const Path & path) = 0;
+
+    /* Queries the hash of a valid path. */ 
+    virtual Hash queryPathHash(const Path & path) = 0;
+
+    /* Queries the set of outgoing FS references for a store path.
+       The result is not cleared. */
+    virtual void queryReferences(const Path & path,
+        PathSet & references) = 0;
+
+    /* Queries the set of incoming FS references for a store path.
+       The result is not cleared. */
+    virtual void queryReferrers(const Path & path,
+        PathSet & referrers) = 0;
+
+    /* Query the deriver of a store path.  Return the empty string if
+       no deriver has been set. */
+    virtual Path queryDeriver(const Path & path) = 0;
+
+    /* Query the set of substitutable paths. */
+    virtual PathSet querySubstitutablePaths() = 0;
+
+    /* More efficient variant if we just want to know if a path has
+       substitutes. */
+    virtual bool hasSubstitutes(const Path & path);
+    
+    /* Copy the contents of a path to the store and register the
+       validity the resulting path.  The resulting path is returned.
+       If `fixed' is true, then the output of a fixed-output
+       derivation is pre-loaded into the Nix store.  The function
+       object `filter' can be used to exclude files (see
+       libutil/archive.hh). */
+    virtual Path addToStore(const Path & srcPath, bool fixed = false,
+        bool recursive = false, string hashAlgo = "",
+        PathFilter & filter = defaultPathFilter) = 0;
+
+    /* Like addToStore, but the contents written to the output path is
+       a regular file containing the given string. */
+    virtual Path addTextToStore(const string & suffix, const string & s,
+        const PathSet & references) = 0;
+
+    /* Export a store path, that is, create a NAR dump of the store
+       path and append its references and its deriver.  Optionally, a
+       cryptographic signature (created by OpenSSL) of the preceding
+       data is attached. */
+    virtual void exportPath(const Path & path, bool sign,
+        Sink & sink) = 0;
+
+    /* Import a NAR dump created by exportPath() into the Nix
+       store. */
+    virtual Path importPath(bool requireSignature, Source & source) = 0;
+
+    /* Ensure that the output paths of the derivation are valid.  If
+       they are already valid, this is a no-op.  Otherwise, validity
+       can be reached in two ways.  First, if the output paths is
+       substitutable, then build the path that way.  Second, the
+       output paths can be created by running the builder, after
+       recursively building any sub-derivations. */
+    virtual void buildDerivations(const PathSet & drvPaths) = 0;
+
+    /* Ensure that a path is valid.  If it is not currently valid, it
+       may be made valid by running a substitute (if defined for the
+       path). */
+    virtual void ensurePath(const Path & path) = 0;
+
+    /* Add a store path as a temporary root of the garbage collector.
+       The root disappears as soon as we exit. */
+    virtual void addTempRoot(const Path & path) = 0;
+
+    /* Add an indirect root, which is merely a symlink to `path' from
+       /nix/var/nix/gcroots/auto/<hash of `path'>.  `path' is supposed
+       to be a symlink to a store path.  The garbage collector will
+       automatically remove the indirect root when it finds that
+       `path' has disappeared. */
+    virtual void addIndirectRoot(const Path & path) = 0;
+
+    /* Acquire the global GC lock, then immediately release it.  This
+       function must be called after registering a new permanent root,
+       but before exiting.  Otherwise, it is possible that a running
+       garbage collector doesn't see the new root and deletes the
+       stuff we've just built.  By acquiring the lock briefly, we
+       ensure that either:
+
+       - The collector is already running, and so we block until the
+         collector is finished.  The collector will know about our
+         *temporary* locks, which should include whatever it is we
+         want to register as a permanent lock.
+
+       - The collector isn't running, or it's just started but hasn't
+         acquired the GC lock yet.  In that case we get and release
+         the lock right away, then exit.  The collector scans the
+         permanent root and sees our's.
+
+       In either case the permanent root is seen by the collector. */
+    virtual void syncWithGC() = 0;
+
+    /* Find the roots of the garbage collector.  Each root is a pair
+       (link, storepath) where `link' is the path of the symlink
+       outside of the Nix store that point to `storePath'.  */
+    virtual Roots findRoots() = 0;
+
+    /* Depending on `action', this function does the following:
+
+       - `gcReturnRoots': find and return the set of roots for the
+         garbage collector.  These are the store paths symlinked to in
+         the `gcroots' directory.
+
+       - `gcReturnLive': return the set of paths reachable from
+         (i.e. in the closure of) the roots.
+
+       - `gcReturnDead': return the set of paths not reachable from
+         the roots.
+
+       - `gcDeleteDead': actually delete the latter set.
+
+       - `gcDeleteSpecific': delete the paths listed in
+         `pathsToDelete', insofar as they are not reachable.
+
+       If `ignoreLiveness' is set, then reachability from the roots is
+       ignored (dangerous!).  However, the paths must still be
+       unreferenced *within* the store (i.e., there can be no other
+       store paths that depend on them).
+
+       For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
+       number of bytes that would be or was freed is returned in
+       `bytesFreed'. */
+    virtual void collectGarbage(GCAction action, const PathSet & pathsToDelete,
+        bool ignoreLiveness, PathSet & result, unsigned long long & bytesFreed) = 0;
+};
+
+
+/* !!! These should be part of the store API, I guess. */
+
+/* Throw an exception if `path' is not directly in the Nix store. */
+void assertStorePath(const Path & path);
+
+bool isInStore(const Path & path);
+bool isStorePath(const Path & path);
+
+void checkStoreName(const string & name);
+
+/* Chop off the parts after the top-level store name, e.g.,
+   /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
+Path toStorePath(const Path & path);
+
+
+/* Constructs a unique store path name. */
+Path makeStorePath(const string & type,
+    const Hash & hash, const string & suffix);
+    
+Path makeFixedOutputPath(bool recursive,
+    string hashAlgo, Hash hash, string name);
+
+
+/* This is the preparatory part of addToStore() and addToStoreFixed();
+   it computes the store path to which srcPath is to be copied.
+   Returns the store path and the cryptographic hash of the
+   contents of srcPath. */
+std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
+    bool fixed = false, bool recursive = false, string hashAlgo = "",
+    PathFilter & filter = defaultPathFilter);
+
+/* Preparatory part of addTextToStore().
+
+   !!! Computation of the path should take the references given to
+   addTextToStore() into account, otherwise we have a (relatively
+   minor) security hole: a caller can register a source file with
+   bogus references.  If there are too many references, the path may
+   not be garbage collected when it has to be (not really a problem,
+   the caller could create a root anyway), or it may be garbage
+   collected when it shouldn't be (more serious).
+
+   Hashing the references would solve this (bogus references would
+   simply yield a different store path, so other users wouldn't be
+   affected), but it has some backwards compatibility issues (the
+   hashing scheme changes), so I'm not doing that for now. */
+Path computeStorePathForText(const string & suffix, const string & s,
+    const PathSet & references);
+
+
+/* Remove the temporary roots file for this process.  Any temporary
+   root becomes garbage after this point unless it has been registered
+   as a (permanent) root. */
+void removeTempRoots();
+
+
+/* Register a permanent GC root. */
+Path addPermRoot(const Path & storePath, const Path & gcRoot,
+    bool indirect, bool allowOutsideRootsDir = false);
+
+
+/* Sort a set of paths topologically under the references relation.
+   If p refers to q, then p follows q in this list. */
+Paths topoSortPaths(const PathSet & paths);
+
+
+/* For now, there is a single global store API object, but we'll
+   purify that in the future. */
+extern boost::shared_ptr<StoreAPI> store;
+
+
+/* Factory method: open the Nix database, either through the local or
+   remote implementation. */
+boost::shared_ptr<StoreAPI> openStore(bool reserveSpace = true);
+
+
+struct ValidPathInfo 
+{
+    Path path;
+    Path deriver;
+    Hash hash;
+    PathSet references;
+};
+
+ValidPathInfo decodeValidPathInfo(std::istream & str);
+
+
+}
+
+
+#endif /* !__STOREAPI_H */

src/libstore/store.hh

@@ -1,178 +0,0 @@
-#ifndef __STORE_H
-#define __STORE_H
-
-#include <string>
-
-#include "hash.hh"
-
-
-namespace nix {
-
-
-class Transaction;
-
-
-/* Nix store and database schema version.  Version 1 (or 0) was Nix <=
-   0.7.  Version 2 was Nix 0.8 and 0.8.  Version 3 is Nix 0.10 and
-   up. */
-const int nixSchemaVersion = 3;
-
-
-/* A substitute is a program invocation that constructs some store
-   path (typically by fetching it from somewhere, e.g., from the
-   network). */
-struct Substitute
-{       
-    /* The derivation that built this store path (empty if none). */
-    Path deriver;
-    
-    /* Program to be executed to create the store path.  Must be in
-       the output path of `storeExpr'. */
-    Path program;
-
-    /* Extra arguments to be passed to the program (the first argument
-       is the store path to be substituted). */
-    Strings args;
-
-    bool operator == (const Substitute & sub) const;
-};
-
-typedef list<Substitute> Substitutes;
-
-
-/* Open the database environment.  If `reserveSpace' is true, make
-   sure that a big empty file exists in /nix/var/nix/db/reserved.  If
-   `reserveSpace' is false, delete this file if it exists.  The idea
-   is that on normal operation, the file exists; but when we run the
-   garbage collector, it is deleted.  This is to ensure that the
-   garbage collector has a small amount of disk space available, which
-   is required to open the Berkeley DB environment. */
-void openDB(bool reserveSpace = true);
-
-/* Create the required database tables. */
-void initDB();
-
-/* Close the database. */
-void closeDB();
-
-/* Get a transaction object. */
-void createStoreTransaction(Transaction & txn);
-
-/* Copy a path recursively. */
-void copyPath(const Path & src, const Path & dst);
-
-/* Register a substitute. */
-void registerSubstitute(const Transaction & txn,
-    const Path & srcPath, const Substitute & sub);
-
-/* Return the substitutes for the given path. */
-Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath);
-
-/* Deregister all substitutes. */
-void clearSubstitutes();
-
-/* Register the validity of a path, i.e., that `path' exists, that the
-   paths referenced by it exists, and in the case of an output path of
-   a derivation, that it has been produced by a succesful execution of
-   the derivation (or something equivalent).  Also register the hash
-   of the file system contents of the path.  The hash must be a
-   SHA-256 hash. */
-void registerValidPath(const Transaction & txn,
-    const Path & path, const Hash & hash, const PathSet & references,
-    const Path & deriver);
-
-struct ValidPathInfo 
-{
-    Path path;
-    Path deriver;
-    Hash hash;
-    PathSet references;
-};
-
-typedef list<ValidPathInfo> ValidPathInfos;
-
-void registerValidPaths(const Transaction & txn,
-    const ValidPathInfos & infos);
-
-/* Throw an exception if `path' is not directly in the Nix store. */
-void assertStorePath(const Path & path);
-
-bool isInStore(const Path & path);
-bool isStorePath(const Path & path);
-
-void checkStoreName(const string & name);
-
-/* Chop off the parts after the top-level store name, e.g.,
-   /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
-Path toStorePath(const Path & path);
-
-/* "Fix", or canonicalise, the meta-data of the files in a store path
-   after it has been built.  In particular:
-   - the last modification date on each file is set to 0 (i.e.,
-     00:00:00 1/1/1970 UTC)
-   - the permissions are set of 444 or 555 (i.e., read-only with or
-     without execute permission; setuid bits etc. are cleared)
-   - the owner and group are set to the Nix user and group, if we're
-     in a setuid Nix installation. */
-void canonicalisePathMetaData(const Path & path);
-
-/* Checks whether a path is valid. */ 
-bool isValidPathTxn(const Transaction & txn, const Path & path);
-bool isValidPath(const Path & path);
-
-/* Queries the hash of a valid path. */ 
-Hash queryPathHash(const Path & path);
-
-/* Sets the set of outgoing FS references for a store path.  Use with
-   care! */
-void setReferences(const Transaction & txn, const Path & storePath,
-    const PathSet & references);
-
-/* Queries the set of outgoing FS references for a store path.  The
-   result is not cleared. */
-void queryReferences(const Transaction & txn,
-    const Path & storePath, PathSet & references);
-
-/* Queries the set of incoming FS references for a store path.  The
-   result is not cleared. */
-void queryReferrers(const Transaction & txn,
-    const Path & storePath, PathSet & referrers);
-
-/* Sets the deriver of a store path.  Use with care! */
-void setDeriver(const Transaction & txn, const Path & storePath,
-    const Path & deriver);
-
-/* Query the deriver of a store path.  Return the empty string if no
-   deriver has been set. */
-Path queryDeriver(const Transaction & txn, const Path & storePath);
-
-/* Constructs a unique store path name. */
-Path makeStorePath(const string & type,
-    const Hash & hash, const string & suffix);
-    
-/* Copy the contents of a path to the store and register the validity
-   the resulting path.  The resulting path is returned. */
-Path addToStore(const Path & srcPath);
-
-/* Like addToStore(), but for pre-adding the outputs of fixed-output
-   derivations. */
-Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath);
-
-Path makeFixedOutputPath(bool recursive,
-    string hashAlgo, Hash hash, string name);
-
-/* Like addToStore, but the contents written to the output path is a
-   regular file containing the given string. */
-Path addTextToStore(const string & suffix, const string & s,
-    const PathSet & references);
-
-/* Delete a value from the nixStore directory. */
-void deleteFromStore(const Path & path, unsigned long long & bytesFreed);
-
-void verifyStore(bool checkContents);
-
- 
-}
-
-
-#endif /* !__STORE_H */

src/libstore/test-builder-1.sh

@@ -1,3 +0,0 @@
-#! /bin/sh
-
-echo "Hello World" > $out