nix eval-hydra-jobs: Support parallel evaluation

Example usage:

  $ nix eval-hydra-jobs -f '<nixpkgs/pkgs/top-level/release.nix>' '' \
      --max-memory-size 2048 --workers 8

.gitignore

@@ -4,9 +4,10 @@ perl/Makefile.config
 # /
 /aclocal.m4
 /autom4te.cache
+/precompiled-headers.h.gch
+/precompiled-headers.h.pch
 /config.*
 /configure
-/nix.spec
 /stamp-h1
 /svn-revision
 /libtool
@@ -84,6 +85,7 @@ perl/Makefile.config
 /tests/restricted-innocent
 /tests/shell
 /tests/shell.drv
+/tests/config.nix
 
 # /tests/lang/
 /tests/lang/*.out
@@ -117,3 +119,5 @@ GPATH
 GRTAGS
 GSYMS
 GTAGS
+
+nix-rust/target

.travis.yml

@@ -1,2 +1,8 @@
-os: osx
-script: ./tests/install-darwin.sh
+matrix:
+  include:
+    - language: osx
+      script: ./tests/install-darwin.sh
+    - language: nix
+      script: nix-build release.nix -A build.x86_64-linux
+notifications:
+  email: false

.version

@@ -1 +1 @@
-2.3
+2.4

Makefile

@@ -1,5 +1,7 @@
 makefiles = \
+  mk/precompiled-headers.mk \
   local.mk \
+  nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
   src/libmain/local.mk \
@@ -15,8 +17,16 @@ makefiles = \
   tests/local.mk \
   tests/plugins/local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h
-
 -include Makefile.config
 
+OPTIMIZE = 1
+
+ifeq ($(OPTIMIZE), 1)
+  GLOBAL_CXXFLAGS += -O3
+else
+  GLOBAL_CXXFLAGS += -O0
+endif
+
 include mk/lib.mk
+
+GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17

Makefile.config.in

@@ -1,5 +1,4 @@
 AR = @AR@
-BDW_GC_LIBS = @BDW_GC_LIBS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
@@ -18,6 +17,7 @@ SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@

configure.ac

@@ -50,14 +50,11 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
-# Set default flags for nix (as per AC_PROG_CC/CXX docs),
-# while still allowing the user to override them from the command line.
-: ${CFLAGS="-O3"}
-: ${CXXFLAGS="-O3"}
+CFLAGS=
+CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_17([noext], [mandatory])
 
 AC_CHECK_TOOL([AR], [ar])
 
@@ -120,26 +117,15 @@ fi
 ])
 
 NEED_PROG(bash, bash)
-NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
-NEED_PROG(sed, sed)
-NEED_PROG(tar, tar)
-NEED_PROG(bzip2, bzip2)
-NEED_PROG(gzip, gzip)
-NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
-NEED_PROG(cat, cat)
-NEED_PROG(tr, tr)
-AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
-  [path of cat, mkdir, etc.]),
-  coreutils=$withval, coreutils=$(dirname $cat))
-AC_SUBST(coreutils)
+AC_SUBST(coreutils, [$(dirname $(type -p cat))])
 
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
@@ -157,8 +143,33 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix
 # ends up with LDFLAGS being empty, so we set it afterwards.
 LDFLAGS="$BOOST_LDFLAGS $LDFLAGS"
 
+# On some platforms, new-style atomics need a helper library
+AC_MSG_CHECKING(whether -latomic is needed)
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+#include <stdint.h>
+uint64_t v;
+int main() {
+    return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE);
+}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
+AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
+if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
+    LIBS="-latomic $LIBS"
+fi
 
-# Look for OpenSSL, a required dependency.
+PKG_PROG_PKG_CONFIG
+
+AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
+  [Build shared libraries for Nix [default=yes]]),
+  shared=$enableval, shared=yes)
+if test "$shared" = yes; then
+  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
+else
+  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
+  PKG_CONFIG="$PKG_CONFIG --static"
+fi
+
+# Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
+# used by S3BinaryCacheStore.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
@@ -167,12 +178,12 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
-
+# Checks for libarchive
+PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
 
-
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
@@ -195,12 +206,15 @@ PKG_CHECK_MODULES([SODIUM], [libsodium],
    have_sodium=1], [have_sodium=])
 AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
-
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
+# Look for zlib, a required dependency.
+PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
+AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
+LDFLAGS="-lz $LDFLAGS"
 
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
@@ -241,17 +255,6 @@ if test -n "$enable_s3"; then
 fi
 
 
-# Whether to use the Boehm garbage collector.
-AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
-  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]),
-  gc=$enableval, gc=no)
-if test "$gc" = yes; then
-  PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
-  CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
-  AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
-fi
-
-
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),
@@ -290,16 +293,6 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
 
-AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
-  [Build shared libraries for Nix [default=yes]]),
-  shared=$enableval, shared=yes)
-if test "$shared" = yes; then
-  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
-else
-  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
-fi
-
-
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'

contrib/stack-collapse.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env nix-shell
 #!nix-shell -i python3 -p python3 --pure
 
-# To be used with `--trace-function-calls` and `-vvvv` and
-# `flamegraph.pl`.
+# To be used with `--trace-function-calls` and `flamegraph.pl`.
 #
 # For example:
 #
-# nix-instantiate --trace-function-calls -vvvv '<nixpkgs>' -A hello 2> nix-function-calls.trace
+# nix-instantiate --trace-function-calls '<nixpkgs>' -A hello 2> nix-function-calls.trace
 # ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded
 # nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg"
 

corepkgs/config.nix.in

@@ -1,29 +1,13 @@
+# FIXME: remove this file?
 let
   fromEnv = var: def:
     let val = builtins.getEnv var; in
     if val != "" then val else def;
 in rec {
-  shell = "@bash@";
-  coreutils = "@coreutils@";
-  bzip2 = "@bzip2@";
-  gzip = "@gzip@";
-  xz = "@xz@";
-  tar = "@tar@";
-  tarFlags = "@tarFlags@";
-  tr = "@tr@";
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
   nixPrefix = "@prefix@";
   nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
   nixLocalstateDir = "@localstatedir@";
   nixSysconfDir = "@sysconfdir@";
   nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
-
-  # If Nix is installed in the Nix store, then automatically add it as
-  # a dependency to the core packages. This ensures that they work
-  # properly in a chroot.
-  chrootDeps =
-    if dirOf nixPrefix == builtins.storeDir then
-      [ (builtins.storePath nixPrefix) ]
-    else
-      [ ];
 }

corepkgs/unpack-channel.nix

@@ -1,39 +1,12 @@
-with import <nix/config.nix>;
-
-let
-
-  builder = builtins.toFile "unpack-channel.sh"
-    ''
-      mkdir $out
-      cd $out
-      xzpat="\.xz\$"
-      gzpat="\.gz\$"
-      if [[ "$src" =~ $xzpat ]]; then
-        ${xz} -d < $src | ${tar} xf - ${tarFlags}
-      elif [[ "$src" =~ $gzpat ]]; then
-        ${gzip} -d < $src | ${tar} xf - ${tarFlags}
-      else
-        ${bzip2} -d < $src | ${tar} xf - ${tarFlags}
-      fi
-      if [ * != $channelName ]; then
-        mv * $out/$channelName
-      fi
-    '';
-
-in
-
 { name, channelName, src }:
 
 derivation {
-  system = builtins.currentSystem;
-  builder = shell;
-  args = [ "-e" builder ];
-  inherit name channelName src;
+  builder = "builtin:unpack-channel";
 
-  PATH = "${nixBinDir}:${coreutils}";
+  system = "builtin";
+
+  inherit name channelName src;
 
   # No point in doing this remotely.
   preferLocalBuild = true;
-
-  inherit chrootDeps;
 }

doc/manual/advanced-topics/cores-vs-jobs.xml

@@ -36,8 +36,8 @@ to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" />
 equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar>
 will be the total number of cores in the system.</para>
 
-<para>The total number of consumed cores is a simple multiplication,
-<xref linkend="conf-cores" /> * <envar>NIX_BUILD_CORES</envar>.</para>
+<para>The maximum number of consumed cores is a simple multiplication,
+<xref linkend="conf-max-jobs" /> * <envar>NIX_BUILD_CORES</envar>.</para>
 
 <para>The balance on how to set these two independent variables depends
 upon each builder's workload and hardware. Here are a few example

doc/manual/advanced-topics/post-build-hook.xml

@@ -5,7 +5,7 @@
       version="5.0"
       >
 
-<title>Using the <xref linkend="conf-post-build-hook" /></title>
+<title>Using the <option linkend="conf-post-build-hook">post-build-hook</option></title>
 <subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
 
 

doc/manual/command-ref/conf-file.xml

@@ -433,7 +433,7 @@ builtins.fetchurl {
   <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term>
 
     <listitem><para>If <literal>false</literal> (default), derivations
-    are not stored in Nix user environments.  That is, the derivation
+    are not stored in Nix user environments.  That is, the derivations of
     any build-time-only dependencies may be garbage-collected.</para>
 
     <para>If <literal>true</literal>, when you add a Nix derivation to

doc/manual/command-ref/env-common.xml

@@ -122,7 +122,7 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen>
 <varlistentry><term><envar>NIX_LOG_DIR</envar></term>
 
   <listitem><para>Overrides the location of the Nix log directory
-  (default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
+  (default <filename><replaceable>prefix</replaceable>/var/log/nix</filename>).</para></listitem>
 
 </varlistentry>
 

doc/manual/command-ref/nix-build.xml

@@ -30,6 +30,7 @@
       <replaceable>attrPath</replaceable>
     </arg>
     <arg><option>--no-out-link</option></arg>
+    <arg><option>--dry-run</option></arg>
     <arg>
       <group choice='req'>
         <arg choice='plain'><option>--out-link</option></arg>
@@ -98,6 +99,10 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--dry-run</option></term>
+   <listitem><para>Show what store paths would be built or downloaded.</para></listitem>
+  </varlistentry>
+
   <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> /
   <option>-o</option> <replaceable>outlink</replaceable></term>
 

doc/manual/command-ref/nix-channel.xml

@@ -36,6 +36,9 @@ stay up-to-date with a set of pre-built Nix expressions.  A Nix
 channel is just a URL that points to a place containing a set of Nix
 expressions.  <phrase condition="manual">See also <xref
 linkend="sec-channels" />.</phrase></para>
+      
+<para>To see the list of official NixOS channels, visit <link
+xlink:href="https://nixos.org/channels" />.</para>
 
 <para>This command has the following operations:
 
@@ -111,13 +114,13 @@ $ nix-env -iA nixpkgs.hello</screen>
 <para>You can revert channel updates using <option>--rollback</option>:</para>
 
 <screen>
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
+$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
 "14.04.527.0e935f1"
 
 $ nix-channel --rollback
 switching from generation 483 to 482
 
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
+$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
 "14.04.526.dbadfad"
 </screen>
 

doc/manual/command-ref/nix-env.xml

@@ -659,7 +659,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4'</screen>
 <literal>gcc-3.3.1</literal> are split into two parts: the package
 name (<literal>gcc</literal>), and the version
 (<literal>3.3.1</literal>).  The version part starts after the first
-dash not following by a letter.  <varname>x</varname> is considered an
+dash not followed by a letter.  <varname>x</varname> is considered an
 upgrade of <varname>y</varname> if their package names match, and the
 version of <varname>y</varname> is higher that that of
 <varname>x</varname>.</para>

doc/manual/command-ref/nix-prefetch-url.xml

@@ -53,7 +53,7 @@ avoided.</para>
 <para>If <replaceable>hash</replaceable> is specified, then a download
 is not performed if the Nix store already contains a file with the
 same hash and base name.  Otherwise, the file is downloaded, and an
-error if signaled if the actual hash of the file does not match the
+error is signaled if the actual hash of the file does not match the
 specified hash.</para>
 
 <para>This command prints the hash on standard output.  Additionally,

doc/manual/command-ref/nix-shell.xml

@@ -39,7 +39,12 @@
           <arg choice='plain'><option>--packages</option></arg>
           <arg choice='plain'><option>-p</option></arg>
         </group>
-        <arg choice='plain' rep='repeat'><replaceable>packages</replaceable></arg>
+        <arg choice='plain' rep='repeat'>
+          <group choice='req'>
+            <arg choice="plain"><replaceable>packages</replaceable></arg>
+            <arg choice="plain"><replaceable>expressions</replaceable></arg>
+          </group>
+        </arg>
       </arg>
       <arg><replaceable>path</replaceable></arg>
     </group>
@@ -189,8 +194,8 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 <variablelist>
 
   <varlistentry><term><envar>NIX_BUILD_SHELL</envar></term>
-    
-    <listitem><para>Shell used to start the interactive environment. 
+
+    <listitem><para>Shell used to start the interactive environment.
     Defaults to the <command>bash</command> found in <envar>PATH</envar>.</para></listitem>
 
   </varlistentry>
@@ -222,8 +227,9 @@ $ nix-shell '&lt;nixpkgs>' -A pan --pure \
     --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
 </screen>
 
-Nix expressions can also be given on the command line.  For instance,
-the following starts a shell containing the packages
+Nix expressions can also be given on the command line using the
+<command>-E</command> and <command>-p</command> flags.
+For instance, the following starts a shell containing the packages
 <literal>sqlite</literal> and <literal>libX11</literal>:
 
 <screen>
@@ -238,6 +244,14 @@ $ nix-shell -p sqlite xorg.libX11
 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
 </screen>
 
+Note that <command>-p</command> accepts multiple full nix expressions that
+are valid in the <literal>buildInputs = [ ... ]</literal> shown above,
+not only package names. So the following is also legal:
+
+<screen>
+$ nix-shell -p sqlite 'git.override { withManual = false; }'
+</screen>
+
 The <command>-p</command> flag looks up Nixpkgs in the Nix search
 path. You can override it by passing <option>-I</option> or setting
 <envar>NIX_PATH</envar>. For example, the following gives you a shell

doc/manual/command-ref/opt-common.xml

@@ -243,9 +243,10 @@
 <varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term>
 
   <listitem><para>This option is accepted by
-  <command>nix-env</command>, <command>nix-instantiate</command> and
-  <command>nix-build</command>.  When evaluating Nix expressions, the
-  expression evaluator will automatically try to call functions that
+  <command>nix-env</command>, <command>nix-instantiate</command>,
+  <command>nix-shell</command> and <command>nix-build</command>.
+  When evaluating Nix expressions, the expression evaluator will
+  automatically try to call functions that
   it encounters.  It can automatically call functions for which every
   argument has a <link linkend='ss-functions'>default value</link>
   (e.g., <literal>{ <replaceable>argName</replaceable> ?
@@ -322,7 +323,14 @@
   Nix expressions to be parsed and evaluated, rather than as a list
   of file names of Nix expressions.
   (<command>nix-instantiate</command>, <command>nix-build</command>
-  and <command>nix-shell</command> only.)</para></listitem>
+  and <command>nix-shell</command> only.)</para>
+
+  <para>For <command>nix-shell</command>, this option is commonly used
+  to give you a shell in which you can build the packages returned
+  by the expression. If you want to get a shell which contain the
+  <emphasis>built</emphasis> packages ready for use, give your
+  expression to the <command>nix-shell -p</command> convenience flag
+  instead.</para></listitem>
 
 </varlistentry>
 

doc/manual/expressions/advanced-attributes.xml

@@ -11,7 +11,7 @@ attributes.</para>
 
 <variablelist>
 
-  <varlistentry><term><varname>allowedReferences</varname></term>
+  <varlistentry xml:id="adv-attr-allowedReferences"><term><varname>allowedReferences</varname></term>
 
     <listitem><para>The optional attribute
     <varname>allowedReferences</varname> specifies a list of legal
@@ -32,7 +32,7 @@ allowedReferences = [];
   </varlistentry>
 
 
-  <varlistentry><term><varname>allowedRequisites</varname></term>
+  <varlistentry xml:id="adv-attr-allowedRequisites"><term><varname>allowedRequisites</varname></term>
 
     <listitem><para>This attribute is similar to
     <varname>allowedReferences</varname>, but it specifies the legal
@@ -50,7 +50,7 @@ allowedRequisites = [ foobar ];
 
   </varlistentry>
 
-  <varlistentry><term><varname>disallowedReferences</varname></term>
+  <varlistentry xml:id="adv-attr-disallowedReferences"><term><varname>disallowedReferences</varname></term>
 
     <listitem><para>The optional attribute
     <varname>disallowedReferences</varname> specifies a list of illegal
@@ -67,7 +67,7 @@ disallowedReferences = [ foo ];
   </varlistentry>
 
 
-  <varlistentry><term><varname>disallowedRequisites</varname></term>
+  <varlistentry xml:id="adv-attr-disallowedRequisites"><term><varname>disallowedRequisites</varname></term>
 
     <listitem><para>This attribute is similar to
     <varname>disallowedReferences</varname>, but it specifies illegal
@@ -85,7 +85,7 @@ disallowedRequisites = [ foobar ];
   </varlistentry>
 
 
-  <varlistentry><term><varname>exportReferencesGraph</varname></term>
+  <varlistentry xml:id="adv-attr-exportReferencesGraph"><term><varname>exportReferencesGraph</varname></term>
 
     <listitem><para>This attribute allows builders access to the
     references graph of their inputs.  The attribute is a list of
@@ -124,7 +124,7 @@ derivation {
   </varlistentry>
 
 
-  <varlistentry><term><varname>impureEnvVars</varname></term>
+  <varlistentry xml:id="adv-attr-impureEnvVars"><term><varname>impureEnvVars</varname></term>
 
     <listitem><para>This attribute allows you to specify a list of
     environment variables that should be passed from the environment
@@ -158,9 +158,9 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 
   <varlistentry xml:id="fixed-output-drvs">
-    <term><varname>outputHash</varname></term>
-    <term><varname>outputHashAlgo</varname></term>
-    <term><varname>outputHashMode</varname></term>
+    <term xml:id="adv-attr-outputHash"><varname>outputHash</varname></term>
+    <term xml:id="adv-attr-outputHashAlgo"><varname>outputHashAlgo</varname></term>
+    <term xml:id="adv-attr-outputHashMode"><varname>outputHashMode</varname></term>
 
     <listitem><para>These attributes declare that the derivation is a
     so-called <emphasis>fixed-output derivation</emphasis>, which
@@ -282,7 +282,7 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry><term><varname>passAsFile</varname></term>
+  <varlistentry xml:id="adv-attr-passAsFile"><term><varname>passAsFile</varname></term>
 
     <listitem><para>A list of names of attributes that should be
     passed via files rather than environment variables.  For example,
@@ -309,7 +309,7 @@ big = "a very long string";
   </varlistentry>
 
 
-  <varlistentry><term><varname>preferLocalBuild</varname></term>
+  <varlistentry xml:id="adv-attr-preferLocalBuild"><term><varname>preferLocalBuild</varname></term>
 
     <listitem><para>If this attribute is set to
     <literal>true</literal> and <link
@@ -323,14 +323,25 @@ big = "a very long string";
   </varlistentry>
 
 
-  <varlistentry><term><varname>allowSubstitutes</varname></term>
+  <varlistentry xml:id="adv-attr-allowSubstitutes"><term><varname>allowSubstitutes</varname></term>
 
-    <listitem><para>If this attribute is set to
+    <listitem>
+    <para>If this attribute is set to
     <literal>false</literal>, then Nix will always build this
     derivation; it will not try to substitute its outputs. This is
     useful for very trivial derivations (such as
     <function>writeText</function> in Nixpkgs) that are cheaper to
-    build than to substitute from a binary cache.</para></listitem>
+    build than to substitute from a binary cache.</para>
+
+    <note><para>You need to have a builder configured which satisfies
+    the derivation’s <literal>system</literal> attribute, since the
+    derivation cannot be substituted. Thus it is usually a good idea
+    to align <literal>system</literal> with
+    <literal>builtins.currentSystem</literal> when setting
+    <literal>allowSubstitutes</literal> to <literal>false</literal>.
+    For most trivial derivations this should be the case.
+    </para></note>
+    </listitem>
 
   </varlistentry>
 

doc/manual/expressions/builtins.xml

@@ -170,18 +170,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
   </varlistentry>
 
 
-  <varlistentry xml:id='builtin-splitVersion'>
-    <term><function>builtins.splitVersion</function>
-    <replaceable>s</replaceable></term>
-
-    <listitem><para>Split a string representing a version into its
-    components, by the same version splitting logic underlying the
-    version comparison in <link linkend="ssec-version-comparisons">
-    <command>nix-env -u</command></link>.</para></listitem>
-
-  </varlistentry>
-
-
   <varlistentry xml:id='builtin-concatLists'>
     <term><function>builtins.concatLists</function>
     <replaceable>lists</replaceable></term>
@@ -301,7 +289,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting>
 
     <listitem><para>Return element <replaceable>n</replaceable> from
     the list <replaceable>xs</replaceable>.  Elements are counted
-    starting from 0.  A fatal error occurs in the index is out of
+    starting from 0.  A fatal error occurs if the index is out of
     bounds.</para></listitem>
 
   </varlistentry>
@@ -448,7 +436,7 @@ stdenv.mkDerivation { … }
       <example>
         <title>Fetching an arbitrary ref</title>
         <programlisting>builtins.fetchGit {
-  url = "https://gitub.com/NixOS/nix.git";
+  url = "https://github.com/NixOS/nix.git";
   ref = "refs/heads/0.5-release";
 }</programlisting>
       </example>
@@ -499,11 +487,8 @@ stdenv.mkDerivation { … }
         <title>Fetching a tag</title>
         <programlisting>builtins.fetchGit {
   url = "https://github.com/nixos/nix.git";
-  ref = "tags/1.9";
+  ref = "refs/tags/1.9";
 }</programlisting>
-        <note><para>Due to a bug (<link
-        xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>),
-        only non-annotated tags can be fetched.</para></note>
       </example>
 
       <example>
@@ -761,6 +746,11 @@ builtins.genList (x: x * x) 5
     separate file, and use it from Nix expressions in other
     files.</para>
 
+    <note><para>Unlike some languages, <function>import</function> is a regular
+    function in Nix. Paths using the angle bracket syntax (e.g., <function>
+    import</function> <replaceable>&lt;foo&gt;</replaceable>) are normal path
+    values (see <xref linkend='ssec-values' />).</para></note>
+
     <para>A Nix expression loaded by <function>import</function> must
     not contain any <emphasis>free variables</emphasis> (identifiers
     that are not defined in the Nix expression itself and are not
@@ -1130,6 +1120,16 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
+  <varlistentry xml:id='builtin-placeholder'>
+    <term><function>builtins.placeholder</function>
+    <replaceable>output</replaceable></term>
+
+    <listitem><para>Return a placeholder string for the specified
+    <replaceable>output</replaceable> that will be substituted by the
+    corresponding output path at build time. Typical outputs would be
+    <literal>"out"</literal>, <literal>"bin"</literal> or
+    <literal>"dev"</literal>.</para></listitem>
+  </varlistentry>
 
   <varlistentry xml:id='builtin-readDir'>
     <term><function>builtins.readDir</function>
@@ -1275,6 +1275,19 @@ Evaluates to <literal>[ "  " [ "FOO" ] "   " ]</literal>.
     </para></listitem>
   </varlistentry>
 
+
+  <varlistentry xml:id='builtin-splitVersion'>
+    <term><function>builtins.splitVersion</function>
+    <replaceable>s</replaceable></term>
+
+    <listitem><para>Split a string representing a version into its
+    components, by the same version splitting logic underlying the
+    version comparison in <link linkend="ssec-version-comparisons">
+    <command>nix-env -u</command></link>.</para></listitem>
+
+  </varlistentry>
+
+
   <varlistentry xml:id='builtin-stringLength'>
     <term><function>builtins.stringLength</function>
     <replaceable>e</replaceable></term>
@@ -1468,7 +1481,7 @@ in foo</programlisting>
       <listitem><para>A set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem>
       <listitem><para>An integer.</para></listitem>
       <listitem><para>A list, in which case the string representations of its elements are joined with spaces.</para></listitem>
-      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>.</para></listitem>
+      <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>).</para></listitem>
       <listitem><para><literal>null</literal>, which yields the empty string.</para></listitem>
     </itemizedlist>
     </listitem>
@@ -1607,12 +1620,18 @@ stdenv.mkDerivation (rec {
     <term><function>builtins.tryEval</function>
     <replaceable>e</replaceable></term>
 
-    <listitem><para>Try to evaluate <replaceable>e</replaceable>.
+    <listitem><para>Try to shallowly evaluate <replaceable>e</replaceable>.
     Return a set containing the attributes <literal>success</literal>
     (<literal>true</literal> if <replaceable>e</replaceable> evaluated
     successfully, <literal>false</literal> if an error was thrown) and
     <literal>value</literal>, equalling <replaceable>e</replaceable>
-    if successful and <literal>false</literal> otherwise.
+    if successful and <literal>false</literal> otherwise. Note that this
+    doesn't evaluate <replaceable>e</replaceable> deeply, so
+    <literal>let e = { x = throw ""; }; in (builtins.tryEval e).success
+    </literal> will be <literal>true</literal>. Using <literal>builtins.deepSeq
+    </literal> one can get the expected result: <literal>let e = { x = throw "";
+    }; in (builtins.tryEval (builtins.deepSeq e e)).success</literal> will be
+    <literal>false</literal>.
     </para></listitem>
 
   </varlistentry>

doc/manual/expressions/simple-building-testing.xml

@@ -43,7 +43,7 @@ use <command>nix-build</command>’s <option
 linkend='opt-out-link'>-o</option> switch to give the symlink another
 name.</para>
 
-<para>Nix has a transactional semantics.  Once a build finishes
+<para>Nix has transactional semantics.  Once a build finishes
 successfully, Nix makes a note of this in its database: it registers
 that the path denoted by <envar>out</envar> is now
 <quote>valid</quote>.  If you try to build the derivation again, Nix

doc/manual/installation/prerequisites-source.xml

@@ -50,14 +50,6 @@
   or higher.  If your distribution does not provide it, please install
   it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>
 
-  <listitem><para>The <link
-  xlink:href="http://www.hboehm.info/gc/">Boehm
-  garbage collector</link> to reduce the evaluator’s memory
-  consumption (optional).  To enable it, install
-  <literal>pkgconfig</literal> and the Boehm garbage collector, and
-  pass the flag <option>--enable-gc</option> to
-  <command>configure</command>.</para></listitem>
-
   <listitem><para>The <literal>boost</literal> library of version
   1.66.0 or higher. It can be obtained from the official web site
   <link xlink:href="https://www.boost.org/" />.</para></listitem>

doc/manual/packages/channels.xml

@@ -17,6 +17,9 @@ a set of Nix expressions and a manifest.  Using the command <link
 linkend="sec-nix-channel"><command>nix-channel</command></link> you
 can automatically stay up to date with whatever is available at that
 URL.</para>
+      
+<para>To see the list of official NixOS channels, visit <link
+xlink:href="https://nixos.org/channels" />.</para>
 
 <para>You can “subscribe” to a channel using
 <command>nix-channel --add</command>, e.g.,

doc/manual/packages/garbage-collection.xml

@@ -52,12 +52,13 @@ garbage collector as follows:
 <screen>
 $ nix-store --gc</screen>
 
-The behaviour of the gargage collector is affected by the <literal>keep-
-derivations</literal> (default: true) and <literal>keep-outputs</literal>
+The behaviour of the gargage collector is affected by the 
+<literal>keep-derivations</literal> (default: true) and <literal>keep-outputs</literal>
 (default: false) options in the Nix configuration file. The defaults will ensure
-that all derivations that are not build-time dependencies of garbage collector roots
-will be collected but that all output paths that are not runtime dependencies
-will be collected. (This is usually what you want, but while you are developing
+that all derivations that are build-time dependencies of garbage collector roots
+will be kept and that all output paths that are runtime dependencies
+will be kept as well. All other derivations or paths will be collected. 
+(This is usually what you want, but while you are developing
 it may make sense to keep outputs to ensure that rebuild times are quick.)
 
 If you are feeling uncertain, you can also first view what files would

doc/manual/packages/s3-substituter.xml

@@ -159,7 +159,6 @@ the S3 URL:</para>
         "s3:ListBucket",
         "s3:ListBucketMultipartUploads",
         "s3:ListMultipartUploadParts",
-        "s3:ListObjects",
         "s3:PutObject"
       ],
       "Resource": [

doc/manual/release-notes/rl-2.3.xml

@@ -13,9 +13,8 @@ incompatible changes:</para>
 
   <listitem>
     <para>Nix now uses BSD file locks instead of POSIX file
-    locks. Since previous releases used POSIX file locks, you should
-    not use Nix 2.2 and previous releases at the same time on a Nix
-    store.</para>
+    locks. Because of this, you should not use Nix 2.3 and previous
+    releases at the same time on a Nix store.</para>
   </listitem>
 
 </itemizedlist>
@@ -34,9 +33,13 @@ incompatible changes:</para>
   </listitem>
 
   <listitem>
-    <para>The installer now enables sandboxing by default on
-    Linux. The <literal>max-jobs</literal> setting now defaults to
-    1.</para>
+    <para>The installer now enables sandboxing by default on Linux when the
+    system has the necessary kernel support.
+    </para>
+  </listitem>
+
+  <listitem>
+    <para>The <literal>max-jobs</literal> setting now defaults to 1.</para>
   </listitem>
 
   <listitem>
@@ -47,9 +50,9 @@ incompatible changes:</para>
   </listitem>
 
   <listitem>
-    <para><command>nix</command>: Add
+    <para>The <command>nix</command> command has a new
     <option>--print-build-logs</option> (<option>-L</option>) flag to
-    print build log output to stderr rather than showing the last log
+    print build log output to stderr, rather than showing the last log
     line in the progress bar. To distinguish between concurrent
     builds, log lines are prefixed by the name of the package.
     </para>
@@ -57,7 +60,7 @@ incompatible changes:</para>
 
   <listitem>
     <para>Builds are now executed in a pseudo-terminal, and the
-    <envar>TERM</envar> evnironment variable is set to
+    <envar>TERM</envar> environment variable is set to
     <literal>xterm-256color</literal>. This allows many programs
     (e.g. <command>gcc</command>, <command>clang</command>,
     <command>cmake</command>) to print colorized log output.</para>
@@ -83,11 +86,6 @@ incompatible changes:</para>
     the duration of Nix function calls to stderr.</para>
   </listitem>
 
-  <listitem>
-    <para>On Linux, sandboxing is now disabled by default on systems
-    that don’t have the necessary kernel support.</para>
-  </listitem>
-
 </itemizedlist>
 
 </section>

local.mk

@@ -2,11 +2,13 @@ ifeq ($(MAKECMDGOALS), dist)
   dist-files += $(shell cat .dist-files)
 endif
 
-dist-files += configure config.h.in nix.spec perl/configure
+dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix
+GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
+
+$(GCH) $(PCH): src/libutil/util.hh config.h

maintainers/upload-release.pl

@@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n";
 
 File::Path::make_path($releasesDir);
 if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die;
+    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
 }
 
 my $releaseDir = "$releasesDir/nix/$releaseName";

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -17,7 +17,7 @@
     <array>
       <string>/bin/sh</string>
       <string>-c</string>
-      <string>/bin/wait4path @bindir@/nix-daemon &amp;&amp; @bindir@/nix-daemon</string>
+      <string>/bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon &amp;&amp; /nix/var/nix/profiles/default/bin/nix-daemon</string>
     </array>
     <key>StandardErrorPath</key>
     <string>/var/log/nix-daemon.log</string>

mk/patterns.mk

@@ -1,10 +1,10 @@
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"

mk/precompiled-headers.mk

@@ -0,0 +1,42 @@
+PRECOMPILE_HEADERS ?= 1
+
+print-var-help += \
+  echo "  PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build";
+
+GCH = $(buildprefix)precompiled-headers.h.gch
+
+$(GCH): precompiled-headers.h
+	@rm -f $@
+	@mkdir -p "$(dir $@)"
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+
+PCH = $(buildprefix)precompiled-headers.h.pch
+
+$(PCH): precompiled-headers.h
+	@rm -f $@
+	@mkdir -p "$(dir $@)"
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+
+clean-files += $(GCH) $(PCH)
+
+ifeq ($(PRECOMPILE_HEADERS), 1)
+
+  ifeq ($(CXX), g++)
+
+    GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
+
+    GLOBAL_ORDER_AFTER += $(GCH)
+
+  else ifeq ($(CXX), clang++)
+
+    GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
+
+    GLOBAL_ORDER_AFTER += $(PCH)
+
+  else
+
+    $(error Don't know how to precompile headers on $(CXX))
+
+  endif
+
+endif

nix-rust/Cargo.lock

@@ -0,0 +1,399 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "assert_matches"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "autocfg"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "bit-set"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "bitflags"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "byteorder"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "c2-chacha"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "cloudabi"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "getrandom"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hex"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "libc"
+version = "0.2.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "nix-rust"
+version = "0.1.0"
+dependencies = [
+ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "proptest"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "quick-error"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "rand"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_isaac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_jitter"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_os"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.1.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "remove_dir_all"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rusty-fork"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasi"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[metadata]
+"checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
+"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
+"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
+"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
+"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
+"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
+"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
+"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
+"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
+"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
+"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
+"checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4"
+"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
+"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
+"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
+"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
+"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
+"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
+"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
+"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+"checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
+"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
+"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
+"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
+"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
+"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
+"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
+"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
+"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
+"checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
+"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
+"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
+"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
+"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

nix-rust/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "nix-rust"
+version = "0.1.0"
+authors = ["Eelco Dolstra <edolstra@gmail.com>"]
+edition = "2018"
+
+[lib]
+name = "nixrust"
+crate-type = ["cdylib"]
+
+[dependencies]
+libc = "0.2"
+#futures-preview = { version = "=0.3.0-alpha.19" }
+#hyper = "0.13.0-alpha.4"
+#http = "0.1"
+#tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] }
+lazy_static = "1.4"
+#byteorder = "1.3"
+
+[dev-dependencies]
+hex = "0.3"
+assert_matches = "1.3"
+proptest = "0.9"

nix-rust/local.mk

@@ -0,0 +1,45 @@
+ifeq ($(OPTIMIZE), 1)
+  RUST_MODE = --release
+  RUST_DIR = release
+else
+  RUST_MODE =
+  RUST_DIR = debug
+endif
+
+libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
+
+ifeq ($(OS), Darwin)
+libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
+else
+libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
+libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
+endif
+
+$(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
+	$(libnixrust_BUILD_FLAGS) \
+	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+
+$(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
+	$(target-gen) cp $^ $@
+ifeq ($(OS), Darwin)
+	install_name_tool -id $@ $@
+endif
+
+dist-files += $(d)/vendor
+
+clean: clean-rust
+
+clean-rust:
+	$(suppress) rm -rfv nix-rust/target
+
+ifneq ($(OS), Darwin)
+check: rust-tests
+
+rust-tests:
+	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+endif

nix-rust/src/c.rs

@@ -0,0 +1,77 @@
+use super::{error, store::path, store::StorePath, util};
+
+#[no_mangle]
+pub unsafe extern "C" fn ffi_String_new(s: &str, out: *mut String) {
+    // FIXME: check whether 's' is valid UTF-8?
+    out.write(s.to_string())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn ffi_String_drop(self_: *mut String) {
+    std::ptr::drop_in_place(self_);
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_new(
+    path: &str,
+    store_dir: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::new(std::path::Path::new(path), std::path::Path::new(store_dir))
+        .map_err(|err| err.into())
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_new2(
+    hash: &[u8; crate::store::path::STORE_PATH_HASH_BYTES],
+    name: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::from_parts(*hash, name).map_err(|err| err.into())
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_fromBaseName(
+    base_name: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::new_from_base_name(base_name).map_err(|err| err.into())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn ffi_StorePath_drop(self_: *mut StorePath) {
+    std::ptr::drop_in_place(self_);
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_to_string(self_: &StorePath) -> Vec<u8> {
+    let mut buf = vec![0; path::STORE_PATH_HASH_CHARS + 1 + self_.name.name().len()];
+    util::base32::encode_into(self_.hash.hash(), &mut buf[0..path::STORE_PATH_HASH_CHARS]);
+    buf[path::STORE_PATH_HASH_CHARS] = b'-';
+    buf[path::STORE_PATH_HASH_CHARS + 1..].clone_from_slice(self_.name.name().as_bytes());
+    buf
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_less_than(a: &StorePath, b: &StorePath) -> bool {
+    a < b
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_eq(a: &StorePath, b: &StorePath) -> bool {
+    a == b
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_clone(self_: &StorePath) -> StorePath {
+    self_.clone()
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_name(self_: &StorePath) -> &str {
+    self_.name.name()
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_hash_data(
+    self_: &StorePath,
+) -> &[u8; crate::store::path::STORE_PATH_HASH_BYTES] {
+    self_.hash.hash()
+}

nix-rust/src/error.rs

@@ -0,0 +1,118 @@
+use std::fmt;
+
+#[derive(Debug)]
+pub enum Error {
+    InvalidPath(crate::store::StorePath),
+    BadStorePath(std::path::PathBuf),
+    NotInStore(std::path::PathBuf),
+    BadNarInfo,
+    BadBase32,
+    StorePathNameEmpty,
+    StorePathNameTooLong,
+    BadStorePathName,
+    NarSizeFieldTooBig,
+    BadNarString,
+    BadNarPadding,
+    BadNarVersionMagic,
+    MissingNarOpenTag,
+    MissingNarCloseTag,
+    MissingNarField,
+    BadNarField(String),
+    BadExecutableField,
+    IOError(std::io::Error),
+    #[cfg(unused)]
+    HttpError(hyper::error::Error),
+    Misc(String),
+    #[cfg(not(test))]
+    Foreign(CppException),
+    BadTarFileMemberName(String),
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Self {
+        Error::IOError(err)
+    }
+}
+
+#[cfg(unused)]
+impl From<hyper::error::Error> for Error {
+    fn from(err: hyper::error::Error) -> Self {
+        Error::HttpError(err)
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::InvalidPath(_) => write!(f, "invalid path"),
+            Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
+            Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
+            Error::NotInStore(path) => {
+                write!(f, "path '{}' is not in the Nix store", path.display())
+            }
+            Error::BadBase32 => write!(f, "invalid base32 string"),
+            Error::StorePathNameEmpty => write!(f, "store path name is empty"),
+            Error::StorePathNameTooLong => {
+                write!(f, "store path name is longer than 211 characters")
+            }
+            Error::BadStorePathName => write!(f, "store path name contains forbidden character"),
+            Error::NarSizeFieldTooBig => write!(f, "size field in NAR is too big"),
+            Error::BadNarString => write!(f, "NAR string is not valid UTF-8"),
+            Error::BadNarPadding => write!(f, "NAR padding is not zero"),
+            Error::BadNarVersionMagic => write!(f, "unsupported NAR version"),
+            Error::MissingNarOpenTag => write!(f, "NAR open tag is missing"),
+            Error::MissingNarCloseTag => write!(f, "NAR close tag is missing"),
+            Error::MissingNarField => write!(f, "expected NAR field is missing"),
+            Error::BadNarField(s) => write!(f, "unrecognized NAR field '{}'", s),
+            Error::BadExecutableField => write!(f, "bad 'executable' field in NAR"),
+            Error::IOError(err) => write!(f, "I/O error: {}", err),
+            #[cfg(unused)]
+            Error::HttpError(err) => write!(f, "HTTP error: {}", err),
+            #[cfg(not(test))]
+            Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
+            Error::Misc(s) => write!(f, "{}", s),
+            Error::BadTarFileMemberName(s) => {
+                write!(f, "tar archive contains illegal file name '{}'", s)
+            }
+        }
+    }
+}
+
+#[cfg(not(test))]
+impl From<Error> for CppException {
+    fn from(err: Error) -> Self {
+        match err {
+            Error::Foreign(ex) => ex,
+            _ => CppException::new(&err.to_string()),
+        }
+    }
+}
+
+#[cfg(not(test))]
+#[repr(C)]
+#[derive(Debug)]
+pub struct CppException(*const libc::c_void); // == std::exception_ptr*
+
+#[cfg(not(test))]
+impl CppException {
+    fn new(s: &str) -> Self {
+        Self(unsafe { make_error(s) })
+    }
+}
+
+#[cfg(not(test))]
+impl Drop for CppException {
+    fn drop(&mut self) {
+        unsafe {
+            destroy_error(self.0);
+        }
+    }
+}
+
+#[cfg(not(test))]
+extern "C" {
+    #[allow(improper_ctypes)] // YOLO
+    fn make_error(s: &str) -> *const libc::c_void;
+
+    fn destroy_error(exc: *const libc::c_void);
+}

nix-rust/src/lib.rs

@@ -0,0 +1,9 @@
+#[cfg(not(test))]
+mod c;
+mod error;
+#[cfg(unused)]
+mod nar;
+mod store;
+mod util;
+
+pub use error::Error;

nix-rust/src/nar.rs

@@ -0,0 +1,126 @@
+use crate::Error;
+use byteorder::{LittleEndian, ReadBytesExt};
+use std::convert::TryFrom;
+use std::io::Read;
+
+pub fn parse<R: Read>(input: &mut R) -> Result<(), Error> {
+    if String::read(input)? != NAR_VERSION_MAGIC {
+        return Err(Error::BadNarVersionMagic);
+    }
+
+    parse_file(input)
+}
+
+const NAR_VERSION_MAGIC: &str = "nix-archive-1";
+
+fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
+    if String::read(input)? != "(" {
+        return Err(Error::MissingNarOpenTag);
+    }
+
+    if String::read(input)? != "type" {
+        return Err(Error::MissingNarField);
+    }
+
+    match String::read(input)?.as_ref() {
+        "regular" => {
+            let mut _executable = false;
+            let mut tag = String::read(input)?;
+            if tag == "executable" {
+                _executable = true;
+                if String::read(input)? != "" {
+                    return Err(Error::BadExecutableField);
+                }
+                tag = String::read(input)?;
+            }
+            if tag != "contents" {
+                return Err(Error::MissingNarField);
+            }
+            let _contents = Vec::<u8>::read(input)?;
+            if String::read(input)? != ")" {
+                return Err(Error::MissingNarCloseTag);
+            }
+        }
+        "directory" => loop {
+            match String::read(input)?.as_ref() {
+                "entry" => {
+                    if String::read(input)? != "(" {
+                        return Err(Error::MissingNarOpenTag);
+                    }
+                    if String::read(input)? != "name" {
+                        return Err(Error::MissingNarField);
+                    }
+                    let _name = String::read(input)?;
+                    if String::read(input)? != "node" {
+                        return Err(Error::MissingNarField);
+                    }
+                    parse_file(input)?;
+                    let tag = String::read(input)?;
+                    if tag != ")" {
+                        return Err(Error::MissingNarCloseTag);
+                    }
+                }
+                ")" => break,
+                s => return Err(Error::BadNarField(s.into())),
+            }
+        },
+        "symlink" => {
+            if String::read(input)? != "target" {
+                return Err(Error::MissingNarField);
+            }
+            let _target = String::read(input)?;
+            if String::read(input)? != ")" {
+                return Err(Error::MissingNarCloseTag);
+            }
+        }
+        s => return Err(Error::BadNarField(s.into())),
+    }
+
+    Ok(())
+}
+
+trait Deserialize: Sized {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error>;
+}
+
+impl Deserialize for String {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let buf = Deserialize::read(input)?;
+        Ok(String::from_utf8(buf).map_err(|_| Error::BadNarString)?)
+    }
+}
+
+impl Deserialize for Vec<u8> {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let n: usize = Deserialize::read(input)?;
+        let mut buf = vec![0; n];
+        input.read_exact(&mut buf)?;
+        skip_padding(input, n)?;
+        Ok(buf)
+    }
+}
+
+fn skip_padding<R: Read>(input: &mut R, len: usize) -> Result<(), Error> {
+    if len % 8 != 0 {
+        let mut buf = [0; 8];
+        let buf = &mut buf[0..8 - (len % 8)];
+        input.read_exact(buf)?;
+        if !buf.iter().all(|b| *b == 0) {
+            return Err(Error::BadNarPadding);
+        }
+    }
+    Ok(())
+}
+
+impl Deserialize for u64 {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        Ok(input.read_u64::<LittleEndian>()?)
+    }
+}
+
+impl Deserialize for usize {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let n: u64 = Deserialize::read(input)?;
+        Ok(usize::try_from(n).map_err(|_| Error::NarSizeFieldTooBig)?)
+    }
+}

nix-rust/src/store/binary_cache_store.rs

@@ -0,0 +1,48 @@
+use super::{PathInfo, Store, StorePath};
+use crate::Error;
+use hyper::client::Client;
+
+pub struct BinaryCacheStore {
+    base_uri: String,
+    client: Client<hyper::client::HttpConnector, hyper::Body>,
+}
+
+impl BinaryCacheStore {
+    pub fn new(base_uri: String) -> Self {
+        Self {
+            base_uri,
+            client: Client::new(),
+        }
+    }
+}
+
+impl Store for BinaryCacheStore {
+    fn query_path_info(
+        &self,
+        path: &StorePath,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>> {
+        let uri = format!("{}/{}.narinfo", self.base_uri.clone(), path.hash);
+        let path = path.clone();
+        let client = self.client.clone();
+        let store_dir = self.store_dir().to_string();
+
+        Box::pin(async move {
+            let response = client.get(uri.parse::<hyper::Uri>().unwrap()).await?;
+
+            if response.status() == hyper::StatusCode::NOT_FOUND
+                || response.status() == hyper::StatusCode::FORBIDDEN
+            {
+                return Err(Error::InvalidPath(path));
+            }
+
+            let mut body = response.into_body();
+
+            let mut bytes = Vec::new();
+            while let Some(next) = body.next().await {
+                bytes.extend(next?);
+            }
+
+            PathInfo::parse_nar_info(std::str::from_utf8(&bytes).unwrap(), &store_dir)
+        })
+    }
+}

nix-rust/src/store/mod.rs

@@ -0,0 +1,17 @@
+pub mod path;
+
+#[cfg(unused)]
+mod binary_cache_store;
+#[cfg(unused)]
+mod path_info;
+#[cfg(unused)]
+mod store;
+
+pub use path::{StorePath, StorePathHash, StorePathName};
+
+#[cfg(unused)]
+pub use binary_cache_store::BinaryCacheStore;
+#[cfg(unused)]
+pub use path_info::PathInfo;
+#[cfg(unused)]
+pub use store::Store;

nix-rust/src/store/path.rs

@@ -0,0 +1,225 @@
+use crate::error::Error;
+use crate::util::base32;
+use std::fmt;
+use std::path::Path;
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePath {
+    pub hash: StorePathHash,
+    pub name: StorePathName,
+}
+
+pub const STORE_PATH_HASH_BYTES: usize = 20;
+pub const STORE_PATH_HASH_CHARS: usize = 32;
+
+impl StorePath {
+    pub fn new(path: &Path, store_dir: &Path) -> Result<Self, Error> {
+        if path.parent() != Some(store_dir) {
+            return Err(Error::NotInStore(path.into()));
+        }
+        Self::new_from_base_name(
+            path.file_name()
+                .ok_or(Error::BadStorePath(path.into()))?
+                .to_str()
+                .ok_or(Error::BadStorePath(path.into()))?,
+        )
+    }
+
+    pub fn from_parts(hash: [u8; STORE_PATH_HASH_BYTES], name: &str) -> Result<Self, Error> {
+        Ok(StorePath {
+            hash: StorePathHash(hash),
+            name: StorePathName::new(name)?,
+        })
+    }
+
+    pub fn new_from_base_name(base_name: &str) -> Result<Self, Error> {
+        if base_name.len() < STORE_PATH_HASH_CHARS + 1
+            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
+        {
+            return Err(Error::BadStorePath(base_name.into()));
+        }
+
+        Ok(StorePath {
+            hash: StorePathHash::new(&base_name[0..STORE_PATH_HASH_CHARS])?,
+            name: StorePathName::new(&base_name[STORE_PATH_HASH_CHARS + 1..])?,
+        })
+    }
+}
+
+impl fmt::Display for StorePath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}-{}", self.hash, self.name)
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct StorePathHash([u8; STORE_PATH_HASH_BYTES]);
+
+impl StorePathHash {
+    pub fn new(s: &str) -> Result<Self, Error> {
+        assert_eq!(s.len(), STORE_PATH_HASH_CHARS);
+        let v = base32::decode(s)?;
+        assert_eq!(v.len(), STORE_PATH_HASH_BYTES);
+        let mut bytes: [u8; 20] = Default::default();
+        bytes.copy_from_slice(&v[0..STORE_PATH_HASH_BYTES]);
+        Ok(Self(bytes))
+    }
+
+    pub fn hash<'a>(&'a self) -> &'a [u8; STORE_PATH_HASH_BYTES] {
+        &self.0
+    }
+}
+
+impl fmt::Display for StorePathHash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut buf = vec![0; STORE_PATH_HASH_CHARS];
+        base32::encode_into(&self.0, &mut buf);
+        f.write_str(std::str::from_utf8(&buf).unwrap())
+    }
+}
+
+impl Ord for StorePathHash {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        // Historically we've sorted store paths by their base32
+        // serialization, but our base32 encodes bytes in reverse
+        // order. So compare them in reverse order as well.
+        self.0.iter().rev().cmp(other.0.iter().rev())
+    }
+}
+
+impl PartialOrd for StorePathHash {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePathName(String);
+
+impl StorePathName {
+    pub fn new(s: &str) -> Result<Self, Error> {
+        if s.len() == 0 {
+            return Err(Error::StorePathNameEmpty);
+        }
+
+        if s.len() > 211 {
+            return Err(Error::StorePathNameTooLong);
+        }
+
+        if s.starts_with('.')
+            || !s.chars().all(|c| {
+                c.is_ascii_alphabetic()
+                    || c.is_ascii_digit()
+                    || c == '+'
+                    || c == '-'
+                    || c == '.'
+                    || c == '_'
+                    || c == '?'
+                    || c == '='
+            })
+        {
+            return Err(Error::BadStorePathName);
+        }
+
+        Ok(Self(s.to_string()))
+    }
+
+    pub fn name<'a>(&'a self) -> &'a str {
+        &self.0
+    }
+}
+
+impl fmt::Display for StorePathName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use assert_matches::assert_matches;
+
+    #[test]
+    fn test_parse() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
+        let p = StorePath::new_from_base_name(&s).unwrap();
+        assert_eq!(p.name.0, "konsole-18.12.3");
+        assert_eq!(
+            p.hash.0,
+            [
+                0x9f, 0x76, 0x49, 0x20, 0xf6, 0x5d, 0xe9, 0x71, 0xc4, 0xca, 0x46, 0x21, 0xab, 0xff,
+                0x9b, 0x44, 0xef, 0x87, 0x0f, 0x3c
+            ]
+        );
+    }
+
+    #[test]
+    fn test_no_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::StorePathNameEmpty)
+        );
+    }
+
+    #[test]
+    fn test_no_dash() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePath(_))
+        );
+    }
+
+    #[test]
+    fn test_short_hash() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxl-konsole-18.12.3";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePath(_))
+        );
+    }
+
+    #[test]
+    fn test_invalid_hash() {
+        let s = "7h7qgvs4kgzsn8e6rb273saxyqh4jxlz-konsole-18.12.3";
+        assert_matches!(StorePath::new_from_base_name(&s), Err(Error::BadBase32));
+    }
+
+    #[test]
+    fn test_long_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+        assert_matches!(StorePath::new_from_base_name(&s), Ok(_));
+    }
+
+    #[test]
+    fn test_too_long_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::StorePathNameTooLong)
+        );
+    }
+
+    #[test]
+    fn test_bad_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-foo bar";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePathName)
+        );
+
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-kónsole";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePathName)
+        );
+    }
+
+    #[test]
+    fn test_roundtrip() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
+        assert_eq!(StorePath::new_from_base_name(&s).unwrap().to_string(), s);
+    }
+}

nix-rust/src/store/path_info.rs

@@ -0,0 +1,70 @@
+use crate::store::StorePath;
+use crate::Error;
+use std::collections::BTreeSet;
+
+#[derive(Clone, Debug)]
+pub struct PathInfo {
+    pub path: StorePath,
+    pub references: BTreeSet<StorePath>,
+    pub nar_size: u64,
+    pub deriver: Option<StorePath>,
+
+    // Additional binary cache info.
+    pub url: Option<String>,
+    pub compression: Option<String>,
+    pub file_size: Option<u64>,
+}
+
+impl PathInfo {
+    pub fn parse_nar_info(nar_info: &str, store_dir: &str) -> Result<Self, Error> {
+        let mut path = None;
+        let mut references = BTreeSet::new();
+        let mut nar_size = None;
+        let mut deriver = None;
+        let mut url = None;
+        let mut compression = None;
+        let mut file_size = None;
+
+        for line in nar_info.lines() {
+            let colon = line.find(':').ok_or(Error::BadNarInfo)?;
+
+            let (name, value) = line.split_at(colon);
+
+            if !value.starts_with(": ") {
+                return Err(Error::BadNarInfo);
+            }
+
+            let value = &value[2..];
+
+            if name == "StorePath" {
+                path = Some(StorePath::new(std::path::Path::new(value), store_dir)?);
+            } else if name == "NarSize" {
+                nar_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
+            } else if name == "References" {
+                if !value.is_empty() {
+                    for r in value.split(' ') {
+                        references.insert(StorePath::new_from_base_name(r)?);
+                    }
+                }
+            } else if name == "Deriver" {
+                deriver = Some(StorePath::new_from_base_name(value)?);
+            } else if name == "URL" {
+                url = Some(value.into());
+            } else if name == "Compression" {
+                compression = Some(value.into());
+            } else if name == "FileSize" {
+                file_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
+            }
+        }
+
+        Ok(PathInfo {
+            path: path.ok_or(Error::BadNarInfo)?,
+            references,
+            nar_size: nar_size.ok_or(Error::BadNarInfo)?,
+            deriver,
+            url: Some(url.ok_or(Error::BadNarInfo)?),
+            compression,
+            file_size,
+        })
+    }
+}

nix-rust/src/store/store.rs

@@ -0,0 +1,53 @@
+use super::{PathInfo, StorePath};
+use crate::Error;
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
+
+pub trait Store: Send + Sync {
+    fn store_dir(&self) -> &str {
+        "/nix/store"
+    }
+
+    fn query_path_info(
+        &self,
+        store_path: &StorePath,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>>;
+}
+
+impl dyn Store {
+    pub fn parse_store_path(&self, path: &Path) -> Result<StorePath, Error> {
+        StorePath::new(path, self.store_dir())
+    }
+
+    pub async fn compute_path_closure(
+        &self,
+        roots: BTreeSet<StorePath>,
+    ) -> Result<BTreeMap<StorePath, PathInfo>, Error> {
+        let mut done = BTreeSet::new();
+        let mut result = BTreeMap::new();
+        let mut pending = vec![];
+
+        for root in roots {
+            pending.push(self.query_path_info(&root));
+            done.insert(root);
+        }
+
+        while !pending.is_empty() {
+            let (info, _, remaining) = futures::future::select_all(pending).await;
+            pending = remaining;
+
+            let info = info?;
+
+            for path in &info.references {
+                if !done.contains(path) {
+                    pending.push(self.query_path_info(&path));
+                    done.insert(path.clone());
+                }
+            }
+
+            result.insert(info.path.clone(), info);
+        }
+
+        Ok(result)
+    }
+}

nix-rust/src/util/base32.rs

@@ -0,0 +1,160 @@
+use crate::error::Error;
+use lazy_static::lazy_static;
+
+pub fn encoded_len(input_len: usize) -> usize {
+    if input_len == 0 {
+        0
+    } else {
+        (input_len * 8 - 1) / 5 + 1
+    }
+}
+
+pub fn decoded_len(input_len: usize) -> usize {
+    input_len * 5 / 8
+}
+
+static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz";
+
+lazy_static! {
+    static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = {
+        let mut xs = [0xffu8; 256];
+        for (n, c) in BASE32_CHARS.iter().enumerate() {
+            xs[*c as usize] = n as u8;
+        }
+        Box::new(xs)
+    };
+}
+
+pub fn encode(input: &[u8]) -> String {
+    let mut buf = vec![0; encoded_len(input.len())];
+    encode_into(input, &mut buf);
+    std::str::from_utf8(&buf).unwrap().to_string()
+}
+
+pub fn encode_into(input: &[u8], output: &mut [u8]) {
+    let len = encoded_len(input.len());
+    assert_eq!(len, output.len());
+
+    let mut nr_bits_left: usize = 0;
+    let mut bits_left: u16 = 0;
+    let mut pos = len;
+
+    for b in input {
+        bits_left |= (*b as u16) << nr_bits_left;
+        nr_bits_left += 8;
+        while nr_bits_left > 5 {
+            output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
+            pos -= 1;
+            bits_left >>= 5;
+            nr_bits_left -= 5;
+        }
+    }
+
+    if nr_bits_left > 0 {
+        output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
+        pos -= 1;
+    }
+
+    assert_eq!(pos, 0);
+}
+
+pub fn decode(input: &str) -> Result<Vec<u8>, crate::Error> {
+    let mut res = Vec::with_capacity(decoded_len(input.len()));
+
+    let mut nr_bits_left: usize = 0;
+    let mut bits_left: u16 = 0;
+
+    for c in input.chars().rev() {
+        let b = BASE32_CHARS_REVERSE[c as usize];
+        if b == 0xff {
+            return Err(Error::BadBase32);
+        }
+        bits_left |= (b as u16) << nr_bits_left;
+        nr_bits_left += 5;
+        if nr_bits_left >= 8 {
+            res.push((bits_left & 0xff) as u8);
+            bits_left >>= 8;
+            nr_bits_left -= 8;
+        }
+    }
+
+    if nr_bits_left > 0 && bits_left != 0 {
+        return Err(Error::BadBase32);
+    }
+
+    Ok(res)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use assert_matches::assert_matches;
+    use hex;
+    use proptest::proptest;
+
+    #[test]
+    fn test_encode() {
+        assert_eq!(encode(&[]), "");
+
+        assert_eq!(
+            encode(&hex::decode("0839703786356bca59b0f4a32987eb2e6de43ae8").unwrap()),
+            "x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88"
+        );
+
+        assert_eq!(
+            encode(
+                &hex::decode("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+                    .unwrap()
+            ),
+            "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
+        );
+
+        assert_eq!(
+            encode(
+                &hex::decode("ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f")
+                    .unwrap()
+            ),
+            "2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx"
+        );
+    }
+
+    #[test]
+    fn test_decode() {
+        assert_eq!(hex::encode(decode("").unwrap()), "");
+
+        assert_eq!(
+            hex::encode(decode("x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88").unwrap()),
+            "0839703786356bca59b0f4a32987eb2e6de43ae8"
+        );
+
+        assert_eq!(
+            hex::encode(decode("1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s").unwrap()),
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
+        );
+
+        assert_eq!(
+            hex::encode(decode("2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx").unwrap()),
+            "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
+        );
+
+        assert_matches!(
+            decode("xoxf8v9fxf3jk8zln1cwlsrmhqvp0f88"),
+            Err(Error::BadBase32)
+        );
+        assert_matches!(
+            decode("2b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"),
+            Err(Error::BadBase32)
+        );
+        assert_matches!(decode("2"), Err(Error::BadBase32));
+        assert_matches!(decode("2gs"), Err(Error::BadBase32));
+        assert_matches!(decode("2gs8"), Err(Error::BadBase32));
+    }
+
+    proptest! {
+
+        #[test]
+        fn roundtrip(s: Vec<u8>) {
+            assert_eq!(s, decode(&encode(&s)).unwrap());
+        }
+    }
+}

nix-rust/src/util/mod.rs

@@ -0,0 +1 @@
+pub mod base32;

nix.spec.in

@@ -1,173 +0,0 @@
-%undefine _hardened_build
-
-%global nixbld_user "nix-builder-"
-%global nixbld_group "nixbld"
-
-# NOTE: BUILD on EL7 requires
-# - Centos / RHEL7 software collection repository 
-#   yum install centos-release-scl
-#
-# - Recent boost backport
-#   curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo
-#
-
-# Disable documentation generation
-# necessary on some platforms
-%bcond_without docgen
-
-Summary: The Nix software deployment system
-Name: nix
-Version: @PACKAGE_VERSION@
-Release: 2%{?dist}
-License: LGPLv2+
-Group: Applications/System
-URL: http://nixos.org/
-Source0: %{name}-%{version}.tar.bz2
-
-Requires: curl
-Requires: bzip2
-Requires: gzip
-Requires: xz
-BuildRequires: bison
-BuildRequires: boost-devel >= 1.60
-BuildRequires: bzip2-devel
-
-# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler
-%if 0%{?rhel}
-BuildRequires: devtoolset-7-gcc
-BuildRequires: devtoolset-7-gcc-c++
-%endif
-
-BuildRequires: flex
-BuildRequires: libcurl-devel
-BuildRequires: libseccomp-devel
-BuildRequires: openssl-devel
-BuildRequires: sqlite-devel
-BuildRequires: xz-devel
-
-%description
-Nix is a purely functional package manager. It allows multiple
-versions of a package to be installed side-by-side, ensures that
-dependency specifications are complete, supports atomic upgrades and
-rollbacks, allows non-root users to install software, and has many
-other features. It is the basis of the NixOS Linux distribution, but
-it can be used equally well under other Unix systems.
-
-%package        devel
-Summary:        Development files for %{name}
-Requires:       %{name}%{?_isa} = %{version}-%{release}
-
-%description   devel
-The %{name}-devel package contains libraries and header files for
-developing applications that use %{name}.
-
-
-%package doc
-Summary:        Documentation files for %{name}
-BuildArch:      noarch
-Requires:       %{name} = %{version}-%{release}
-
-%description   doc
-The %{name}-doc package contains documentation files for %{name}.
-
-%prep
-%setup -q
-
-
-%build
-%if 0%{?rhel}
-source /opt/rh/devtoolset-7/enable 
-%endif
-extraFlags=
-# - override docdir so large documentation files are owned by the
-#   -doc subpackage
-# - set localstatedir by hand to the preferred nix value
-%configure --localstatedir=/nix/var \
-	   %{!?without_docgen:--disable-doc-gen} \
-           --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
-           $extraFlags
-make V=1 %{?_smp_mflags}
-
-
-%install
-%if 0%{?rhel}
-source /opt/rh/devtoolset-7/enable 
-%endif
-
-make DESTDIR=$RPM_BUILD_ROOT install
-
-find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';'
-
-# make the store
-mkdir -p $RPM_BUILD_ROOT/nix/store
-chmod 1775 $RPM_BUILD_ROOT/nix/store
-
-# make per-user directories
-for d in profiles gcroots;
-do
-  mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-  chmod 1777 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-done
-
-# fix permission of nix profile
-# (until this is fixed in the relevant Makefile)
-chmod -x $RPM_BUILD_ROOT%{_sysconfdir}/profile.d/nix.sh
-
-# we ship this file in the base package
-rm -f $RPM_BUILD_ROOT%{_defaultdocdir}/%{name}-doc-%{version}/README
-
-# Get rid of Upstart job.
-rm -rf $RPM_BUILD_ROOT%{_sysconfdir}/init
-
-
-%clean
-rm -rf $RPM_BUILD_ROOT
-
-
-%pre
-getent group %{nixbld_group} >/dev/null || groupadd -r %{nixbld_group}
-for i in $(seq 10);
-do
-  getent passwd %{nixbld_user}$i >/dev/null || \
-    useradd -r -g %{nixbld_group} -G %{nixbld_group} -d /var/empty \
-      -s %{_sbindir}/nologin \
-      -c "Nix build user $i" %{nixbld_user}$i
-done
-
-%post
-chgrp %{nixbld_group} /nix/store
-%if ! 0%{?rhel} || 0%{?rhel} >= 7
-# Enable and start Nix worker
-systemctl enable nix-daemon.socket nix-daemon.service
-systemctl start  nix-daemon.socket
-%endif
-
-%files
-%license COPYING
-%{_bindir}/nix*
-%{_libdir}/*.so
-%{_prefix}/libexec/*
-%if ! 0%{?rhel} || 0%{?rhel} >= 7
-%{_prefix}/lib/systemd/system/nix-daemon.socket
-%{_prefix}/lib/systemd/system/nix-daemon.service
-%endif
-%{_datadir}/nix
-#%if ! %{without docgen}
-#%{_mandir}/man1/*.1*
-#%{_mandir}/man5/*.5*
-#%{_mandir}/man8/*.8*
-#%endif
-%config(noreplace) %{_sysconfdir}/profile.d/nix.sh
-%config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
-/nix
-
-%files devel
-%{_includedir}/nix
-%{_prefix}/lib/pkgconfig/*.pc
-
-
-#%if ! %{without docgen}
-#%files doc
-#%docdir %{_defaultdocdir}/%{name}-doc-%{version}
-#%{_defaultdocdir}/%{name}-doc-%{version}
-#%endif

perl/Makefile

@@ -4,4 +4,12 @@ GLOBAL_CXXFLAGS += -g -Wall
 
 -include Makefile.config
 
+OPTIMIZE = 1
+
+ifeq ($(OPTIMIZE), 1)
+  GLOBAL_CXXFLAGS += -O3
+else
+  GLOBAL_CXXFLAGS += -O0
+endif
+
 include mk/lib.mk

perl/configure.ac

@@ -2,13 +2,10 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX
 AC_CONFIG_SRCDIR(MANIFEST)
 AC_CONFIG_AUX_DIR(../config)
 
-# Set default flags for nix (as per AC_PROG_CC/CXX docs),
-# while still allowing the user to override them from the command line.
-: ${CFLAGS="-O3"}
-: ${CXXFLAGS="-O3"}
+CFLAGS=
+CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
-AX_CXX_COMPILE_STDCXX_11
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
@@ -71,14 +68,15 @@ AC_SUBST(perlFlags)
 
 PKG_CHECK_MODULES([NIX], [nix-store])
 
-NEED_PROG([NIX_INSTANTIATE_PROGRAM], [nix-instantiate])
+NEED_PROG([NIX], [nix])
 
 # Get nix configure values
-nixbindir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixBinDir | tr -d \")
-nixlibexecdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLibexecDir | tr -d \")
-nixlocalstatedir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLocalstateDir | tr -d \")
-nixsysconfdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixSysconfDir | tr -d \")
-nixstoredir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixStoreDir | tr -d \")
+export NIX_REMOTE=daemon
+nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixBinDir)
+nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLibexecDir)
+nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLocalstateDir)
+nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixSysconfDir)
+nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixStoreDir)
 AC_SUBST(nixbindir)
 AC_SUBST(nixlibexecdir)
 AC_SUBST(nixlocalstatedir)

perl/lib/Nix/Config.pm.in

@@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
 $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
 $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
 
-$bzip2 = "@bzip2@";
-$xz = "@xz@";
-$curl = "@curl@";
-
 $useBindings = 1;
 
 %config = ();

perl/lib/Nix/Store.xs

@@ -59,7 +59,7 @@ void setVerbosity(int level)
 int isValidPath(char * path)
     CODE:
         try {
-            RETVAL = store()->isValidPath(path);
+            RETVAL = store()->isValidPath(store()->parseStorePath(path));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -70,9 +70,8 @@ int isValidPath(char * path)
 SV * queryReferences(char * path)
     PPCODE:
         try {
-            PathSet paths = store()->queryPathInfo(path)->references;
-            for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+            for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -81,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(path)->narHash.to_string();
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -91,9 +90,9 @@ SV * queryPathHash(char * path)
 SV * queryDeriver(char * path)
     PPCODE:
         try {
-            auto deriver = store()->queryPathInfo(path)->deriver;
-            if (deriver == "") XSRETURN_UNDEF;
-            XPUSHs(sv_2mortal(newSVpv(deriver.c_str(), 0)));
+            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            if (!info->deriver) XSRETURN_UNDEF;
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -102,18 +101,18 @@ SV * queryDeriver(char * path)
 SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
-            auto info = store()->queryPathInfo(path);
-            if (info->deriver == "")
+            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            if (info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
-                XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0)));
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
             auto s = info->narHash.to_string(base32 ? Base32 : Base16);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
             AV * arr = newAV();
-            for (PathSet::iterator i = info->references.begin(); i != info->references.end(); ++i)
-                av_push(arr, newSVpv(i->c_str(), 0));
+            for (auto & i : info->references)
+                av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
             XPUSHs(sv_2mortal(newRV((SV *) arr)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -123,8 +122,8 @@ SV * queryPathInfo(char * path, int base32)
 SV * queryPathFromHashPart(char * hashPart)
     PPCODE:
         try {
-            Path path = store()->queryPathFromHashPart(hashPart);
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->queryPathFromHashPart(hashPart);
+            XPUSHs(sv_2mortal(newSVpv(path ? store()->printStorePath(*path).c_str() : "", 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -133,11 +132,11 @@ SV * queryPathFromHashPart(char * hashPart)
 SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
     PPCODE:
         try {
-            PathSet paths;
+            StorePathSet paths;
             for (int n = 2; n < items; ++n)
-                store()->computeFSClosure(SvPV_nolen(ST(n)), paths, flipDirection, includeOutputs);
-            for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+                store()->computeFSClosure(store()->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs);
+            for (auto & i : paths)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -146,11 +145,11 @@ SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
 SV * topoSortPaths(...)
     PPCODE:
         try {
-            PathSet paths;
-            for (int n = 0; n < items; ++n) paths.insert(SvPV_nolen(ST(n)));
-            Paths sorted = store()->topoSortPaths(paths);
-            for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+            StorePathSet paths;
+            for (int n = 0; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n))));
+            auto sorted = store()->topoSortPaths(paths);
+            for (auto & i : sorted)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -159,7 +158,7 @@ SV * topoSortPaths(...)
 SV * followLinksToStorePath(char * path)
     CODE:
         try {
-            RETVAL = newSVpv(store()->followLinksToStorePath(path).c_str(), 0);
+            RETVAL = newSVpv(store()->printStorePath(store()->followLinksToStorePath(path)).c_str(), 0);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -170,8 +169,8 @@ SV * followLinksToStorePath(char * path)
 void exportPaths(int fd, ...)
     PPCODE:
         try {
-            Paths paths;
-            for (int n = 1; n < items; ++n) paths.push_back(SvPV_nolen(ST(n)));
+            StorePathSet paths;
+            for (int n = 1; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n))));
             FdSink sink(fd);
             store()->exportPaths(paths, sink);
         } catch (Error & e) {
@@ -275,8 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            Path path = store()->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo));
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -286,8 +285,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            Path path = store()->makeFixedOutputPath(recursive, h, name);
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->makeFixedOutputPath(recursive, h, name);
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -298,35 +297,35 @@ SV * derivationFromPath(char * drvPath)
         HV *hash;
     CODE:
         try {
-            Derivation drv = store()->derivationFromPath(drvPath);
+            Derivation drv = store()->derivationFromPath(store()->parseStorePath(drvPath));
             hash = newHV();
 
             HV * outputs = newHV();
-            for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
-                hv_store(outputs, i->first.c_str(), i->first.size(), newSVpv(i->second.path.c_str(), 0), 0);
+            for (auto & i : drv.outputs)
+                hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0);
             hv_stores(hash, "outputs", newRV((SV *) outputs));
 
             AV * inputDrvs = newAV();
-            for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i)
-                av_push(inputDrvs, newSVpv(i->first.c_str(), 0)); // !!! ignores i->second
+            for (auto & i : drv.inputDrvs)
+                av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second
             hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
 
             AV * inputSrcs = newAV();
-            for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i)
-                av_push(inputSrcs, newSVpv(i->c_str(), 0));
+            for (auto & i : drv.inputSrcs)
+                av_push(inputSrcs, newSVpv(store()->printStorePath(i).c_str(), 0));
             hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs));
 
             hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0));
             hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0));
 
             AV * args = newAV();
-            for (Strings::iterator i = drv.args.begin(); i != drv.args.end(); ++i)
-                av_push(args, newSVpv(i->c_str(), 0));
+            for (auto & i : drv.args)
+                av_push(args, newSVpv(i.c_str(), 0));
             hv_stores(hash, "args", newRV((SV *) args));
 
             HV * env = newHV();
-            for (StringPairs::iterator i = drv.env.begin(); i != drv.env.end(); ++i)
-                hv_store(env, i->first.c_str(), i->first.size(), newSVpv(i->second.c_str(), 0), 0);
+            for (auto & i : drv.env)
+                hv_store(env, i.first.c_str(), i.first.size(), newSVpv(i.second.c_str(), 0), 0);
             hv_stores(hash, "env", newRV((SV *) env));
 
             RETVAL = newRV_noinc((SV *)hash);
@@ -340,7 +339,7 @@ SV * derivationFromPath(char * drvPath)
 void addTempRoot(char * storePath)
     PPCODE:
         try {
-            store()->addTempRoot(storePath);
+            store()->addTempRoot(store()->parseStorePath(storePath));
         } catch (Error & e) {
             croak("%s", e.what());
         }

precompiled-headers.h

@@ -0,0 +1,61 @@
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cassert>
+#include <cctype>
+#include <chrono>
+#include <climits>
+#include <cmath>
+#include <condition_variable>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <functional>
+#include <future>
+#include <iostream>
+#include <limits>
+#include <list>
+#include <locale>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <optional>
+#include <queue>
+#include <random>
+#include <regex>
+#include <set>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include <boost/format.hpp>
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <grp.h>
+#include <netdb.h>
+#include <pwd.h>
+#include <signal.h>
+#include <sys/resource.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <sys/wait.h>
+#include <termios.h>
+#include <unistd.h>
+
+#include "util.hh"
+#include "args.hh"

release-common.nix

@@ -30,9 +30,7 @@ rec {
   });
 
   configureFlags =
-    [
-      "--enable-gc"
-    ] ++ lib.optionals stdenv.isLinux [
+    lib.optionals stdenv.isLinux [
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
@@ -49,9 +47,12 @@ rec {
 
   buildDeps =
     [ curl
-      bzip2 xz brotli editline
-      openssl pkgconfig sqlite boehmgc
+      bzip2 xz brotli zlib editline
+      openssl pkgconfig sqlite
+      libarchive
       boost
+      nlohmann_json
+      rustc cargo
 
       # Tests
       git

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -10,6 +10,50 @@ let
 
   jobs = rec {
 
+    # Create a "vendor" directory that contains the crates listed in
+    # Cargo.lock, and include it in the Nix tarball. This allows Nix
+    # to be built without network access.
+    vendoredCrates =
+      let
+        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
+
+        files = map (pkg: import <nix/fetchurl.nix> {
+          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
+
+      in pkgs.runCommand "cargo-vendor-dir" {}
+        ''
+          mkdir -p $out/vendor
+
+          cat > $out/vendor/config <<EOF
+          [source.crates-io]
+          replace-with = "vendored-sources"
+
+          [source.vendored-sources]
+          directory = "vendor"
+          EOF
+
+          ${toString (builtins.map (file: ''
+            mkdir $out/vendor/tmp
+            tar xvf ${file} -C $out/vendor/tmp
+            dir=$(echo $out/vendor/tmp/*)
+
+            # Add just enough metadata to keep Cargo happy.
+            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+            # Clean up some cruft from the winapi crates. FIXME: find
+            # a way to remove winapi* from our dependencies.
+            if [[ $dir =~ /winapi ]]; then
+              find $dir -name "*.a" -print0 | xargs -0 rm -f --
+            fi
+
+            mv "$dir" $out/vendor/
+
+            rm -rf $out/vendor/tmp
+          '') files)}
+        '';
+
 
     tarball =
       with pkgs;
@@ -25,8 +69,6 @@ let
 
         buildInputs = tarballDeps ++ buildDeps;
 
-        configureFlags = "--enable-gc";
-
         postUnpack = ''
           (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
           cat $sourceRoot/.dist-files
@@ -40,6 +82,8 @@ let
 
         distPhase =
           ''
+            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
+
             runHook preDist
             make dist
             mkdir -p $out/tarballs
@@ -128,7 +172,7 @@ let
       in
 
       runCommand "nix-binary-tarball-${version}"
-        { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
           meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
         }
         ''
@@ -196,31 +240,26 @@ let
         name = "nix-build";
         src = tarball;
 
+        enableParallelBuilding = true;
+
         buildInputs = buildDeps;
 
         dontInstall = false;
 
         doInstallCheck = true;
 
-        lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
+        lcovFilter = [ "*/boost/*" "*-tab.*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some
         # fonts.  So provide those.
         FONTCONFIG_FILE = texFunctions.fontsConf;
+
+        # To test building without precompiled headers.
+        makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
       };
 
 
-    #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
-
-
-    #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
-    #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
-
-    #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ];
-    #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ];
-
-
     # System tests.
     tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
       inherit nixpkgs;
@@ -262,7 +301,7 @@ let
             x86_64-linux = "${build.x86_64-linux}";
           }
           EOF
-          su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+          su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
           (! [ -L /home/alice/.profile-1-link ])
           su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
 
@@ -271,6 +310,7 @@ let
           umount /nix
         ''); # */
 
+    /*
     tests.evalNixpkgs =
       import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
         inherit nixpkgs;
@@ -289,6 +329,7 @@ let
 
           touch $out
         '';
+    */
 
 
     installerScript =
@@ -300,7 +341,7 @@ let
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
-              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
+              (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
               [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
             } \
             --replace '@nixVersion@' ${build.x86_64-linux.src.version}
@@ -326,8 +367,8 @@ let
           tests.remoteBuilds
           tests.nix-copy-closure
           tests.binaryTarball
-          tests.evalNixpkgs
-          tests.evalNixOS
+          #tests.evalNixpkgs
+          #tests.evalNixOS
           installerScript
         ];
     };
@@ -335,55 +376,4 @@ let
   };
 
 
-  makeRPM_i686 = makeRPM "i686-linux";
-  makeRPM_x86_64 = makeRPM "x86_64-linux";
-
-  makeRPM =
-    system: diskImageFun: extraPackages:
-
-    with import nixpkgs { inherit system; };
-
-    releaseTools.rpmBuild rec {
-      name = "nix-rpm";
-      src = jobs.tarball;
-      diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
-            ++ extraPackages; };
-      # At most 2047MB can be simulated in qemu-system-i386
-      memSize = 2047;
-      meta.schedulingPriority = 50;
-      postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
-      #enableParallelBuilding = true;
-    };
-
-
-  makeDeb_i686 = makeDeb "i686-linux";
-  makeDeb_x86_64 = makeDeb "x86_64-linux";
-
-  makeDeb =
-    system: diskImageFun: extraPackages: extraDebPackages:
-
-    with import nixpkgs { inherit system; };
-
-    releaseTools.debBuild {
-      name = "nix-deb";
-      src = jobs.tarball;
-      diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
-            ++ extraPackages; };
-      memSize = 2047;
-      meta.schedulingPriority = 50;
-      postInstall = "make installcheck";
-      configureFlags = "--sysconfdir=/etc";
-      debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ]
-        ++ extraDebPackages;
-      debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
-      doInstallCheck = true;
-      #enableParallelBuilding = true;
-    };
-
-
 in jobs

scripts/install-darwin-multi-user.sh

@@ -39,7 +39,7 @@ EOF
 
 poly_configure_nix_daemon_service() {
     _sudo "to set up the nix-daemon as a LaunchDaemon" \
-          ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
+          cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST"
 
     _sudo "to load the LaunchDaemon plist for nix-daemon" \
           launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist

scripts/install-multi-user.sh

@@ -19,9 +19,6 @@ readonly BLUE_UL='\033[38;4;34m'
 readonly GREEN='\033[38;32m'
 readonly GREEN_UL='\033[38;4;32m'
 readonly RED='\033[38;31m'
-readonly RED_UL='\033[38;4;31m'
-readonly YELLOW='\033[38;33m'
-readonly YELLOW_UL='\033[38;4;33m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
@@ -278,73 +275,9 @@ EOF
     fi
 
     if type nix-env 2> /dev/null >&2; then
-        failure <<EOF
-Nix already appears to be installed, and this tool assumes it is
-_not_ yet installed.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ "${NIX_REMOTE:-}" != "" ]; then
-        failure <<EOF
-For some reason, \$NIX_REMOTE is set. It really should not be set
-before this installer runs, and it hints that Nix is currently
-installed. Please delete the old Nix installation and start again.
-
-Note: You might need to close your shell window and open a new shell
-to clear the variable.
-EOF
-    fi
-
-    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
-        failure <<EOF
-It looks like \$SSL_CERT_FILE is set to a path that used to be part of
-the old Nix installation. Please unset that variable and try again:
-
-  $ unset SSL_CERT_FILE
-
-EOF
-    fi
-
-    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
-        if [ -f "$file" ]; then
-            if grep -l "^[^#].*.nix-profile" "$file"; then
-                failure <<EOF
-I found a reference to a ".nix-profile" in $file.
-This has a high chance of breaking a new nix installation. It was most
-likely put there by a previous Nix installer.
-
-Please remove this reference and try running this again. You should
-also look for similar references in:
-
- - ~/.bash_profile
- - ~/.bash_login
- - ~/.profile
-
-or other shell init files that you may have.
-
-$(uninstall_directions)
-EOF
-            fi
-        fi
-    done
-
-    if [ -d /nix/store ] || [ -d /nix/var ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ -d /etc/nix ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /etc/nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
+        warning <<EOF
+Nix already appears to be installed. This installer may run into issues.
+If an error occurs, try manually uninstalling, then rerunning this script.
 
 $(uninstall_directions)
 EOF
@@ -352,7 +285,7 @@ EOF
 
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
-        failure <<EOF
+            failure <<EOF
 When this script runs, it backs up the current $profile_target to
 $profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
 
@@ -364,38 +297,10 @@ in case.
 2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
 it has anything nix-related in it. If it does, something is probably
 quite wrong. Please open an issue or get in touch immediately.
-
-3. Take care to make sure that $profile_target doesn't look like it has
-anything nix-related in it. If it does, and $profile_target _did not_,
-run:
-
-  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
-
-and try again.
-EOF
-        fi
-
-        if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then
-            failure <<EOF
-It looks like $profile_target already has some Nix configuration in
-there. There should be no reason to run this again. If you're having
-trouble, please open an issue.
 EOF
         fi
     done
 
-    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
-    for danger_path in "${danger_paths[@]}"; do
-        if _sudo "making sure that $danger_path doesn't exist" \
-           test -e "$danger_path"; then
-            failure <<EOF
-I found a file at $danger_path, which is a relic of a previous
-installation. You must first delete this file before continuing.
-
-$(uninstall_directions)
-EOF
-        fi
-    done
 }
 
 setup_report() {
@@ -529,24 +434,17 @@ create_build_users() {
 }
 
 create_directories() {
+    # FIXME: remove all of this because it duplicates LocalStore::LocalStore().
+
     _sudo "to make the basic directory structure of Nix (part 1)" \
-          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool}
+          mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
 
     _sudo "to make the basic directory structure of Nix (part 2)" \
-          mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user
-
-    _sudo "to make the basic directory structure of Nix (part 3)" \
           mkdir -pv -m 1775 /nix/store
 
-    _sudo "to make the basic directory structure of Nix (part 4)" \
+    _sudo "to make the basic directory structure of Nix (part 3)" \
           chgrp "$NIX_BUILD_GROUP_NAME" /nix/store
 
-    _sudo "to set up the root user's profile (part 1)" \
-          mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root
-
-    _sudo "to set up the root user's profile (part 2)" \
-          mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr"
-
     _sudo "to place the default nix daemon configuration (part 1)" \
           mkdir -pv -m 0555 /etc/nix
 }
@@ -589,7 +487,7 @@ EOF
 We will:
 
  - make sure your computer doesn't already have Nix files
-   (if it does, I  will tell you how to clean them up.)
+   (if it does, I will tell you how to clean them up.)
  - create local users (see the list above for the users we'll make)
  - create a local group ($NIX_BUILD_GROUP_NAME)
  - install Nix in to $NIX_ROOT
@@ -772,9 +670,7 @@ main() {
     welcome_to_nix
     chat_about_sudo
 
-    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
-        validate_starting_assumptions
-    fi
+    validate_starting_assumptions
 
     setup_report
 

scripts/install-nix-from-closure.sh

@@ -102,7 +102,7 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do
         rm -rf "$i_tmp"
     fi
     if ! [ -e "$dest/store/$i" ]; then
-        cp -Rp "$self/store/$i" "$i_tmp"
+        cp -RPp "$self/store/$i" "$i_tmp"
         chmod -R a-w "$i_tmp"
         chmod +w "$i_tmp"
         mv "$i_tmp" "$dest/store/$i"
@@ -141,11 +141,9 @@ if [ -z "$_NIX_INSTALLER_TEST" ]; then
 fi
 
 added=
+p=$HOME/.nix-profile/etc/profile.d/nix.sh
 if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
-
     # Make the shell source nix.sh during login.
-    p=$HOME/.nix-profile/etc/profile.d/nix.sh
-
     for i in .bash_profile .bash_login .profile; do
         fn="$HOME/$i"
         if [ -w "$fn" ]; then
@@ -157,7 +155,6 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
             break
         fi
     done
-
 fi
 
 if [ -z "$added" ]; then

scripts/install.in

@@ -56,7 +56,7 @@ fi
 
 unpack=$tmpDir/unpack
 mkdir -p "$unpack"
-tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
+tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
 
 script=$(echo "$unpack"/*/install)
 

scripts/nix-profile-daemon.sh.in

@@ -2,48 +2,8 @@
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
-export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
-# Set up the per-user profile.
-mkdir -m 0755 -p $NIX_USER_PROFILE_DIR
-if ! test -O "$NIX_USER_PROFILE_DIR"; then
-    echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2
-fi
-
-if test -w $HOME; then
-  if ! test -L $HOME/.nix-profile; then
-      if test "$USER" != root; then
-          ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile
-      else
-          # Root installs in the system-wide profile by default.
-          ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile
-      fi
-  fi
-
-  # Subscribe the root user to the NixOS channel by default.
-  if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then
-      echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels
-  fi
-
-  # Create the per-user garbage collector roots directory.
-  NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER
-  mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR
-  if ! test -O "$NIX_USER_GCROOTS_DIR"; then
-      echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2
-  fi
-
-  # Set up a default Nix expression from which to install stuff.
-  if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then
-      rm -f $HOME/.nix-defexpr
-      mkdir -p $HOME/.nix-defexpr
-      if [ "$USER" != root ]; then
-          ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root
-      fi
-  fi
-fi
-
-
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
 if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE
@@ -64,5 +24,4 @@ else
   done
 fi
 
-export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
 export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"

scripts/nix-profile.sh.in

@@ -1,60 +1,10 @@
 if [ -n "$HOME" ] && [ -n "$USER" ]; then
-    __savedpath="$PATH"
-    export PATH=@coreutils@
 
     # Set up the per-user profile.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix
 
     NIX_LINK=$HOME/.nix-profile
 
-    NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER
-
-    mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR"
-
-    if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then
-        echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2
-    fi
-
-    if [ -w "$HOME" ]; then
-        if ! [ -L "$NIX_LINK" ]; then
-            echo "Nix: creating $NIX_LINK" >&2
-            if [ "$USER" != root ]; then
-                if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then
-                    echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2
-                fi
-            else
-                # Root installs in the system-wide profile by default.
-                ln -s @localstatedir@/nix/profiles/default "$NIX_LINK"
-            fi
-        fi
-
-        # Subscribe the user to the unstable Nixpkgs channel by default.
-        if [ ! -e "$HOME/.nix-channels" ]; then
-            echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels"
-        fi
-
-        # Create the per-user garbage collector roots directory.
-        __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER"
-        mkdir -m 0755 -p "$__user_gcroots"
-        if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then
-            echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2
-        fi
-        unset __user_gcroots
-
-        # Set up a default Nix expression from which to install stuff.
-        __nix_defexpr="$HOME"/.nix-defexpr
-        [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr"
-        mkdir -m 0755 -p "$__nix_defexpr"
-        if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then
-            ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root
-        fi
-        unset __nix_defexpr
-    fi
-
-    # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs>
-    # paths work when the user has fetched the Nixpkgs channel.
-    export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels
-
     # Set up environment.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
@@ -78,6 +28,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi
 
-    export PATH="$NIX_LINK/bin:$__savedpath"
-    unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR
+    export PATH="$NIX_LINK/bin:$PATH"
+    unset NIX_LINK
 fi

shell.nix

@@ -1,13 +1,13 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps;
+  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -88,7 +88,7 @@ static int _main(int argc, char * * argv)
             return 0;
         }
 
-        string drvPath;
+        std::optional<StorePath> drvPath;
         string storeUri;
 
         while (true) {
@@ -100,7 +100,7 @@ static int _main(int argc, char * * argv)
 
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
-            source >> drvPath;
+            drvPath = store->parseStorePath(readString(source));
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
              auto canBuildLocally = amWilling
@@ -188,7 +188,7 @@ static int _main(int argc, char * * argv)
 
                     Store::Params storeParams;
                     if (hasPrefix(bestMachine->storeUri, "ssh://")) {
-                        storeParams["max-connections"] ="1";
+                        storeParams["max-connections"] = "1";
                         storeParams["log-fd"] = "4";
                         if (bestMachine->sshKey != "")
                             storeParams["ssh-key"] = bestMachine->sshKey;
@@ -236,26 +236,27 @@ connected:
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
+            copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
-        drv.inputSrcs = inputs;
+        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        drv.inputSrcs = store->parseStorePathSet(inputs);
 
-        auto result = sshStore->buildDerivation(drvPath, drv);
+        auto result = sshStore->buildDerivation(*drvPath, drv);
 
         if (!result.success())
-            throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg);
+            throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
 
-        PathSet missing;
+        StorePathSet missing;
         for (auto & path : outputs)
-            if (!store->isValidPath(path)) missing.insert(path);
+            if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
 
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
-            store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
+            for (auto & i : missing)
+                store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 

src/libexpr/attr-path.cc

@@ -32,7 +32,7 @@ static Strings parseAttrPath(const string & s)
 }
 
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
@@ -40,7 +40,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
     Error attrError =
         Error(format("attribute selection path '%1%' does not match expression") % attrPath);
 
-    Value * v = &vIn;
+    Ptr<Value> v(&vIn);
 
     for (auto & attr : tokens) {
 
@@ -50,7 +50,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
         if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
-        Value * vNew = state.allocValue();
+        auto vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
         state.forceValue(*v);
@@ -71,7 +71,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
                 throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
-            v = &*a->value;
+            v = a->value;
         }
 
         else if (apType == apIndex) {
@@ -93,4 +93,35 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 }
 
 
+Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
+{
+    Value * v2;
+    try {
+        v2 = findAlongAttrPath(state, "meta.position", *state.emptyBindings, v);
+    } catch (Error &) {
+        throw Error("package '%s' has no source location information", what);
+    }
+
+    // FIXME: is it possible to extract the Pos object instead of doing this
+    //        toString + parsing?
+    auto pos = state.forceString(*v2);
+
+    auto colon = pos.rfind(':');
+    if (colon == std::string::npos)
+        throw Error("cannot parse meta.position attribute '%s'", pos);
+
+    std::string filename(pos, 0, colon);
+    unsigned int lineno;
+    try {
+        lineno = std::stoi(std::string(pos, colon + 1));
+    } catch (std::invalid_argument & e) {
+        throw Error("cannot parse line number '%s'", pos);
+    }
+
+    Symbol file = state.symbols.create(filename);
+
+    return { file, lineno, 0 };
+}
+
+
 }

src/libexpr/attr-path.hh

@@ -7,7 +7,10 @@
 
 namespace nix {
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
+/* Heuristic to find the filename and lineno or a nix value. */
+Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
+
 }

src/libexpr/attr-set.cc

@@ -7,28 +7,28 @@
 namespace nix {
 
 
-/* Allocate a new array of attributes for an attribute set with a specific
-   capacity. The space is implicitly reserved after the Bindings
-   structure. */
-Bindings * EvalState::allocBindings(size_t capacity)
+/* Allocate a new array of attributes for an attribute set with a
+   specific capacity. The space is implicitly reserved after the
+   Bindings structure. */
+Ptr<Bindings> Bindings::allocBindings(size_t capacity)
 {
-    if (capacity > std::numeric_limits<Bindings::size_t>::max())
+    if (capacity >= 1UL << Object::miscBytes * 8)
         throw Error("attribute set of size %d is too big", capacity);
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
+    return gc.alloc<Bindings>(Bindings::wordsFor(capacity), capacity);
 }
 
 
 void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
-        v = vEmptySet;
-        return;
+        v.attrs = emptyBindings;
+        v.type = tAttrs;
+    } else {
+        v.attrs = Bindings::allocBindings(capacity);
+        v.type = tAttrs;
+        nrAttrsets++;
+        nrAttrsInAttrsets += capacity;
     }
-    clearValue(v);
-    v.type = tAttrs;
-    v.attrs = allocBindings(capacity);
-    nrAttrsets++;
-    nrAttrsInAttrsets += capacity;
 }
 
 
@@ -37,7 +37,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
    this attribute. */
 Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 {
-    Value * v = allocValue();
+    auto v = allocValue();
     vAttrs.attrs->push_back(Attr(name, v));
     return v;
 }

src/libexpr/attr-set.hh

@@ -2,8 +2,10 @@
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
+#include "gc.hh"
 
 #include <algorithm>
+#include <optional>
 
 namespace nix {
 
@@ -30,19 +32,22 @@ struct Attr
    by its size and its capacity, the capacity being the number of Attr
    elements allocated after this structure, while the size corresponds to
    the number of elements already inserted in this structure. */
-class Bindings
+class Bindings : public Object
 {
 public:
     typedef uint32_t size_t;
 
 private:
-    size_t size_, capacity_;
+    // FIXME: eliminate size_. We can just rely on capacity by sorting
+    // null entries towards the end of the vector.
+    size_t size_;
     Attr attrs[0];
 
-    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
+    Bindings(size_t capacity) : Object(tBindings, capacity), size_(0) {}
     Bindings(const Bindings & bindings) = delete;
 
 public:
+
     size_t size() const { return size_; }
 
     bool empty() const { return !size_; }
@@ -51,7 +56,8 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(size_ < capacity_);
+        assert(size_ < capacity());
+        gc.assertObject(attr.value);
         attrs[size_++] = attr;
     }
 
@@ -63,6 +69,22 @@ public:
         return end();
     }
 
+    Attr * get(const Symbol & name)
+    {
+        Attr key(name, 0);
+        iterator i = std::lower_bound(begin(), end(), key);
+        if (i != end() && i->name == name) return &*i;
+        return nullptr;
+    }
+
+    Attr & need(const Symbol & name, const Pos & pos = noPos)
+    {
+        auto a = get(name);
+        if (!a)
+            throw Error("attribute '%s' missing, at %s", name, pos);
+        return *a;
+    }
+
     iterator begin() { return &attrs[0]; }
     iterator end() { return &attrs[size_]; }
 
@@ -73,7 +95,7 @@ public:
 
     void sort();
 
-    size_t capacity() { return capacity_; }
+    size_t capacity() const { return getMisc(); }
 
     /* Returns the attributes in lexicographically sorted order. */
     std::vector<const Attr *> lexicographicOrder() const
@@ -88,7 +110,19 @@ public:
         return res;
     }
 
-    friend class EvalState;
+    size_t words() const
+    {
+        return wordsFor(capacity());
+    }
+
+    static size_t wordsFor(size_t capacity)
+    {
+        return 2 + 3 * capacity; // FIXME
+    }
+
+    static Ptr<Bindings> allocBindings(size_t capacity);
+
+    friend class GC;
 };
 
 

src/libexpr/common-eval-args.cc

@@ -28,19 +28,21 @@ MixEvalArgs::MixEvalArgs()
         .handler([&](std::string s) { searchPath.push_back(s); });
 }
 
-Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
+Ptr<Bindings> MixEvalArgs::getAutoArgs(EvalState & state)
 {
-    Bindings * res = state.allocBindings(autoArgs.size());
-    for (auto & i : autoArgs) {
-        Value * v = state.allocValue();
-        if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
-        else
-            mkString(*v, string(i.second, 1));
-        res->push_back(Attr(state.symbols.create(i.first), v));
+    if (!bindings) {
+        bindings = Bindings::allocBindings(autoArgs.size());
+        for (auto & i : autoArgs) {
+            Value * v = state.allocValue();
+            if (i.second[0] == 'E')
+                state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
+            else
+                mkString(*v, string(i.second, 1));
+            bindings->push_back(Attr(state.symbols.create(i.first), v));
+        }
+        bindings->sort();
     }
-    res->sort();
-    return res;
+    return bindings;
 }
 
 Path lookupFileArg(EvalState & state, string s)

src/libexpr/common-eval-args.hh

@@ -1,24 +1,28 @@
 #pragma once
 
 #include "args.hh"
+#include "eval.hh"
 
 namespace nix {
 
 class Store;
-class EvalState;
-class Bindings;
+
+template<class T>
+struct Ptr;
 
 struct MixEvalArgs : virtual Args
 {
     MixEvalArgs();
 
-    Bindings * getAutoArgs(EvalState & state);
+    Ptr<Bindings> getAutoArgs(EvalState & state);
 
     Strings searchPath;
 
 private:
 
     std::map<std::string, std::string> autoArgs;
+
+    Ptr<Bindings> bindings;
 };
 
 Path lookupFileArg(EvalState & state, string s);

src/libexpr/eval-inline.hh

@@ -27,21 +27,27 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const
 void EvalState::forceValue(Value & v, const Pos & pos)
 {
     if (v.type == tThunk) {
-        Env * env = v.thunk.env;
+        // FIXME: this is necessary because some values (like vList2)
+        // are created non-atomically.
+        Ptr<Env> env(v.thunk.env);
         Expr * expr = v.thunk.expr;
         try {
             v.type = tBlackhole;
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
-            v.type = tThunk;
             v.thunk.env = env;
             v.thunk.expr = expr;
+            v.type = tThunk;
             throw;
         }
     }
-    else if (v.type == tApp)
+    else if (v.type == tApp) {
+        // FIXME: idem.
+        Ptr<Value> left(v.app.left);
+        Ptr<Value> right(v.app.right);
         callFunction(*v.app.left, *v.app.right, v, noPos);
+    }
     else if (v.type == tBlackhole)
         throwEvalError("infinite recursion encountered, at %1%", pos);
 }
@@ -78,18 +84,5 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-inline void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_MALLOC(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
 
 }

src/libexpr/eval.hh

@@ -6,9 +6,10 @@
 #include "symbol-table.hh"
 #include "hash.hh"
 #include "config.hh"
-#include "function-trace.hh"
+#include "gc.hh"
 
 #include <map>
+#include <optional>
 #include <unordered_map>
 
 
@@ -17,6 +18,8 @@ namespace nix {
 
 class Store;
 class EvalState;
+struct Derivation; // FIXME: remove
+struct StorePath;
 enum RepairFlag : bool;
 
 
@@ -33,24 +36,61 @@ struct PrimOp
 };
 
 
-struct Env
+struct Env : Object
 {
     Env * up;
-    unsigned short size; // used by ‘valueSize’
-    unsigned short prevWith:14; // nr of levels up to next `with' environment
-    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
+
+private:
+
+    constexpr static size_t maxSize = 1 << 16;
+    constexpr static size_t maxPrevWith = 1 << 10;
+
+    Env(Tag type, size_t size, size_t prevWith)
+        : Object(type, size | (prevWith << 16))
+    {
+        if (size >= maxSize)
+            throw Error("environment size %d is too big", size);
+
+        if (prevWith >= maxPrevWith)
+            throw Error("too many nesting levels");
+
+        for (size_t i = 0; i < size; i++)
+            values[i] = nullptr;
+    }
+
+    friend class GC;
+
+public:
+
+    unsigned short getPrevWith() const
+    {
+        return getMisc() >> 16;
+    }
+
+    unsigned short getSize() const
+    {
+        return getMisc() & 0xffff;
+    }
+
+    size_t words() const
+    {
+        return wordsFor(getSize());
+    }
+
+    static size_t wordsFor(unsigned short size)
+    {
+        return 2 + size;
+    }
 };
 
 
-Value & mkString(Value & v, const string & s, const PathSet & context = PathSet());
-
-void copyContext(const Value & v, PathSet & context);
+Value & mkString(Value & v, std::string_view s, const PathSet & context);
 
 
 /* Cache for calls to addToStore(); maps source paths to the store
    paths. */
-typedef std::map<Path, Path> SrcToStore;
+typedef std::map<Path, StorePath> SrcToStore;
 
 
 std::ostream & operator << (std::ostream & str, const Value & v);
@@ -60,14 +100,10 @@ typedef std::pair<std::string, std::string> SearchPathElem;
 typedef std::list<SearchPathElem> SearchPath;
 
 
-/* Initialise the Boehm GC, if applicable. */
-void initGC();
-
-
 class EvalState
 {
 public:
-    SymbolTable symbols;
+    SymbolTable & symbols{nix::symbols}; // FIXME: remove
 
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
@@ -84,7 +120,7 @@ public:
        mode. */
     std::optional<PathSet> allowedPaths;
 
-    Value vEmptySet;
+    Ptr<Bindings> emptyBindings;
 
     const ref<Store> store;
 
@@ -92,19 +128,11 @@ private:
     SrcToStore srcToStore;
 
     /* A cache from path names to parse trees. */
-#if HAVE_BOEHMGC
-    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
-#else
     typedef std::map<Path, Expr *> FileParseCache;
-#endif
     FileParseCache fileParseCache;
 
     /* A cache from path names to values. */
-#if HAVE_BOEHMGC
-    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
-#else
-    typedef std::map<Path, Value> FileEvalCache;
-#endif
+    typedef std::map<Path, Ptr<Value>> FileEvalCache;
     FileEvalCache fileEvalCache;
 
     SearchPath searchPath;
@@ -196,6 +224,9 @@ public:
        set with attribute `type = "derivation"'). */
     bool isDerivation(Value & v);
 
+    std::optional<string> tryAttrsToString(const Pos & pos, Value & v,
+        PathSet & context, bool coerceMore = false, bool copyToStore = true);
+
     /* String coercion.  Converts strings, paths and derivations to a
        string.  If `coerceMore' is set, also converts nulls, integers,
        booleans and lists to a string.  If `copyToStore' is set,
@@ -214,7 +245,7 @@ public:
 
     /* The base environment, containing the builtin functions and
        values. */
-    Env & baseEnv;
+    Ptr<Env> baseEnv;
 
     /* The same, but used during parsing to resolve variables. */
     StaticEnv staticBaseEnv; // !!! should be private
@@ -261,13 +292,18 @@ public:
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
 
     /* Allocation primitives. */
-    Value * allocValue();
-    Env & allocEnv(size_t size);
+    Ptr<Value> allocValue()
+    {
+        nrValues++;
+        return gc.alloc<Value>(Value::words());
+    }
 
+    Ptr<Env> allocEnv(size_t size, size_t prevWith = 0, Tag type = tEnv);
+
+    // Note: the resulting Value is only reachable as long as vAttrs
+    // is reachable.
     Value * allocAttr(Value & vAttrs, const Symbol & name);
 
-    Bindings * allocBindings(size_t capacity);
-
     void mkList(Value & v, size_t length);
     void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
@@ -311,6 +347,10 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
+public:
+
+    std::function<void(const StorePath & drvPath, const Derivation & drv)> derivationHook;
 };
 
 
@@ -335,9 +375,16 @@ struct InvalidPathError : EvalError
 
 struct EvalSettings : Config
 {
+    EvalSettings();
+
+    static Strings getDefaultNixPath();
+
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
+    Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path",
+        "List of directories to be searched for <...> file references."};
+
     Setting<bool> restrictEval{this, false, "restrict-eval",
         "Whether to restrict file system access to paths in $NIX_PATH, "
         "and network access to the URI prefixes listed in 'allowed-uris'."};

src/libexpr/function-trace.cc

@@ -0,0 +1,17 @@
+#include "function-trace.hh"
+
+namespace nix {
+
+FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) {
+    auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+    printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count());
+}
+
+FunctionCallTrace::~FunctionCallTrace() {
+    auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+    auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
+    printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count());
+}
+
+}

src/libexpr/function-trace.hh

@@ -1,24 +1,15 @@
 #pragma once
 
 #include "eval.hh"
-#include <sys/time.h>
+
+#include <chrono>
 
 namespace nix {
 
 struct FunctionCallTrace
 {
     const Pos & pos;
-
-    FunctionCallTrace(const Pos & pos) : pos(pos) {
-        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
-        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-        vomit("function-trace entered %1% at %2%", pos, ns.count());
-    }
-
-    ~FunctionCallTrace() {
-        auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
-        auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration);
-        vomit("function-trace exited %1% at %2%", pos, ns.count());
-    }
+    FunctionCallTrace(const Pos & pos);
+    ~FunctionCallTrace();
 };
 }

src/libexpr/gc.cc

@@ -0,0 +1,537 @@
+#include "gc.hh"
+#include "value.hh"
+#include "attr-set.hh"
+#include "eval.hh"
+
+#include <chrono>
+
+namespace nix {
+
+GC gc;
+
+GC::GC()
+{
+    nextSize = std::max((size_t) 2, parseSize<size_t>(getEnv("GC_INITIAL_HEAP_SIZE").value_or("131072")) / WORD_SIZE);
+
+    // FIXME: placement new
+    frontPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
+    backPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
+
+    frontPtrSentinel->prev = nullptr;
+    frontPtrSentinel->next = backPtrSentinel;
+
+    backPtrSentinel->prev = frontPtrSentinel;
+    backPtrSentinel->next = nullptr;
+
+    frontRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
+    backRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
+
+    frontRootSentinel->prev = nullptr;
+    frontRootSentinel->next = backRootSentinel;
+
+    backRootSentinel->prev = frontRootSentinel;
+    backRootSentinel->next = nullptr;
+
+    freeLists[0].minSize = 2;
+    freeLists[1].minSize = 3;
+    freeLists[2].minSize = 4;
+    freeLists[3].minSize = 8;
+    freeLists[4].minSize = 16;
+    freeLists[5].minSize = 32;
+    freeLists[6].minSize = 64;
+    freeLists[7].minSize = 128;
+
+    addArena(nextSize);
+}
+
+GC::~GC()
+{
+    debug("%d bytes in arenas, %d bytes allocated, %d bytes reclaimed, in %d ms",
+        totalSize * WORD_SIZE,
+        allTimeWordsAllocated * WORD_SIZE,
+        allTimeWordsFreed * WORD_SIZE,
+        totalDurationMs);
+
+    size_t n = 0;
+    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next)
+        n++;
+    if (n)
+        warn("%d GC root pointers still exist on exit", n);
+
+    n = 0;
+    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next)
+        n++;
+    if (n)
+        warn("%d GC root objects still exist on exit", n);
+
+    assert(!frontPtrSentinel->prev);
+    assert(!backPtrSentinel->next);
+    assert(!frontRootSentinel->prev);
+    assert(!backRootSentinel->next);
+}
+
+void GC::addArena(size_t arenaSize)
+{
+    debug("allocating arena of %d bytes", arenaSize * WORD_SIZE);
+
+    auto arena = Arena(arenaSize);
+
+    // Add this arena to a freelist as a single block.
+    addToFreeList(new (arena.start) Free(arenaSize));
+
+    arenas.emplace_back(std::move(arena));
+
+    totalSize += arenaSize;
+
+    nextSize = arenaSize * 1.5; // FIXME: overflow, clamp
+}
+
+void GC::addToFreeList(Free * obj)
+{
+    auto size = obj->words();
+    for (auto i = freeLists.rbegin(); i != freeLists.rend(); ++i)
+        if (size >= i->minSize) {
+            obj->next = i->front;
+            i->front = obj;
+            return;
+        }
+    abort();
+}
+
+void GC::gc()
+{
+    typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
+
+    auto before = steady_time_point::clock::now();
+
+    size_t marked = 0;
+
+    std::stack<Object *> stack;
+
+    // FIXME: ensure this gets inlined.
+    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
+
+    auto pushPointers = [&](Object * obj) {
+        switch (obj->type) {
+
+        case tFree:
+            printError("reached a freed object at %x", obj);
+            abort();
+
+        case tBindings: {
+            auto obj2 = (Bindings *) obj;
+            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
+                push(i->value);
+            break;
+        }
+
+        case tValueList: {
+            auto obj2 = (PtrList<Object> *) obj;
+            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tWithExprEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            break;
+        }
+
+        case tWithAttrsEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            push(obj2->values[0]);
+            break;
+        }
+
+        case tString:
+        case tContext:
+        case tInt:
+        case tBool:
+        case tNull:
+        case tList0:
+        case tFloat:
+        case tShortString:
+        case tStaticString:
+            break;
+
+        case tLongString: {
+            auto obj2 = (Value *) obj;
+            push(obj2->string.s);
+            // See setContext().
+            if (!(((ptrdiff_t) obj2->string.context) & 1))
+                push(obj2->string.context);
+            break;
+        }
+
+        case tPath:
+            push(((Value *) obj)->path);
+            break;
+
+        case tAttrs:
+            push(((Value *) obj)->attrs);
+            break;
+
+        case tList1:
+            push(((Value *) obj)->smallList[0]);
+            break;
+
+        case tList2:
+            push(((Value *) obj)->smallList[0]);
+            push(((Value *) obj)->smallList[1]);
+            break;
+
+        case tListN:
+            push(((Value *) obj)->bigList);
+            break;
+
+        case tThunk:
+        case tBlackhole:
+            push(((Value *) obj)->thunk.env);
+            break;
+
+        case tApp:
+        case tPrimOpApp:
+            push(((Value *) obj)->app.left);
+            push(((Value *) obj)->app.right);
+            break;
+
+        case tLambda:
+            push(((Value *) obj)->lambda.env);
+            break;
+
+        case tPrimOp:
+            // FIXME: GC primops?
+            break;
+
+        default:
+            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
+            abort();
+        }
+    };
+
+    auto processStack = [&]() {
+        while (!stack.empty()) {
+            auto obj = stack.top();
+            stack.pop();
+
+            //printError("MARK %x", obj);
+
+            if (!obj->isMarked()) {
+                marked++;
+                obj->mark();
+                pushPointers(obj);
+            }
+        }
+    };
+
+    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next) {
+        pushPointers(&p->value);
+        processStack();
+    }
+
+    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next) {
+        if (!p->value) continue;
+        stack.push(p->value);
+        processStack();
+    }
+
+    auto afterMark = steady_time_point::clock::now();
+
+    // Reset all the freelists.
+    for (auto & freeList : freeLists)
+        freeList.front = nullptr;
+
+    // Go through all the arenas and add free objects to the
+    // appropriate freelists.
+    size_t totalObjectsFreed = 0;
+    size_t totalWordsFreed = 0;
+    size_t totalObjectsKept = 0;
+    size_t totalWordsKept = 0;
+
+    for (auto & arena : arenas) {
+        auto [objectsFreed, wordsFreed, objectsKept, wordsKept] = freeUnmarked(arena);
+        totalObjectsFreed += objectsFreed;
+        totalWordsFreed += wordsFreed;
+        totalObjectsKept += objectsKept;
+        totalWordsKept += wordsKept;
+    }
+
+    auto after = steady_time_point::clock::now();
+
+    auto markDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(afterMark - before).count();
+    auto sweepDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(after - afterMark).count();
+
+    debug("freed %d dead objects (%d bytes), keeping %d/%d objects (%d bytes), marked in %d ms, swept in %d ms",
+        totalObjectsFreed, totalWordsFreed * WORD_SIZE,
+        marked, totalObjectsKept, totalWordsKept * WORD_SIZE,
+        markDurationMs, sweepDurationMs);
+
+    allTimeWordsFreed += totalWordsFreed;
+    totalDurationMs += markDurationMs + sweepDurationMs;
+}
+
+size_t GC::getObjectSize(Object * obj)
+{
+    auto tag = obj->type;
+    if (tag >= tInt && tag <= tFloat) {
+        return ((Value *) obj)->words();
+    } else {
+        switch (tag) {
+        case tFree:
+            return ((Free *) obj)->words();
+            break;
+        case tString:
+            return ((String *) obj)->words();
+            break;
+        case tBindings:
+            return ((Bindings *) obj)->words();
+            break;
+        case tValueList:
+            return ((PtrList<Value> *) obj)->words();
+            break;
+        case tEnv:
+        case tWithExprEnv:
+        case tWithAttrsEnv:
+            return ((Env *) obj)->words();
+            break;
+        case tContext:
+            return ((Context *) obj)->getSize() + 1;
+            break;
+        default:
+            printError("GC encountered invalid object with tag %d", tag);
+            abort();
+        }
+    }
+}
+
+std::tuple<size_t, size_t, size_t, size_t> GC::freeUnmarked(Arena & arena)
+{
+    size_t objectsFreed = 0;
+    size_t wordsFreed = 0;
+    size_t objectsKept = 0;
+    size_t wordsKept = 0;
+
+    auto end = arena.start + arena.size;
+    auto pos = arena.start;
+
+    Free * curFree = nullptr;
+
+    auto linkCurFree = [&]() {
+        if (curFree && curFree->words() > 1)
+            addToFreeList(curFree);
+        curFree = nullptr;
+    };
+
+    while (pos < end) {
+        auto obj = (Object *) pos;
+
+        auto objSize = getObjectSize(obj);
+
+        // Merge current object into the previous free object.
+        auto mergeFree = [&]() {
+            //printError("MERGE %x %x %d", curFree, obj, curFree->size() + objSize);
+            assert(curFree->words() >= 1);
+            curFree->setSize(curFree->words() + objSize);
+        };
+
+        if (obj->type == tFree) {
+            //debug("KEEP FREE %x %d", obj, obj->getMisc());
+            if (curFree) {
+                // Merge this object into the previous free
+                // object.
+                mergeFree();
+            } else {
+                curFree = (Free *) obj;
+            }
+        } else {
+
+            if (obj->isMarked()) {
+                // Unmark to prepare for the next GC run.
+                //debug("KEEP OBJECT %x %d %d", obj, obj->type, objSize);
+                linkCurFree();
+                obj->unmark();
+                objectsKept += 1;
+                wordsKept += objSize;
+            } else {
+                //debug("FREE OBJECT %x %d %d", obj, obj->type, objSize);
+                #if GC_DEBUG
+                for (size_t i = 0; i < objSize; ++i)
+                    ((Word *) obj)[i] = 0xdeadc0dedeadbeefULL;
+                #endif
+                objectsFreed += 1;
+                wordsFreed += objSize;
+                if (curFree) {
+                    mergeFree();
+                } else {
+                    // Convert to a free object.
+                    curFree = (Free *) obj;
+                    curFree->type = tFree;
+                    curFree->setSize(objSize);
+                }
+            }
+        }
+
+        pos += objSize;
+    }
+
+    linkCurFree();
+
+    assert(pos == end);
+
+    return {objectsFreed, wordsFreed, objectsKept, wordsKept};
+}
+
+bool GC::isObject(void * p)
+{
+    for (auto & arena : arenas)
+        if (p >= arena.start && p < arena.start + arena.size)
+            return true;
+    return false;
+}
+
+std::tuple<size_t, size_t> GC::getObjectClosureSize(Object * p)
+{
+    std::unordered_set<Object *> seen;
+
+    // FIXME: cut&paste.
+
+    std::stack<Object *> stack;
+
+    // FIXME: ensure this gets inlined.
+    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
+
+    auto pushPointers = [&](Object * obj) {
+        switch (obj->type) {
+
+        case tFree:
+            printError("reached a freed object at %x", obj);
+            abort();
+
+        case tBindings: {
+            auto obj2 = (Bindings *) obj;
+            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
+                push(i->value);
+            break;
+        }
+
+        case tValueList: {
+            auto obj2 = (PtrList<Object> *) obj;
+            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
+                push(*i);
+            break;
+        }
+
+        case tWithExprEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            break;
+        }
+
+        case tWithAttrsEnv: {
+            auto obj2 = (Env *) obj;
+            push(obj2->up);
+            push(obj2->values[0]);
+            break;
+        }
+
+        case tString:
+        case tContext:
+        case tInt:
+        case tBool:
+        case tNull:
+        case tList0:
+        case tFloat:
+        case tShortString:
+        case tStaticString:
+            break;
+
+        case tLongString: {
+            auto obj2 = (Value *) obj;
+            push(obj2->string.s);
+            // See setContext().
+            if (!(((ptrdiff_t) obj2->string.context) & 1))
+                push(obj2->string.context);
+            break;
+        }
+
+        case tPath:
+            push(((Value *) obj)->path);
+            break;
+
+        case tAttrs:
+            push(((Value *) obj)->attrs);
+            break;
+
+        case tList1:
+            push(((Value *) obj)->smallList[0]);
+            break;
+
+        case tList2:
+            push(((Value *) obj)->smallList[0]);
+            push(((Value *) obj)->smallList[1]);
+            break;
+
+        case tListN:
+            push(((Value *) obj)->bigList);
+            break;
+
+        case tThunk:
+        case tBlackhole:
+            push(((Value *) obj)->thunk.env);
+            break;
+
+        case tApp:
+        case tPrimOpApp:
+            push(((Value *) obj)->app.left);
+            push(((Value *) obj)->app.right);
+            break;
+
+        case tLambda:
+            push(((Value *) obj)->lambda.env);
+            break;
+
+        case tPrimOp:
+            // FIXME: GC primops?
+            break;
+
+        default:
+            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
+            abort();
+        }
+    };
+
+    stack.push(p);
+
+    size_t totalSize = 0;
+
+    while (!stack.empty()) {
+        auto obj = stack.top();
+        stack.pop();
+        if (seen.insert(obj).second) {
+            pushPointers(obj);
+            totalSize += getObjectSize(obj);
+        }
+    }
+
+    return {seen.size(), totalSize};
+}
+
+}

src/libexpr/gc.hh

@@ -0,0 +1,482 @@
+#pragma once
+
+#include "logging.hh"
+
+#include <stack>
+#include <limits>
+#include <cassert>
+#include <cstring>
+
+//#define GC_DEBUG 1
+
+namespace nix {
+
+typedef unsigned long Word;
+
+enum Tag {
+    tFree = 3,
+
+    // Misc types
+    tString,
+    tBindings,
+    tValueList,
+    tEnv,
+    tWithExprEnv,
+    tWithAttrsEnv,
+    tContext,
+
+    // Value tags
+    tInt,
+    tBool,
+    tShortString,
+    tStaticString,
+    tLongString,
+    tPath,
+    tNull,
+    tAttrs,
+    tList0,
+    tList1,
+    tList2,
+    tListN,
+    tThunk,
+    tApp,
+    tLambda,
+    tBlackhole,
+    tPrimOp,
+    tPrimOpApp,
+    tExternal,
+    tFloat
+};
+
+constexpr size_t WORD_SIZE = 8;
+
+struct Object
+{
+    friend class GC;
+
+public:
+    constexpr static size_t miscBytes = 7;
+
+public: // FIXME
+    Tag type:7;
+
+private:
+    bool marked:1;
+
+    unsigned long misc:56;
+
+    void unmark()
+    {
+        marked = false;
+    }
+
+protected:
+
+    Object(Tag type, unsigned long misc) : type(type), marked(false), misc(misc) { }
+
+    bool isMarked()
+    {
+        return marked;
+    }
+
+    void mark()
+    {
+        marked = true;
+    }
+
+    void setMisc(unsigned int misc)
+    {
+        this->misc = misc;
+    }
+
+    unsigned int getMisc() const
+    {
+        return misc;
+    }
+
+    char * getMiscData() const
+    {
+        return ((char *) this) + 1;
+    }
+};
+
+template<class T>
+struct PtrList : Object
+{
+    T * elems[0];
+
+    PtrList(Tag type, size_t size) : Object(type, size)
+    {
+        for (size_t i = 0; i < size; i++)
+            elems[i] = nullptr;
+    }
+
+    size_t size() const { return getMisc(); }
+    size_t words() const { return wordsFor(size()); }
+    static size_t wordsFor(size_t size) { return 1 + size; }
+};
+
+struct Free : Object
+{
+    Free * next;
+
+    Free(size_t size) : Object(tFree, size), next(nullptr) { }
+
+    // return size in words
+    size_t words() const { return getMisc(); }
+
+    void setSize(size_t size) { assert(size >= 1); setMisc(size); }
+};
+
+template<class T>
+struct Ptr;
+
+template<class T>
+struct Root;
+
+struct GC
+{
+
+private:
+
+    Ptr<Object> * frontPtrSentinel;
+    Ptr<Object> * backPtrSentinel;
+
+    Root<Object> * frontRootSentinel;
+    Root<Object> * backRootSentinel;
+
+    template<class T>
+    friend class Ptr;
+
+    template<class T>
+    friend class Root;
+
+    struct Arena
+    {
+        size_t size; // in words
+        Word * start;
+
+        Arena(size_t size)
+            : size(size)
+            , start(new Word[size])
+        {
+            assert(size >= 2);
+        }
+
+        Arena(const Arena & arena) = delete;
+
+        Arena(Arena && arena)
+        {
+            size = arena.size;
+            start = arena.start;
+            arena.start = nullptr;
+        }
+
+        ~Arena()
+        {
+            delete[] start;
+        }
+    };
+
+    size_t totalSize = 0;
+    size_t nextSize;
+
+    std::vector<Arena> arenas;
+
+    struct FreeList
+    {
+        size_t minSize;
+        Free * front = nullptr;
+    };
+
+    std::array<FreeList, 8> freeLists;
+
+public:
+
+    size_t getHeapSize() { return totalSize * WORD_SIZE; }
+
+    size_t allTimeWordsAllocated = 0;
+    size_t allTimeWordsFreed = 0;
+    uint64_t totalDurationMs = 0;
+
+private:
+
+    Object * allocObject(size_t size)
+    {
+        assert(size >= 2);
+
+        for (int attempt = 0; attempt < 3; attempt++) {
+
+            for (size_t i = 0; i < freeLists.size(); ++i) {
+                auto & freeList = freeLists[i];
+
+                if ((size <= freeList.minSize || i == freeLists.size() - 1) && freeList.front) {
+                    //printError("TRY %d %d %d", size, i, freeList.minSize);
+
+                    Free * * prev = &freeList.front;
+
+                    while (Free * freeObj = *prev) {
+                        //printError("LOOK %x %d %x", freeObj, freeObj->words(), freeObj->next);
+                        assert(freeObj->words() >= freeList.minSize);
+                        if (freeObj->words() == size) {
+                            // Convert the free object.
+                            *prev = freeObj->next;
+                            return (Object *) freeObj;
+                        } else if (freeObj->words() >= size + 2) {
+                            // Split the free object.
+                            auto newSize = freeObj->words() - size;
+                            freeObj->setSize(newSize);
+                            if (newSize < freeList.minSize) {
+                                /* The free object is now smaller than
+                                   the minimum size for this freelist,
+                                   so move it to another one. */
+                                //printError("MOVE %x %d -> %d", freeObj, newSize + size, newSize);
+                                *prev = freeObj->next;
+                                addToFreeList(freeObj);
+                            }
+                            return (Object *) (((Word *) freeObj) + newSize);
+                        } else if (freeObj->words() == size + 1) {
+                            // Return the free object and add a padding word.
+                            *prev = freeObj->next;
+                            freeObj->setSize(1);
+                            return (Object *) (((Word *) freeObj) + 1);
+                        } else {
+                            assert(freeObj->words() < size);
+                            prev = &freeObj->next;
+                        }
+                    }
+                }
+            }
+
+            if (attempt == 0) {
+                debug("allocation of %d bytes failed, GCing...", size * WORD_SIZE);
+                gc();
+            } else if (attempt == 1) {
+                addArena(std::max(nextSize, size));
+            }
+        }
+
+        throw Error("allocation of %d bytes failed", size);
+    }
+
+public:
+
+    GC();
+    ~GC();
+
+    template<typename T, typename... Args>
+    Ptr<T> alloc(size_t size, const Args & ... args)
+    {
+        auto raw = allocObject(size);
+        allTimeWordsAllocated += size;
+        return new (raw) T(args...);
+    }
+
+    void gc();
+
+    bool isObject(void * p);
+
+    void assertObject(void * p)
+    {
+        #if GC_DEBUG
+        if (!isObject(p)) {
+            printError("object %p is not an object", p);
+            abort();
+        }
+        #endif
+    }
+
+    /* Return the size in words of object 'obj'. */
+    static size_t getObjectSize(Object * obj);
+
+    /* Return the number and size in words of the objects reachable
+       from object 'obj'. */
+    std::tuple<size_t, size_t> getObjectClosureSize(Object * obj);
+
+private:
+
+    void addArena(size_t arenaSize);
+
+    void addToFreeList(Free * obj);
+
+    std::tuple<size_t, size_t, size_t, size_t> freeUnmarked(Arena & arena);
+};
+
+extern GC gc;
+
+template<class T>
+struct Ptr
+{
+private:
+
+    friend class GC;
+
+    Ptr * prev = nullptr, * next = nullptr;
+    T * value = nullptr;
+
+    void link()
+    {
+        prev = (Ptr *) gc.frontPtrSentinel;
+        next = prev->next;
+        next->prev = this;
+        prev->next = this;
+    }
+
+public:
+
+    Ptr() {
+        link();
+    }
+
+    Ptr(T * value) : value(value)
+    {
+        link();
+    }
+
+    Ptr(const Ptr & p)
+    {
+        auto & p2 = const_cast<Ptr &>(p);
+        value = p2.value;
+        next = &p2;
+        prev = p2.prev;
+        prev->next = this;
+        p2.prev = this;
+    }
+
+    Ptr(Ptr && p)
+    {
+        link();
+        value = p.value;
+        p.value = nullptr;
+    }
+
+    Ptr & operator =(const Ptr & p)
+    {
+        value = p.value;
+        return *this;
+    }
+
+    Ptr & operator =(Ptr && p)
+    {
+        value = p.value;
+        p.value = nullptr;
+        return *this;
+    }
+
+    Ptr & operator =(T * v)
+    {
+        value = v;
+        return *this;
+    }
+
+    ~Ptr()
+    {
+        assert(next);
+        assert(prev);
+        assert(next->prev == this);
+        next->prev = prev;
+        assert(prev->next == this);
+        prev->next = next;
+    }
+
+    T * operator ->()
+    {
+        return value;
+    }
+
+    T * operator ->() const // FIXME
+    {
+        return value;
+    }
+
+    operator T * ()
+    {
+        return value;
+    }
+
+    operator T & ()
+    {
+        assert(value);
+        return *value;
+    }
+
+    operator bool() const
+    {
+        return value != nullptr;
+    }
+};
+
+template<class T>
+struct Root
+{
+    Root * prev = nullptr, * next = nullptr;
+    T value;
+
+    template<typename... Args>
+    Root(const Args & ... args)
+        : value{args... }
+    {
+        prev = (Root *) gc.frontRootSentinel;
+        next = prev->next;
+        next->prev = this;
+        prev->next = this;
+    }
+
+    Root(const Root & p) = delete;
+    Root(Root && p) = delete;
+
+    Root & operator =(const T & v) { value = v; return *this; }
+
+    ~Root()
+    {
+        assert(next);
+        assert(prev);
+        assert(next->prev == this);
+        next->prev = prev;
+        assert(prev->next == this);
+        prev->next = next;
+    }
+
+    T * operator ->()
+    {
+        return &value;
+    }
+
+    operator T * ()
+    {
+        return &value;
+    }
+
+    operator T & ()
+    {
+        return value;
+    }
+};
+
+struct String : Object
+{
+    char s[0];
+
+    String(size_t len, const char * src)
+        : Object(tString, len)
+    {
+        std::memcpy(s, src, len + 1);
+    }
+
+    size_t words() const { return wordsFor(getMisc()); }
+
+    /* Return the number of words needed to store a string of 'len'
+       characters (where 'len' excludes the terminator). */
+    static size_t wordsFor(size_t len)
+    {
+        return len / WORD_SIZE + 2;
+    }
+
+    static String * alloc(const char * src)
+    {
+        auto len = strlen(src);
+        return gc.alloc<String>(wordsFor(len), len, src);
+    }
+};
+
+}

src/libexpr/get-drvs.cc

@@ -19,27 +19,27 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
-    auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
+    auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
 
-    drvPath = spec.first;
+    this->drvPath = store->printStorePath(drvPath);
 
     auto drv = store->derivationFromPath(drvPath);
 
-    name = storePathToName(drvPath);
+    name = drvPath.name();
 
-    if (spec.second.size() > 1)
+    if (selectedOutputs.size() > 1)
         throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
 
     outputName =
-        spec.second.empty()
-        ? get(drv.env, "outputName", "out")
-        : *spec.second.begin();
+        selectedOutputs.empty()
+        ? get(drv.env, "outputName").value_or("out")
+        : *selectedOutputs.begin();
 
     auto i = drv.outputs.find(outputName);
     if (i == drv.outputs.end())
-        throw Error("derivation '%s' does not have output '%s'", drvPath, outputName);
+        throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
 
-    outPath = i->second.path;
+    outPath = store->printStorePath(i->second.path);
 }
 
 
@@ -115,15 +115,15 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
         return outputs;
 
     /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
-    const Value * outTI = queryMeta("outputsToInstall");
+    const auto outTI = queryMeta("outputsToInstall");
     if (!outTI) return outputs;
     const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'");
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
     for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if ((*i)->type != tString) throw errMsg;
-        auto out = outputs.find((*i)->string.s);
+        if (!(*i)->isString()) throw errMsg;
+        auto out = outputs.find((*i)->getString());
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
     }
@@ -178,8 +178,7 @@ bool DrvInfo::checkMeta(Value & v)
             if (!checkMeta(*i.value)) return false;
         return true;
     }
-    else return v.type == tInt || v.type == tBool || v.type == tString ||
-                v.type == tFloat;
+    else return v.type == tInt || v.type == tBool || v.isString() || v.type == tFloat;
 }
 
 
@@ -194,36 +193,36 @@ Value * DrvInfo::queryMeta(const string & name)
 
 string DrvInfo::queryMetaString(const string & name)
 {
-    Value * v = queryMeta(name);
-    if (!v || v->type != tString) return "";
-    return v->string.s;
+    auto v = queryMeta(name);
+    if (!v || !v->isString()) return "";
+    return v->getString();
 }
 
 
 NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
 {
-    Value * v = queryMeta(name);
+    auto v = queryMeta(name);
     if (!v) return def;
     if (v->type == tInt) return v->integer;
-    if (v->type == tString) {
+    if (v->isString()) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
         NixInt n;
-        if (string2Int(v->string.s, n)) return n;
+        if (string2Int(v->getString(), n)) return n;
     }
     return def;
 }
 
 NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 {
-    Value * v = queryMeta(name);
+    auto v = queryMeta(name);
     if (!v) return def;
     if (v->type == tFloat) return v->fpoint;
-    if (v->type == tString) {
+    if (v->isString()) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
         NixFloat n;
-        if (string2Float(v->string.s, n)) return n;
+        if (string2Float(v->getString(), n)) return n;
     }
     return def;
 }
@@ -231,14 +230,15 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 
 bool DrvInfo::queryMetaBool(const string & name, bool def)
 {
-    Value * v = queryMeta(name);
+    auto v = queryMeta(name);
     if (!v) return def;
     if (v->type == tBool) return v->boolean;
-    if (v->type == tString) {
+    if (v->isString()) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
-        if (strcmp(v->string.s, "true") == 0) return true;
-        if (strcmp(v->string.s, "false") == 0) return false;
+        auto s = v->getString();
+        if (strcmp(s, "true") == 0) return true;
+        if (strcmp(s, "false") == 0) return false;
     }
     return def;
 }
@@ -247,8 +247,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 void DrvInfo::setMeta(const string & name, Value * v)
 {
     getMeta();
-    Bindings * old = meta;
-    meta = state->allocBindings(1 + (old ? old->size() : 0));
+    Ptr<Bindings> old = meta;
+    meta = Bindings::allocBindings(1 + (old ? old->size() : 0));
     Symbol sym = state->symbols.create(name);
     if (old)
         for (auto i : *old)
@@ -260,6 +260,7 @@ void DrvInfo::setMeta(const string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
+// FIXME: Use Ptr?
 typedef set<Bindings *> Done;
 
 
@@ -277,8 +278,7 @@ static bool getDerivation(EvalState & state, Value & v,
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
            derivation {...}; y = x;}'. */
-        if (done.find(v.attrs) != done.end()) return false;
-        done.insert(v.attrs);
+        if (!done.insert(v.attrs).second) return false;
 
         DrvInfo drv(state, attrPath, v.attrs);
 
@@ -320,24 +320,24 @@ static void getDerivations(EvalState & state, Value & vIn,
     DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
-    Value v;
+    Root<Value> v;
     state.autoCallFunction(autoArgs, vIn, v);
 
     /* Process the expression. */
     if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
 
-    else if (v.type == tAttrs) {
+    else if (v->type == tAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
+        bool combineChannels = v->attrs->find(state.symbols.create("_combineChannels")) != v->attrs->end();
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        for (auto & i : v.attrs->lexicographicOrder()) {
+        for (auto & i : v->attrs->lexicographicOrder()) {
             debug("evaluating attribute '%1%'", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
@@ -357,11 +357,11 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else if (v.isList()) {
-        for (unsigned int n = 0; n < v.listSize(); ++n) {
+    else if (v->isList()) {
+        for (unsigned int n = 0; n < v->listSize(); ++n) {
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
-                getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+            if (getDerivation(state, *v->listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
+                getDerivations(state, *v->listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
     }
 

src/libexpr/get-drvs.hh

@@ -26,7 +26,8 @@ private:
 
     bool failed = false; // set if we get an AssertionError
 
-    Bindings * attrs = nullptr, * meta = nullptr;
+    Ptr<Bindings> attrs;
+    Ptr<Bindings> meta;
 
     Bindings * getMeta();
 
@@ -69,11 +70,7 @@ public:
 };
 
 
-#if HAVE_BOEHMGC
-typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
-#else
-typedef list<DrvInfo> DrvInfos;
-#endif
+typedef std::list<DrvInfo> DrvInfos;
 
 
 /* If value `v' denotes a derivation, return a DrvInfo object

src/libexpr/json-to-value.cc

@@ -1,149 +1,162 @@
 #include "json-to-value.hh"
 
-#include <cstring>
+#include <variant>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+using std::unique_ptr;
 
 namespace nix {
 
+// for more information, refer to
+// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
+class JSONSax : nlohmann::json_sax<json> {
 
-static void skipWhitespace(const char * & s)
-{
-    while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
-}
+    class JSONState {
+    protected:
+        unique_ptr<JSONState> parent;
+        Ptr<Value> v;
+    public:
+        virtual unique_ptr<JSONState> resolve(EvalState &)
+        {
+            throw std::logic_error("tried to close toplevel json parser state");
+        };
+        explicit JSONState(unique_ptr<JSONState> && p) : parent(std::move(p)), v(nullptr) {};
+        explicit JSONState(Value * v) : v(v) {};
+        JSONState(JSONState & p) = delete;
+        Value & value(EvalState & state)
+        {
+            if (!v)
+                v = state.allocValue();
+            return *v;
+        };
+        virtual ~JSONState() {};
+        virtual void add() {};
+    };
 
-
-static string parseJSONString(const char * & s)
-{
-    string res;
-    if (*s++ != '"') throw JSONParseError("expected JSON string");
-    while (*s != '"') {
-        if (!*s) throw JSONParseError("got end-of-string in JSON string");
-        if (*s == '\\') {
-            s++;
-            if (*s == '"') res += '"';
-            else if (*s == '\\') res += '\\';
-            else if (*s == '/') res += '/';
-            else if (*s == '/') res += '/';
-            else if (*s == 'b') res += '\b';
-            else if (*s == 'f') res += '\f';
-            else if (*s == 'n') res += '\n';
-            else if (*s == 'r') res += '\r';
-            else if (*s == 't') res += '\t';
-            else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported");
-            else throw JSONParseError("invalid escaped character in JSON string");
-            s++;
-        } else
-            res += *s++;
-    }
-    s++;
-    return res;
-}
-
-
-static void parseJSON(EvalState & state, const char * & s, Value & v)
-{
-    skipWhitespace(s);
-
-    if (!*s) throw JSONParseError("expected JSON value");
-
-    if (*s == '[') {
-        s++;
-        ValueVector values;
-        values.reserve(128);
-        skipWhitespace(s);
-        while (1) {
-            if (values.empty() && *s == ']') break;
-            Value * v2 = state.allocValue();
-            parseJSON(state, s, *v2);
-            values.push_back(v2);
-            skipWhitespace(s);
-            if (*s == ']') break;
-            if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
-            s++;
+    class JSONObjectState : public JSONState {
+        using JSONState::JSONState;
+        ValueMap attrs = ValueMap();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        {
+            Value & v = parent->value(state);
+            state.mkAttrs(v, attrs.size());
+            for (auto & i : attrs)
+                v.attrs->push_back(Attr(i.first, i.second));
+            return std::move(parent);
         }
-        s++;
-        state.mkList(v, values.size());
-        for (size_t n = 0; n < values.size(); ++n)
-            v.listElems()[n] = values[n];
-    }
-
-    else if (*s == '{') {
-        s++;
-        ValueMap attrs;
-        while (1) {
-            skipWhitespace(s);
-            if (attrs.empty() && *s == '}') break;
-            string name = parseJSONString(s);
-            skipWhitespace(s);
-            if (*s != ':') throw JSONParseError("expected ':' in JSON object");
-            s++;
-            Value * v2 = state.allocValue();
-            parseJSON(state, s, *v2);
-            attrs[state.symbols.create(name)] = v2;
-            skipWhitespace(s);
-            if (*s == '}') break;
-            if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
-            s++;
+        virtual void add() override { v = nullptr; };
+    public:
+        void key(string_t & name, EvalState & state)
+        {
+            attrs[state.symbols.create(name)] = &value(state);
         }
-        state.mkAttrs(v, attrs.size());
-        for (auto & i : attrs)
-            v.attrs->push_back(Attr(i.first, i.second));
-        v.attrs->sort();
-        s++;
-    }
+    };
 
-    else if (*s == '"') {
-        mkString(v, parseJSONString(s));
-    }
-
-    else if (isdigit(*s) || *s == '-' || *s == '.' ) {
-        // Buffer into a string first, then use built-in C++ conversions
-        std::string tmp_number;
-        ValueType number_type = tInt;
-
-        while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
-            if (*s == '.' || *s == 'e' || *s == 'E')
-                number_type = tFloat;
-            tmp_number += *s++;
+    class JSONListState : public JSONState {
+        ValueVector values = ValueVector();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        {
+            Value & v = parent->value(state);
+            state.mkList(v, values.size());
+            for (size_t n = 0; n < values.size(); ++n) {
+                v.listElems()[n] = values[n];
+            }
+            return std::move(parent);
         }
-
-        try {
-            if (number_type == tFloat)
-                mkFloat(v, stod(tmp_number));
-            else
-                mkInt(v, stol(tmp_number));
-        } catch (std::invalid_argument e) {
-            throw JSONParseError("invalid JSON number");
-        } catch (std::out_of_range e) {
-            throw JSONParseError("out-of-range JSON number");
+        virtual void add() override {
+            values.push_back(v);
+            v = nullptr;
+        };
+    public:
+        JSONListState(unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
+        {
+            values.reserve(reserve);
         }
+    };
+
+    EvalState & state;
+    unique_ptr<JSONState> rs;
+
+    template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
+    {
+        f(rs->value(state), args...);
+        rs->add();
+        return true;
     }
 
-    else if (strncmp(s, "true", 4) == 0) {
-        s += 4;
-        mkBool(v, true);
+public:
+    JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
+
+    bool null()
+    {
+        return handle_value(mkNull);
     }
 
-    else if (strncmp(s, "false", 5) == 0) {
-        s += 5;
-        mkBool(v, false);
+    bool boolean(bool val)
+    {
+        return handle_value(mkBool, val);
     }
 
-    else if (strncmp(s, "null", 4) == 0) {
-        s += 4;
-        mkNull(v);
+    bool number_integer(number_integer_t val)
+    {
+        return handle_value(mkInt, val);
     }
 
-    else throw JSONParseError("unrecognised JSON value");
-}
+    bool number_unsigned(number_unsigned_t val)
+    {
+        return handle_value(mkInt, val);
+    }
 
+    bool number_float(number_float_t val, const string_t& s)
+    {
+        return handle_value(mkFloat, val);
+    }
+
+    bool string(string_t& val)
+    {
+        return handle_value<Value & (Value &, std::string_view)>(mkString, (std::string_view) val);
+    }
+
+    bool start_object(std::size_t len)
+    {
+        rs = std::make_unique<JSONObjectState>(std::move(rs));
+        return true;
+    }
+
+    bool key(string_t& name)
+    {
+        dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
+        return true;
+    }
+
+    bool end_object() {
+        rs = rs->resolve(state);
+        rs->add();
+        return true;
+    }
+
+    bool end_array() {
+        return end_object();
+    }
+
+    bool start_array(size_t len) {
+        rs = std::make_unique<JSONListState>(std::move(rs),
+            len != std::numeric_limits<size_t>::max() ? len : 128);
+        return true;
+    }
+
+    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
+        throw JSONParseError(ex.what());
+    }
+};
 
 void parseJSON(EvalState & state, const string & s_, Value & v)
 {
-    const char * s = s_.c_str();
-    parseJSON(state, s, v);
-    skipWhitespace(s);
-    if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s);
+    JSONSax parser(state, v);
+    bool res = json::sax_parse(s_, &parser);
+    if (!res)
+        throw JSONParseError("Invalid JSON Value");
 }
 
-
 }

src/libexpr/json-to-value.hh

@@ -6,7 +6,7 @@
 
 namespace nix {
 
-MakeError(JSONParseError, EvalError)
+MakeError(JSONParseError, EvalError);
 
 void parseJSON(EvalState & state, const string & s, Value & v);
 

src/libexpr/local.mk

@@ -6,18 +6,13 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore
+libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
 endif
 
-# The dependency on libgc must be propagated (i.e. meaning that
-# programs/libraries that use libexpr must explicitly pass -lgc),
-# because inline functions in libexpr's header files call libgc.
-libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
-
 libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y

src/libexpr/names.cc

@@ -16,14 +16,14 @@ DrvName::DrvName()
    a letter.  The `version' part is the rest (excluding the separating
    dash).  E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
    '2.0.48'). */
-DrvName::DrvName(const string & s) : hits(0)
+DrvName::DrvName(std::string_view s) : hits(0)
 {
-    name = fullName = s;
+    name = fullName = std::string(s);
     for (unsigned int i = 0; i < s.size(); ++i) {
         /* !!! isalpha/isdigit are affected by the locale. */
         if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) {
-            name = string(s, 0, i);
-            version = string(s, i + 1);
+            name = s.substr(0, i);
+            version = s.substr(i + 1);
             break;
         }
     }

src/libexpr/names.hh

@@ -15,7 +15,7 @@ struct DrvName
     unsigned int hits;
 
     DrvName();
-    DrvName(const string & s);
+    DrvName(std::string_view s);
     bool matches(DrvName & n);
 
 private:

src/libexpr/nixexpr.cc

@@ -80,7 +80,7 @@ void ExprString::show(std::ostream & str) const
 
 void ExprPath::show(std::ostream & str) const
 {
-    str << s;
+    str << v->path->s;
 }
 
 void ExprVar::show(std::ostream & str) const

src/libexpr/nixexpr.hh

@@ -9,14 +9,14 @@
 namespace nix {
 
 
-MakeError(EvalError, Error)
-MakeError(ParseError, Error)
-MakeError(AssertionError, EvalError)
-MakeError(ThrownError, AssertionError)
-MakeError(Abort, EvalError)
-MakeError(TypeError, EvalError)
-MakeError(UndefinedVarError, Error)
-MakeError(RestrictedPathError, Error)
+MakeError(EvalError, Error);
+MakeError(ParseError, Error);
+MakeError(AssertionError, EvalError);
+MakeError(ThrownError, AssertionError);
+MakeError(Abort, EvalError);
+MakeError(TypeError, EvalError);
+MakeError(UndefinedVarError, Error);
+MakeError(RestrictedPathError, Error);
 
 
 /* Position objects. */
@@ -78,7 +78,7 @@ struct Expr
     virtual void show(std::ostream & str) const;
     virtual void bindVars(const StaticEnv & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
-    virtual Value * maybeThunk(EvalState & state, Env & env);
+    virtual Ptr<Value> maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol & name);
 };
 
@@ -92,28 +92,37 @@ std::ostream & operator << (std::ostream & str, const Expr & e);
 struct ExprInt : Expr
 {
     NixInt n;
-    Value v;
-    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
+    Ptr<Value> v;
+    ExprInt(NixInt n) : n(n) {
+        v = gc.alloc<Value>(Value::words());
+        mkInt(v, n);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprFloat : Expr
 {
     NixFloat nf;
-    Value v;
-    ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
+    Ptr<Value> v;
+    ExprFloat(NixFloat nf) : nf(nf) {
+        v = gc.alloc<Value>(Value::words());
+        mkFloat(v, nf);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprString : Expr
 {
     Symbol s;
-    Value v;
-    ExprString(const Symbol & s) : s(s) { mkString(v, s); };
+    Ptr<Value> v;
+    ExprString(const Symbol & s) : s(s) {
+        v = gc.alloc<Value>(Value::words());
+        mkString(v, s);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 /* Temporary class used during parsing of indented strings. */
@@ -125,11 +134,13 @@ struct ExprIndStr : Expr
 
 struct ExprPath : Expr
 {
-    string s;
-    Value v;
-    ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
+    Ptr<Value> v;
+    ExprPath(const string & s) {
+        v = gc.alloc<Value>(Value::words());
+        mkPath(v, s);
+    };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprVar : Expr
@@ -153,7 +164,7 @@ struct ExprVar : Expr
     ExprVar(const Symbol & name) : name(name) { };
     ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
     COMMON_METHODS
-    Value * maybeThunk(EvalState & state, Env & env);
+    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
 };
 
 struct ExprSelect : Expr

src/libexpr/parser.y

@@ -1,5 +1,5 @@
 %glr-parser
-%pure-parser
+%define api.pure
 %locations
 %define parse.error verbose
 %defines
@@ -20,6 +20,7 @@
 
 #include "nixexpr.hh"
 #include "eval.hh"
+#include "globals.hh"
 
 namespace nix {
 
@@ -138,11 +139,10 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
-    if (formals->argNames.find(formal.name) != formals->argNames.end())
+    if (!formals->argNames.insert(formal.name).second)
         throw ParseError(format("duplicate formal function argument '%1%' at %2%")
             % formal.name % pos);
     formals->formals.push_front(formal);
-    formals->argNames.insert(formal.name);
 }
 
 
@@ -402,7 +402,12 @@ expr_simple
               new ExprVar(data->symbols.create("__nixPath"))),
           new ExprString(data->symbols.create(path)));
   }
-  | URI { $$ = new ExprString(data->symbols.create($1)); }
+  | URI {
+      static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
+      if (noURLLiterals)
+          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+      $$ = new ExprString(data->symbols.create($1));
+  }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
@@ -571,12 +576,17 @@ Path resolveExprPath(Path path)
 {
     assert(path[0] == '/');
 
+    unsigned int followCount = 0, maxFollow = 1024;
+
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
     struct stat st;
     while (true) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
         if (lstat(path.c_str(), &st))
-            throw SysError(format("getting status of '%1%'") % path);
+            throw SysError("getting status of '%s'", path);
         if (!S_ISLNK(st.st_mode)) break;
         path = absPath(readLink(path), dirOf(path));
     }

src/libexpr/primops.cc

@@ -44,29 +44,28 @@ std::pair<string, string> decodeContext(const string & s)
 
 
 InvalidPathError::InvalidPathError(const Path & path) :
-    EvalError(format("path '%1%' is not valid") % path), path(path) {}
+    EvalError("path '%s' is not valid", path), path(path) {}
 
 void EvalState::realiseContext(const PathSet & context)
 {
-    PathSet drvs;
+    std::vector<StorePathWithOutputs> drvs;
 
     for (auto & i : context) {
         std::pair<string, string> decoded = decodeContext(i);
-        Path ctx = decoded.first;
-        assert(store->isStorePath(ctx));
+        auto ctx = store->parseStorePath(decoded.first);
         if (!store->isValidPath(ctx))
-            throw InvalidPathError(ctx);
-        if (!decoded.second.empty() && nix::isDerivation(ctx)) {
-            drvs.insert(decoded.first + "!" + decoded.second);
+            throw InvalidPathError(store->printStorePath(ctx));
+        if (!decoded.second.empty() && ctx.isDerivation()) {
+            drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}});
 
             /* Add the output of this derivation to the allowed
                paths. */
             if (allowedPaths) {
-                auto drv = store->derivationFromPath(decoded.first);
+                auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
                 DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
                 if (i == drv.outputs.end())
                     throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
-                allowedPaths->insert(i->second.path);
+                allowedPaths->insert(store->printStorePath(i->second.path));
             }
         }
     }
@@ -74,10 +73,11 @@ void EvalState::realiseContext(const PathSet & context)
     if (drvs.empty()) return;
 
     if (!evalSettings.enableImportFromDerivation)
-        throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
+        throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
+            store->printStorePath(drvs.begin()->path));
 
     /* For performance, prefetch all substitute info. */
-    PathSet willBuild, willSubstitute, unknown;
+    StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
     store->buildPaths(drvs);
@@ -100,27 +100,27 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
 
-    if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) {
-        Derivation drv = readDerivation(realPath);
-        Value & w = *state.allocValue();
+    // FIXME
+    if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
+        Derivation drv = readDerivation(*state.store, realPath);
+        auto w = state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
-        Value * v2 = state.allocAttr(w, state.sDrvPath);
+        auto v2 = state.allocAttr(w, state.sDrvPath);
         mkString(*v2, path, {"=" + path});
         v2 = state.allocAttr(w, state.sName);
         mkString(*v2, drv.env["name"]);
-        Value * outputsVal =
-            state.allocAttr(w, state.symbols.create("outputs"));
+        auto outputsVal = state.allocAttr(w, state.symbols.create("outputs"));
         state.mkList(*outputsVal, drv.outputs.size());
         unsigned int outputs_index = 0;
 
         for (const auto & o : drv.outputs) {
             v2 = state.allocAttr(w, state.symbols.create(o.first));
-            mkString(*v2, o.second.path, {"!" + o.first + "!" + path});
+            mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path});
             outputsVal->listElems()[outputs_index] = state.allocValue();
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
-        w.attrs->sort();
-        Value fun;
+        w->attrs->sort();
+        auto fun = state.allocValue();
         state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
         state.forceFunction(fun, pos);
         mkApp(v, fun, w);
@@ -130,8 +130,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         if (args[0]->attrs->empty())
             state.evalFile(realPath, v);
         else {
-            Env * env = &state.allocEnv(args[0]->attrs->size());
-            env->up = &state.baseEnv;
+            auto env = state.allocEnv(args[0]->attrs->size());
+            env->up = state.baseEnv;
 
             StaticEnv staticEnv(false, &state.staticBaseEnv);
 
@@ -239,19 +239,24 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
     switch (args[0]->type) {
         case tInt: t = "int"; break;
         case tBool: t = "bool"; break;
-        case tString: t = "string"; break;
+        case tShortString:
+        case tStaticString:
+        case tLongString:
+            t = "string"; break;
         case tPath: t = "path"; break;
         case tNull: t = "null"; break;
         case tAttrs: t = "set"; break;
-        case tList1: case tList2: case tListN: t = "list"; break;
+        case tList0: case tList1: case tList2: case tListN: t = "list"; break;
         case tLambda:
         case tPrimOp:
         case tPrimOpApp:
             t = "lambda";
             break;
+#if 0
         case tExternal:
             t = args[0]->external->typeOf();
             break;
+#endif
         case tFloat: t = "float"; break;
         default: abort();
     }
@@ -304,7 +309,7 @@ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    mkBool(v, args[0]->type == tString);
+    mkBool(v, args[0]->isString());
 }
 
 
@@ -330,6 +335,8 @@ struct CompareValues
             return v1->fpoint < v2->integer;
         if (v1->type == tInt && v2->type == tFloat)
             return v1->integer < v2->fpoint;
+        if (v1->isString() && v2->isString())
+            return strcmp(v1->getString(), v2->getString()) < 0;
         if (v1->type != v2->type)
             throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         switch (v1->type) {
@@ -337,10 +344,8 @@ struct CompareValues
                 return v1->integer < v2->integer;
             case tFloat:
                 return v1->fpoint < v2->fpoint;
-            case tString:
-                return strcmp(v1->string.s, v2->string.s) < 0;
             case tPath:
-                return strcmp(v1->path, v2->path) < 0;
+                return strcmp(v1->path->s, v2->path->s) < 0;
             default:
                 throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         }
@@ -348,11 +353,7 @@ struct CompareValues
 };
 
 
-#if HAVE_BOEHMGC
-typedef list<Value *, gc_allocator<Value *> > ValueList;
-#else
-typedef list<Value *> ValueList;
-#endif
+typedef list<Ptr<Value>> ValueList;
 
 
 static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -382,10 +383,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
        no new elements are found. */
     ValueList res;
     // `doneKeys' doesn't need to be a GC root, because its values are
-    // reachable from res.
+    // reachable from res. FIXME: dubious.
     set<Value *, CompareValues> doneKeys;
     while (!workSet.empty()) {
-        Value * e = *(workSet.begin());
+        auto e = *(workSet.begin());
         workSet.pop_front();
 
         state.forceAttrs(*e, pos);
@@ -396,19 +397,18 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
             throw EvalError(format("attribute 'key' required, at %1%") % pos);
         state.forceValue(*key->value);
 
-        if (doneKeys.find(key->value) != doneKeys.end()) continue;
-        doneKeys.insert(key->value);
+        if (!doneKeys.insert(key->value).second) continue;
         res.push_back(e);
 
         /* Call the `operator' function with `e' as argument. */
-        Value call;
+        Root<Value> call;
         mkApp(call, *op->value, *e);
         state.forceList(call, pos);
 
         /* Add the values returned by the operator to the work set. */
-        for (unsigned int n = 0; n < call.listSize(); ++n) {
-            state.forceValue(*call.listElems()[n]);
-            workSet.push_back(call.listElems()[n]);
+        for (unsigned int n = 0; n < call->listSize(); ++n) {
+            state.forceValue(*call->listElems()[n]);
+            workSet.push_back(call->listElems()[n]);
         }
     }
 
@@ -470,7 +470,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name));
+    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or(""));
 }
 
 
@@ -498,22 +498,15 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    if (args[0]->type == tString)
-        printError(format("trace: %1%") % args[0]->string.s);
+    if (args[0]->isString())
+        printError("trace: %s", args[0]->getString());
     else
-        printError(format("trace: %1%") % *args[0]);
+        printError("trace: %s", *args[0]);
     state.forceValue(*args[1]);
     v = *args[1];
 }
 
 
-void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    /* We're not forcing the argument on purpose. */
-    mkInt(v, valueSize(*args[0]));
-}
-
-
 /*************************************************************
  * Derivations
  *************************************************************/
@@ -684,24 +677,24 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            runs. */
         if (path.at(0) == '=') {
             /* !!! This doesn't work if readOnlyMode is set. */
-            PathSet refs;
-            state.store->computeFSClosure(string(path, 1), refs);
+            StorePathSet refs;
+            state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
             for (auto & j : refs) {
-                drv.inputSrcs.insert(j);
-                if (isDerivation(j))
-                    drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j);
+                drv.inputSrcs.insert(j.clone());
+                if (j.isDerivation())
+                    drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j);
             }
         }
 
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
-            drv.inputDrvs[ctx.first].insert(ctx.second);
+            drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second);
         }
 
         /* Otherwise it's a source file. */
         else
-            drv.inputSrcs.insert(path);
+            drv.inputSrcs.insert(state.store->parseStorePath(path));
     }
 
     /* Do we have all required attributes? */
@@ -711,10 +704,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName);
 
     /* Check whether the derivation name is valid. */
-    checkStoreName(drvName);
     if (isDerivation(drvName))
-        throw EvalError(format("derivation names are not allowed to end in '%1%', at %2%")
-            % drvExtension % posDrvName);
+        throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName);
 
     if (outputHash) {
         /* Handle fixed-output derivations. */
@@ -724,52 +715,55 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
 
-        Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
-        if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath,
-            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
-            h.to_string(Base16, false));
+        auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
+        if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
+        drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
+                (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+                h.to_string(Base16, false)));
     }
 
     else {
-        /* Construct the "masked" store derivation, which is the final
-           one except that in the list of outputs, the output paths
-           are empty, and the corresponding environment variables have
-           an empty value.  This ensures that changes in the set of
-           output names do get reflected in the hash. */
+        /* Compute a hash over the "masked" store derivation, which is
+           the final one except that in the list of outputs, the
+           output paths are empty strings, and the corresponding
+           environment variables have an empty value.  This ensures
+           that changes in the set of output names do get reflected in
+           the hash. */
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
-            drv.outputs[i] = DerivationOutput("", "", "");
+            drv.outputs.insert_or_assign(i,
+                DerivationOutput(StorePath::dummy.clone(), "", ""));
         }
 
-        /* Use the masked derivation expression to compute the output
-           path. */
-        Hash h = hashDerivationModulo(*state.store, drv);
+        Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
 
-        for (auto & i : drv.outputs)
-            if (i.second.path == "") {
-                Path outPath = state.store->makeOutputPath(i.first, h, drvName);
-                if (!jsonObject) drv.env[i.first] = outPath;
-                i.second.path = outPath;
-            }
+        for (auto & i : outputs) {
+            auto outPath = state.store->makeOutputPath(i, h, drvName);
+            if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
+            drv.outputs.insert_or_assign(i,
+                DerivationOutput(std::move(outPath), "", ""));
+        }
     }
 
     /* Write the resulting term into the Nix store directory. */
-    Path drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+    auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+    auto drvPathS = state.store->printStorePath(drvPath);
 
-    printMsg(lvlChatty, format("instantiated '%1%' -> '%2%'")
-        % drvName % drvPath);
+    if (state.derivationHook) state.derivationHook(drvPath, drv);
+
+    printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
 
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store derivations, so we can't
        read them later. */
-    drvHashes[drvPath] = hashDerivationModulo(*state.store, drv);
+    drvHashes.insert_or_assign(drvPath.clone(),
+        hashDerivationModulo(*state.store, Derivation(drv), false));
 
     state.mkAttrs(v, 1 + drv.outputs.size());
-    mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath});
+    mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
     for (auto & i : drv.outputs) {
         mkString(*state.allocAttr(v, state.symbols.create(i.first)),
-            i.second.path, {"!" + i.first + "!" + drvPath});
+            state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS});
     }
     v.attrs->sort();
 }
@@ -822,7 +816,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
         throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos);
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
-        state.store->ensurePath(path2);
+        state.store->ensurePath(state.store->parseStorePath(path2));
     context.insert(path2);
     mkString(v, path, context);
 }
@@ -958,15 +952,20 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     DirEntries entries = readDirectory(state.checkSourcePath(path));
     state.mkAttrs(v, entries.size());
 
+    auto sRegular = state.symbols.create("regular");
+    auto sDirectory = state.symbols.create("directory");
+    auto sSymlink = state.symbols.create("symlink");
+    auto sUnknown = state.symbols.create("unknown");
+
     for (auto & ent : entries) {
-        Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name));
+        auto ent_val = state.allocAttr(v, state.symbols.create(ent.name));
         if (ent.type == DT_UNKNOWN)
             ent.type = getFileType(path + "/" + ent.name);
-        mkStringNoCopy(*ent_val,
-            ent.type == DT_REG ? "regular" :
-            ent.type == DT_DIR ? "directory" :
-            ent.type == DT_LNK ? "symlink" :
-            "unknown");
+        mkString(*ent_val,
+            ent.type == DT_REG ? sRegular :
+            ent.type == DT_DIR ? sDirectory :
+            ent.type == DT_LNK ? sSymlink :
+            sUnknown);
     }
 
     v.attrs->sort();
@@ -1018,17 +1017,17 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     string name = state.forceStringNoCtx(*args[0], pos);
     string contents = state.forceString(*args[1], context, pos);
 
-    PathSet refs;
+    StorePathSet refs;
 
     for (auto path : context) {
         if (path.at(0) != '/')
             throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-        refs.insert(path);
+        refs.insert(state.store->parseStorePath(path));
     }
 
-    Path storePath = settings.readOnlyMode
+    auto storePath = state.store->printStorePath(settings.readOnlyMode
         ? state.store->computeStorePathForText(name, contents, refs)
-        : state.store->addTextToStore(name, contents, refs, state.repair);
+        : state.store->addTextToStore(name, contents, refs, state.repair));
 
     /* Note: we don't need to add `context' to the context of the
        result, since `storePath' itself has references to the paths
@@ -1049,40 +1048,37 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
 
         /* Call the filter function.  The first argument is the path,
            the second is a string indicating the type of the file. */
-        Value arg1;
+        auto arg1 = state.allocValue();
         mkString(arg1, path);
 
-        Value fun2;
+        Root<Value> fun2;
         state.callFunction(*filterFun, arg1, fun2, noPos);
 
-        Value arg2;
+        auto arg2 = state.allocValue();
         mkString(arg2,
             S_ISREG(st.st_mode) ? "regular" :
             S_ISDIR(st.st_mode) ? "directory" :
             S_ISLNK(st.st_mode) ? "symlink" :
             "unknown" /* not supported, will fail! */);
 
-        Value res;
+        Root<Value> res;
         state.callFunction(fun2, arg2, res, noPos);
 
         return state.forceBool(res, pos);
     }) : defaultPathFilter;
 
-    Path expectedStorePath;
-    if (expectedHash) {
-        expectedStorePath =
-            state.store->makeFixedOutputPath(recursive, expectedHash, name);
-    }
+    std::optional<StorePath> expectedStorePath;
+    if (expectedHash)
+        expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name);
     Path dstPath;
-    if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
-        dstPath = settings.readOnlyMode
+    if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+        dstPath = state.store->printStorePath(settings.readOnlyMode
             ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
-            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
-        if (expectedHash && expectedStorePath != dstPath) {
-            throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
-        }
+            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair));
+        if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+            throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
     } else
-        dstPath = expectedStorePath;
+        dstPath = state.store->printStorePath(*expectedStorePath);
 
     mkString(v, dstPath, {dstPath});
 }
@@ -1099,7 +1095,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (args[0]->type != tLambda)
         throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
-    addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
 }
 
 static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1157,7 +1153,7 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V
         mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
 
     std::sort(v.listElems(), v.listElems() + n,
-              [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
+        [](Value * v1, Value * v2) { return strcmp(v1->getString(), v2->getString()) < 0; });
 }
 
 
@@ -1234,10 +1230,9 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
 
     /* Get the attribute names to be removed. */
     std::set<Symbol> names;
-    for (unsigned int i = 0; i < args[1]->listSize(); ++i) {
-        state.forceStringNoCtx(*args[1]->listElems()[i], pos);
-        names.insert(state.symbols.create(args[1]->listElems()[i]->string.s));
-    }
+    for (unsigned int i = 0; i < args[1]->listSize(); ++i)
+        names.insert(state.symbols.create(
+                state.forceStringNoCtx(*args[1]->listElems()[i], pos)));
 
     /* Copy all attributes not in that set.  Note that we don't need
        to sort v.attrs because it's a subset of an already sorted
@@ -1273,13 +1268,12 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
-        if (seen.find(sym) == seen.end()) {
+        if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
                 throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos);
 
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
-            seen.insert(sym);
         }
     }
 
@@ -1374,8 +1368,8 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
     state.mkAttrs(v, args[1]->attrs->size());
 
     for (auto & i : *args[1]->attrs) {
-        Value * vName = state.allocValue();
-        Value * vFun2 = state.allocValue();
+        auto vName = state.allocValue();
+        auto vFun2 = state.allocValue();
         mkString(*vName, i.name);
         mkApp(*vFun2, *args[0], *vName);
         mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
@@ -1462,7 +1456,7 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu
 
     bool same = true;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
-        Value res;
+        Root<Value> res;
         state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos);
         if (state.forceBool(res, pos))
             vs[k++] = args[1]->listElems()[n];
@@ -1517,12 +1511,12 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceList(*args[2], pos);
 
     if (args[2]->listSize()) {
-        Value * vCur = args[1];
+        Ptr<Value> vCur = args[1];
 
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
-            Value vTmp;
+            Root<Value> vTmp; // FIXME: correct?
             state.callFunction(*args[0], *vCur, vTmp, pos);
-            vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
+            vCur = n == args[2]->listSize() - 1 ? Ptr(&v) : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
         state.forceValue(v);
@@ -1538,7 +1532,7 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg
     state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
-    Value vTmp;
+    Root<Value> vTmp;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
         state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos);
         bool res = state.forceBool(vTmp, pos);
@@ -1574,7 +1568,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     state.mkList(v, len);
 
     for (unsigned int n = 0; n < (unsigned int) len; ++n) {
-        Value * arg = state.allocValue();
+        auto arg = state.allocValue();
         mkInt(*arg, n);
         mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg);
     }
@@ -1603,7 +1597,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
         if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan)
             return CompareValues()(a, b);
 
-        Value vTmp1, vTmp2;
+        Root<Value> vTmp1, vTmp2; // FIXME
         state.callFunction(*args[0], *a, vTmp1, pos);
         state.callFunction(vTmp1, *b, vTmp2, pos);
         return state.forceBool(vTmp2, pos);
@@ -1623,12 +1617,13 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     auto len = args[1]->listSize();
 
-    ValueVector right, wrong;
+    // Note: these Values are reachable via args[0].
+    std::vector<Value *> right, wrong;
 
     for (unsigned int n = 0; n < len; ++n) {
         auto vElem = args[1]->listElems()[n];
         state.forceValue(*vElem);
-        Value res;
+        Root<Value> res;
         state.callFunction(*args[0], *vElem, res, pos);
         if (state.forceBool(res, pos))
             right.push_back(vElem);
@@ -1638,13 +1633,13 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     state.mkAttrs(v, 2);
 
-    Value * vRight = state.allocAttr(v, state.sRight);
+    auto vRight = state.allocAttr(v, state.sRight);
     auto rsize = right.size();
     state.mkList(*vRight, rsize);
     if (rsize)
         memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize);
 
-    Value * vWrong = state.allocAttr(v, state.sWrong);
+    auto vWrong = state.allocAttr(v, state.sWrong);
     auto wsize = wrong.size();
     state.mkList(*vWrong, wsize);
     if (wsize)
@@ -1662,22 +1657,23 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
     state.forceList(*args[1], pos);
     auto nrLists = args[1]->listSize();
 
-    Value lists[nrLists];
+    // FIXME: Root<>[] is inefficient
+    Root<Value> lists[nrLists];
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
         state.forceList(lists[n], pos);
-        len += lists[n].listSize();
+        len += lists[n]->listSize();
     }
 
     state.mkList(v, len);
     auto out = v.listElems();
     for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
-        auto l = lists[n].listSize();
+        auto l = lists[n]->listSize();
         if (l)
-            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
+            memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
         pos += l;
     }
 }
@@ -2089,12 +2085,12 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     if (evalSettings.pureEval && !request.expectedHash)
         throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who);
 
-    Path res = getDownloader()->downloadCached(state.store, request).path;
+    auto res = getDownloader()->downloadCached(state.store, request);
 
     if (state.allowedPaths)
-        state.allowedPaths->insert(res);
+        state.allowedPaths->insert(res.path);
 
-    mkString(v, res, PathSet({res}));
+    mkString(v, res.storePath, PathSet({res.storePath}));
 }
 
 
@@ -2127,10 +2123,10 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
 
 void EvalState::createBaseEnv()
 {
-    baseEnv.up = 0;
+    baseEnv->up = 0;
 
     /* Add global constants such as `true' to the base environment. */
-    Value v;
+    auto v = allocValue();
 
     /* `builtins' must be first! */
     mkAttrs(v, 128);
@@ -2148,7 +2144,7 @@ void EvalState::createBaseEnv()
     auto vThrow = addPrimOp("throw", 1, prim_throw);
 
     auto addPurityError = [&](const std::string & name) {
-        Value * v2 = allocValue();
+        auto v2 = allocValue();
         mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name));
         mkApp(v, *vThrow, *v2);
         addConstant(name, v);
@@ -2160,7 +2156,7 @@ void EvalState::createBaseEnv()
     }
 
     if (!evalSettings.pureEval) {
-        mkString(v, settings.thisSystem);
+        mkString(v, settings.thisSystem.get());
         addConstant("__currentSystem", v);
     }
 
@@ -2179,7 +2175,7 @@ void EvalState::createBaseEnv()
 
     // Miscellaneous
     auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport);
-    Value * v2 = allocValue();
+    auto v2 = allocValue();
     mkAttrs(*v2, 0);
     mkApp(v, *vScopedImport, *v2);
     forceValue(v);
@@ -2208,7 +2204,6 @@ void EvalState::createBaseEnv()
 
     // Debugging
     addPrimOp("__trace", 2, prim_trace);
-    addPrimOp("__valueSize", 1, prim_valueSize);
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);
@@ -2308,7 +2303,7 @@ void EvalState::createBaseEnv()
     mkList(v, searchPath.size());
     int n = 0;
     for (auto & i : searchPath) {
-        v2 = v.listElems()[n++] = allocValue();
+        v2 = v->listElems()[n++] = allocValue();
         mkAttrs(*v2, 2);
         mkString(*allocAttr(*v2, symbols.create("path")), i.second);
         mkString(*allocAttr(*v2, symbols.create("prefix")), i.first);
@@ -2322,7 +2317,7 @@ void EvalState::createBaseEnv()
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
-    baseEnv.values[0]->attrs->sort();
+    baseEnv->values[0]->attrs->sort();
 }
 
 

src/libexpr/primops/context.cc

@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (!state.store->isStorePath(i.name))
             throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
         if (!settings.readOnlyMode)
-            state.store->ensurePath(i.name);
+            state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
         auto iter = i.value->attrs->find(sPath);
         if (iter != i.value->attrs->end()) {
@@ -160,7 +160,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                    throw EvalError("tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +170,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
+                throw EvalError("tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fetchGit.cc

@@ -4,6 +4,7 @@
 #include "store-api.hh"
 #include "pathlocks.hh"
 #include "hash.hh"
+#include "tarfile.hh"
 
 #include <sys/time.h>
 
@@ -38,16 +39,14 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         try {
             runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
-        } catch (ExecError e) {
+        } catch (ExecError & e) {
             if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
             clean = false;
         }
 
         if (!clean) {
 
-            /* This is an unclean working tree. So copy all tracked
-               files. */
-
+            /* This is an unclean working tree. So copy all tracked files. */
             GitInfo gitInfo;
             gitInfo.rev = "0000000000000000000000000000000000000000";
             gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
@@ -70,7 +69,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
 
             return gitInfo;
         }
@@ -157,7 +156,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         gitInfo.storePath = json["storePath"];
 
-        if (store->isValidPath(gitInfo.storePath)) {
+        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
             gitInfo.revCount = json["revCount"];
             return gitInfo;
         }
@@ -166,16 +165,18 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (e.errNo != ENOENT) throw;
     }
 
-    // FIXME: should pipe this, or find some better way to extract a
-    // revision.
-    auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev });
+    auto source = sinkToSource([&](Sink & sink) {
+        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+        gitOptions.standardOut = &sink;
+        runProgram2(gitOptions);
+    });
 
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
 
-    runProgram("tar", true, { "x", "-C", tmpDir }, tar);
+    unpackTarfile(*source, tmpDir);
 
-    gitInfo.storePath = store->addToStore(name, tmpDir);
+    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
 
     gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
 

src/libexpr/primops/fetchMercurial.cc

@@ -63,7 +63,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
 
             return hgInfo;
         }
@@ -134,7 +134,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
 
         hgInfo.storePath = json["storePath"];
 
-        if (store->isValidPath(hgInfo.storePath)) {
+        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
             printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
             return hgInfo;
         }
@@ -150,7 +150,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
 
     deletePath(tmpDir + "/.hg_archival.txt");
 
-    hgInfo.storePath = store->addToStore(name, tmpDir);
+    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
 
     nlohmann::json json;
     json["storePath"] = hgInfo.storePath;

src/libexpr/symbol-table.hh

@@ -19,6 +19,7 @@ private:
     const string * s; // pointer into SymbolTable
     Symbol(const string * s) : s(s) { };
     friend class SymbolTable;
+    friend class Value;
 
 public:
     Symbol() : s(0) { };
@@ -38,7 +39,12 @@ public:
         return s < s2.s;
     }
 
-    operator const string & () const
+    operator const std::string & () const
+    {
+        return *s;
+    }
+
+    operator const std::string_view () const
     {
         return *s;
     }
@@ -84,4 +90,6 @@ public:
     }
 };
 
+extern SymbolTable symbols;
+
 }

src/libexpr/value-to-json.cc

@@ -26,13 +26,15 @@ void printValueAsJSON(EvalState & state, bool strict,
             out.write(v.boolean);
             break;
 
-        case tString:
-            copyContext(v, context);
-            out.write(v.string.s);
+        case tShortString:
+        case tStaticString:
+        case tLongString:
+            v.getContext(context);
+            out.write(v.getString());
             break;
 
         case tPath:
-            out.write(state.copyPathToStore(context, v.path));
+            out.write(state.copyPathToStore(context, v.path->s));
             break;
 
         case tNull:
@@ -40,7 +42,12 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case tAttrs: {
-            Bindings::iterator i = v.attrs->find(state.sOutPath);
+            auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
+            if (maybeString) {
+                out.write(*maybeString);
+                break;
+            }
+            auto i = v.attrs->find(state.sOutPath);
             if (i == v.attrs->end()) {
                 auto obj(out.object());
                 StringSet names;
@@ -56,7 +63,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-        case tList1: case tList2: case tListN: {
+        case tList0: case tList1: case tList2: case tListN: {
             auto list(out.list());
             for (unsigned int n = 0; n < v.listSize(); ++n) {
                 auto placeholder(list.placeholder());
@@ -65,9 +72,11 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
+#if 0
         case tExternal:
             v.external->printValueAsJSON(state, strict, out, context);
             break;
+#endif
 
         case tFloat:
             out.write(v.fpoint);
@@ -85,11 +94,13 @@ void printValueAsJSON(EvalState & state, bool strict,
     printValueAsJSON(state, strict, v, out, context);
 }
 
+#if 0
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
     throw TypeError(format("cannot convert %1% to JSON") % showType());
 }
+#endif
 
 
 }

src/libexpr/value-to-xml.cc

@@ -68,14 +68,16 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
             break;
 
-        case tString:
+        case tShortString:
+        case tStaticString:
+        case tLongString:
             /* !!! show the context? */
-            copyContext(v, context);
-            doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
+            v.getContext(context);
+            doc.writeEmptyElement("string", singletonAttrs("value", v.getString()));
             break;
 
         case tPath:
-            doc.writeEmptyElement("path", singletonAttrs("value", v.path));
+            doc.writeEmptyElement("path", singletonAttrs("value", v.path->s));
             break;
 
         case tNull:
@@ -92,23 +94,22 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
-                        xmlAttrs["drvPath"] = drvPath = a->value->string.s;
+                    if (a->value->isString())
+                        xmlAttrs["drvPath"] = drvPath = a->value->getString();
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->type == tString)
-                        xmlAttrs["outPath"] = a->value->string.s;
+                    if (a->value->isString())
+                        xmlAttrs["outPath"] = a->value->getString();
                 }
 
                 XMLOpenElement _(doc, "derivation", xmlAttrs);
 
-                if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) {
-                    drvsSeen.insert(drvPath);
+                if (drvPath != "" && drvsSeen.insert(drvPath).second)
                     showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
-                } else
+                else
                     doc.writeEmptyElement("repeated");
             }
 
@@ -119,7 +120,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
             break;
 
-        case tList1: case tList2: case tListN: {
+        case tList0: case tList1: case tList2: case tListN: {
             XMLOpenElement _(doc, "list");
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
@@ -144,9 +145,11 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             break;
         }
 
+#if 0
         case tExternal:
             v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
             break;
+#endif
 
         case tFloat:
             doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
@@ -158,11 +161,13 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 }
 
 
+#if 0
 void ExternalValueBase::printValueAsXML(EvalState & state, bool strict,
     bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const
 {
     doc.writeEmptyElement("unevaluated");
 }
+#endif
 
 
 void printValueAsXML(EvalState & state, bool strict, bool location,

src/libexpr/value.hh

@@ -1,41 +1,17 @@
 #pragma once
 
 #include "symbol-table.hh"
+#include "gc.hh"
 
-#if HAVE_BOEHMGC
-#include <gc/gc_allocator.h>
-#endif
+#include <cstring>
 
 namespace nix {
 
-
-typedef enum {
-    tInt = 1,
-    tBool,
-    tString,
-    tPath,
-    tNull,
-    tAttrs,
-    tList1,
-    tList2,
-    tListN,
-    tThunk,
-    tApp,
-    tLambda,
-    tBlackhole,
-    tPrimOp,
-    tPrimOpApp,
-    tExternal,
-    tFloat
-} ValueType;
-
-
 class Bindings;
 struct Env;
 struct Expr;
 struct ExprLambda;
 struct PrimOp;
-struct PrimOp;
 class Symbol;
 struct Pos;
 class EvalState;
@@ -46,13 +22,15 @@ class JSONPlaceholder;
 typedef int64_t NixInt;
 typedef double NixFloat;
 
+#if 0
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
  */
 class ExternalValueBase
 {
     friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
-    protected:
+
+protected:
     /* Print out the value */
     virtual std::ostream & print(std::ostream & str) const = 0;
 
@@ -63,9 +41,6 @@ class ExternalValueBase
     /* Return a string to be used in builtins.typeOf */
     virtual string typeOf() const = 0;
 
-    /* How much space does this value take up */
-    virtual size_t valueSize(std::set<const void *> & seen) const = 0;
-
     /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an
      * error
      */
@@ -90,11 +65,29 @@ class ExternalValueBase
 };
 
 std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
+#endif
 
 
-struct Value
+class Context : Object
+{
+    friend class Value;
+    friend class GC;
+
+    Symbol members[0];
+
+    Context(const PathSet & context) : Object(tContext, context.size())
+    {
+        size_t n = 0;
+        for (auto & i : context)
+            members[n++] = symbols.create(i);
+    }
+
+    size_t getSize() { return getMisc(); }
+};
+
+
+struct Value : Object
 {
-    ValueType type;
     union
     {
         NixInt integer;
@@ -117,20 +110,15 @@ struct Value
            with context C is used as a derivation attribute, then the
            derivations in C will be added to the inputDrvs of the
            derivation, and the other store paths in C will be added to
-           the inputSrcs of the derivations.
-
-           For canonicity, the store paths should be in sorted order. */
+           the inputSrcs of the derivations. */
         struct {
-            const char * s;
-            const char * * context; // must be in sorted order
+            String * s;
+            Context * context;
         } string;
-
-        const char * path;
+        const char * staticString;
+        String * path;
         Bindings * attrs;
-        struct {
-            size_t size;
-            Value * * elems;
-        } bigList;
+        PtrList<Value> * bigList;
         Value * smallList[2];
         struct {
             Env * env;
@@ -144,46 +132,99 @@ struct Value
             ExprLambda * fun;
         } lambda;
         PrimOp * primOp;
-        struct {
-            Value * left, * right;
-        } primOpApp;
-        ExternalValueBase * external;
+        //ExternalValueBase * external;
         NixFloat fpoint;
     };
 
+private:
+
+    Value() : Object(tNull, 0) {}
+
+    friend class GC;
+    template<typename T> friend class Root;
+
+public:
+
     bool isList() const
     {
-        return type == tList1 || type == tList2 || type == tListN;
+        return type >= tList0 && type <= tListN;
     }
 
     Value * * listElems()
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return type == tList0 || type == tList1 || type == tList2 ? smallList : bigList->elems;
     }
 
     const Value * const * listElems() const
     {
-        return type == tList1 || type == tList2 ? smallList : bigList.elems;
+        return type == tList0 || type == tList1 || type == tList2 ? smallList : bigList->elems;
     }
 
     size_t listSize() const
     {
-        return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
+        return type == tList0 ? 0 : type == tList1 ? 1 : type == tList2 ? 2 : bigList->size();
+    }
+
+    constexpr static size_t words() { return 3; } // FIXME
+
+    void setContext(const PathSet & context)
+    {
+        if (context.size() == 0)
+            string.context = nullptr;
+        else if (context.size() == 1) {
+            // If we have a single context, then store it
+            // directly. This saves allocating a Context object (16
+            // bytes).
+            auto symbol = symbols.create(*context.begin());
+            string.context = (Context *) (((ptrdiff_t) symbol.s) | 1);
+        } else {
+            string.context = gc.alloc<Context>(1 + context.size(), context);
+        }
+    }
+
+    void getContext(PathSet & context)
+    {
+        if (type == tLongString && string.context) {
+            if (((ptrdiff_t) string.context) & 1) {
+                auto s = (const std::string *) (((ptrdiff_t) string.context) & ~1UL);
+                context.insert(*s);
+            } else {
+                auto size = string.context->getSize();
+                for (size_t i = 0; i < size; ++i)
+                    context.insert(string.context->members[i]);
+            }
+        }
+    }
+
+    bool isString() const
+    {
+        return type == tShortString || type == tStaticString || type == tLongString;
+    }
+
+    void setShortString(std::string_view s)
+    {
+        // FIXME: can't use strcpy here because gcc flags it as a
+        // buffer overflow on 'misc'.
+        auto p = getMiscData();
+        memcpy(p, s.data(), s.size());
+        p[s.size()] = 0;
+        type = tShortString;
+    }
+
+    const char * getString() const
+    {
+        if (type == tShortString)
+            return getMiscData();
+        else if (type == tStaticString)
+            return staticString;
+        else
+            return string.s->s;
     }
 };
 
 
-/* After overwriting an app node, be sure to clear pointers in the
-   Value to ensure that the target isn't kept alive unnecessarily. */
-static inline void clearValue(Value & v)
-{
-    v.app.left = v.app.right = 0;
-}
-
-
 static inline void mkInt(Value & v, NixInt n)
 {
-    clearValue(v);
     v.type = tInt;
     v.integer = n;
 }
@@ -191,7 +232,6 @@ static inline void mkInt(Value & v, NixInt n)
 
 static inline void mkFloat(Value & v, NixFloat n)
 {
-    clearValue(v);
     v.type = tFloat;
     v.fpoint = n;
 }
@@ -199,7 +239,6 @@ static inline void mkFloat(Value & v, NixFloat n)
 
 static inline void mkBool(Value & v, bool b)
 {
-    clearValue(v);
     v.type = tBool;
     v.boolean = b;
 }
@@ -207,7 +246,6 @@ static inline void mkBool(Value & v, bool b)
 
 static inline void mkNull(Value & v)
 {
-    clearValue(v);
     v.type = tNull;
 }
 
@@ -228,47 +266,35 @@ static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
 }
 
 
-static inline void mkStringNoCopy(Value & v, const char * s)
+static inline Value & mkString(Value & v, std::string_view s)
 {
-    v.type = tString;
-    v.string.s = s;
-    v.string.context = 0;
+    if (s.size() < WORD_SIZE * 2 + Object::miscBytes)
+        v.setShortString(s);
+    else {
+        v.string.s = gc.alloc<String>(String::wordsFor(s.size()), s.size(), s.data());
+        v.string.context = 0;
+        v.type = tLongString;
+    }
+    return v;
 }
 
 
 static inline void mkString(Value & v, const Symbol & s)
 {
-    mkStringNoCopy(v, ((const string &) s).c_str());
+    v.staticString = ((const string &) s).c_str();
+    v.type = tStaticString;
 }
 
 
-void mkString(Value & v, const char * s);
-
-
-static inline void mkPathNoCopy(Value & v, const char * s)
+static inline void mkPath(Value & v, const std::string & s)
 {
-    clearValue(v);
+    v.path = String::alloc(s.c_str());
     v.type = tPath;
-    v.path = s;
 }
 
 
-void mkPath(Value & v, const char * s);
-
-
-/* Compute the size in bytes of the given value, including all values
-   and environments reachable from it. Static expressions (Exprs) are
-   not included. */
-size_t valueSize(Value & v);
-
-
-#if HAVE_BOEHMGC
-typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
-typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
-#else
-typedef std::vector<Value *> ValueVector;
-typedef std::map<Symbol, Value *> ValueMap;
-#endif
+typedef std::vector<Ptr<Value>> ValueVector; // FIXME: make more efficient
+typedef std::map<Symbol, Ptr<Value>> ValueMap; // FIXME: use Bindings?
 
 
 }

src/libmain/shared.cc

@@ -33,25 +33,25 @@ void printGCWarning()
 }
 
 
-void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl)
+void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
 {
     unsigned long long downloadSize, narSize;
-    PathSet willBuild, willSubstitute, unknown;
+    StorePathSet willBuild, willSubstitute, unknown;
     store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
     printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
 }
 
 
-void printMissing(ref<Store> store, const PathSet & willBuild,
-    const PathSet & willSubstitute, const PathSet & unknown,
+void printMissing(ref<Store> store, const StorePathSet & willBuild,
+    const StorePathSet & willSubstitute, const StorePathSet & unknown,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
         printMsg(lvl, "these derivations will be built:");
-        Paths sorted = store->topoSortPaths(willBuild);
+        auto sorted = store->topoSortPaths(willBuild);
         reverse(sorted.begin(), sorted.end());
         for (auto & i : sorted)
-            printMsg(lvl, fmt("  %s", i));
+            printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
 
     if (!willSubstitute.empty()) {
@@ -59,14 +59,14 @@ void printMissing(ref<Store> store, const PathSet & willBuild,
                 downloadSize / (1024.0 * 1024.0),
                 narSize / (1024.0 * 1024.0)));
         for (auto & i : willSubstitute)
-            printMsg(lvl, fmt("  %s", i));
+            printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
 
     if (!unknown.empty()) {
         printMsg(lvl, fmt("don't know how to build these paths%s:",
                 (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")));
         for (auto & i : unknown)
-            printMsg(lvl, fmt("  %s", i));
+            printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }
 }
 
@@ -80,6 +80,7 @@ string getArg(const string & opt,
 }
 
 
+#if OPENSSL_VERSION_NUMBER < 0x10101000L
 /* OpenSSL is not thread-safe by default - it will randomly crash
    unless the user supplies a mutex locking function. So let's do
    that. */
@@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
     else
         opensslLocks[type].unlock();
 }
+#endif
 
 
 static void sigHandler(int signo) { }
@@ -105,9 +107,11 @@ void initNix()
     std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));
 #endif
 
+#if OPENSSL_VERSION_NUMBER < 0x10101000L
     /* Initialise OpenSSL locking. */
     opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks());
     CRYPTO_set_locking_callback(opensslLockCallback);
+#endif
 
     loadConfFile();
 
@@ -151,7 +155,7 @@ void initNix()
        sshd). This breaks build users because they don't have access
        to the TMPDIR, in particular in ‘nix-store --serve’. */
 #if __APPLE__
-    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/"))
+    if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
 }
@@ -233,7 +237,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish)
 void parseCmdLine(int argc, char * * argv,
     std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg)
 {
-    parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg);
+    parseCmdLine(std::string(baseNameOf(argv[0])), argvToStrings(argc, argv), parseArg);
 }
 
 
@@ -249,9 +253,6 @@ void printVersion(const string & programName)
     std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
     if (verbosity > lvlInfo) {
         Strings cfg;
-#if HAVE_BOEHMGC
-        cfg.push_back("gc");
-#endif
 #if HAVE_SODIUM
         cfg.push_back("signed-caches");
 #endif

src/libmain/shared.hh

@@ -3,6 +3,7 @@
 #include "util.hh"
 #include "args.hh"
 #include "common-args.hh"
+#include "path.hh"
 
 #include <signal.h>
 
@@ -37,11 +38,15 @@ void printVersion(const string & programName);
 void printGCWarning();
 
 class Store;
+struct StorePathWithOutputs;
 
-void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo);
+void printMissing(
+    ref<Store> store,
+    const std::vector<StorePathWithOutputs> & paths,
+    Verbosity lvl = lvlInfo);
 
-void printMissing(ref<Store> store, const PathSet & willBuild,
-    const PathSet & willSubstitute, const PathSet & unknown,
+void printMissing(ref<Store> store, const StorePathSet & willBuild,
+    const StorePathSet & willSubstitute, const StorePathSet & unknown,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
 
 string getArg(const string & opt,
@@ -52,23 +57,7 @@ template<class N> N getIntArg(const string & opt,
 {
     ++i;
     if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
-    string s = *i;
-    N multiplier = 1;
-    if (allowUnit && !s.empty()) {
-        char u = std::toupper(*s.rbegin());
-        if (std::isalpha(u)) {
-            if (u == 'K') multiplier = 1ULL << 10;
-            else if (u == 'M') multiplier = 1ULL << 20;
-            else if (u == 'G') multiplier = 1ULL << 30;
-            else if (u == 'T') multiplier = 1ULL << 40;
-            else throw UsageError(format("invalid unit specifier '%1%'") % u);
-            s.resize(s.size() - 1);
-        }
-    }
-    N n;
-    if (!string2Int(s, n))
-        throw UsageError(format("'%1%' requires an integer argument") % opt);
-    return n * multiplier;
+    return parseSize<N>(*i, allowUnit);
 }
 
 

src/libstore/binary-cache-store.cc

@@ -10,10 +10,13 @@
 #include "nar-info-disk-cache.hh"
 #include "nar-accessor.hh"
 #include "json.hh"
+#include "thread-pool.hh"
 
 #include <chrono>
-
 #include <future>
+#include <regex>
+
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -46,9 +49,9 @@ void BinaryCacheStore::init()
                     throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'")
                         % getUri() % value % storeDir);
             } else if (name == "WantMassQuery") {
-                wantMassQuery_ = value == "1";
+                wantMassQuery.setDefault(value == "1" ? "true" : "false");
             } else if (name == "Priority") {
-                string2Int(value, priority);
+                priority.setDefault(fmt("%d", std::stoi(value)));
             }
         }
     }
@@ -88,19 +91,18 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
     return sink.s;
 }
 
-Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
+std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
 {
-    assertStorePath(storePath);
-    return storePathToHash(storePath) + ".narinfo";
+    return storePathToHash(printStorePath(storePath)) + ".narinfo";
 }
 
 void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 {
     auto narInfoFile = narInfoFileFor(narInfo->path);
 
-    upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo");
+    upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    auto hashPart = storePathToHash(narInfo->path);
+    auto hashPart = storePathToHash(printStorePath(narInfo->path));
 
     {
         auto state_(state.lock());
@@ -123,8 +125,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             if (ref != info.path)
                 queryPathInfo(ref);
         } catch (InvalidPath &) {
-            throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid")
-                % info.path % ref);
+            throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid",
+                printStorePath(info.path), printStorePath(ref));
         }
 
     assert(nar->compare(0, narMagic.size(), narMagic) == 0);
@@ -135,10 +137,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     narInfo->narHash = hashString(htSHA256, *nar);
 
     if (info.narHash && info.narHash != narInfo->narHash)
-        throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path);
+        throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
 
     auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
 
+    auto narAccessor = makeNarAccessor(nar);
+
+    if (accessor_)
+        accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
+
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -148,23 +155,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             JSONObject jsonRoot(jsonOut);
             jsonRoot.attr("version", 1);
 
-            auto narAccessor = makeNarAccessor(nar);
-
-            if (accessor_)
-                accessor_->addToCache(info.path, *nar, narAccessor);
-
             {
                 auto res = jsonRoot.placeholder("root");
                 listNar(res, narAccessor, "", true);
             }
         }
 
-        upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
-    }
-
-    else {
-        if (accessor_)
-            accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
+        upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json");
     }
 
     /* Compress the NAR. */
@@ -176,17 +173,75 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     narInfo->fileSize = narCompressed->size();
 
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
-    printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
-        % narInfo->path % narInfo->narSize
-        % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
-        % duration);
+    printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+        printStorePath(narInfo->path), narInfo->narSize,
+        ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
+        duration);
 
-    /* Atomically write the NAR file. */
     narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
            compression == "br" ? ".br" :
            "");
+
+    /* Optionally maintain an index of DWARF debug info files
+       consisting of JSON files named 'debuginfo/<build-id>' that
+       specify the NAR file and member containing the debug info. */
+    if (writeDebugInfo) {
+
+        std::string buildIdDir = "/lib/debug/.build-id";
+
+        if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
+
+            ThreadPool threadPool(25);
+
+            auto doFile = [&](std::string member, std::string key, std::string target) {
+                checkInterrupt();
+
+                nlohmann::json json;
+                json["archive"] = target;
+                json["member"] = member;
+
+                // FIXME: or should we overwrite? The previous link may point
+                // to a GC'ed file, so overwriting might be useful...
+                if (fileExists(key)) return;
+
+                printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
+
+                upsertFile(key, json.dump(), "application/json");
+            };
+
+            std::regex regex1("^[0-9a-f]{2}$");
+            std::regex regex2("^[0-9a-f]{38}\\.debug$");
+
+            for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
+                auto dir = buildIdDir + "/" + s1;
+
+                if (narAccessor->stat(dir).type != FSAccessor::tDirectory
+                    || !std::regex_match(s1, regex1))
+                    continue;
+
+                for (auto & s2 : narAccessor->readDirectory(dir)) {
+                    auto debugPath = dir + "/" + s2;
+
+                    if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
+                        || !std::regex_match(s2, regex2))
+                        continue;
+
+                    auto buildId = s1 + s2;
+
+                    std::string key = "debuginfo/" + buildId;
+                    std::string target = "../" + narInfo->url;
+
+                    threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
+                }
+            }
+
+            threadPool.process();
+        }
+    }
+
+    /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
         upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
@@ -198,14 +253,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
     stats.narWriteCompressionTimeMs += duration;
 
     /* Atomically write the NAR info file.*/
-    if (secretKey) narInfo->sign(*secretKey);
+    if (secretKey) narInfo->sign(*this, *secretKey);
 
     writeNarInfo(narInfo);
 
     stats.narInfoWrite++;
 }
 
-bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
+bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath)
 {
     // FIXME: this only checks whether a .narinfo with a matching hash
     // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
@@ -213,7 +268,7 @@ bool BinaryCacheStore::isValidPathUncached(const Path & storePath)
     return fileExists(narInfoFileFor(storePath));
 }
 
-void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
+void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
@@ -239,12 +294,13 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
     stats.narReadBytes += narSize;
 }
 
-void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
-    Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept
+void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     auto uri = getUri();
+    auto storePathS = printStorePath(storePath);
     auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo,
-        fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri});
+        fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri});
     PushActivity pact(act->id);
 
     auto narInfoFile = narInfoFileFor(storePath);
@@ -270,7 +326,7 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath,
         }});
 }
 
-Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
+StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
     bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
 {
     // FIXME: some cut&paste from LocalStore::addToStore().
@@ -289,20 +345,18 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
         h = hashString(hashAlgo, s);
     }
 
-    ValidPathInfo info;
-    info.path = makeFixedOutputPath(recursive, h, name);
+    ValidPathInfo info(makeFixedOutputPath(recursive, h, name));
 
     addToStore(info, sink.s, repair, CheckSigs, nullptr);
 
-    return info.path;
+    return std::move(info.path);
 }
 
-Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
-    const PathSet & references, RepairFlag repair)
+StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
+    const StorePathSet & references, RepairFlag repair)
 {
-    ValidPathInfo info;
-    info.path = computeStorePathForText(name, s, references);
-    info.references = references;
+    ValidPathInfo info(computeStorePathForText(name, s, references));
+    info.references = cloneStorePathSet(references);
 
     if (repair || !isValidPath(info.path)) {
         StringSink sink;
@@ -310,7 +364,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
         addToStore(info, sink.s, repair, CheckSigs, nullptr);
     }
 
-    return info.path;
+    return std::move(info.path);
 }
 
 ref<FSAccessor> BinaryCacheStore::getFSAccessor()
@@ -318,7 +372,7 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
     return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
 }
 
-void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs)
+void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
 {
     /* Note: this is inherently racy since there is no locking on
        binary caches. In particular, with S3 this unreliable, even
@@ -334,24 +388,22 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s
     writeNarInfo(narInfo);
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
+std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
-    Path drvPath;
+    auto drvPath = path.clone();
 
-    if (isDerivation(path))
-        drvPath = path;
-    else {
+    if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
-            if (info->deriver == "") return nullptr;
-            drvPath = info->deriver;
+            if (!info->deriver) return nullptr;
+            drvPath = info->deriver->clone();
         } catch (InvalidPath &) {
             return nullptr;
         }
     }
 
-    auto logPath = "log/" + baseNameOf(drvPath);
+    auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath)));
 
     debug("fetching build log from binary cache '%s/%s'", getUri(), logPath);
 

src/libstore/binary-cache-store.hh

@@ -17,6 +17,7 @@ public:
 
     const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+    const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
     const Setting<bool> parallelCompression{this, false, "parallel-compression",
@@ -51,11 +52,6 @@ public:
 
     std::shared_ptr<std::string> getFile(const std::string & path);
 
-protected:
-
-    bool wantMassQuery_ = false;
-    int priority = 50;
-
 public:
 
     virtual void init();
@@ -64,49 +60,45 @@ private:
 
     std::string narMagic;
 
-    std::string narInfoFileFor(const Path & storePath);
+    std::string narInfoFileFor(const StorePath & storePath);
 
     void writeNarInfo(ref<NarInfo> narInfo);
 
 public:
 
-    bool isValidPathUncached(const Path & path) override;
+    bool isValidPathUncached(const StorePath & path) override;
 
-    void queryPathInfoUncached(const Path & path,
-        Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override;
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
-    Path queryPathFromHashPart(const string & hashPart) override
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
-    bool wantMassQuery() override { return wantMassQuery_; }
-
     void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,
         std::shared_ptr<FSAccessor> accessor) override;
 
-    Path addToStore(const string & name, const Path & srcPath,
+    StorePath addToStore(const string & name, const Path & srcPath,
         bool recursive, HashType hashAlgo,
         PathFilter & filter, RepairFlag repair) override;
 
-    Path addTextToStore(const string & name, const string & s,
-        const PathSet & references, RepairFlag repair) override;
+    StorePath addTextToStore(const string & name, const string & s,
+        const StorePathSet & references, RepairFlag repair) override;
 
-    void narFromPath(const Path & path, Sink & sink) override;
+    void narFromPath(const StorePath & path, Sink & sink) override;
 
-    BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+    BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override
     { unsupported("buildDerivation"); }
 
-    void ensurePath(const Path & path) override
+    void ensurePath(const StorePath & path) override
     { unsupported("ensurePath"); }
 
     ref<FSAccessor> getFSAccessor() override;
 
-    void addSignatures(const Path & storePath, const StringSet & sigs) override;
+    void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
-    std::shared_ptr<std::string> getBuildLog(const Path & path) override;
-
-    int getPriority() override { return priority; }
+    std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
 
 };
 

src/libstore/builtins.hh

@@ -7,5 +7,6 @@ namespace nix {
 // TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
 void builtinBuildenv(const BasicDerivation & drv);
+void builtinUnpackChannel(const BasicDerivation & drv);
 
 }

src/libstore/builtins/buildenv.cc

@@ -123,8 +123,7 @@ static Path out;
 
 static void addPkg(const Path & pkgDir, int priority)
 {
-    if (done.count(pkgDir)) return;
-    done.insert(pkgDir);
+    if (!done.insert(pkgDir).second) return;
     createLinks(pkgDir, out, priority);
 
     try {