downloader: when retry but can't resume, retry the whole file

(cherry picked from commit 2c692a3b14)

.dir-locals.el

@@ -13,4 +13,5 @@
   (eval . (c-set-offset 'arglist-cont-nonempty '+))
   (eval . (c-set-offset 'substatement-open 0))
   (eval . (c-set-offset 'access-label '-))
+  (eval . (c-set-offset 'inlambda 0))
   )))

.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: "Test"
+on:
+  pull_request:
+  push:
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v8
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

.gitignore

@@ -4,9 +4,10 @@ perl/Makefile.config
 # /
 /aclocal.m4
 /autom4te.cache
+/precompiled-headers.h.gch
+/precompiled-headers.h.pch
 /config.*
 /configure
-/nix.spec
 /stamp-h1
 /svn-revision
 /libtool
@@ -46,7 +47,7 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/*.gen.hh
+*.gen.*
 
 /src/nix/nix
 
@@ -84,6 +85,7 @@ perl/Makefile.config
 /tests/restricted-innocent
 /tests/shell
 /tests/shell.drv
+/tests/config.nix
 
 # /tests/lang/
 /tests/lang/*.out
@@ -117,3 +119,5 @@ GPATH
 GRTAGS
 GSYMS
 GTAGS
+
+nix-rust/target

.travis.yml

@@ -1,2 +1,8 @@
-os: osx
-script: ./tests/install-darwin.sh
+matrix:
+  include:
+    - language: osx
+      script: ./tests/install-darwin.sh
+    - language: nix
+      script: nix-build release.nix -A build.x86_64-linux
+notifications:
+  email: false

.version

@@ -1 +1 @@
-2.3.3
+2.4

Makefile

@@ -1,5 +1,7 @@
 makefiles = \
+  mk/precompiled-headers.mk \
   local.mk \
+  nix-rust/local.mk \
   src/libutil/local.mk \
   src/libstore/local.mk \
   src/libmain/local.mk \
@@ -15,8 +17,16 @@ makefiles = \
   tests/local.mk \
   tests/plugins/local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h
-
 -include Makefile.config
 
+OPTIMIZE = 1
+
+ifeq ($(OPTIMIZE), 1)
+  GLOBAL_CXXFLAGS += -O3
+else
+  GLOBAL_CXXFLAGS += -O0
+endif
+
 include mk/lib.mk
+
+GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17

Makefile.config.in

@@ -18,6 +18,7 @@ SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@
@@ -36,6 +37,7 @@ prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
+system = @system@
 doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

README.md

@@ -9,16 +9,13 @@ appear with Nix.
 
 To find out more about the tool, usage and installation instructions, please
 read the manual, which is available on the Nix website at
-<http://nixos.org/nix/manual>.
+<https://nixos.org/nix/manual>.
 
 ## Contributing
 
-Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking)
+Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking)
 of the manual. It helps you to get started with building Nix from source.
 
 ## License
 
 Nix is released under the LGPL v2.1
-
-This product includes software developed by the OpenSSL Project for
-use in the [OpenSSL Toolkit](http://www.OpenSSL.org/).

configure.ac

@@ -50,14 +50,11 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
 test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var
 
 
-# Set default flags for nix (as per AC_PROG_CC/CXX docs),
-# while still allowing the user to override them from the command line.
-: ${CFLAGS="-O3"}
-: ${CXXFLAGS="-O3"}
+CFLAGS=
+CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 AC_PROG_CPP
-AX_CXX_COMPILE_STDCXX_17([noext], [mandatory])
 
 AC_CHECK_TOOL([AR], [ar])
 
@@ -120,26 +117,15 @@ fi
 ])
 
 NEED_PROG(bash, bash)
-NEED_PROG(patch, patch)
 AC_PATH_PROG(xmllint, xmllint, false)
 AC_PATH_PROG(xsltproc, xsltproc, false)
 AC_PATH_PROG(flex, flex, false)
 AC_PATH_PROG(bison, bison, false)
-NEED_PROG(sed, sed)
-NEED_PROG(tar, tar)
-NEED_PROG(bzip2, bzip2)
-NEED_PROG(gzip, gzip)
-NEED_PROG(xz, xz)
 AC_PATH_PROG(dot, dot)
 AC_PATH_PROG(lsof, lsof, lsof)
 
 
-NEED_PROG(cat, cat)
-NEED_PROG(tr, tr)
-AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
-  [path of cat, mkdir, etc.]),
-  coreutils=$withval, coreutils=$(dirname $cat))
-AC_SUBST(coreutils)
+AC_SUBST(coreutils, [$(dirname $(type -p cat))])
 
 
 AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH],
@@ -157,8 +143,33 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix
 # ends up with LDFLAGS being empty, so we set it afterwards.
 LDFLAGS="$BOOST_LDFLAGS $LDFLAGS"
 
+# On some platforms, new-style atomics need a helper library
+AC_MSG_CHECKING(whether -latomic is needed)
+AC_LINK_IFELSE([AC_LANG_SOURCE([[
+#include <stdint.h>
+uint64_t v;
+int main() {
+    return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE);
+}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
+AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
+if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
+    LIBS="-latomic $LIBS"
+fi
 
-# Look for OpenSSL, a required dependency.
+PKG_PROG_PKG_CONFIG
+
+AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
+  [Build shared libraries for Nix [default=yes]]),
+  shared=$enableval, shared=yes)
+if test "$shared" = yes; then
+  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
+else
+  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
+  PKG_CONFIG="$PKG_CONFIG --static"
+fi
+
+# Look for OpenSSL, a required dependency. FIXME: this is only (maybe)
+# used by S3BinaryCacheStore.
 PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"])
 
 
@@ -167,12 +178,12 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
-
+# Checks for libarchive
+PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])
 
-
 # Look for libcurl, a required dependency.
 PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"])
 
@@ -195,12 +206,15 @@ PKG_CHECK_MODULES([SODIUM], [libsodium],
    have_sodium=1], [have_sodium=])
 AC_SUBST(HAVE_SODIUM, [$have_sodium])
 
-
 # Look for liblzma, a required dependency.
 PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
 AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
   [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
 
+# Look for zlib, a required dependency.
+PKG_CHECK_MODULES([ZLIB], [zlib], [CXXFLAGS="$ZLIB_CFLAGS $CXXFLAGS"])
+AC_CHECK_HEADER([zlib.h],[:],[AC_MSG_ERROR([could not find the zlib.h header])])
+LDFLAGS="-lz $LDFLAGS"
 
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
@@ -243,8 +257,8 @@ fi
 
 # Whether to use the Boehm garbage collector.
 AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
-  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]),
-  gc=$enableval, gc=no)
+  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]),
+  gc=$enableval, gc=yes)
 if test "$gc" = yes; then
   PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
   CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
@@ -290,16 +304,6 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH],
   sandbox_shell=$withval)
 AC_SUBST(sandbox_shell)
 
-AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared],
-  [Build shared libraries for Nix [default=yes]]),
-  shared=$enableval, shared=yes)
-if test "$shared" = yes; then
-  AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.])
-else
-  AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.])
-fi
-
-
 # Expand all variables in config.status.
 test "$prefix" = NONE && prefix=$ac_default_prefix
 test "$exec_prefix" = NONE && exec_prefix='${prefix}'

corepkgs/config.nix.in

@@ -1,29 +1,13 @@
+# FIXME: remove this file?
 let
   fromEnv = var: def:
     let val = builtins.getEnv var; in
     if val != "" then val else def;
 in rec {
-  shell = "@bash@";
-  coreutils = "@coreutils@";
-  bzip2 = "@bzip2@";
-  gzip = "@gzip@";
-  xz = "@xz@";
-  tar = "@tar@";
-  tarFlags = "@tarFlags@";
-  tr = "@tr@";
   nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@";
   nixPrefix = "@prefix@";
   nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@";
   nixLocalstateDir = "@localstatedir@";
   nixSysconfDir = "@sysconfdir@";
   nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@";
-
-  # If Nix is installed in the Nix store, then automatically add it as
-  # a dependency to the core packages. This ensures that they work
-  # properly in a chroot.
-  chrootDeps =
-    if dirOf nixPrefix == builtins.storeDir then
-      [ (builtins.storePath nixPrefix) ]
-    else
-      [ ];
 }

corepkgs/local.mk

@@ -1,4 +1,7 @@
-corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix
+corepkgs_FILES = \
+  unpack-channel.nix \
+  derivation.nix \
+  fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
 

corepkgs/unpack-channel.nix

@@ -1,39 +1,12 @@
-with import <nix/config.nix>;
-
-let
-
-  builder = builtins.toFile "unpack-channel.sh"
-    ''
-      mkdir $out
-      cd $out
-      xzpat="\.xz\$"
-      gzpat="\.gz\$"
-      if [[ "$src" =~ $xzpat ]]; then
-        ${xz} -d < $src | ${tar} xf - ${tarFlags}
-      elif [[ "$src" =~ $gzpat ]]; then
-        ${gzip} -d < $src | ${tar} xf - ${tarFlags}
-      else
-        ${bzip2} -d < $src | ${tar} xf - ${tarFlags}
-      fi
-      if [ * != $channelName ]; then
-        mv * $out/$channelName
-      fi
-    '';
-
-in
-
 { name, channelName, src }:
 
 derivation {
-  system = builtins.currentSystem;
-  builder = shell;
-  args = [ "-e" builder ];
-  inherit name channelName src;
+  builder = "builtin:unpack-channel";
 
-  PATH = "${nixBinDir}:${coreutils}";
+  system = "builtin";
+
+  inherit name channelName src;
 
   # No point in doing this remotely.
   preferLocalBuild = true;
-
-  inherit chrootDeps;
 }

doc/manual/advanced-topics/cores-vs-jobs.xml

@@ -36,8 +36,8 @@ to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" />
 equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar>
 will be the total number of cores in the system.</para>
 
-<para>The total number of consumed cores is a simple multiplication,
-<xref linkend="conf-cores" /> * <envar>NIX_BUILD_CORES</envar>.</para>
+<para>The maximum number of consumed cores is a simple multiplication,
+<xref linkend="conf-max-jobs" /> * <envar>NIX_BUILD_CORES</envar>.</para>
 
 <para>The balance on how to set these two independent variables depends
 upon each builder's workload and hardware. Here are a few example

doc/manual/advanced-topics/post-build-hook.xml

@@ -5,7 +5,7 @@
       version="5.0"
       >
 
-<title>Using the <xref linkend="conf-post-build-hook" /></title>
+<title>Using the <option linkend="conf-post-build-hook">post-build-hook</option></title>
 <subtitle>Uploading to an S3-compatible binary cache after each build</subtitle>
 
 

doc/manual/command-ref/nix-build.xml

@@ -100,7 +100,7 @@ also <xref linkend="sec-common-options" />.</phrase></para>
   </varlistentry>
 
   <varlistentry><term><option>--dry-run</option></term>
-   <listitem><para>Show what store paths would be built or downloaded</para></listitem>
+   <listitem><para>Show what store paths would be built or downloaded.</para></listitem>
   </varlistentry>
 
   <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> /

doc/manual/command-ref/nix-channel.xml

@@ -36,6 +36,9 @@ stay up-to-date with a set of pre-built Nix expressions.  A Nix
 channel is just a URL that points to a place containing a set of Nix
 expressions.  <phrase condition="manual">See also <xref
 linkend="sec-channels" />.</phrase></para>
+      
+<para>To see the list of official NixOS channels, visit <link
+xlink:href="https://nixos.org/channels" />.</para>
 
 <para>This command has the following operations:
 
@@ -111,13 +114,13 @@ $ nix-env -iA nixpkgs.hello</screen>
 <para>You can revert channel updates using <option>--rollback</option>:</para>
 
 <screen>
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
+$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
 "14.04.527.0e935f1"
 
 $ nix-channel --rollback
 switching from generation 483 to 482
 
-$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.nixpkgsVersion'
+$ nix-instantiate --eval -E '(import &lt;nixpkgs> {}).lib.version'
 "14.04.526.dbadfad"
 </screen>
 

doc/manual/command-ref/nix-env.xml

@@ -1066,7 +1066,8 @@ user environment elements, etc. -->
     the derivation, which can be used to unambiguously select it using
     the <link linkend="opt-attr"><option>--attr</option> option</link>
     available in commands that install derivations like
-    <literal>nix-env --install</literal>.</para></listitem>
+    <literal>nix-env --install</literal>. This option only works
+    together with <option>--available</option></para></listitem>
 
   </varlistentry>
 

doc/manual/command-ref/nix-shell.xml

@@ -39,7 +39,12 @@
           <arg choice='plain'><option>--packages</option></arg>
           <arg choice='plain'><option>-p</option></arg>
         </group>
-        <arg choice='plain' rep='repeat'><replaceable>packages</replaceable></arg>
+        <arg choice='plain' rep='repeat'>
+          <group choice='req'>
+            <arg choice="plain"><replaceable>packages</replaceable></arg>
+            <arg choice="plain"><replaceable>expressions</replaceable></arg>
+          </group>
+        </arg>
       </arg>
       <arg><replaceable>path</replaceable></arg>
     </group>
@@ -189,8 +194,8 @@ also <xref linkend="sec-common-options" />.</phrase></para>
 <variablelist>
 
   <varlistentry><term><envar>NIX_BUILD_SHELL</envar></term>
-    
-    <listitem><para>Shell used to start the interactive environment. 
+
+    <listitem><para>Shell used to start the interactive environment.
     Defaults to the <command>bash</command> found in <envar>PATH</envar>.</para></listitem>
 
   </varlistentry>
@@ -222,8 +227,9 @@ $ nix-shell '&lt;nixpkgs>' -A pan --pure \
     --command 'export NIX_DEBUG=1; export NIX_CORES=8; return'
 </screen>
 
-Nix expressions can also be given on the command line.  For instance,
-the following starts a shell containing the packages
+Nix expressions can also be given on the command line using the
+<command>-E</command> and <command>-p</command> flags.
+For instance, the following starts a shell containing the packages
 <literal>sqlite</literal> and <literal>libX11</literal>:
 
 <screen>
@@ -238,6 +244,14 @@ $ nix-shell -p sqlite xorg.libX11
 … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib …
 </screen>
 
+Note that <command>-p</command> accepts multiple full nix expressions that
+are valid in the <literal>buildInputs = [ ... ]</literal> shown above,
+not only package names. So the following is also legal:
+
+<screen>
+$ nix-shell -p sqlite 'git.override { withManual = false; }'
+</screen>
+
 The <command>-p</command> flag looks up Nixpkgs in the Nix search
 path. You can override it by passing <option>-I</option> or setting
 <envar>NIX_PATH</envar>. For example, the following gives you a shell

doc/manual/command-ref/nix-store.xml

@@ -360,7 +360,6 @@ EOF
     <arg choice='plain'><option>--print-roots</option></arg>
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
-    <arg choice='plain'><option>--delete</option></arg>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
 </cmdsynopsis>
@@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of
 
   </varlistentry>
 
-  <varlistentry><term><option>--delete</option></term>
-
-    <listitem><para>This operation performs an actual garbage
-    collection.  All dead paths are removed from the
-    store.  This is the default.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 <para>By default, all unreachable paths are deleted.  The following
@@ -444,10 +435,10 @@ and <link
 linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
-<para>With <option>--delete</option>, the collector prints the total
-number of freed bytes when it finishes (or when it is interrupted).
-With <option>--print-dead</option>, it prints the number of bytes that
-would be freed.</para>
+<para>By default, the collector prints the total number of freed bytes
+when it finishes (or when it is interrupted). With
+<option>--print-dead</option>, it prints the number of bytes that would
+be freed.</para>
 
 </refsection>
 
@@ -1148,7 +1139,7 @@ the information that Nix considers important.  For instance,
 timestamps are elided because all files in the Nix store have their
 timestamp set to 0 anyway.  Likewise, all permissions are left out
 except for the execute bit, because all files in the Nix store have
-644 or 755 permission.</para>
+444 or 555 permission.</para>
 
 <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
 that “equal” paths always produce the same NAR archive.  For instance,

doc/manual/command-ref/opt-common.xml

@@ -243,9 +243,10 @@
 <varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term>
 
   <listitem><para>This option is accepted by
-  <command>nix-env</command>, <command>nix-instantiate</command> and
-  <command>nix-build</command>.  When evaluating Nix expressions, the
-  expression evaluator will automatically try to call functions that
+  <command>nix-env</command>, <command>nix-instantiate</command>,
+  <command>nix-shell</command> and <command>nix-build</command>.
+  When evaluating Nix expressions, the expression evaluator will
+  automatically try to call functions that
   it encounters.  It can automatically call functions for which every
   argument has a <link linkend='ss-functions'>default value</link>
   (e.g., <literal>{ <replaceable>argName</replaceable> ?
@@ -322,7 +323,14 @@
   Nix expressions to be parsed and evaluated, rather than as a list
   of file names of Nix expressions.
   (<command>nix-instantiate</command>, <command>nix-build</command>
-  and <command>nix-shell</command> only.)</para></listitem>
+  and <command>nix-shell</command> only.)</para>
+
+  <para>For <command>nix-shell</command>, this option is commonly used
+  to give you a shell in which you can build the packages returned
+  by the expression. If you want to get a shell which contain the
+  <emphasis>built</emphasis> packages ready for use, give your
+  expression to the <command>nix-shell -p</command> convenience flag
+  instead.</para></listitem>
 
 </varlistentry>
 

doc/manual/expressions/advanced-attributes.xml

@@ -11,7 +11,7 @@ attributes.</para>
 
 <variablelist>
 
-  <varlistentry><term><varname>allowedReferences</varname></term>
+  <varlistentry xml:id="adv-attr-allowedReferences"><term><varname>allowedReferences</varname></term>
 
     <listitem><para>The optional attribute
     <varname>allowedReferences</varname> specifies a list of legal
@@ -32,7 +32,7 @@ allowedReferences = [];
   </varlistentry>
 
 
-  <varlistentry><term><varname>allowedRequisites</varname></term>
+  <varlistentry xml:id="adv-attr-allowedRequisites"><term><varname>allowedRequisites</varname></term>
 
     <listitem><para>This attribute is similar to
     <varname>allowedReferences</varname>, but it specifies the legal
@@ -50,7 +50,7 @@ allowedRequisites = [ foobar ];
 
   </varlistentry>
 
-  <varlistentry><term><varname>disallowedReferences</varname></term>
+  <varlistentry xml:id="adv-attr-disallowedReferences"><term><varname>disallowedReferences</varname></term>
 
     <listitem><para>The optional attribute
     <varname>disallowedReferences</varname> specifies a list of illegal
@@ -67,7 +67,7 @@ disallowedReferences = [ foo ];
   </varlistentry>
 
 
-  <varlistentry><term><varname>disallowedRequisites</varname></term>
+  <varlistentry xml:id="adv-attr-disallowedRequisites"><term><varname>disallowedRequisites</varname></term>
 
     <listitem><para>This attribute is similar to
     <varname>disallowedReferences</varname>, but it specifies illegal
@@ -85,7 +85,7 @@ disallowedRequisites = [ foobar ];
   </varlistentry>
 
 
-  <varlistentry><term><varname>exportReferencesGraph</varname></term>
+  <varlistentry xml:id="adv-attr-exportReferencesGraph"><term><varname>exportReferencesGraph</varname></term>
 
     <listitem><para>This attribute allows builders access to the
     references graph of their inputs.  The attribute is a list of
@@ -124,7 +124,7 @@ derivation {
   </varlistentry>
 
 
-  <varlistentry><term><varname>impureEnvVars</varname></term>
+  <varlistentry xml:id="adv-attr-impureEnvVars"><term><varname>impureEnvVars</varname></term>
 
     <listitem><para>This attribute allows you to specify a list of
     environment variables that should be passed from the environment
@@ -158,9 +158,9 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ];
 
 
   <varlistentry xml:id="fixed-output-drvs">
-    <term><varname>outputHash</varname></term>
-    <term><varname>outputHashAlgo</varname></term>
-    <term><varname>outputHashMode</varname></term>
+    <term xml:id="adv-attr-outputHash"><varname>outputHash</varname></term>
+    <term xml:id="adv-attr-outputHashAlgo"><varname>outputHashAlgo</varname></term>
+    <term xml:id="adv-attr-outputHashMode"><varname>outputHashMode</varname></term>
 
     <listitem><para>These attributes declare that the derivation is a
     so-called <emphasis>fixed-output derivation</emphasis>, which
@@ -282,7 +282,7 @@ stdenv.mkDerivation {
   </varlistentry>
 
 
-  <varlistentry><term><varname>passAsFile</varname></term>
+  <varlistentry xml:id="adv-attr-passAsFile"><term><varname>passAsFile</varname></term>
 
     <listitem><para>A list of names of attributes that should be
     passed via files rather than environment variables.  For example,
@@ -309,7 +309,7 @@ big = "a very long string";
   </varlistentry>
 
 
-  <varlistentry><term><varname>preferLocalBuild</varname></term>
+  <varlistentry xml:id="adv-attr-preferLocalBuild"><term><varname>preferLocalBuild</varname></term>
 
     <listitem><para>If this attribute is set to
     <literal>true</literal> and <link
@@ -323,14 +323,25 @@ big = "a very long string";
   </varlistentry>
 
 
-  <varlistentry><term><varname>allowSubstitutes</varname></term>
+  <varlistentry xml:id="adv-attr-allowSubstitutes"><term><varname>allowSubstitutes</varname></term>
 
-    <listitem><para>If this attribute is set to
+    <listitem>
+    <para>If this attribute is set to
     <literal>false</literal>, then Nix will always build this
     derivation; it will not try to substitute its outputs. This is
     useful for very trivial derivations (such as
     <function>writeText</function> in Nixpkgs) that are cheaper to
-    build than to substitute from a binary cache.</para></listitem>
+    build than to substitute from a binary cache.</para>
+
+    <note><para>You need to have a builder configured which satisfies
+    the derivation’s <literal>system</literal> attribute, since the
+    derivation cannot be substituted. Thus it is usually a good idea
+    to align <literal>system</literal> with
+    <literal>builtins.currentSystem</literal> when setting
+    <literal>allowSubstitutes</literal> to <literal>false</literal>.
+    For most trivial derivations this should be the case.
+    </para></note>
+    </listitem>
 
   </varlistentry>
 

doc/manual/installation/installing-binary.xml

@@ -142,7 +142,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
   </para>
 
   <para>

doc/manual/installation/prerequisites-source.xml

@@ -8,6 +8,14 @@
 
 <itemizedlist>
 
+  <listitem><para>GNU Autoconf
+  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
+  and the autoconf-archive macro collection
+  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
+  These are only needed to run the bootstrap script, and are not necessary
+  if your source distribution came with a pre-built
+  <literal>./configure</literal> script.</para></listitem>
+
   <listitem><para>GNU Make.</para></listitem>
   
   <listitem><para>Bash Shell. The <literal>./configure</literal> script

doc/manual/local.mk

@@ -4,11 +4,10 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
-  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
-  --param callout.graphics.extension \'.gif\' \
+  --param callout.graphics 0 \
   --param contrib.inline.enabled 0 \
   --stringparam generate.toc "book toc" \
   --param keep.relative.image.uris 0
@@ -66,12 +65,10 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
-$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts)))
-
 $(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
 
 

doc/manual/packages/channels.xml

@@ -17,6 +17,9 @@ a set of Nix expressions and a manifest.  Using the command <link
 linkend="sec-nix-channel"><command>nix-channel</command></link> you
 can automatically stay up to date with whatever is available at that
 URL.</para>
+      
+<para>To see the list of official NixOS channels, visit <link
+xlink:href="https://nixos.org/channels" />.</para>
 
 <para>You can “subscribe” to a channel using
 <command>nix-channel --add</command>, e.g.,

doc/manual/release-notes/rl-2.0.xml

@@ -503,14 +503,14 @@
   </listitem>
 
   <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
-    of the existing restricted evaluation mode. In pure mode, the Nix
-    evaluator forbids access to anything that could cause different
-    evaluations of the same command line arguments to produce a
+    <para><emphasis>Pure evaluation mode</emphasis>. With the
+    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
+    restricted evaluation mode that forbids access to anything that could cause
+    different evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as
     <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a
-    content hash or commit hash is specified. For example, calls to
+    <emphasis>all</emphasis> filesystem or network access unless a content hash
+    or commit hash is specified. For example, calls to
     <function>builtins.fetchGit</function> are only allowed if a
     <varname>rev</varname> attribute is specified.</para>
 

doc/manual/style.css

@@ -1,263 +0,0 @@
-/* Copied from http://bakefile.sourceforge.net/, which appears
-   licensed under the GNU GPL. */
-
-
-/***************************************************************************
-                             Basic headers and text:
- ***************************************************************************/
-
-body
-{
-    font-family: "Nimbus Sans L", sans-serif;
-    background: white;
-    margin: 2em 1em 2em 1em;
-}
-
-h1, h2, h3, h4
-{
-    color: #005aa0;
-}
-
-h1 /* title */
-{
-    font-size: 200%;
-}
-
-div.part h1
-{
-    font-size: 240%;
-}
-
-h2 /* chapters, appendices, subtitle */
-{
-    font-size: 180%;
-}
-
-div.part
-{
-    margin-top: 4em;
-}
-
-/* Extra space between chapters, appendices. */
-div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
-{ 
-    margin-top: 1.5em;
-}
-
-div.section > div.titlepage h2 /* sections */
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-h3 /* subsections */
-{
-    font-size: 125%;
-}
-
-div.simplesect h2
-{
-    font-size: 110%;
-}
-
-div.appendix h3
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-div.refentry\.separator
-{
-    margin-top: 2.5em;
-    margin-bottom: 2em;
-}
-
-div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
-{
-    margin-top: 1.4em;
-    font-size: 125%;
-}
-
-div.refsection h3
-{
-    font-size: 110%;
-}
-
-
-/***************************************************************************
-                               Examples:
- ***************************************************************************/
-
-div.example
-{
-    border: 1px solid #b0b0b0;
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    background: #f4f4f8;
-    border-radius: 0.4em;
-}
-
-div.example p.title
-{
-    margin-top: 0em;
-}
-
-div.example pre
-{
-}
-
-
-/***************************************************************************
-                            Screen dumps:
- ***************************************************************************/
-
-pre.screen, pre.programlisting
-{
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    color: #600000;
-    background: #f4f4f8;
-    font-family: monospace;
-}
-
-div.example pre.programlisting
-{
-    border: 0px;
-    padding: 0 0;
-    margin: 0 0 0 0;
-}
-
-
-/***************************************************************************
-                               Notes, warnings etc:
- ***************************************************************************/
-
-.note, .warning
-{
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    margin-bottom: 1em;
-    padding: 0.3em 0.3em 0.3em 0.3em;
-    background: #fffff5;
-    border-radius: 0.4em;
-}
-
-div.note, div.warning
-{
-    font-style: italic;
-}
-
-div.note h3, div.warning h3
-{
-    color: red;
-    font-size: 100%;
-    padding-right: 0.5em;
-    display: inline;
-}
-
-div.note p, div.warning p
-{
-    margin-bottom: 0em;
-}
-
-div.note h3 + p, div.warning h3 + p
-{
-    display: inline;
-}
-
-div.note h3
-{
-    color: blue;
-    font-size: 100%;
-}
-
-div.navfooter *
-{
-    font-size: 90%;
-}
-
-
-/***************************************************************************
-                        Links colors and highlighting: 
- ***************************************************************************/
-
-a { text-decoration: none; }
-a:hover { text-decoration: underline; }
-a:link { color: #0048b3; }
-a:visited { color: #002a6a; }
-
-
-/***************************************************************************
-                              Table of contents:
- ***************************************************************************/
-
-div.toc
-{
-    font-size: 90%;
-}
-
-div.toc dl
-{
-    margin-top: 0em;
-    margin-bottom: 0em;
-}
-
-
-/***************************************************************************
-                               Special elements:
- ***************************************************************************/
-
-tt, code
-{
-    color: #400000;
-}
-
-.term
-{
-    font-weight: bold;
-    
-}
-
-div.variablelist dd p, div.glosslist dd p
-{
-    margin-top: 0em;
-}
-
-div.variablelist dd, div.glosslist dd
-{
-    margin-left: 1.5em;
-}
-
-div.glosslist dt
-{
-    font-style: italic;
-}
-
-.varname
-{
-    color: #400000;
-}
-
-span.command strong
-{
-    font-weight: normal;
-    color: #400000;
-}
-
-div.calloutlist table
-{
-}
-
-table
-{
-    border-collapse: collapse;
-}
-
-div.affiliation
-{
-    font-style: italic;
-}

local.mk

@@ -2,11 +2,15 @@ ifeq ($(MAKECMDGOALS), dist)
   dist-files += $(shell cat .dist-files)
 endif
 
-dist-files += configure config.h.in nix.spec perl/configure
+dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
+
+$(GCH) $(PCH): src/libutil/util.hh config.h
+
+GCH_CXXFLAGS = -I src/libutil

maintainers/upload-release.pl

@@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n";
 
 File::Path::make_path($releasesDir);
 if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die;
+    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
 }
 
 my $releaseDir = "$releasesDir/nix/$releaseName";

mk/README.md

@@ -1,6 +0,0 @@
-This is a set of helper Makefiles for doing non-recursive builds with
-GNU Make.  The canonical source can be found at
-https://github.com/edolstra/make-rules.  You should copy the files
-into the `mk` subdirectory of your project.
-
-TODO: write more documentation.

mk/patterns.mk

@@ -1,10 +1,10 @@
 $(buildprefix)%.o: %.cc
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.cpp
 	@mkdir -p "$(dir $@)"
-	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
+	$(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS_PCH) $(GLOBAL_CXXFLAGS) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP
 
 $(buildprefix)%.o: %.c
 	@mkdir -p "$(dir $@)"

mk/precompiled-headers.mk

@@ -0,0 +1,42 @@
+PRECOMPILE_HEADERS ?= 1
+
+print-var-help += \
+  echo "  PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build";
+
+GCH = $(buildprefix)precompiled-headers.h.gch
+
+$(GCH): precompiled-headers.h
+	@rm -f $@
+	@mkdir -p "$(dir $@)"
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
+
+PCH = $(buildprefix)precompiled-headers.h.pch
+
+$(PCH): precompiled-headers.h
+	@rm -f $@
+	@mkdir -p "$(dir $@)"
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
+
+clean-files += $(GCH) $(PCH)
+
+ifeq ($(PRECOMPILE_HEADERS), 1)
+
+  ifeq ($(CXX), g++)
+
+    GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
+
+    GLOBAL_ORDER_AFTER += $(GCH)
+
+  else ifeq ($(CXX), clang++)
+
+    GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
+
+    GLOBAL_ORDER_AFTER += $(PCH)
+
+  else
+
+    $(error Don't know how to precompile headers on $(CXX))
+
+  endif
+
+endif

nix-rust/Cargo.lock

@@ -0,0 +1,399 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "assert_matches"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "autocfg"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "bit-set"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "bitflags"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "byteorder"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "c2-chacha"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "cfg-if"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "cloudabi"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "getrandom"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hex"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "libc"
+version = "0.2.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "nix-rust"
+version = "0.1.0"
+dependencies = [
+ "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "proptest"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "quick-error"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "rand"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_isaac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_jitter"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_os"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.1.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "remove_dir_all"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rusty-fork"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
+ "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "wasi"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[metadata]
+"checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
+"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
+"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
+"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
+"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
+"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
+"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
+"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
+"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
+"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
+"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
+"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
+"checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4"
+"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
+"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
+"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
+"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
+"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
+"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
+"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
+"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+"checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
+"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
+"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
+"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
+"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
+"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
+"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
+"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
+"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
+"checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
+"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
+"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
+"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
+"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

nix-rust/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "nix-rust"
+version = "0.1.0"
+authors = ["Eelco Dolstra <edolstra@gmail.com>"]
+edition = "2018"
+
+[lib]
+name = "nixrust"
+crate-type = ["cdylib"]
+
+[dependencies]
+libc = "0.2"
+#futures-preview = { version = "=0.3.0-alpha.19" }
+#hyper = "0.13.0-alpha.4"
+#http = "0.1"
+#tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] }
+lazy_static = "1.4"
+#byteorder = "1.3"
+
+[dev-dependencies]
+hex = "0.3"
+assert_matches = "1.3"
+proptest = "0.9"

nix-rust/local.mk

@@ -0,0 +1,45 @@
+ifeq ($(OPTIMIZE), 1)
+  RUST_MODE = --release
+  RUST_DIR = release
+else
+  RUST_MODE =
+  RUST_DIR = debug
+endif
+
+libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
+libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl
+libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl
+
+ifeq ($(OS), Darwin)
+libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
+else
+libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
+libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
+endif
+
+$(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
+	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
+	$(libnixrust_BUILD_FLAGS) \
+	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
+	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
+
+$(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
+	$(target-gen) cp $^ $@
+ifeq ($(OS), Darwin)
+	install_name_tool -id $@ $@
+endif
+
+dist-files += $(d)/vendor
+
+clean: clean-rust
+
+clean-rust:
+	$(suppress) rm -rfv nix-rust/target
+
+ifneq ($(OS), Darwin)
+check: rust-tests
+
+rust-tests:
+	cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
+endif

nix-rust/src/c.rs

@@ -0,0 +1,77 @@
+use super::{error, store::path, store::StorePath, util};
+
+#[no_mangle]
+pub unsafe extern "C" fn ffi_String_new(s: &str, out: *mut String) {
+    // FIXME: check whether 's' is valid UTF-8?
+    out.write(s.to_string())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn ffi_String_drop(self_: *mut String) {
+    std::ptr::drop_in_place(self_);
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_new(
+    path: &str,
+    store_dir: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::new(std::path::Path::new(path), std::path::Path::new(store_dir))
+        .map_err(|err| err.into())
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_new2(
+    hash: &[u8; crate::store::path::STORE_PATH_HASH_BYTES],
+    name: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::from_parts(*hash, name).map_err(|err| err.into())
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_fromBaseName(
+    base_name: &str,
+) -> Result<StorePath, error::CppException> {
+    StorePath::new_from_base_name(base_name).map_err(|err| err.into())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn ffi_StorePath_drop(self_: *mut StorePath) {
+    std::ptr::drop_in_place(self_);
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_to_string(self_: &StorePath) -> Vec<u8> {
+    let mut buf = vec![0; path::STORE_PATH_HASH_CHARS + 1 + self_.name.name().len()];
+    util::base32::encode_into(self_.hash.hash(), &mut buf[0..path::STORE_PATH_HASH_CHARS]);
+    buf[path::STORE_PATH_HASH_CHARS] = b'-';
+    buf[path::STORE_PATH_HASH_CHARS + 1..].clone_from_slice(self_.name.name().as_bytes());
+    buf
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_less_than(a: &StorePath, b: &StorePath) -> bool {
+    a < b
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_eq(a: &StorePath, b: &StorePath) -> bool {
+    a == b
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_clone(self_: &StorePath) -> StorePath {
+    self_.clone()
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_name(self_: &StorePath) -> &str {
+    self_.name.name()
+}
+
+#[no_mangle]
+pub extern "C" fn ffi_StorePath_hash_data(
+    self_: &StorePath,
+) -> &[u8; crate::store::path::STORE_PATH_HASH_BYTES] {
+    self_.hash.hash()
+}

nix-rust/src/error.rs

@@ -0,0 +1,118 @@
+use std::fmt;
+
+#[derive(Debug)]
+pub enum Error {
+    InvalidPath(crate::store::StorePath),
+    BadStorePath(std::path::PathBuf),
+    NotInStore(std::path::PathBuf),
+    BadNarInfo,
+    BadBase32,
+    StorePathNameEmpty,
+    StorePathNameTooLong,
+    BadStorePathName,
+    NarSizeFieldTooBig,
+    BadNarString,
+    BadNarPadding,
+    BadNarVersionMagic,
+    MissingNarOpenTag,
+    MissingNarCloseTag,
+    MissingNarField,
+    BadNarField(String),
+    BadExecutableField,
+    IOError(std::io::Error),
+    #[cfg(unused)]
+    HttpError(hyper::error::Error),
+    Misc(String),
+    #[cfg(not(test))]
+    Foreign(CppException),
+    BadTarFileMemberName(String),
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Self {
+        Error::IOError(err)
+    }
+}
+
+#[cfg(unused)]
+impl From<hyper::error::Error> for Error {
+    fn from(err: hyper::error::Error) -> Self {
+        Error::HttpError(err)
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::InvalidPath(_) => write!(f, "invalid path"),
+            Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
+            Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
+            Error::NotInStore(path) => {
+                write!(f, "path '{}' is not in the Nix store", path.display())
+            }
+            Error::BadBase32 => write!(f, "invalid base32 string"),
+            Error::StorePathNameEmpty => write!(f, "store path name is empty"),
+            Error::StorePathNameTooLong => {
+                write!(f, "store path name is longer than 211 characters")
+            }
+            Error::BadStorePathName => write!(f, "store path name contains forbidden character"),
+            Error::NarSizeFieldTooBig => write!(f, "size field in NAR is too big"),
+            Error::BadNarString => write!(f, "NAR string is not valid UTF-8"),
+            Error::BadNarPadding => write!(f, "NAR padding is not zero"),
+            Error::BadNarVersionMagic => write!(f, "unsupported NAR version"),
+            Error::MissingNarOpenTag => write!(f, "NAR open tag is missing"),
+            Error::MissingNarCloseTag => write!(f, "NAR close tag is missing"),
+            Error::MissingNarField => write!(f, "expected NAR field is missing"),
+            Error::BadNarField(s) => write!(f, "unrecognized NAR field '{}'", s),
+            Error::BadExecutableField => write!(f, "bad 'executable' field in NAR"),
+            Error::IOError(err) => write!(f, "I/O error: {}", err),
+            #[cfg(unused)]
+            Error::HttpError(err) => write!(f, "HTTP error: {}", err),
+            #[cfg(not(test))]
+            Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
+            Error::Misc(s) => write!(f, "{}", s),
+            Error::BadTarFileMemberName(s) => {
+                write!(f, "tar archive contains illegal file name '{}'", s)
+            }
+        }
+    }
+}
+
+#[cfg(not(test))]
+impl From<Error> for CppException {
+    fn from(err: Error) -> Self {
+        match err {
+            Error::Foreign(ex) => ex,
+            _ => CppException::new(&err.to_string()),
+        }
+    }
+}
+
+#[cfg(not(test))]
+#[repr(C)]
+#[derive(Debug)]
+pub struct CppException(*const libc::c_void); // == std::exception_ptr*
+
+#[cfg(not(test))]
+impl CppException {
+    fn new(s: &str) -> Self {
+        Self(unsafe { make_error(s) })
+    }
+}
+
+#[cfg(not(test))]
+impl Drop for CppException {
+    fn drop(&mut self) {
+        unsafe {
+            destroy_error(self.0);
+        }
+    }
+}
+
+#[cfg(not(test))]
+extern "C" {
+    #[allow(improper_ctypes)] // YOLO
+    fn make_error(s: &str) -> *const libc::c_void;
+
+    fn destroy_error(exc: *const libc::c_void);
+}

nix-rust/src/lib.rs

@@ -0,0 +1,9 @@
+#[cfg(not(test))]
+mod c;
+mod error;
+#[cfg(unused)]
+mod nar;
+mod store;
+mod util;
+
+pub use error::Error;

nix-rust/src/nar.rs

@@ -0,0 +1,126 @@
+use crate::Error;
+use byteorder::{LittleEndian, ReadBytesExt};
+use std::convert::TryFrom;
+use std::io::Read;
+
+pub fn parse<R: Read>(input: &mut R) -> Result<(), Error> {
+    if String::read(input)? != NAR_VERSION_MAGIC {
+        return Err(Error::BadNarVersionMagic);
+    }
+
+    parse_file(input)
+}
+
+const NAR_VERSION_MAGIC: &str = "nix-archive-1";
+
+fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
+    if String::read(input)? != "(" {
+        return Err(Error::MissingNarOpenTag);
+    }
+
+    if String::read(input)? != "type" {
+        return Err(Error::MissingNarField);
+    }
+
+    match String::read(input)?.as_ref() {
+        "regular" => {
+            let mut _executable = false;
+            let mut tag = String::read(input)?;
+            if tag == "executable" {
+                _executable = true;
+                if String::read(input)? != "" {
+                    return Err(Error::BadExecutableField);
+                }
+                tag = String::read(input)?;
+            }
+            if tag != "contents" {
+                return Err(Error::MissingNarField);
+            }
+            let _contents = Vec::<u8>::read(input)?;
+            if String::read(input)? != ")" {
+                return Err(Error::MissingNarCloseTag);
+            }
+        }
+        "directory" => loop {
+            match String::read(input)?.as_ref() {
+                "entry" => {
+                    if String::read(input)? != "(" {
+                        return Err(Error::MissingNarOpenTag);
+                    }
+                    if String::read(input)? != "name" {
+                        return Err(Error::MissingNarField);
+                    }
+                    let _name = String::read(input)?;
+                    if String::read(input)? != "node" {
+                        return Err(Error::MissingNarField);
+                    }
+                    parse_file(input)?;
+                    let tag = String::read(input)?;
+                    if tag != ")" {
+                        return Err(Error::MissingNarCloseTag);
+                    }
+                }
+                ")" => break,
+                s => return Err(Error::BadNarField(s.into())),
+            }
+        },
+        "symlink" => {
+            if String::read(input)? != "target" {
+                return Err(Error::MissingNarField);
+            }
+            let _target = String::read(input)?;
+            if String::read(input)? != ")" {
+                return Err(Error::MissingNarCloseTag);
+            }
+        }
+        s => return Err(Error::BadNarField(s.into())),
+    }
+
+    Ok(())
+}
+
+trait Deserialize: Sized {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error>;
+}
+
+impl Deserialize for String {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let buf = Deserialize::read(input)?;
+        Ok(String::from_utf8(buf).map_err(|_| Error::BadNarString)?)
+    }
+}
+
+impl Deserialize for Vec<u8> {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let n: usize = Deserialize::read(input)?;
+        let mut buf = vec![0; n];
+        input.read_exact(&mut buf)?;
+        skip_padding(input, n)?;
+        Ok(buf)
+    }
+}
+
+fn skip_padding<R: Read>(input: &mut R, len: usize) -> Result<(), Error> {
+    if len % 8 != 0 {
+        let mut buf = [0; 8];
+        let buf = &mut buf[0..8 - (len % 8)];
+        input.read_exact(buf)?;
+        if !buf.iter().all(|b| *b == 0) {
+            return Err(Error::BadNarPadding);
+        }
+    }
+    Ok(())
+}
+
+impl Deserialize for u64 {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        Ok(input.read_u64::<LittleEndian>()?)
+    }
+}
+
+impl Deserialize for usize {
+    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
+        let n: u64 = Deserialize::read(input)?;
+        Ok(usize::try_from(n).map_err(|_| Error::NarSizeFieldTooBig)?)
+    }
+}

nix-rust/src/store/binary_cache_store.rs

@@ -0,0 +1,48 @@
+use super::{PathInfo, Store, StorePath};
+use crate::Error;
+use hyper::client::Client;
+
+pub struct BinaryCacheStore {
+    base_uri: String,
+    client: Client<hyper::client::HttpConnector, hyper::Body>,
+}
+
+impl BinaryCacheStore {
+    pub fn new(base_uri: String) -> Self {
+        Self {
+            base_uri,
+            client: Client::new(),
+        }
+    }
+}
+
+impl Store for BinaryCacheStore {
+    fn query_path_info(
+        &self,
+        path: &StorePath,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>> {
+        let uri = format!("{}/{}.narinfo", self.base_uri.clone(), path.hash);
+        let path = path.clone();
+        let client = self.client.clone();
+        let store_dir = self.store_dir().to_string();
+
+        Box::pin(async move {
+            let response = client.get(uri.parse::<hyper::Uri>().unwrap()).await?;
+
+            if response.status() == hyper::StatusCode::NOT_FOUND
+                || response.status() == hyper::StatusCode::FORBIDDEN
+            {
+                return Err(Error::InvalidPath(path));
+            }
+
+            let mut body = response.into_body();
+
+            let mut bytes = Vec::new();
+            while let Some(next) = body.next().await {
+                bytes.extend(next?);
+            }
+
+            PathInfo::parse_nar_info(std::str::from_utf8(&bytes).unwrap(), &store_dir)
+        })
+    }
+}

nix-rust/src/store/mod.rs

@@ -0,0 +1,17 @@
+pub mod path;
+
+#[cfg(unused)]
+mod binary_cache_store;
+#[cfg(unused)]
+mod path_info;
+#[cfg(unused)]
+mod store;
+
+pub use path::{StorePath, StorePathHash, StorePathName};
+
+#[cfg(unused)]
+pub use binary_cache_store::BinaryCacheStore;
+#[cfg(unused)]
+pub use path_info::PathInfo;
+#[cfg(unused)]
+pub use store::Store;

nix-rust/src/store/path.rs

@@ -0,0 +1,225 @@
+use crate::error::Error;
+use crate::util::base32;
+use std::fmt;
+use std::path::Path;
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePath {
+    pub hash: StorePathHash,
+    pub name: StorePathName,
+}
+
+pub const STORE_PATH_HASH_BYTES: usize = 20;
+pub const STORE_PATH_HASH_CHARS: usize = 32;
+
+impl StorePath {
+    pub fn new(path: &Path, store_dir: &Path) -> Result<Self, Error> {
+        if path.parent() != Some(store_dir) {
+            return Err(Error::NotInStore(path.into()));
+        }
+        Self::new_from_base_name(
+            path.file_name()
+                .ok_or(Error::BadStorePath(path.into()))?
+                .to_str()
+                .ok_or(Error::BadStorePath(path.into()))?,
+        )
+    }
+
+    pub fn from_parts(hash: [u8; STORE_PATH_HASH_BYTES], name: &str) -> Result<Self, Error> {
+        Ok(StorePath {
+            hash: StorePathHash(hash),
+            name: StorePathName::new(name)?,
+        })
+    }
+
+    pub fn new_from_base_name(base_name: &str) -> Result<Self, Error> {
+        if base_name.len() < STORE_PATH_HASH_CHARS + 1
+            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != '-' as u8
+        {
+            return Err(Error::BadStorePath(base_name.into()));
+        }
+
+        Ok(StorePath {
+            hash: StorePathHash::new(&base_name[0..STORE_PATH_HASH_CHARS])?,
+            name: StorePathName::new(&base_name[STORE_PATH_HASH_CHARS + 1..])?,
+        })
+    }
+}
+
+impl fmt::Display for StorePath {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}-{}", self.hash, self.name)
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct StorePathHash([u8; STORE_PATH_HASH_BYTES]);
+
+impl StorePathHash {
+    pub fn new(s: &str) -> Result<Self, Error> {
+        assert_eq!(s.len(), STORE_PATH_HASH_CHARS);
+        let v = base32::decode(s)?;
+        assert_eq!(v.len(), STORE_PATH_HASH_BYTES);
+        let mut bytes: [u8; 20] = Default::default();
+        bytes.copy_from_slice(&v[0..STORE_PATH_HASH_BYTES]);
+        Ok(Self(bytes))
+    }
+
+    pub fn hash<'a>(&'a self) -> &'a [u8; STORE_PATH_HASH_BYTES] {
+        &self.0
+    }
+}
+
+impl fmt::Display for StorePathHash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut buf = vec![0; STORE_PATH_HASH_CHARS];
+        base32::encode_into(&self.0, &mut buf);
+        f.write_str(std::str::from_utf8(&buf).unwrap())
+    }
+}
+
+impl Ord for StorePathHash {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        // Historically we've sorted store paths by their base32
+        // serialization, but our base32 encodes bytes in reverse
+        // order. So compare them in reverse order as well.
+        self.0.iter().rev().cmp(other.0.iter().rev())
+    }
+}
+
+impl PartialOrd for StorePathHash {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
+pub struct StorePathName(String);
+
+impl StorePathName {
+    pub fn new(s: &str) -> Result<Self, Error> {
+        if s.len() == 0 {
+            return Err(Error::StorePathNameEmpty);
+        }
+
+        if s.len() > 211 {
+            return Err(Error::StorePathNameTooLong);
+        }
+
+        if s.starts_with('.')
+            || !s.chars().all(|c| {
+                c.is_ascii_alphabetic()
+                    || c.is_ascii_digit()
+                    || c == '+'
+                    || c == '-'
+                    || c == '.'
+                    || c == '_'
+                    || c == '?'
+                    || c == '='
+            })
+        {
+            return Err(Error::BadStorePathName);
+        }
+
+        Ok(Self(s.to_string()))
+    }
+
+    pub fn name<'a>(&'a self) -> &'a str {
+        &self.0
+    }
+}
+
+impl fmt::Display for StorePathName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(&self.0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use assert_matches::assert_matches;
+
+    #[test]
+    fn test_parse() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
+        let p = StorePath::new_from_base_name(&s).unwrap();
+        assert_eq!(p.name.0, "konsole-18.12.3");
+        assert_eq!(
+            p.hash.0,
+            [
+                0x9f, 0x76, 0x49, 0x20, 0xf6, 0x5d, 0xe9, 0x71, 0xc4, 0xca, 0x46, 0x21, 0xab, 0xff,
+                0x9b, 0x44, 0xef, 0x87, 0x0f, 0x3c
+            ]
+        );
+    }
+
+    #[test]
+    fn test_no_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::StorePathNameEmpty)
+        );
+    }
+
+    #[test]
+    fn test_no_dash() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePath(_))
+        );
+    }
+
+    #[test]
+    fn test_short_hash() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxl-konsole-18.12.3";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePath(_))
+        );
+    }
+
+    #[test]
+    fn test_invalid_hash() {
+        let s = "7h7qgvs4kgzsn8e6rb273saxyqh4jxlz-konsole-18.12.3";
+        assert_matches!(StorePath::new_from_base_name(&s), Err(Error::BadBase32));
+    }
+
+    #[test]
+    fn test_long_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+        assert_matches!(StorePath::new_from_base_name(&s), Ok(_));
+    }
+
+    #[test]
+    fn test_too_long_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::StorePathNameTooLong)
+        );
+    }
+
+    #[test]
+    fn test_bad_name() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-foo bar";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePathName)
+        );
+
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-kónsole";
+        assert_matches!(
+            StorePath::new_from_base_name(&s),
+            Err(Error::BadStorePathName)
+        );
+    }
+
+    #[test]
+    fn test_roundtrip() {
+        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
+        assert_eq!(StorePath::new_from_base_name(&s).unwrap().to_string(), s);
+    }
+}

nix-rust/src/store/path_info.rs

@@ -0,0 +1,70 @@
+use crate::store::StorePath;
+use crate::Error;
+use std::collections::BTreeSet;
+
+#[derive(Clone, Debug)]
+pub struct PathInfo {
+    pub path: StorePath,
+    pub references: BTreeSet<StorePath>,
+    pub nar_size: u64,
+    pub deriver: Option<StorePath>,
+
+    // Additional binary cache info.
+    pub url: Option<String>,
+    pub compression: Option<String>,
+    pub file_size: Option<u64>,
+}
+
+impl PathInfo {
+    pub fn parse_nar_info(nar_info: &str, store_dir: &str) -> Result<Self, Error> {
+        let mut path = None;
+        let mut references = BTreeSet::new();
+        let mut nar_size = None;
+        let mut deriver = None;
+        let mut url = None;
+        let mut compression = None;
+        let mut file_size = None;
+
+        for line in nar_info.lines() {
+            let colon = line.find(':').ok_or(Error::BadNarInfo)?;
+
+            let (name, value) = line.split_at(colon);
+
+            if !value.starts_with(": ") {
+                return Err(Error::BadNarInfo);
+            }
+
+            let value = &value[2..];
+
+            if name == "StorePath" {
+                path = Some(StorePath::new(std::path::Path::new(value), store_dir)?);
+            } else if name == "NarSize" {
+                nar_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
+            } else if name == "References" {
+                if !value.is_empty() {
+                    for r in value.split(' ') {
+                        references.insert(StorePath::new_from_base_name(r)?);
+                    }
+                }
+            } else if name == "Deriver" {
+                deriver = Some(StorePath::new_from_base_name(value)?);
+            } else if name == "URL" {
+                url = Some(value.into());
+            } else if name == "Compression" {
+                compression = Some(value.into());
+            } else if name == "FileSize" {
+                file_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
+            }
+        }
+
+        Ok(PathInfo {
+            path: path.ok_or(Error::BadNarInfo)?,
+            references,
+            nar_size: nar_size.ok_or(Error::BadNarInfo)?,
+            deriver,
+            url: Some(url.ok_or(Error::BadNarInfo)?),
+            compression,
+            file_size,
+        })
+    }
+}

nix-rust/src/store/store.rs

@@ -0,0 +1,53 @@
+use super::{PathInfo, StorePath};
+use crate::Error;
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::Path;
+
+pub trait Store: Send + Sync {
+    fn store_dir(&self) -> &str {
+        "/nix/store"
+    }
+
+    fn query_path_info(
+        &self,
+        store_path: &StorePath,
+    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>>;
+}
+
+impl dyn Store {
+    pub fn parse_store_path(&self, path: &Path) -> Result<StorePath, Error> {
+        StorePath::new(path, self.store_dir())
+    }
+
+    pub async fn compute_path_closure(
+        &self,
+        roots: BTreeSet<StorePath>,
+    ) -> Result<BTreeMap<StorePath, PathInfo>, Error> {
+        let mut done = BTreeSet::new();
+        let mut result = BTreeMap::new();
+        let mut pending = vec![];
+
+        for root in roots {
+            pending.push(self.query_path_info(&root));
+            done.insert(root);
+        }
+
+        while !pending.is_empty() {
+            let (info, _, remaining) = futures::future::select_all(pending).await;
+            pending = remaining;
+
+            let info = info?;
+
+            for path in &info.references {
+                if !done.contains(path) {
+                    pending.push(self.query_path_info(&path));
+                    done.insert(path.clone());
+                }
+            }
+
+            result.insert(info.path.clone(), info);
+        }
+
+        Ok(result)
+    }
+}

nix-rust/src/util/base32.rs

@@ -0,0 +1,160 @@
+use crate::error::Error;
+use lazy_static::lazy_static;
+
+pub fn encoded_len(input_len: usize) -> usize {
+    if input_len == 0 {
+        0
+    } else {
+        (input_len * 8 - 1) / 5 + 1
+    }
+}
+
+pub fn decoded_len(input_len: usize) -> usize {
+    input_len * 5 / 8
+}
+
+static BASE32_CHARS: &'static [u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz";
+
+lazy_static! {
+    static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = {
+        let mut xs = [0xffu8; 256];
+        for (n, c) in BASE32_CHARS.iter().enumerate() {
+            xs[*c as usize] = n as u8;
+        }
+        Box::new(xs)
+    };
+}
+
+pub fn encode(input: &[u8]) -> String {
+    let mut buf = vec![0; encoded_len(input.len())];
+    encode_into(input, &mut buf);
+    std::str::from_utf8(&buf).unwrap().to_string()
+}
+
+pub fn encode_into(input: &[u8], output: &mut [u8]) {
+    let len = encoded_len(input.len());
+    assert_eq!(len, output.len());
+
+    let mut nr_bits_left: usize = 0;
+    let mut bits_left: u16 = 0;
+    let mut pos = len;
+
+    for b in input {
+        bits_left |= (*b as u16) << nr_bits_left;
+        nr_bits_left += 8;
+        while nr_bits_left > 5 {
+            output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
+            pos -= 1;
+            bits_left >>= 5;
+            nr_bits_left -= 5;
+        }
+    }
+
+    if nr_bits_left > 0 {
+        output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
+        pos -= 1;
+    }
+
+    assert_eq!(pos, 0);
+}
+
+pub fn decode(input: &str) -> Result<Vec<u8>, crate::Error> {
+    let mut res = Vec::with_capacity(decoded_len(input.len()));
+
+    let mut nr_bits_left: usize = 0;
+    let mut bits_left: u16 = 0;
+
+    for c in input.chars().rev() {
+        let b = BASE32_CHARS_REVERSE[c as usize];
+        if b == 0xff {
+            return Err(Error::BadBase32);
+        }
+        bits_left |= (b as u16) << nr_bits_left;
+        nr_bits_left += 5;
+        if nr_bits_left >= 8 {
+            res.push((bits_left & 0xff) as u8);
+            bits_left >>= 8;
+            nr_bits_left -= 8;
+        }
+    }
+
+    if nr_bits_left > 0 && bits_left != 0 {
+        return Err(Error::BadBase32);
+    }
+
+    Ok(res)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use assert_matches::assert_matches;
+    use hex;
+    use proptest::proptest;
+
+    #[test]
+    fn test_encode() {
+        assert_eq!(encode(&[]), "");
+
+        assert_eq!(
+            encode(&hex::decode("0839703786356bca59b0f4a32987eb2e6de43ae8").unwrap()),
+            "x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88"
+        );
+
+        assert_eq!(
+            encode(
+                &hex::decode("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+                    .unwrap()
+            ),
+            "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
+        );
+
+        assert_eq!(
+            encode(
+                &hex::decode("ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f")
+                    .unwrap()
+            ),
+            "2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx"
+        );
+    }
+
+    #[test]
+    fn test_decode() {
+        assert_eq!(hex::encode(decode("").unwrap()), "");
+
+        assert_eq!(
+            hex::encode(decode("x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88").unwrap()),
+            "0839703786356bca59b0f4a32987eb2e6de43ae8"
+        );
+
+        assert_eq!(
+            hex::encode(decode("1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s").unwrap()),
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
+        );
+
+        assert_eq!(
+            hex::encode(decode("2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx").unwrap()),
+            "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
+        );
+
+        assert_matches!(
+            decode("xoxf8v9fxf3jk8zln1cwlsrmhqvp0f88"),
+            Err(Error::BadBase32)
+        );
+        assert_matches!(
+            decode("2b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"),
+            Err(Error::BadBase32)
+        );
+        assert_matches!(decode("2"), Err(Error::BadBase32));
+        assert_matches!(decode("2gs"), Err(Error::BadBase32));
+        assert_matches!(decode("2gs8"), Err(Error::BadBase32));
+    }
+
+    proptest! {
+
+        #[test]
+        fn roundtrip(s: Vec<u8>) {
+            assert_eq!(s, decode(&encode(&s)).unwrap());
+        }
+    }
+}

nix-rust/src/util/mod.rs

@@ -0,0 +1 @@
+pub mod base32;

nix.spec.in

@@ -1,173 +0,0 @@
-%undefine _hardened_build
-
-%global nixbld_user "nix-builder-"
-%global nixbld_group "nixbld"
-
-# NOTE: BUILD on EL7 requires
-# - Centos / RHEL7 software collection repository 
-#   yum install centos-release-scl
-#
-# - Recent boost backport
-#   curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo
-#
-
-# Disable documentation generation
-# necessary on some platforms
-%bcond_without docgen
-
-Summary: The Nix software deployment system
-Name: nix
-Version: @PACKAGE_VERSION@
-Release: 2%{?dist}
-License: LGPLv2+
-Group: Applications/System
-URL: http://nixos.org/
-Source0: %{name}-%{version}.tar.bz2
-
-Requires: curl
-Requires: bzip2
-Requires: gzip
-Requires: xz
-BuildRequires: bison
-BuildRequires: boost-devel >= 1.60
-BuildRequires: bzip2-devel
-
-# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler
-%if 0%{?rhel}
-BuildRequires: devtoolset-7-gcc
-BuildRequires: devtoolset-7-gcc-c++
-%endif
-
-BuildRequires: flex
-BuildRequires: libcurl-devel
-BuildRequires: libseccomp-devel
-BuildRequires: openssl-devel
-BuildRequires: sqlite-devel
-BuildRequires: xz-devel
-
-%description
-Nix is a purely functional package manager. It allows multiple
-versions of a package to be installed side-by-side, ensures that
-dependency specifications are complete, supports atomic upgrades and
-rollbacks, allows non-root users to install software, and has many
-other features. It is the basis of the NixOS Linux distribution, but
-it can be used equally well under other Unix systems.
-
-%package        devel
-Summary:        Development files for %{name}
-Requires:       %{name}%{?_isa} = %{version}-%{release}
-
-%description   devel
-The %{name}-devel package contains libraries and header files for
-developing applications that use %{name}.
-
-
-%package doc
-Summary:        Documentation files for %{name}
-BuildArch:      noarch
-Requires:       %{name} = %{version}-%{release}
-
-%description   doc
-The %{name}-doc package contains documentation files for %{name}.
-
-%prep
-%setup -q
-
-
-%build
-%if 0%{?rhel}
-source /opt/rh/devtoolset-7/enable 
-%endif
-extraFlags=
-# - override docdir so large documentation files are owned by the
-#   -doc subpackage
-# - set localstatedir by hand to the preferred nix value
-%configure --localstatedir=/nix/var \
-	   %{!?without_docgen:--disable-doc-gen} \
-           --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \
-           $extraFlags
-make V=1 %{?_smp_mflags}
-
-
-%install
-%if 0%{?rhel}
-source /opt/rh/devtoolset-7/enable 
-%endif
-
-make DESTDIR=$RPM_BUILD_ROOT install
-
-find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';'
-
-# make the store
-mkdir -p $RPM_BUILD_ROOT/nix/store
-chmod 1775 $RPM_BUILD_ROOT/nix/store
-
-# make per-user directories
-for d in profiles gcroots;
-do
-  mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-  chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user
-done
-
-# fix permission of nix profile
-# (until this is fixed in the relevant Makefile)
-chmod -x $RPM_BUILD_ROOT%{_sysconfdir}/profile.d/nix.sh
-
-# we ship this file in the base package
-rm -f $RPM_BUILD_ROOT%{_defaultdocdir}/%{name}-doc-%{version}/README
-
-# Get rid of Upstart job.
-rm -rf $RPM_BUILD_ROOT%{_sysconfdir}/init
-
-
-%clean
-rm -rf $RPM_BUILD_ROOT
-
-
-%pre
-getent group %{nixbld_group} >/dev/null || groupadd -r %{nixbld_group}
-for i in $(seq 10);
-do
-  getent passwd %{nixbld_user}$i >/dev/null || \
-    useradd -r -g %{nixbld_group} -G %{nixbld_group} -d /var/empty \
-      -s %{_sbindir}/nologin \
-      -c "Nix build user $i" %{nixbld_user}$i
-done
-
-%post
-chgrp %{nixbld_group} /nix/store
-%if ! 0%{?rhel} || 0%{?rhel} >= 7
-# Enable and start Nix worker
-systemctl enable nix-daemon.socket nix-daemon.service
-systemctl start  nix-daemon.socket
-%endif
-
-%files
-%license COPYING
-%{_bindir}/nix*
-%{_libdir}/*.so
-%{_prefix}/libexec/*
-%if ! 0%{?rhel} || 0%{?rhel} >= 7
-%{_prefix}/lib/systemd/system/nix-daemon.socket
-%{_prefix}/lib/systemd/system/nix-daemon.service
-%endif
-%{_datadir}/nix
-#%if ! %{without docgen}
-#%{_mandir}/man1/*.1*
-#%{_mandir}/man5/*.5*
-#%{_mandir}/man8/*.8*
-#%endif
-%config(noreplace) %{_sysconfdir}/profile.d/nix.sh
-%config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh
-/nix
-
-%files devel
-%{_includedir}/nix
-%{_prefix}/lib/pkgconfig/*.pc
-
-
-#%if ! %{without docgen}
-#%files doc
-#%docdir %{_defaultdocdir}/%{name}-doc-%{version}
-#%{_defaultdocdir}/%{name}-doc-%{version}
-#%endif

perl/Makefile

@@ -4,4 +4,12 @@ GLOBAL_CXXFLAGS += -g -Wall
 
 -include Makefile.config
 
+OPTIMIZE = 1
+
+ifeq ($(OPTIMIZE), 1)
+  GLOBAL_CXXFLAGS += -O3
+else
+  GLOBAL_CXXFLAGS += -O0
+endif
+
 include mk/lib.mk

perl/configure.ac

@@ -2,13 +2,10 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX
 AC_CONFIG_SRCDIR(MANIFEST)
 AC_CONFIG_AUX_DIR(../config)
 
-# Set default flags for nix (as per AC_PROG_CC/CXX docs),
-# while still allowing the user to override them from the command line.
-: ${CFLAGS="-O3"}
-: ${CXXFLAGS="-O3"}
+CFLAGS=
+CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
-AX_CXX_COMPILE_STDCXX_11
 
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
@@ -71,14 +68,15 @@ AC_SUBST(perlFlags)
 
 PKG_CHECK_MODULES([NIX], [nix-store])
 
-NEED_PROG([NIX_INSTANTIATE_PROGRAM], [nix-instantiate])
+NEED_PROG([NIX], [nix])
 
 # Get nix configure values
-nixbindir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixBinDir | tr -d \")
-nixlibexecdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLibexecDir | tr -d \")
-nixlocalstatedir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixLocalstateDir | tr -d \")
-nixsysconfdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixSysconfDir | tr -d \")
-nixstoredir=$("$NIX_INSTANTIATE_PROGRAM" --eval '<nix/config.nix>' -A nixStoreDir | tr -d \")
+export NIX_REMOTE=daemon
+nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixBinDir)
+nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLibexecDir)
+nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixLocalstateDir)
+nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixSysconfDir)
+nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '<nix/config.nix>' nixStoreDir)
 AC_SUBST(nixbindir)
 AC_SUBST(nixlibexecdir)
 AC_SUBST(nixlocalstatedir)

perl/lib/Nix/Config.pm.in

@@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix";
 $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix";
 $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@";
 
-$bzip2 = "@bzip2@";
-$xz = "@xz@";
-$curl = "@curl@";
-
 $useBindings = 1;
 
 %config = ();

perl/lib/Nix/Store.xs

@@ -59,7 +59,7 @@ void setVerbosity(int level)
 int isValidPath(char * path)
     CODE:
         try {
-            RETVAL = store()->isValidPath(path);
+            RETVAL = store()->isValidPath(store()->parseStorePath(path));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -70,9 +70,8 @@ int isValidPath(char * path)
 SV * queryReferences(char * path)
     PPCODE:
         try {
-            PathSet paths = store()->queryPathInfo(path)->references;
-            for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+            for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -81,7 +80,7 @@ SV * queryReferences(char * path)
 SV * queryPathHash(char * path)
     PPCODE:
         try {
-            auto s = store()->queryPathInfo(path)->narHash.to_string();
+            auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string();
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -91,9 +90,9 @@ SV * queryPathHash(char * path)
 SV * queryDeriver(char * path)
     PPCODE:
         try {
-            auto deriver = store()->queryPathInfo(path)->deriver;
-            if (deriver == "") XSRETURN_UNDEF;
-            XPUSHs(sv_2mortal(newSVpv(deriver.c_str(), 0)));
+            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            if (!info->deriver) XSRETURN_UNDEF;
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -102,18 +101,18 @@ SV * queryDeriver(char * path)
 SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
-            auto info = store()->queryPathInfo(path);
-            if (info->deriver == "")
+            auto info = store()->queryPathInfo(store()->parseStorePath(path));
+            if (!info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
-                XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0)));
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
             auto s = info->narHash.to_string(base32 ? Base32 : Base16);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
             AV * arr = newAV();
-            for (PathSet::iterator i = info->references.begin(); i != info->references.end(); ++i)
-                av_push(arr, newSVpv(i->c_str(), 0));
+            for (auto & i : info->references)
+                av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0));
             XPUSHs(sv_2mortal(newRV((SV *) arr)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -123,8 +122,8 @@ SV * queryPathInfo(char * path, int base32)
 SV * queryPathFromHashPart(char * hashPart)
     PPCODE:
         try {
-            Path path = store()->queryPathFromHashPart(hashPart);
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->queryPathFromHashPart(hashPart);
+            XPUSHs(sv_2mortal(newSVpv(path ? store()->printStorePath(*path).c_str() : "", 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -133,11 +132,11 @@ SV * queryPathFromHashPart(char * hashPart)
 SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
     PPCODE:
         try {
-            PathSet paths;
+            StorePathSet paths;
             for (int n = 2; n < items; ++n)
-                store()->computeFSClosure(SvPV_nolen(ST(n)), paths, flipDirection, includeOutputs);
-            for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+                store()->computeFSClosure(store()->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs);
+            for (auto & i : paths)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -146,11 +145,11 @@ SV * computeFSClosure(int flipDirection, int includeOutputs, ...)
 SV * topoSortPaths(...)
     PPCODE:
         try {
-            PathSet paths;
-            for (int n = 0; n < items; ++n) paths.insert(SvPV_nolen(ST(n)));
-            Paths sorted = store()->topoSortPaths(paths);
-            for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i)
-                XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0)));
+            StorePathSet paths;
+            for (int n = 0; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n))));
+            auto sorted = store()->topoSortPaths(paths);
+            for (auto & i : sorted)
+                XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -159,7 +158,7 @@ SV * topoSortPaths(...)
 SV * followLinksToStorePath(char * path)
     CODE:
         try {
-            RETVAL = newSVpv(store()->followLinksToStorePath(path).c_str(), 0);
+            RETVAL = newSVpv(store()->printStorePath(store()->followLinksToStorePath(path)).c_str(), 0);
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -170,8 +169,8 @@ SV * followLinksToStorePath(char * path)
 void exportPaths(int fd, ...)
     PPCODE:
         try {
-            Paths paths;
-            for (int n = 1; n < items; ++n) paths.push_back(SvPV_nolen(ST(n)));
+            StorePathSet paths;
+            for (int n = 1; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n))));
             FdSink sink(fd);
             store()->exportPaths(paths, sink);
         } catch (Error & e) {
@@ -275,8 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg)
 SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
-            Path path = store()->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo));
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo));
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -286,8 +285,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
     PPCODE:
         try {
             Hash h(hash, parseHashType(algo));
-            Path path = store()->makeFixedOutputPath(recursive, h, name);
-            XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0)));
+            auto path = store()->makeFixedOutputPath(recursive, h, name);
+            XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
         }
@@ -298,35 +297,35 @@ SV * derivationFromPath(char * drvPath)
         HV *hash;
     CODE:
         try {
-            Derivation drv = store()->derivationFromPath(drvPath);
+            Derivation drv = store()->derivationFromPath(store()->parseStorePath(drvPath));
             hash = newHV();
 
             HV * outputs = newHV();
-            for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i)
-                hv_store(outputs, i->first.c_str(), i->first.size(), newSVpv(i->second.path.c_str(), 0), 0);
+            for (auto & i : drv.outputs)
+                hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0);
             hv_stores(hash, "outputs", newRV((SV *) outputs));
 
             AV * inputDrvs = newAV();
-            for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i)
-                av_push(inputDrvs, newSVpv(i->first.c_str(), 0)); // !!! ignores i->second
+            for (auto & i : drv.inputDrvs)
+                av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second
             hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs));
 
             AV * inputSrcs = newAV();
-            for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i)
-                av_push(inputSrcs, newSVpv(i->c_str(), 0));
+            for (auto & i : drv.inputSrcs)
+                av_push(inputSrcs, newSVpv(store()->printStorePath(i).c_str(), 0));
             hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs));
 
             hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0));
             hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0));
 
             AV * args = newAV();
-            for (Strings::iterator i = drv.args.begin(); i != drv.args.end(); ++i)
-                av_push(args, newSVpv(i->c_str(), 0));
+            for (auto & i : drv.args)
+                av_push(args, newSVpv(i.c_str(), 0));
             hv_stores(hash, "args", newRV((SV *) args));
 
             HV * env = newHV();
-            for (StringPairs::iterator i = drv.env.begin(); i != drv.env.end(); ++i)
-                hv_store(env, i->first.c_str(), i->first.size(), newSVpv(i->second.c_str(), 0), 0);
+            for (auto & i : drv.env)
+                hv_store(env, i.first.c_str(), i.first.size(), newSVpv(i.second.c_str(), 0), 0);
             hv_stores(hash, "env", newRV((SV *) env));
 
             RETVAL = newRV_noinc((SV *)hash);
@@ -340,7 +339,7 @@ SV * derivationFromPath(char * drvPath)
 void addTempRoot(char * storePath)
     PPCODE:
         try {
-            store()->addTempRoot(storePath);
+            store()->addTempRoot(store()->parseStorePath(storePath));
         } catch (Error & e) {
             croak("%s", e.what());
         }

precompiled-headers.h

@@ -0,0 +1,61 @@
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cassert>
+#include <cctype>
+#include <chrono>
+#include <climits>
+#include <cmath>
+#include <condition_variable>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <functional>
+#include <future>
+#include <iostream>
+#include <limits>
+#include <list>
+#include <locale>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <numeric>
+#include <optional>
+#include <queue>
+#include <random>
+#include <regex>
+#include <set>
+#include <sstream>
+#include <stack>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+#include <boost/format.hpp>
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <grp.h>
+#include <netdb.h>
+#include <pwd.h>
+#include <signal.h>
+#include <sys/resource.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/utsname.h>
+#include <sys/wait.h>
+#include <termios.h>
+#include <unistd.h>
+
+#include "util.hh"
+#include "args.hh"

release-common.nix

@@ -30,13 +30,11 @@ rec {
   });
 
   configureFlags =
-    [
-      "--enable-gc"
-    ] ++ lib.optionals stdenv.isLinux [
+    lib.optionals stdenv.isLinux [
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
-  tarballDeps =
+  buildDeps =
     [ bison
       flex
       libxml2
@@ -45,13 +43,14 @@ rec {
       docbook_xsl_ns
       autoconf-archive
       autoreconfHook
-    ];
 
-  buildDeps =
-    [ curl
-      bzip2 xz brotli editline
+      curl
+      bzip2 xz brotli zlib editline
       openssl pkgconfig sqlite
+      libarchive
       boost
+      nlohmann_json
+      rustc cargo
 
       # Tests
       git

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -8,50 +8,55 @@ let
 
   pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
 
+  version =
+    builtins.readFile ./.version
+    + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
+
   jobs = rec {
 
+    # Create a "vendor" directory that contains the crates listed in
+    # Cargo.lock. This allows Nix to be built without network access.
+    vendoredCrates =
+      let
+        lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
 
-    tarball =
-      with pkgs;
+        files = map (pkg: import <nix/fetchurl.nix> {
+          url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download";
+          sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)";
+        }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package);
 
-      with import ./release-common.nix { inherit pkgs; };
+      in pkgs.runCommand "cargo-vendor-dir" {}
+        ''
+          mkdir -p $out/vendor
 
-      releaseTools.sourceTarball {
-        name = "nix-tarball";
-        version = builtins.readFile ./.version;
-        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
-        src = nix;
-        inherit officialRelease;
+          cat > $out/vendor/config <<EOF
+          [source.crates-io]
+          replace-with = "vendored-sources"
 
-        buildInputs = tarballDeps ++ buildDeps ++ propagatedDeps;
+          [source.vendored-sources]
+          directory = "vendor"
+          EOF
 
-        configureFlags = "--enable-gc";
+          ${toString (builtins.map (file: ''
+            mkdir $out/vendor/tmp
+            tar xvf ${file} -C $out/vendor/tmp
+            dir=$(echo $out/vendor/tmp/*)
 
-        postUnpack = ''
-          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
-          cat $sourceRoot/.dist-files
+            # Add just enough metadata to keep Cargo happy.
+            printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json"
+
+            # Clean up some cruft from the winapi crates. FIXME: find
+            # a way to remove winapi* from our dependencies.
+            if [[ $dir =~ /winapi ]]; then
+              find $dir -name "*.a" -print0 | xargs -0 rm -f --
+            fi
+
+            mv "$dir" $out/vendor/
+
+            rm -rf $out/vendor/tmp
+          '') files)}
         '';
 
-        preConfigure = ''
-          (cd perl ; autoreconf --install --force --verbose)
-          # TeX needs a writable font cache.
-          export VARTEXFONTS=$TMPDIR/texfonts
-        '';
-
-        distPhase =
-          ''
-            runHook preDist
-            make dist
-            mkdir -p $out/tarballs
-            cp *.tar.* $out/tarballs
-          '';
-
-        preDist = ''
-          make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk
-          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
-        '';
-      };
-
 
     build = pkgs.lib.genAttrs systems (system:
 
@@ -61,18 +66,21 @@ let
 
       with import ./release-common.nix { inherit pkgs; };
 
-      releaseTools.nixBuild {
-        name = "nix";
-        src = tarball;
+      stdenv.mkDerivation {
+        name = "nix-${version}";
+
+        src = nix;
+
+        outputs = [ "out" "dev" "doc" ];
 
         buildInputs = buildDeps;
 
         propagatedBuildInputs = propagatedDeps;
 
         preConfigure =
-          # Copy libboost_context so we don't get all of Boost in our closure.
-          # https://github.com/NixOS/nixpkgs/issues/45462
           ''
+            # Copy libboost_context so we don't get all of Boost in our closure.
+            # https://github.com/NixOS/nixpkgs/issues/45462
             mkdir -p $out/lib
             cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
             rm -f $out/lib/*.a
@@ -80,6 +88,10 @@ let
               chmod u+w $out/lib/*.so.*
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
+
+            ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor
+
+            (cd perl; autoreconf --install --force --verbose)
           '';
 
         configureFlags = configureFlags ++
@@ -91,10 +103,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
+        doCheck = true;
+
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
 
         separateDebugInfo = true;
+
+        preDist = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
       });
 
 
@@ -103,11 +122,21 @@ let
       let pkgs = import nixpkgs { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
-        name = "nix-perl";
-        src = tarball;
+        name = "nix-perl-${version}";
+
+        src = nix;
 
         buildInputs =
-          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
+          [ autoconf-archive
+            autoreconfHook
+            jobs.build.${system}
+            curl
+            bzip2
+            xz
+            pkgconfig
+            pkgs.perl
+            boost
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
@@ -127,7 +156,6 @@ let
 
       let
         toplevel = builtins.getAttr system jobs.build;
-        version = toplevel.src.version;
         installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
       in
 
@@ -197,8 +225,9 @@ let
       with import ./release-common.nix { inherit pkgs; };
 
       releaseTools.coverageAnalysis {
-        name = "nix-build";
-        src = tarball;
+        name = "nix-coverage-${version}";
+
+        src = nix;
 
         enableParallelBuilding = true;
 
@@ -208,25 +237,18 @@ let
 
         doInstallCheck = true;
 
-        lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ];
+        lcovFilter = [ "*/boost/*" "*-tab.*" ];
 
         # We call `dot', and even though we just use it to
         # syntax-check generated dot files, it still requires some
         # fonts.  So provide those.
         FONTCONFIG_FILE = texFunctions.fontsConf;
+
+        # To test building without precompiled headers.
+        makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
       };
 
 
-    #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ];
-
-
-    #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ];
-    #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ];
-
-    #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ];
-    #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ];
-
-
     # System tests.
     tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
       inherit nixpkgs;
@@ -268,7 +290,7 @@ let
             x86_64-linux = "${build.x86_64-linux}";
           }
           EOF
-          su - alice -c 'nix upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+          su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
           (! [ -L /home/alice/.profile-1-link ])
           su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
 
@@ -301,97 +323,21 @@ let
 
     installerScript =
       pkgs.runCommand "installer-script"
-        { buildInputs = [ build.x86_64-linux ];
-        }
+        { buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
         ''
           mkdir -p $out/nix-support
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
-              (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
-              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+              (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
+              systems
             } \
-            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+            --replace '@nixVersion@' ${version}
 
           echo "file installer $out/install" >> $out/nix-support/hydra-build-products
         '';
 
-
-    # Aggregate job containing the release-critical jobs.
-    release = pkgs.releaseTools.aggregate {
-      name = "nix-${tarball.version}";
-      meta.description = "Release-critical builds";
-      constituents =
-        [ tarball
-          build.i686-linux
-          build.x86_64-darwin
-          build.x86_64-linux
-          build.aarch64-linux
-          binaryTarball.i686-linux
-          binaryTarball.x86_64-darwin
-          binaryTarball.x86_64-linux
-          binaryTarball.aarch64-linux
-          tests.remoteBuilds
-          tests.nix-copy-closure
-          tests.binaryTarball
-          #tests.evalNixpkgs
-          #tests.evalNixOS
-          installerScript
-        ];
-    };
-
   };
 
 
-  makeRPM_i686 = makeRPM "i686-linux";
-  makeRPM_x86_64 = makeRPM "x86_64-linux";
-
-  makeRPM =
-    system: diskImageFun: extraPackages:
-
-    with import nixpkgs { inherit system; };
-
-    releaseTools.rpmBuild rec {
-      name = "nix-rpm";
-      src = jobs.tarball;
-      diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
-            ++ extraPackages; };
-      # At most 2047MB can be simulated in qemu-system-i386
-      memSize = 2047;
-      meta.schedulingPriority = 50;
-      postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
-      #enableParallelBuilding = true;
-    };
-
-
-  makeDeb_i686 = makeDeb "i686-linux";
-  makeDeb_x86_64 = makeDeb "x86_64-linux";
-
-  makeDeb =
-    system: diskImageFun: extraPackages: extraDebPackages:
-
-    with import nixpkgs { inherit system; };
-
-    releaseTools.debBuild {
-      name = "nix-deb";
-      src = jobs.tarball;
-      diskImage = (diskImageFun vmTools.diskImageFuns)
-        { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
-            ++ extraPackages; };
-      memSize = 2047;
-      meta.schedulingPriority = 50;
-      postInstall = "make installcheck";
-      configureFlags = "--sysconfdir=/etc";
-      debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ]
-        ++ extraDebPackages;
-      debMaintainer = "Eelco Dolstra <eelco.dolstra@logicblox.com>";
-      doInstallCheck = true;
-      #enableParallelBuilding = true;
-    };
-
-
 in jobs

scripts/install-multi-user.sh

@@ -13,12 +13,12 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
+readonly BOLD='\033[1m'
+readonly BLUE='\033[34m'
+readonly BLUE_UL='\033[4;34m'
+readonly GREEN='\033[32m'
+readonly GREEN_UL='\033[4;32m'
+readonly RED='\033[31m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
@@ -275,73 +275,9 @@ EOF
     fi
 
     if type nix-env 2> /dev/null >&2; then
-        failure <<EOF
-Nix already appears to be installed, and this tool assumes it is
-_not_ yet installed.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ "${NIX_REMOTE:-}" != "" ]; then
-        failure <<EOF
-For some reason, \$NIX_REMOTE is set. It really should not be set
-before this installer runs, and it hints that Nix is currently
-installed. Please delete the old Nix installation and start again.
-
-Note: You might need to close your shell window and open a new shell
-to clear the variable.
-EOF
-    fi
-
-    if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then
-        failure <<EOF
-It looks like \$SSL_CERT_FILE is set to a path that used to be part of
-the old Nix installation. Please unset that variable and try again:
-
-  $ unset SSL_CERT_FILE
-
-EOF
-    fi
-
-    for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do
-        if [ -f "$file" ]; then
-            if grep -l "^[^#].*.nix-profile" "$file"; then
-                failure <<EOF
-I found a reference to a ".nix-profile" in $file.
-This has a high chance of breaking a new nix installation. It was most
-likely put there by a previous Nix installer.
-
-Please remove this reference and try running this again. You should
-also look for similar references in:
-
- - ~/.bash_profile
- - ~/.bash_login
- - ~/.profile
-
-or other shell init files that you may have.
-
-$(uninstall_directions)
-EOF
-            fi
-        fi
-    done
-
-    if [ -d /nix/store ] || [ -d /nix/var ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
-
-$(uninstall_directions)
-EOF
-    fi
-
-    if [ -d /etc/nix ]; then
-        failure <<EOF
-There are some relics of a previous installation of Nix at /etc/nix, and
-this scripts assumes Nix is _not_ yet installed. Please delete the old
-Nix installation and start again.
+        warning <<EOF
+Nix already appears to be installed. This installer may run into issues.
+If an error occurs, try manually uninstalling, then rerunning this script.
 
 $(uninstall_directions)
 EOF
@@ -349,7 +285,7 @@ EOF
 
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
-        failure <<EOF
+            failure <<EOF
 When this script runs, it backs up the current $profile_target to
 $profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
 
@@ -361,38 +297,10 @@ in case.
 2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
 it has anything nix-related in it. If it does, something is probably
 quite wrong. Please open an issue or get in touch immediately.
-
-3. Take care to make sure that $profile_target doesn't look like it has
-anything nix-related in it. If it does, and $profile_target _did not_,
-run:
-
-  $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
-
-and try again.
-EOF
-        fi
-
-        if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then
-            failure <<EOF
-It looks like $profile_target already has some Nix configuration in
-there. There should be no reason to run this again. If you're having
-trouble, please open an issue.
 EOF
         fi
     done
 
-    danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile")
-    for danger_path in "${danger_paths[@]}"; do
-        if _sudo "making sure that $danger_path doesn't exist" \
-           test -e "$danger_path"; then
-            failure <<EOF
-I found a file at $danger_path, which is a relic of a previous
-installation. You must first delete this file before continuing.
-
-$(uninstall_directions)
-EOF
-        fi
-    done
 }
 
 setup_report() {
@@ -659,7 +567,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
@@ -762,9 +670,7 @@ main() {
     welcome_to_nix
     chat_about_sudo
 
-    if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then
-        validate_starting_assumptions
-    fi
+    validate_starting_assumptions
 
     setup_report
 

scripts/install-nix-from-closure.sh

@@ -87,7 +87,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 
@@ -102,7 +102,7 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do
         rm -rf "$i_tmp"
     fi
     if ! [ -e "$dest/store/$i" ]; then
-        cp -Rp "$self/store/$i" "$i_tmp"
+        cp -RPp "$self/store/$i" "$i_tmp"
         chmod -R a-w "$i_tmp"
         chmod +w "$i_tmp"
         mv "$i_tmp" "$dest/store/$i"

scripts/install-systemd-multi-user.sh

@@ -88,7 +88,7 @@ poly_configure_nix_daemon_service() {
           systemctl start nix-daemon.socket
 
     _sudo "to start the nix-daemon.service" \
-          systemctl start nix-daemon.service
+          systemctl restart nix-daemon.service
 
 }
 

scripts/install.in

@@ -30,12 +30,13 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
 tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
+require_util xz "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
@@ -56,7 +57,7 @@ fi
 
 unpack=$tmpDir/unpack
 mkdir -p "$unpack"
-tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
+tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'"
 
 script=$(echo "$unpack"/*/install)
 

scripts/nix-profile-daemon.sh.in

@@ -2,7 +2,6 @@
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
 __ETC_PROFILE_NIX_SOURCED=1
 
-export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER"
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
@@ -25,5 +24,4 @@ else
   done
 fi
 
-export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels"
 export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH"

scripts/nix-profile.sh.in

@@ -5,12 +5,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
 
     NIX_LINK=$HOME/.nix-profile
 
-    NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER
-
-    # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs>
-    # paths work when the user has fetched the Nixpkgs channel.
-    export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels
-
     # Set up environment.
     # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix
     export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
@@ -35,5 +29,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
     fi
 
     export PATH="$NIX_LINK/bin:$PATH"
-    unset NIX_LINK NIX_USER_PROFILE_DIR
+    unset NIX_LINK
 fi

shell.nix

@@ -1,13 +1,13 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ propagatedDeps ++ tarballDeps ++ perlDeps;
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using std::cin;
@@ -88,7 +88,7 @@ static int _main(int argc, char * * argv)
             return 0;
         }
 
-        string drvPath;
+        std::optional<StorePath> drvPath;
         string storeUri;
 
         while (true) {
@@ -100,7 +100,7 @@ static int _main(int argc, char * * argv)
 
             auto amWilling = readInt(source);
             auto neededSystem = readString(source);
-            source >> drvPath;
+            drvPath = store->parseStorePath(readString(source));
             auto requiredFeatures = readStrings<std::set<std::string>>(source);
 
              auto canBuildLocally = amWilling
@@ -188,7 +188,7 @@ static int _main(int argc, char * * argv)
 
                     Store::Params storeParams;
                     if (hasPrefix(bestMachine->storeUri, "ssh://")) {
-                        storeParams["max-connections"] ="1";
+                        storeParams["max-connections"] = "1";
                         storeParams["log-fd"] = "4";
                         if (bestMachine->sshKey != "")
                             storeParams["ssh-key"] = bestMachine->sshKey;
@@ -236,26 +236,27 @@ connected:
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute);
+            copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
 
-        BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath)));
-        drv.inputSrcs = inputs;
+        BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string())));
+        drv.inputSrcs = store->parseStorePathSet(inputs);
 
-        auto result = sshStore->buildDerivation(drvPath, drv);
+        auto result = sshStore->buildDerivation(*drvPath, drv);
 
         if (!result.success())
-            throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg);
+            throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
 
-        PathSet missing;
+        StorePathSet missing;
         for (auto & path : outputs)
-            if (!store->isValidPath(path)) missing.insert(path);
+            if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
 
         if (!missing.empty()) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
-            store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
+            for (auto & i : missing)
+                store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
             copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
         }
 

src/libexpr/attr-path.cc

@@ -32,7 +32,7 @@ static Strings parseAttrPath(const string & s)
 }
 
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
@@ -41,6 +41,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
         Error(format("attribute selection path '%1%' does not match expression") % attrPath);
 
     Value * v = &vIn;
+    Pos pos = noPos;
 
     for (auto & attr : tokens) {
 
@@ -70,8 +71,9 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
+                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
             v = &*a->value;
+            pos = *a->pos;
         }
 
         else if (apType == apIndex) {
@@ -82,14 +84,47 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
                     % attrPath % showType(*v));
 
             if (attrIndex >= v->listSize())
-                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
             v = v->listElems()[attrIndex];
+            pos = noPos;
         }
 
     }
 
-    return v;
+    return {v, pos};
+}
+
+
+Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
+{
+    Value * v2;
+    try {
+        auto dummyArgs = state.allocBindings(0);
+        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
+    } catch (Error &) {
+        throw NoPositionInfo("package '%s' has no source location information", what);
+    }
+
+    // FIXME: is it possible to extract the Pos object instead of doing this
+    //        toString + parsing?
+    auto pos = state.forceString(*v2);
+
+    auto colon = pos.rfind(':');
+    if (colon == std::string::npos)
+        throw Error("cannot parse meta.position attribute '%s'", pos);
+
+    std::string filename(pos, 0, colon);
+    unsigned int lineno;
+    try {
+        lineno = std::stoi(std::string(pos, colon + 1));
+    } catch (std::invalid_argument & e) {
+        throw Error("cannot parse line number '%s'", pos);
+    }
+
+    Symbol file = state.symbols.create(filename);
+
+    return { file, lineno, 0 };
 }
 
 

src/libexpr/attr-path.hh

@@ -7,7 +7,13 @@
 
 namespace nix {
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+MakeError(AttrPathNotFound, Error);
+MakeError(NoPositionInfo, Error);
+
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
+/* Heuristic to find the filename and lineno or a nix value. */
+Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
+
 }

src/libexpr/attr-set.cc

@@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 }
 
 
+Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+{
+    return allocAttr(vAttrs, symbols.create(name));
+}
+
+
 void Bindings::sort()
 {
     std::sort(begin(), end());

src/libexpr/attr-set.hh

@@ -4,6 +4,7 @@
 #include "symbol-table.hh"
 
 #include <algorithm>
+#include <optional>
 
 namespace nix {
 
@@ -63,6 +64,22 @@ public:
         return end();
     }
 
+    Attr * get(const Symbol & name)
+    {
+        Attr key(name, 0);
+        iterator i = std::lower_bound(begin(), end(), key);
+        if (i != end() && i->name == name) return &*i;
+        return nullptr;
+    }
+
+    Attr & need(const Symbol & name, const Pos & pos = noPos)
+    {
+        auto a = get(name);
+        if (!a)
+            throw Error("attribute '%s' missing, at %s", name, pos);
+        return *a;
+    }
+
     iterator begin() { return &attrs[0]; }
     iterator end() { return &attrs[size_]; }
 

src/libexpr/eval.cc

@@ -43,15 +43,27 @@ static char * dupString(const char * s)
 }
 
 
+static char * dupStringWithLen(const char * s, size_t size)
+{
+    char * t;
+#if HAVE_BOEHMGC
+    t = GC_STRNDUP(s, size);
+#else
+    t = strndup(s, size);
+#endif
+    if (!t) throw std::bad_alloc();
+    return t;
+}
+
+
 static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
 {
     checkInterrupt();
 
-    if (active.find(&v) != active.end()) {
+    if (!active.insert(&v).second) {
         str << "<CYCLE>";
         return;
     }
-    active.insert(&v);
 
     switch (v.type) {
     case tInt:
@@ -219,7 +231,7 @@ void initGC()
        that GC_expand_hp() causes a lot of virtual, but not physical
        (resident) memory to be allocated.  This might be a problem on
        systems that don't overcommit. */
-    if (!getenv("GC_INITIAL_HEAP_SIZE")) {
+    if (!getEnv("GC_INITIAL_HEAP_SIZE")) {
         size_t size = 32 * 1024 * 1024;
 #if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
         size_t maxSize = 384 * 1024 * 1024;
@@ -308,7 +320,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
 {
-    countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
+    countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
     assert(gcInitialised);
 
@@ -316,9 +328,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
     /* Initialise the Nix expression search path. */
     if (!evalSettings.pureEval) {
-        Strings paths = parseNixPath(getEnv("NIX_PATH", ""));
         for (auto & i : _searchPath) addToSearchPath(i);
-        for (auto & i : paths) addToSearchPath(i);
+        for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
     }
     addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
 
@@ -332,10 +343,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
             auto path = r.second;
 
             if (store->isInStore(r.second)) {
-                PathSet closure;
-                store->computeFSClosure(store->toStorePath(r.second), closure);
+                StorePathSet closure;
+                store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
                 for (auto & path : closure)
-                    allowedPaths->insert(path);
+                    allowedPaths->insert(store->printStorePath(path));
             } else
                 allowedPaths->insert(r.second);
         }
@@ -434,7 +445,7 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context)
         !context.empty() && store->isInStore(path)
         ? store->toRealPath(path)
         : path;
-};
+}
 
 
 Value * EvalState::addConstant(const string & name, Value & v)
@@ -520,9 +531,9 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun
     throw TypeError(format(s) % fun.showNamePos() % s2 % pos);
 }
 
-LocalNoInlineNoReturn(void throwAssertionError(const char * s, const Pos & pos))
+LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos))
 {
-    throw AssertionError(format(s) % pos);
+    throw AssertionError(format(s) % s1 % pos);
 }
 
 LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos))
@@ -552,9 +563,11 @@ void mkString(Value & v, const char * s)
 }
 
 
-Value & mkString(Value & v, const string & s, const PathSet & context)
+Value & mkString(Value & v, std::string_view s, const PathSet & context)
 {
-    mkString(v, s.c_str());
+    v.type = tString;
+    v.string.s = dupStringWithLen(s.data(), s.size());
+    v.string.context = 0;
     if (!context.empty()) {
         size_t n = 0;
         v.string.context = (const char * *)
@@ -617,13 +630,9 @@ Value * EvalState::allocValue()
 
 Env & EvalState::allocEnv(size_t size)
 {
-    if (size > std::numeric_limits<decltype(Env::size)>::max())
-        throw Error("environment size %d is too big", size);
-
     nrEnvs++;
     nrValuesInEnvs += size;
     Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
-    env->size = (decltype(Env::size)) size;
     env->type = Env::Plain;
 
     /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
@@ -1253,8 +1262,11 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v)
 
 void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 {
-    if (!state.evalBool(env, cond, pos))
-        throwAssertionError("assertion failed at %1%", pos);
+    if (!state.evalBool(env, cond, pos)) {
+        std::ostringstream out;
+        cond->show(out);
+        throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos);
+    }
     body->eval(state, env, v);
 }
 
@@ -1444,8 +1456,7 @@ void EvalState::forceValueDeep(Value & v)
     std::function<void(Value & v)> recurse;
 
     recurse = [&](Value & v) {
-        if (seen.find(&v) != seen.end()) return;
-        seen.insert(&v);
+        if (!seen.insert(&v).second) return;
 
         forceValue(v);
 
@@ -1644,15 +1655,16 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
         throwEvalError("file names are not allowed to end in '%1%'", drvExtension);
 
     Path dstPath;
-    if (srcToStore[path] != "")
-        dstPath = srcToStore[path];
+    auto i = srcToStore.find(path);
+    if (i != srcToStore.end())
+        dstPath = store->printStorePath(i->second);
     else {
-        dstPath = settings.readOnlyMode
-            ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first
-            : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
-        srcToStore[path] = dstPath;
-        printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")
-            % path % dstPath);
+        auto p = settings.readOnlyMode
+            ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
+            : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
+        dstPath = store->printStorePath(p);
+        srcToStore.insert_or_assign(path, std::move(p));
+        printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
     }
 
     context.insert(dstPath);
@@ -1753,7 +1765,7 @@ bool EvalState::eqValues(Value & v1, Value & v2)
 
 void EvalState::printStats()
 {
-    bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0";
+    bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
 
     struct rusage buf;
     getrusage(RUSAGE_SELF, &buf);
@@ -1769,7 +1781,7 @@ void EvalState::printStats()
     GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
 #endif
     if (showStats) {
-        auto outPath = getEnv("NIX_SHOW_STATS_PATH","-");
+        auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-");
         std::fstream fs;
         if (outPath != "-")
             fs.open(outPath, std::fstream::out);
@@ -1861,7 +1873,7 @@ void EvalState::printStats()
             }
         }
 
-        if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
+        if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") {
             auto list = topObj.list("symbols");
             symbols.dump([&](const std::string & s) { list.elem(s); });
         }
@@ -1869,99 +1881,6 @@ void EvalState::printStats()
 }
 
 
-size_t valueSize(Value & v)
-{
-    std::set<const void *> seen;
-
-    auto doString = [&](const char * s) -> size_t {
-        if (seen.find(s) != seen.end()) return 0;
-        seen.insert(s);
-        return strlen(s) + 1;
-    };
-
-    std::function<size_t(Value & v)> doValue;
-    std::function<size_t(Env & v)> doEnv;
-
-    doValue = [&](Value & v) -> size_t {
-        if (seen.find(&v) != seen.end()) return 0;
-        seen.insert(&v);
-
-        size_t sz = sizeof(Value);
-
-        switch (v.type) {
-        case tString:
-            sz += doString(v.string.s);
-            if (v.string.context)
-                for (const char * * p = v.string.context; *p; ++p)
-                    sz += doString(*p);
-            break;
-        case tPath:
-            sz += doString(v.path);
-            break;
-        case tAttrs:
-            if (seen.find(v.attrs) == seen.end()) {
-                seen.insert(v.attrs);
-                sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity();
-                for (auto & i : *v.attrs)
-                    sz += doValue(*i.value);
-            }
-            break;
-        case tList1:
-        case tList2:
-        case tListN:
-            if (seen.find(v.listElems()) == seen.end()) {
-                seen.insert(v.listElems());
-                sz += v.listSize() * sizeof(Value *);
-                for (size_t n = 0; n < v.listSize(); ++n)
-                    sz += doValue(*v.listElems()[n]);
-            }
-            break;
-        case tThunk:
-            sz += doEnv(*v.thunk.env);
-            break;
-        case tApp:
-            sz += doValue(*v.app.left);
-            sz += doValue(*v.app.right);
-            break;
-        case tLambda:
-            sz += doEnv(*v.lambda.env);
-            break;
-        case tPrimOpApp:
-            sz += doValue(*v.primOpApp.left);
-            sz += doValue(*v.primOpApp.right);
-            break;
-        case tExternal:
-            if (seen.find(v.external) != seen.end()) break;
-            seen.insert(v.external);
-            sz += v.external->valueSize(seen);
-            break;
-        default:
-            ;
-        }
-
-        return sz;
-    };
-
-    doEnv = [&](Env & env) -> size_t {
-        if (seen.find(&env) != seen.end()) return 0;
-        seen.insert(&env);
-
-        size_t sz = sizeof(Env) + sizeof(Value *) * env.size;
-
-        if (env.type != Env::HasWithExpr)
-            for (size_t i = 0; i < env.size; ++i)
-                if (env.values[i])
-                    sz += doValue(*env.values[i]);
-
-        if (env.up) sz += doEnv(*env.up);
-
-        return sz;
-    };
-
-    return doValue(v);
-}
-
-
 string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
 {
     throw TypeError(format("cannot coerce %1% to a string, at %2%") %
@@ -1980,6 +1899,22 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
+EvalSettings::EvalSettings()
+{
+    auto var = getEnv("NIX_PATH");
+    if (var) nixPath = parseNixPath(*var);
+}
+
+Strings EvalSettings::getDefaultNixPath()
+{
+    Strings res;
+    auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
+    add(getHome() + "/.nix-defexpr/channels");
+    add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
+    add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
+    return res;
+}
+
 EvalSettings evalSettings;
 
 static GlobalConfig::Register r1(&evalSettings);

src/libexpr/eval.hh

@@ -7,6 +7,7 @@
 #include "hash.hh"
 #include "config.hh"
 
+#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
@@ -17,6 +18,7 @@ namespace nix {
 
 class Store;
 class EvalState;
+struct StorePath;
 enum RepairFlag : bool;
 
 
@@ -36,21 +38,20 @@ struct PrimOp
 struct Env
 {
     Env * up;
-    unsigned short size; // used by ‘valueSize’
     unsigned short prevWith:14; // nr of levels up to next `with' environment
     enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
 };
 
 
-Value & mkString(Value & v, const string & s, const PathSet & context = PathSet());
+Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
 
 void copyContext(const Value & v, PathSet & context);
 
 
 /* Cache for calls to addToStore(); maps source paths to the store
    paths. */
-typedef std::map<Path, Path> SrcToStore;
+typedef std::map<Path, StorePath> SrcToStore;
 
 
 std::ostream & operator << (std::ostream & str, const Value & v);
@@ -114,6 +115,9 @@ private:
     /* Cache used by checkSourcePath(). */
     std::unordered_map<Path, Path> resolvedPaths;
 
+    /* Cache used by prim_match(). */
+    std::unordered_map<std::string, std::regex> regexCache;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -141,8 +145,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(const string & s, const Path & basePath);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath);
 
     Expr * parseStdin();
 
@@ -268,6 +272,7 @@ public:
     Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
+    Value * allocAttr(Value & vAttrs, const std::string & name);
 
     Bindings * allocBindings(size_t capacity);
 
@@ -314,6 +319,7 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
 };
 
 
@@ -338,9 +344,16 @@ struct InvalidPathError : EvalError
 
 struct EvalSettings : Config
 {
+    EvalSettings();
+
+    static Strings getDefaultNixPath();
+
     Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
         "Whether builtin functions that allow executing native code should be enabled."};
 
+    Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path",
+        "List of directories to be searched for <...> file references."};
+
     Setting<bool> restrictEval{this, false, "restrict-eval",
         "Whether to restrict file system access to paths in $NIX_PATH, "
         "and network access to the URI prefixes listed in 'allowed-uris'."};
@@ -355,7 +368,7 @@ struct EvalSettings : Config
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/function-trace.cc

@@ -1,4 +1,5 @@
 #include "function-trace.hh"
+#include "logging.hh"
 
 namespace nix {
 

src/libexpr/get-drvs.cc

@@ -19,27 +19,27 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
 DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
     : state(&state), attrs(nullptr), attrPath("")
 {
-    auto spec = parseDrvPathWithOutputs(drvPathWithOutputs);
+    auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
 
-    drvPath = spec.first;
+    this->drvPath = store->printStorePath(drvPath);
 
     auto drv = store->derivationFromPath(drvPath);
 
-    name = storePathToName(drvPath);
+    name = drvPath.name();
 
-    if (spec.second.size() > 1)
+    if (selectedOutputs.size() > 1)
         throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs);
 
     outputName =
-        spec.second.empty()
-        ? get(drv.env, "outputName", "out")
-        : *spec.second.begin();
+        selectedOutputs.empty()
+        ? get(drv.env, "outputName").value_or("out")
+        : *selectedOutputs.begin();
 
     auto i = drv.outputs.find(outputName);
     if (i == drv.outputs.end())
-        throw Error("derivation '%s' does not have output '%s'", drvPath, outputName);
+        throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
 
-    outPath = i->second.path;
+    outPath = store->printStorePath(i->second.path);
 }
 
 
@@ -277,8 +277,7 @@ static bool getDerivation(EvalState & state, Value & v,
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
            derivation {...}; y = x;}'. */
-        if (done.find(v.attrs) != done.end()) return false;
-        done.insert(v.attrs);
+        if (!done.insert(v.attrs).second) return false;
 
         DrvInfo drv(state, attrPath, v.attrs);
 

src/libexpr/json-to-value.cc

@@ -1,149 +1,161 @@
 #include "json-to-value.hh"
 
-#include <cstring>
+#include <variant>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+using std::unique_ptr;
 
 namespace nix {
 
+// for more information, refer to
+// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
+class JSONSax : nlohmann::json_sax<json> {
+    class JSONState {
+    protected:
+        unique_ptr<JSONState> parent;
+        Value * v;
+    public:
+        virtual unique_ptr<JSONState> resolve(EvalState &)
+        {
+            throw std::logic_error("tried to close toplevel json parser state");
+        };
+        explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
+        explicit JSONState(Value* v) : v(v) {};
+        JSONState(JSONState& p) = delete;
+        Value& value(EvalState & state)
+        {
+            if (v == nullptr)
+                v = state.allocValue();
+            return *v;
+        };
+        virtual ~JSONState() {};
+        virtual void add() {};
+    };
 
-static void skipWhitespace(const char * & s)
-{
-    while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
-}
-
-
-static string parseJSONString(const char * & s)
-{
-    string res;
-    if (*s++ != '"') throw JSONParseError("expected JSON string");
-    while (*s != '"') {
-        if (!*s) throw JSONParseError("got end-of-string in JSON string");
-        if (*s == '\\') {
-            s++;
-            if (*s == '"') res += '"';
-            else if (*s == '\\') res += '\\';
-            else if (*s == '/') res += '/';
-            else if (*s == '/') res += '/';
-            else if (*s == 'b') res += '\b';
-            else if (*s == 'f') res += '\f';
-            else if (*s == 'n') res += '\n';
-            else if (*s == 'r') res += '\r';
-            else if (*s == 't') res += '\t';
-            else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported");
-            else throw JSONParseError("invalid escaped character in JSON string");
-            s++;
-        } else
-            res += *s++;
-    }
-    s++;
-    return res;
-}
-
-
-static void parseJSON(EvalState & state, const char * & s, Value & v)
-{
-    skipWhitespace(s);
-
-    if (!*s) throw JSONParseError("expected JSON value");
-
-    if (*s == '[') {
-        s++;
-        ValueVector values;
-        values.reserve(128);
-        skipWhitespace(s);
-        while (1) {
-            if (values.empty() && *s == ']') break;
-            Value * v2 = state.allocValue();
-            parseJSON(state, s, *v2);
-            values.push_back(v2);
-            skipWhitespace(s);
-            if (*s == ']') break;
-            if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element");
-            s++;
+    class JSONObjectState : public JSONState {
+        using JSONState::JSONState;
+        ValueMap attrs = ValueMap();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        {
+            Value& v = parent->value(state);
+            state.mkAttrs(v, attrs.size());
+            for (auto & i : attrs)
+                v.attrs->push_back(Attr(i.first, i.second));
+            return std::move(parent);
         }
-        s++;
-        state.mkList(v, values.size());
-        for (size_t n = 0; n < values.size(); ++n)
-            v.listElems()[n] = values[n];
-    }
-
-    else if (*s == '{') {
-        s++;
-        ValueMap attrs;
-        while (1) {
-            skipWhitespace(s);
-            if (attrs.empty() && *s == '}') break;
-            string name = parseJSONString(s);
-            skipWhitespace(s);
-            if (*s != ':') throw JSONParseError("expected ':' in JSON object");
-            s++;
-            Value * v2 = state.allocValue();
-            parseJSON(state, s, *v2);
-            attrs[state.symbols.create(name)] = v2;
-            skipWhitespace(s);
-            if (*s == '}') break;
-            if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member");
-            s++;
+        virtual void add() override { v = nullptr; };
+    public:
+        void key(string_t& name, EvalState & state)
+        {
+            attrs[state.symbols.create(name)] = &value(state);
         }
-        state.mkAttrs(v, attrs.size());
-        for (auto & i : attrs)
-            v.attrs->push_back(Attr(i.first, i.second));
-        v.attrs->sort();
-        s++;
-    }
+    };
 
-    else if (*s == '"') {
-        mkString(v, parseJSONString(s));
-    }
-
-    else if (isdigit(*s) || *s == '-' || *s == '.' ) {
-        // Buffer into a string first, then use built-in C++ conversions
-        std::string tmp_number;
-        ValueType number_type = tInt;
-
-        while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') {
-            if (*s == '.' || *s == 'e' || *s == 'E')
-                number_type = tFloat;
-            tmp_number += *s++;
+    class JSONListState : public JSONState {
+        ValueVector values = ValueVector();
+        virtual unique_ptr<JSONState> resolve(EvalState & state) override
+        {
+            Value& v = parent->value(state);
+            state.mkList(v, values.size());
+            for (size_t n = 0; n < values.size(); ++n) {
+                v.listElems()[n] = values[n];
+            }
+            return std::move(parent);
         }
-
-        try {
-            if (number_type == tFloat)
-                mkFloat(v, stod(tmp_number));
-            else
-                mkInt(v, stol(tmp_number));
-        } catch (std::invalid_argument & e) {
-            throw JSONParseError("invalid JSON number");
-        } catch (std::out_of_range & e) {
-            throw JSONParseError("out-of-range JSON number");
+        virtual void add() override {
+            values.push_back(v);
+            v = nullptr;
+        };
+    public:
+        JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
+        {
+            values.reserve(reserve);
         }
+    };
+
+    EvalState & state;
+    unique_ptr<JSONState> rs;
+
+    template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
+    {
+        f(rs->value(state), args...);
+        rs->add();
+        return true;
     }
 
-    else if (strncmp(s, "true", 4) == 0) {
-        s += 4;
-        mkBool(v, true);
+public:
+    JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
+
+    bool null()
+    {
+        return handle_value(mkNull);
     }
 
-    else if (strncmp(s, "false", 5) == 0) {
-        s += 5;
-        mkBool(v, false);
+    bool boolean(bool val)
+    {
+        return handle_value(mkBool, val);
     }
 
-    else if (strncmp(s, "null", 4) == 0) {
-        s += 4;
-        mkNull(v);
+    bool number_integer(number_integer_t val)
+    {
+        return handle_value(mkInt, val);
     }
 
-    else throw JSONParseError("unrecognised JSON value");
-}
+    bool number_unsigned(number_unsigned_t val)
+    {
+        return handle_value(mkInt, val);
+    }
 
+    bool number_float(number_float_t val, const string_t& s)
+    {
+        return handle_value(mkFloat, val);
+    }
+
+    bool string(string_t& val)
+    {
+        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
+    }
+
+    bool start_object(std::size_t len)
+    {
+        rs = std::make_unique<JSONObjectState>(std::move(rs));
+        return true;
+    }
+
+    bool key(string_t& name)
+    {
+        dynamic_cast<JSONObjectState*>(rs.get())->key(name, state);
+        return true;
+    }
+
+    bool end_object() {
+        rs = rs->resolve(state);
+        rs->add();
+        return true;
+    }
+
+    bool end_array() {
+        return end_object();
+    }
+
+    bool start_array(size_t len) {
+        rs = std::make_unique<JSONListState>(std::move(rs),
+            len != std::numeric_limits<size_t>::max() ? len : 128);
+        return true;
+    }
+
+    bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) {
+        throw JSONParseError(ex.what());
+    }
+};
 
 void parseJSON(EvalState & state, const string & s_, Value & v)
 {
-    const char * s = s_.c_str();
-    parseJSON(state, s, v);
-    skipWhitespace(s);
-    if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s);
+    JSONSax parser(state, v);
+    bool res = json::sax_parse(s_, &parser);
+    if (!res)
+        throw JSONParseError("Invalid JSON Value");
 }
 
-
 }

src/libexpr/json-to-value.hh

@@ -6,7 +6,7 @@
 
 namespace nix {
 
-MakeError(JSONParseError, EvalError)
+MakeError(JSONParseError, EvalError);
 
 void parseJSON(EvalState & state, const string & s, Value & v);
 

src/libexpr/local.mk

@@ -6,7 +6,9 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
-libexpr_LIBS = libutil libstore
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+
+libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
 ifneq ($(OS), FreeBSD)
@@ -31,3 +33,5 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh

src/libexpr/names.cc

@@ -16,14 +16,14 @@ DrvName::DrvName()
    a letter.  The `version' part is the rest (excluding the separating
    dash).  E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd',
    '2.0.48'). */
-DrvName::DrvName(const string & s) : hits(0)
+DrvName::DrvName(std::string_view s) : hits(0)
 {
-    name = fullName = s;
+    name = fullName = std::string(s);
     for (unsigned int i = 0; i < s.size(); ++i) {
         /* !!! isalpha/isdigit are affected by the locale. */
         if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) {
-            name = string(s, 0, i);
-            version = string(s, i + 1);
+            name = s.substr(0, i);
+            version = s.substr(i + 1);
             break;
         }
     }

src/libexpr/names.hh

@@ -15,7 +15,7 @@ struct DrvName
     unsigned int hits;
 
     DrvName();
-    DrvName(const string & s);
+    DrvName(std::string_view s);
     bool matches(DrvName & n);
 
 private:

src/libexpr/nixexpr.hh

@@ -9,14 +9,14 @@
 namespace nix {
 
 
-MakeError(EvalError, Error)
-MakeError(ParseError, Error)
-MakeError(AssertionError, EvalError)
-MakeError(ThrownError, AssertionError)
-MakeError(Abort, EvalError)
-MakeError(TypeError, EvalError)
-MakeError(UndefinedVarError, Error)
-MakeError(RestrictedPathError, Error)
+MakeError(EvalError, Error);
+MakeError(ParseError, Error);
+MakeError(AssertionError, EvalError);
+MakeError(ThrownError, AssertionError);
+MakeError(Abort, EvalError);
+MakeError(TypeError, EvalError);
+MakeError(UndefinedVarError, Error);
+MakeError(RestrictedPathError, Error);
 
 
 /* Position objects. */

src/libexpr/parser.y

@@ -1,5 +1,5 @@
 %glr-parser
-%pure-parser
+%define api.pure
 %locations
 %define parse.error verbose
 %defines
@@ -20,6 +20,7 @@
 
 #include "nixexpr.hh"
 #include "eval.hh"
+#include "globals.hh"
 
 namespace nix {
 
@@ -138,11 +139,10 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 
 static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 {
-    if (formals->argNames.find(formal.name) != formals->argNames.end())
+    if (!formals->argNames.insert(formal.name).second)
         throw ParseError(format("duplicate formal function argument '%1%' at %2%")
             % formal.name % pos);
     formals->formals.push_front(formal);
-    formals->argNames.insert(formal.name);
 }
 
 
@@ -402,7 +402,12 @@ expr_simple
               new ExprVar(data->symbols.create("__nixPath"))),
           new ExprString(data->symbols.create(path)));
   }
-  | URI { $$ = new ExprString(data->symbols.create($1)); }
+  | URI {
+      static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
+      if (noURLLiterals)
+          throw ParseError("URL literals are disabled, at %s", CUR_POS);
+      $$ = new ExprString(data->symbols.create($1));
+  }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
      into `(rec {..., body = ...}).body'. */
@@ -571,12 +576,17 @@ Path resolveExprPath(Path path)
 {
     assert(path[0] == '/');
 
+    unsigned int followCount = 0, maxFollow = 1024;
+
     /* If `path' is a symlink, follow it.  This is so that relative
        path references work. */
     struct stat st;
     while (true) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
         if (lstat(path.c_str(), &st))
-            throw SysError(format("getting status of '%1%'") % path);
+            throw SysError("getting status of '%s'", path);
         if (!S_ISLNK(st.st_mode)) break;
         path = absPath(readLink(path), dirOf(path));
     }
@@ -601,13 +611,13 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.c_str(), "(string)", basePath, staticEnv);
+    return parse(s.data(), "(string)", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 {
     return parseExprFromString(s, basePath, staticBaseEnv);
 }

src/libexpr/primops.cc

@@ -44,29 +44,28 @@ std::pair<string, string> decodeContext(const string & s)
 
 
 InvalidPathError::InvalidPathError(const Path & path) :
-    EvalError(format("path '%1%' is not valid") % path), path(path) {}
+    EvalError("path '%s' is not valid", path), path(path) {}
 
 void EvalState::realiseContext(const PathSet & context)
 {
-    PathSet drvs;
+    std::vector<StorePathWithOutputs> drvs;
 
     for (auto & i : context) {
         std::pair<string, string> decoded = decodeContext(i);
-        Path ctx = decoded.first;
-        assert(store->isStorePath(ctx));
+        auto ctx = store->parseStorePath(decoded.first);
         if (!store->isValidPath(ctx))
-            throw InvalidPathError(ctx);
-        if (!decoded.second.empty() && nix::isDerivation(ctx)) {
-            drvs.insert(decoded.first + "!" + decoded.second);
+            throw InvalidPathError(store->printStorePath(ctx));
+        if (!decoded.second.empty() && ctx.isDerivation()) {
+            drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}});
 
             /* Add the output of this derivation to the allowed
                paths. */
             if (allowedPaths) {
-                auto drv = store->derivationFromPath(decoded.first);
+                auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
                 DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
                 if (i == drv.outputs.end())
                     throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
-                allowedPaths->insert(i->second.path);
+                allowedPaths->insert(store->printStorePath(i->second.path));
             }
         }
     }
@@ -74,10 +73,11 @@ void EvalState::realiseContext(const PathSet & context)
     if (drvs.empty()) return;
 
     if (!evalSettings.enableImportFromDerivation)
-        throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin()));
+        throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
+            store->printStorePath(drvs.begin()->path));
 
     /* For performance, prefetch all substitute info. */
-    PathSet willBuild, willSubstitute, unknown;
+    StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
     store->buildPaths(drvs);
@@ -100,8 +100,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
 
-    if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) {
-        Derivation drv = readDerivation(realPath);
+    // FIXME
+    if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
+        Derivation drv = readDerivation(*state.store, realPath);
         Value & w = *state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
         Value * v2 = state.allocAttr(w, state.sDrvPath);
@@ -115,15 +116,22 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
 
         for (const auto & o : drv.outputs) {
             v2 = state.allocAttr(w, state.symbols.create(o.first));
-            mkString(*v2, o.second.path, {"!" + o.first + "!" + path});
+            mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path});
             outputsVal->listElems()[outputs_index] = state.allocValue();
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
         w.attrs->sort();
-        Value fun;
-        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
-        state.forceFunction(fun, pos);
-        mkApp(v, fun, w);
+
+        static Value * fun = nullptr;
+        if (!fun) {
+            fun = state.allocValue();
+            state.eval(state.parseExprFromString(
+                #include "imported-drv-to-derivation.nix.gen.hh"
+                , "/"), *fun);
+        }
+
+        state.forceFunction(*fun, pos);
+        mkApp(v, *fun, w);
         state.forceAttrs(v, pos);
     } else {
         state.forceAttrs(*args[0]);
@@ -396,8 +404,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
             throw EvalError(format("attribute 'key' required, at %1%") % pos);
         state.forceValue(*key->value);
 
-        if (doneKeys.find(key->value) != doneKeys.end()) continue;
-        doneKeys.insert(key->value);
+        if (!doneKeys.insert(key->value).second) continue;
         res.push_back(e);
 
         /* Call the `operator' function with `e' as argument. */
@@ -470,7 +477,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string name = state.forceStringNoCtx(*args[0], pos);
-    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name));
+    mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or(""));
 }
 
 
@@ -507,13 +514,6 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value
 }
 
 
-void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
-    /* We're not forcing the argument on purpose. */
-    mkInt(v, valueSize(*args[0]));
-}
-
-
 /*************************************************************
  * Derivations
  *************************************************************/
@@ -684,24 +684,24 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
            runs. */
         if (path.at(0) == '=') {
             /* !!! This doesn't work if readOnlyMode is set. */
-            PathSet refs;
-            state.store->computeFSClosure(string(path, 1), refs);
+            StorePathSet refs;
+            state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
             for (auto & j : refs) {
-                drv.inputSrcs.insert(j);
-                if (isDerivation(j))
-                    drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j);
+                drv.inputSrcs.insert(j.clone());
+                if (j.isDerivation())
+                    drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j);
             }
         }
 
         /* Handle derivation outputs of the form ‘!<name>!<path>’. */
         else if (path.at(0) == '!') {
             std::pair<string, string> ctx = decodeContext(path);
-            drv.inputDrvs[ctx.first].insert(ctx.second);
+            drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second);
         }
 
         /* Otherwise it's a source file. */
         else
-            drv.inputSrcs.insert(path);
+            drv.inputSrcs.insert(state.store->parseStorePath(path));
     }
 
     /* Do we have all required attributes? */
@@ -711,10 +711,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName);
 
     /* Check whether the derivation name is valid. */
-    checkStoreName(drvName);
     if (isDerivation(drvName))
-        throw EvalError(format("derivation names are not allowed to end in '%1%', at %2%")
-            % drvExtension % posDrvName);
+        throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName);
 
     if (outputHash) {
         /* Handle fixed-output derivations. */
@@ -724,52 +722,53 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
         Hash h(*outputHash, ht);
 
-        Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
-        if (!jsonObject) drv.env["out"] = outPath;
-        drv.outputs["out"] = DerivationOutput(outPath,
-            (outputHashRecursive ? "r:" : "") + printHashType(h.type),
-            h.to_string(Base16, false));
+        auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
+        if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
+        drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
+                (outputHashRecursive ? "r:" : "") + printHashType(h.type),
+                h.to_string(Base16, false)));
     }
 
     else {
-        /* Construct the "masked" store derivation, which is the final
-           one except that in the list of outputs, the output paths
-           are empty, and the corresponding environment variables have
-           an empty value.  This ensures that changes in the set of
-           output names do get reflected in the hash. */
+        /* Compute a hash over the "masked" store derivation, which is
+           the final one except that in the list of outputs, the
+           output paths are empty strings, and the corresponding
+           environment variables have an empty value.  This ensures
+           that changes in the set of output names do get reflected in
+           the hash. */
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
-            drv.outputs[i] = DerivationOutput("", "", "");
+            drv.outputs.insert_or_assign(i,
+                DerivationOutput(StorePath::dummy.clone(), "", ""));
         }
 
-        /* Use the masked derivation expression to compute the output
-           path. */
-        Hash h = hashDerivationModulo(*state.store, drv);
+        Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
 
-        for (auto & i : drv.outputs)
-            if (i.second.path == "") {
-                Path outPath = state.store->makeOutputPath(i.first, h, drvName);
-                if (!jsonObject) drv.env[i.first] = outPath;
-                i.second.path = outPath;
-            }
+        for (auto & i : outputs) {
+            auto outPath = state.store->makeOutputPath(i, h, drvName);
+            if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
+            drv.outputs.insert_or_assign(i,
+                DerivationOutput(std::move(outPath), "", ""));
+        }
     }
 
     /* Write the resulting term into the Nix store directory. */
-    Path drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+    auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+    auto drvPathS = state.store->printStorePath(drvPath);
 
-    printMsg(lvlChatty, format("instantiated '%1%' -> '%2%'")
-        % drvName % drvPath);
+    printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
 
     /* Optimisation, but required in read-only mode! because in that
        case we don't actually write store derivations, so we can't
        read them later. */
-    drvHashes[drvPath] = hashDerivationModulo(*state.store, drv);
+    drvHashes.insert_or_assign(drvPath.clone(),
+        hashDerivationModulo(*state.store, Derivation(drv), false));
 
     state.mkAttrs(v, 1 + drv.outputs.size());
-    mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath});
+    mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
     for (auto & i : drv.outputs) {
         mkString(*state.allocAttr(v, state.symbols.create(i.first)),
-            i.second.path, {"!" + i.first + "!" + drvPath});
+            state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS});
     }
     v.attrs->sort();
 }
@@ -822,7 +821,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
         throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos);
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
-        state.store->ensurePath(path2);
+        state.store->ensurePath(state.store->parseStorePath(path2));
     context.insert(path2);
     mkString(v, path, context);
 }
@@ -1018,17 +1017,17 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
     string name = state.forceStringNoCtx(*args[0], pos);
     string contents = state.forceString(*args[1], context, pos);
 
-    PathSet refs;
+    StorePathSet refs;
 
     for (auto path : context) {
         if (path.at(0) != '/')
             throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos);
-        refs.insert(path);
+        refs.insert(state.store->parseStorePath(path));
     }
 
-    Path storePath = settings.readOnlyMode
+    auto storePath = state.store->printStorePath(settings.readOnlyMode
         ? state.store->computeStorePathForText(name, contents, refs)
-        : state.store->addTextToStore(name, contents, refs, state.repair);
+        : state.store->addTextToStore(name, contents, refs, state.repair));
 
     /* Note: we don't need to add `context' to the context of the
        result, since `storePath' itself has references to the paths
@@ -1068,21 +1067,18 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
         return state.forceBool(res, pos);
     }) : defaultPathFilter;
 
-    Path expectedStorePath;
-    if (expectedHash) {
-        expectedStorePath =
-            state.store->makeFixedOutputPath(recursive, expectedHash, name);
-    }
+    std::optional<StorePath> expectedStorePath;
+    if (expectedHash)
+        expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name);
     Path dstPath;
-    if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
-        dstPath = settings.readOnlyMode
+    if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+        dstPath = state.store->printStorePath(settings.readOnlyMode
             ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
-            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
-        if (expectedHash && expectedStorePath != dstPath) {
-            throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
-        }
+            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair));
+        if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+            throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
     } else
-        dstPath = expectedStorePath;
+        dstPath = state.store->printStorePath(*expectedStorePath);
 
     mkString(v, dstPath, {dstPath});
 }
@@ -1099,7 +1095,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (args[0]->type != tLambda)
         throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
 
-    addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v);
 }
 
 static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1273,13 +1269,12 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         string name = state.forceStringNoCtx(*j->value, pos);
 
         Symbol sym = state.symbols.create(name);
-        if (seen.find(sym) == seen.end()) {
+        if (seen.insert(sym).second) {
             Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
             if (j2 == v2.attrs->end())
                 throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos);
 
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
-            seen.insert(sym);
         }
     }
 
@@ -1823,19 +1818,21 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
 
 /* Match a regular expression against a string and return either
    ‘null’ or a list containing substring matches. */
-static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto re = state.forceStringNoCtx(*args[0], pos);
 
     try {
 
-        std::regex regex(re, std::regex::extended);
+        auto regex = state.regexCache.find(re);
+        if (regex == state.regexCache.end())
+            regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
 
         PathSet context;
         const std::string str = state.forceString(*args[1], context, pos);
 
         std::smatch match;
-        if (!std::regex_match(str, match, regex)) {
+        if (!std::regex_match(str, match, regex->second)) {
             mkNull(v);
             return;
         }
@@ -2160,7 +2157,7 @@ void EvalState::createBaseEnv()
     }
 
     if (!evalSettings.pureEval) {
-        mkString(v, settings.thisSystem);
+        mkString(v, settings.thisSystem.get());
         addConstant("__currentSystem", v);
     }
 
@@ -2208,7 +2205,6 @@ void EvalState::createBaseEnv()
 
     // Debugging
     addPrimOp("__trace", 2, prim_trace);
-    addPrimOp("__valueSize", 1, prim_valueSize);
 
     // Paths
     addPrimOp("__toPath", 1, prim_toPath);

src/libexpr/primops/context.cc

@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (!state.store->isStorePath(i.name))
             throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos);
         if (!settings.readOnlyMode)
-            state.store->ensurePath(i.name);
+            state.store->ensurePath(state.store->parseStorePath(i.name));
         state.forceAttrs(*i.value, *i.pos);
         auto iter = i.value->attrs->find(sPath);
         if (iter != i.value->attrs->end()) {

src/libexpr/primops/fetchGit.cc

@@ -4,6 +4,7 @@
 #include "store-api.hh"
 #include "pathlocks.hh"
 #include "hash.hh"
+#include "tarfile.hh"
 
 #include <sys/time.h>
 
@@ -45,9 +46,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         if (!clean) {
 
-            /* This is an unclean working tree. So copy all tracked
-               files. */
-
+            /* This is an unclean working tree. So copy all tracked files. */
             GitInfo gitInfo;
             gitInfo.rev = "0000000000000000000000000000000000000000";
             gitInfo.shortRev = std::string(gitInfo.rev, 0, 7);
@@ -70,7 +69,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+            gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
 
             return gitInfo;
         }
@@ -157,7 +156,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
 
         gitInfo.storePath = json["storePath"];
 
-        if (store->isValidPath(gitInfo.storePath)) {
+        if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) {
             gitInfo.revCount = json["revCount"];
             return gitInfo;
         }
@@ -166,16 +165,18 @@ GitInfo exportGit(ref<Store> store, const std::string & uri,
         if (e.errNo != ENOENT) throw;
     }
 
-    // FIXME: should pipe this, or find some better way to extract a
-    // revision.
-    auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev });
+    auto source = sinkToSource([&](Sink & sink) {
+        RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev });
+        gitOptions.standardOut = &sink;
+        runProgram2(gitOptions);
+    });
 
     Path tmpDir = createTempDir();
     AutoDelete delTmpDir(tmpDir, true);
 
-    runProgram("tar", true, { "x", "-C", tmpDir }, tar);
+    unpackTarfile(*source, tmpDir);
 
-    gitInfo.storePath = store->addToStore(name, tmpDir);
+    gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
 
     gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev }));
 

src/libexpr/primops/fetchMercurial.cc

@@ -63,7 +63,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
                 return files.count(file);
             };
 
-            hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
+            hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter));
 
             return hgInfo;
         }
@@ -134,7 +134,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
 
         hgInfo.storePath = json["storePath"];
 
-        if (store->isValidPath(hgInfo.storePath)) {
+        if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) {
             printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath);
             return hgInfo;
         }
@@ -150,7 +150,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri,
 
     deletePath(tmpDir + "/.hg_archival.txt");
 
-    hgInfo.storePath = store->addToStore(name, tmpDir);
+    hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir));
 
     nlohmann::json json;
     json["storePath"] = hgInfo.storePath;

src/libexpr/primops/fromTOML.cc

@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "cpptoml/cpptoml.h"
+#include "../../cpptoml/cpptoml.h"
 
 namespace nix {
 

src/libexpr/symbol-table.hh

@@ -38,7 +38,12 @@ public:
         return s < s2.s;
     }
 
-    operator const string & () const
+    operator const std::string & () const
+    {
+        return *s;
+    }
+
+    operator const std::string_view () const
     {
         return *s;
     }

src/libexpr/value-to-xml.cc

@@ -105,10 +105,9 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
                 XMLOpenElement _(doc, "derivation", xmlAttrs);
 
-                if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) {
-                    drvsSeen.insert(drvPath);
+                if (drvPath != "" && drvsSeen.insert(drvPath).second)
                     showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen);
-                } else
+                else
                     doc.writeEmptyElement("repeated");
             }