downloader: when retry but can't resume, retry the whole file

(cherry picked from commit 2c692a3b14)

.dir-locals.el

@@ -13,4 +13,5 @@
   (eval . (c-set-offset 'arglist-cont-nonempty '+))
   (eval . (c-set-offset 'substatement-open 0))
   (eval . (c-set-offset 'access-label '-))
+  (eval . (c-set-offset 'inlambda 0))
   )))

.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: "Test"
+on:
+  pull_request:
+  push:
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v8
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

.gitignore

@@ -47,7 +47,7 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/*.gen.hh
+*.gen.*
 
 /src/nix/nix
 

Makefile.config.in

@@ -1,4 +1,5 @@
 AR = @AR@
+BDW_GC_LIBS = @BDW_GC_LIBS@
 BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
 CC = @CC@
 CFLAGS = @CFLAGS@
@@ -36,6 +37,7 @@ prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
+system = @system@
 doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

README.md

@@ -9,16 +9,13 @@ appear with Nix.
 
 To find out more about the tool, usage and installation instructions, please
 read the manual, which is available on the Nix website at
-<http://nixos.org/nix/manual>.
+<https://nixos.org/nix/manual>.
 
 ## Contributing
 
-Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking)
+Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking)
 of the manual. It helps you to get started with building Nix from source.
 
 ## License
 
 Nix is released under the LGPL v2.1
-
-This product includes software developed by the OpenSSL Project for
-use in the [OpenSSL Toolkit](http://www.OpenSSL.org/).

configure.ac

@@ -255,6 +255,17 @@ if test -n "$enable_s3"; then
 fi
 
 
+# Whether to use the Boehm garbage collector.
+AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc],
+  [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]),
+  gc=$enableval, gc=yes)
+if test "$gc" = yes; then
+  PKG_CHECK_MODULES([BDW_GC], [bdw-gc])
+  CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS"
+  AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.])
+fi
+
+
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen],
   [disable documentation generation]),

corepkgs/local.mk

@@ -1,4 +1,7 @@
-corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix
+corepkgs_FILES = \
+  unpack-channel.nix \
+  derivation.nix \
+  fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
 

doc/manual/command-ref/nix-env.xml

@@ -1066,7 +1066,8 @@ user environment elements, etc. -->
     the derivation, which can be used to unambiguously select it using
     the <link linkend="opt-attr"><option>--attr</option> option</link>
     available in commands that install derivations like
-    <literal>nix-env --install</literal>.</para></listitem>
+    <literal>nix-env --install</literal>. This option only works
+    together with <option>--available</option></para></listitem>
 
   </varlistentry>
 

doc/manual/command-ref/nix-store.xml

@@ -360,7 +360,6 @@ EOF
     <arg choice='plain'><option>--print-roots</option></arg>
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
-    <arg choice='plain'><option>--delete</option></arg>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
 </cmdsynopsis>
@@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of
 
   </varlistentry>
 
-  <varlistentry><term><option>--delete</option></term>
-
-    <listitem><para>This operation performs an actual garbage
-    collection.  All dead paths are removed from the
-    store.  This is the default.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 <para>By default, all unreachable paths are deleted.  The following
@@ -444,10 +435,10 @@ and <link
 linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
-<para>With <option>--delete</option>, the collector prints the total
-number of freed bytes when it finishes (or when it is interrupted).
-With <option>--print-dead</option>, it prints the number of bytes that
-would be freed.</para>
+<para>By default, the collector prints the total number of freed bytes
+when it finishes (or when it is interrupted). With
+<option>--print-dead</option>, it prints the number of bytes that would
+be freed.</para>
 
 </refsection>
 
@@ -1148,7 +1139,7 @@ the information that Nix considers important.  For instance,
 timestamps are elided because all files in the Nix store have their
 timestamp set to 0 anyway.  Likewise, all permissions are left out
 except for the execute bit, because all files in the Nix store have
-644 or 755 permission.</para>
+444 or 555 permission.</para>
 
 <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
 that “equal” paths always produce the same NAR archive.  For instance,

doc/manual/installation/installing-binary.xml

@@ -142,7 +142,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
   </para>
 
   <para>

doc/manual/installation/prerequisites-source.xml

@@ -8,6 +8,14 @@
 
 <itemizedlist>
 
+  <listitem><para>GNU Autoconf
+  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
+  and the autoconf-archive macro collection
+  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
+  These are only needed to run the bootstrap script, and are not necessary
+  if your source distribution came with a pre-built
+  <literal>./configure</literal> script.</para></listitem>
+
   <listitem><para>GNU Make.</para></listitem>
   
   <listitem><para>Bash Shell. The <literal>./configure</literal> script
@@ -50,6 +58,14 @@
   or higher.  If your distribution does not provide it, please install
   it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem>
 
+  <listitem><para>The <link
+  xlink:href="http://www.hboehm.info/gc/">Boehm
+  garbage collector</link> to reduce the evaluator’s memory
+  consumption (optional).  To enable it, install
+  <literal>pkgconfig</literal> and the Boehm garbage collector, and
+  pass the flag <option>--enable-gc</option> to
+  <command>configure</command>.</para></listitem>
+
   <listitem><para>The <literal>boost</literal> library of version
   1.66.0 or higher. It can be obtained from the official web site
   <link xlink:href="https://www.boost.org/" />.</para></listitem>

doc/manual/local.mk

@@ -4,11 +4,10 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
-  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
-  --param callout.graphics.extension \'.gif\' \
+  --param callout.graphics 0 \
   --param contrib.inline.enabled 0 \
   --stringparam generate.toc "book toc" \
   --param keep.relative.image.uris 0
@@ -66,12 +65,10 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
-$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts)))
-
 $(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
 
 

doc/manual/release-notes/rl-2.0.xml

@@ -503,14 +503,14 @@
   </listitem>
 
   <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
-    of the existing restricted evaluation mode. In pure mode, the Nix
-    evaluator forbids access to anything that could cause different
-    evaluations of the same command line arguments to produce a
+    <para><emphasis>Pure evaluation mode</emphasis>. With the
+    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
+    restricted evaluation mode that forbids access to anything that could cause
+    different evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as
     <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a
-    content hash or commit hash is specified. For example, calls to
+    <emphasis>all</emphasis> filesystem or network access unless a content hash
+    or commit hash is specified. For example, calls to
     <function>builtins.fetchGit</function> are only allowed if a
     <varname>rev</varname> attribute is specified.</para>
 

doc/manual/style.css

@@ -1,263 +0,0 @@
-/* Copied from http://bakefile.sourceforge.net/, which appears
-   licensed under the GNU GPL. */
-
-
-/***************************************************************************
-                             Basic headers and text:
- ***************************************************************************/
-
-body
-{
-    font-family: "Nimbus Sans L", sans-serif;
-    background: white;
-    margin: 2em 1em 2em 1em;
-}
-
-h1, h2, h3, h4
-{
-    color: #005aa0;
-}
-
-h1 /* title */
-{
-    font-size: 200%;
-}
-
-div.part h1
-{
-    font-size: 240%;
-}
-
-h2 /* chapters, appendices, subtitle */
-{
-    font-size: 180%;
-}
-
-div.part
-{
-    margin-top: 4em;
-}
-
-/* Extra space between chapters, appendices. */
-div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
-{ 
-    margin-top: 1.5em;
-}
-
-div.section > div.titlepage h2 /* sections */
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-h3 /* subsections */
-{
-    font-size: 125%;
-}
-
-div.simplesect h2
-{
-    font-size: 110%;
-}
-
-div.appendix h3
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-div.refentry\.separator
-{
-    margin-top: 2.5em;
-    margin-bottom: 2em;
-}
-
-div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
-{
-    margin-top: 1.4em;
-    font-size: 125%;
-}
-
-div.refsection h3
-{
-    font-size: 110%;
-}
-
-
-/***************************************************************************
-                               Examples:
- ***************************************************************************/
-
-div.example
-{
-    border: 1px solid #b0b0b0;
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    background: #f4f4f8;
-    border-radius: 0.4em;
-}
-
-div.example p.title
-{
-    margin-top: 0em;
-}
-
-div.example pre
-{
-}
-
-
-/***************************************************************************
-                            Screen dumps:
- ***************************************************************************/
-
-pre.screen, pre.programlisting
-{
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    color: #600000;
-    background: #f4f4f8;
-    font-family: monospace;
-}
-
-div.example pre.programlisting
-{
-    border: 0px;
-    padding: 0 0;
-    margin: 0 0 0 0;
-}
-
-
-/***************************************************************************
-                               Notes, warnings etc:
- ***************************************************************************/
-
-.note, .warning
-{
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    margin-bottom: 1em;
-    padding: 0.3em 0.3em 0.3em 0.3em;
-    background: #fffff5;
-    border-radius: 0.4em;
-}
-
-div.note, div.warning
-{
-    font-style: italic;
-}
-
-div.note h3, div.warning h3
-{
-    color: red;
-    font-size: 100%;
-    padding-right: 0.5em;
-    display: inline;
-}
-
-div.note p, div.warning p
-{
-    margin-bottom: 0em;
-}
-
-div.note h3 + p, div.warning h3 + p
-{
-    display: inline;
-}
-
-div.note h3
-{
-    color: blue;
-    font-size: 100%;
-}
-
-div.navfooter *
-{
-    font-size: 90%;
-}
-
-
-/***************************************************************************
-                        Links colors and highlighting: 
- ***************************************************************************/
-
-a { text-decoration: none; }
-a:hover { text-decoration: underline; }
-a:link { color: #0048b3; }
-a:visited { color: #002a6a; }
-
-
-/***************************************************************************
-                              Table of contents:
- ***************************************************************************/
-
-div.toc
-{
-    font-size: 90%;
-}
-
-div.toc dl
-{
-    margin-top: 0em;
-    margin-bottom: 0em;
-}
-
-
-/***************************************************************************
-                               Special elements:
- ***************************************************************************/
-
-tt, code
-{
-    color: #400000;
-}
-
-.term
-{
-    font-weight: bold;
-    
-}
-
-div.variablelist dd p, div.glosslist dd p
-{
-    margin-top: 0em;
-}
-
-div.variablelist dd, div.glosslist dd
-{
-    margin-left: 1.5em;
-}
-
-div.glosslist dt
-{
-    font-style: italic;
-}
-
-.varname
-{
-    color: #400000;
-}
-
-span.command strong
-{
-    font-weight: normal;
-    color: #400000;
-}
-
-div.calloutlist table
-{
-}
-
-table
-{
-    border-collapse: collapse;
-}
-
-div.affiliation
-{
-    font-style: italic;
-}

local.mk

@@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(GCH) $(PCH): src/libutil/util.hh config.h
+
+GCH_CXXFLAGS = -I src/libutil

mk/README.md

@@ -1,6 +0,0 @@
-This is a set of helper Makefiles for doing non-recursive builds with
-GNU Make.  The canonical source can be found at
-https://github.com/edolstra/make-rules.  You should copy the files
-into the `mk` subdirectory of your project.
-
-TODO: write more documentation.

mk/precompiled-headers.mk

@@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
 $(GCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 PCH = $(buildprefix)precompiled-headers.h.pch
 
 $(PCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 clean-files += $(GCH) $(PCH)
 

perl/lib/Nix/Store.xs

@@ -102,7 +102,7 @@ SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
             auto info = store()->queryPathInfo(store()->parseStorePath(path));
-            if (info->deriver)
+            if (!info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));

release-common.nix

@@ -34,7 +34,7 @@ rec {
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
-  tarballDeps =
+  buildDeps =
     [ bison
       flex
       libxml2
@@ -43,10 +43,8 @@ rec {
       docbook_xsl_ns
       autoconf-archive
       autoreconfHook
-    ];
 
-  buildDeps =
-    [ curl
+      curl
       bzip2 xz brotli zlib editline
       openssl pkgconfig sqlite
       libarchive
@@ -73,6 +71,10 @@ rec {
         */
       }));
 
+  propagatedDeps =
+    [ (boehmgc.override { enableLargeConfig = true; })
+    ];
+
   perlDeps =
     [ perl
       perlPackages.DBDSQLite

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -8,11 +8,14 @@ let
 
   pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
 
+  version =
+    builtins.readFile ./.version
+    + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
+
   jobs = rec {
 
     # Create a "vendor" directory that contains the crates listed in
-    # Cargo.lock, and include it in the Nix tarball. This allows Nix
-    # to be built without network access.
+    # Cargo.lock. This allows Nix to be built without network access.
     vendoredCrates =
       let
         lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
@@ -55,48 +58,6 @@ let
         '';
 
 
-    tarball =
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      releaseTools.sourceTarball {
-        name = "nix-tarball";
-        version = builtins.readFile ./.version;
-        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
-        src = nix;
-        inherit officialRelease;
-
-        buildInputs = tarballDeps ++ buildDeps;
-
-        postUnpack = ''
-          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
-          cat $sourceRoot/.dist-files
-        '';
-
-        preConfigure = ''
-          (cd perl ; autoreconf --install --force --verbose)
-          # TeX needs a writable font cache.
-          export VARTEXFONTS=$TMPDIR/texfonts
-        '';
-
-        distPhase =
-          ''
-            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
-
-            runHook preDist
-            make dist
-            mkdir -p $out/tarballs
-            cp *.tar.* $out/tarballs
-          '';
-
-        preDist = ''
-          make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk
-          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
-        '';
-      };
-
-
     build = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import nixpkgs { inherit system; }; in
@@ -105,16 +66,21 @@ let
 
       with import ./release-common.nix { inherit pkgs; };
 
-      releaseTools.nixBuild {
-        name = "nix";
-        src = tarball;
+      stdenv.mkDerivation {
+        name = "nix-${version}";
+
+        src = nix;
+
+        outputs = [ "out" "dev" "doc" ];
 
         buildInputs = buildDeps;
 
+        propagatedBuildInputs = propagatedDeps;
+
         preConfigure =
-          # Copy libboost_context so we don't get all of Boost in our closure.
-          # https://github.com/NixOS/nixpkgs/issues/45462
           ''
+            # Copy libboost_context so we don't get all of Boost in our closure.
+            # https://github.com/NixOS/nixpkgs/issues/45462
             mkdir -p $out/lib
             cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
             rm -f $out/lib/*.a
@@ -122,6 +88,10 @@ let
               chmod u+w $out/lib/*.so.*
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
+
+            ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor
+
+            (cd perl; autoreconf --install --force --verbose)
           '';
 
         configureFlags = configureFlags ++
@@ -133,8 +103,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
+        doCheck = true;
+
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
+
+        separateDebugInfo = true;
+
+        preDist = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
       });
 
 
@@ -143,11 +122,21 @@ let
       let pkgs = import nixpkgs { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
-        name = "nix-perl";
-        src = tarball;
+        name = "nix-perl-${version}";
+
+        src = nix;
 
         buildInputs =
-          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
+          [ autoconf-archive
+            autoreconfHook
+            jobs.build.${system}
+            curl
+            bzip2
+            xz
+            pkgconfig
+            pkgs.perl
+            boost
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
@@ -167,7 +156,6 @@ let
 
       let
         toplevel = builtins.getAttr system jobs.build;
-        version = toplevel.src.version;
         installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
       in
 
@@ -237,12 +225,13 @@ let
       with import ./release-common.nix { inherit pkgs; };
 
       releaseTools.coverageAnalysis {
-        name = "nix-build";
-        src = tarball;
+        name = "nix-coverage-${version}";
+
+        src = nix;
 
         enableParallelBuilding = true;
 
-        buildInputs = buildDeps;
+        buildInputs = buildDeps ++ propagatedDeps;
 
         dontInstall = false;
 
@@ -334,45 +323,20 @@ let
 
     installerScript =
       pkgs.runCommand "installer-script"
-        { buildInputs = [ build.x86_64-linux ];
-        }
+        { buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
         ''
           mkdir -p $out/nix-support
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
               (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
-              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+              systems
             } \
-            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+            --replace '@nixVersion@' ${version}
 
           echo "file installer $out/install" >> $out/nix-support/hydra-build-products
         '';
 
-
-    # Aggregate job containing the release-critical jobs.
-    release = pkgs.releaseTools.aggregate {
-      name = "nix-${tarball.version}";
-      meta.description = "Release-critical builds";
-      constituents =
-        [ tarball
-          build.i686-linux
-          build.x86_64-darwin
-          build.x86_64-linux
-          build.aarch64-linux
-          binaryTarball.i686-linux
-          binaryTarball.x86_64-darwin
-          binaryTarball.x86_64-linux
-          binaryTarball.aarch64-linux
-          tests.remoteBuilds
-          tests.nix-copy-closure
-          tests.binaryTarball
-          #tests.evalNixpkgs
-          #tests.evalNixOS
-          installerScript
-        ];
-    };
-
   };
 
 

scripts/install-multi-user.sh

@@ -13,12 +13,12 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
+readonly BOLD='\033[1m'
+readonly BLUE='\033[34m'
+readonly BLUE_UL='\033[4;34m'
+readonly GREEN='\033[32m'
+readonly GREEN_UL='\033[4;32m'
+readonly RED='\033[31m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
@@ -567,7 +567,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

scripts/install-nix-from-closure.sh

@@ -87,7 +87,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

scripts/install-systemd-multi-user.sh

@@ -88,7 +88,7 @@ poly_configure_nix_daemon_service() {
           systemctl start nix-daemon.socket
 
     _sudo "to start the nix-daemon.service" \
-          systemctl start nix-daemon.service
+          systemctl restart nix-daemon.service
 
 }
 

scripts/install.in

@@ -30,12 +30,13 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
 tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
+require_util xz "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -1,13 +1,13 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using std::cin;

src/libexpr/attr-path.cc

@@ -32,7 +32,7 @@ static Strings parseAttrPath(const string & s)
 }
 
 
-Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
@@ -40,7 +40,8 @@ Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
     Error attrError =
         Error(format("attribute selection path '%1%' does not match expression") % attrPath);
 
-    Ptr<Value> v(&vIn);
+    Value * v = &vIn;
+    Pos pos = noPos;
 
     for (auto & attr : tokens) {
 
@@ -50,7 +51,7 @@ Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
         if (string2Int(attr, attrIndex)) apType = apIndex;
 
         /* Evaluate the expression. */
-        auto vNew = state.allocValue();
+        Value * vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
         state.forceValue(*v);
@@ -70,8 +71,9 @@ Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
-            v = a->value;
+                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
+            v = &*a->value;
+            pos = *a->pos;
         }
 
         else if (apType == apIndex) {
@@ -82,14 +84,15 @@ Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
                     % attrPath % showType(*v));
 
             if (attrIndex >= v->listSize())
-                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
             v = v->listElems()[attrIndex];
+            pos = noPos;
         }
 
     }
 
-    return v;
+    return {v, pos};
 }
 
 
@@ -97,9 +100,10 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
-        v2 = findAlongAttrPath(state, "meta.position", *state.emptyBindings, v);
+        auto dummyArgs = state.allocBindings(0);
+        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
     } catch (Error &) {
-        throw Error("package '%s' has no source location information", what);
+        throw NoPositionInfo("package '%s' has no source location information", what);
     }
 
     // FIXME: is it possible to extract the Pos object instead of doing this

src/libexpr/attr-path.hh

@@ -7,7 +7,10 @@
 
 namespace nix {
 
-Ptr<Value> findAlongAttrPath(EvalState & state, const string & attrPath,
+MakeError(AttrPathNotFound, Error);
+MakeError(NoPositionInfo, Error);
+
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */

src/libexpr/attr-set.cc

@@ -7,28 +7,28 @@
 namespace nix {
 
 
-/* Allocate a new array of attributes for an attribute set with a
-   specific capacity. The space is implicitly reserved after the
-   Bindings structure. */
-Ptr<Bindings> Bindings::allocBindings(size_t capacity)
+/* Allocate a new array of attributes for an attribute set with a specific
+   capacity. The space is implicitly reserved after the Bindings
+   structure. */
+Bindings * EvalState::allocBindings(size_t capacity)
 {
-    if (capacity >= 1UL << Object::miscBytes * 8)
+    if (capacity > std::numeric_limits<Bindings::size_t>::max())
         throw Error("attribute set of size %d is too big", capacity);
-    return gc.alloc<Bindings>(Bindings::wordsFor(capacity), capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
 void EvalState::mkAttrs(Value & v, size_t capacity)
 {
     if (capacity == 0) {
-        v.attrs = emptyBindings;
-        v.type = tAttrs;
-    } else {
-        v.attrs = Bindings::allocBindings(capacity);
-        v.type = tAttrs;
-        nrAttrsets++;
-        nrAttrsInAttrsets += capacity;
+        v = vEmptySet;
+        return;
     }
+    clearValue(v);
+    v.type = tAttrs;
+    v.attrs = allocBindings(capacity);
+    nrAttrsets++;
+    nrAttrsInAttrsets += capacity;
 }
 
 
@@ -37,12 +37,18 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
    this attribute. */
 Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 {
-    auto v = allocValue();
+    Value * v = allocValue();
     vAttrs.attrs->push_back(Attr(name, v));
     return v;
 }
 
 
+Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+{
+    return allocAttr(vAttrs, symbols.create(name));
+}
+
+
 void Bindings::sort()
 {
     std::sort(begin(), end());

src/libexpr/attr-set.hh

@@ -2,7 +2,6 @@
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
-#include "gc.hh"
 
 #include <algorithm>
 #include <optional>
@@ -32,22 +31,19 @@ struct Attr
    by its size and its capacity, the capacity being the number of Attr
    elements allocated after this structure, while the size corresponds to
    the number of elements already inserted in this structure. */
-class Bindings : public Object
+class Bindings
 {
 public:
     typedef uint32_t size_t;
 
 private:
-    // FIXME: eliminate size_. We can just rely on capacity by sorting
-    // null entries towards the end of the vector.
-    size_t size_;
+    size_t size_, capacity_;
     Attr attrs[0];
 
-    Bindings(size_t capacity) : Object(tBindings, capacity), size_(0) {}
+    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
     Bindings(const Bindings & bindings) = delete;
 
 public:
-
     size_t size() const { return size_; }
 
     bool empty() const { return !size_; }
@@ -56,8 +52,7 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(size_ < capacity());
-        gc.assertObject(attr.value);
+        assert(size_ < capacity_);
         attrs[size_++] = attr;
     }
 
@@ -95,7 +90,7 @@ public:
 
     void sort();
 
-    size_t capacity() const { return getMisc(); }
+    size_t capacity() { return capacity_; }
 
     /* Returns the attributes in lexicographically sorted order. */
     std::vector<const Attr *> lexicographicOrder() const
@@ -110,19 +105,7 @@ public:
         return res;
     }
 
-    size_t words() const
-    {
-        return wordsFor(capacity());
-    }
-
-    static size_t wordsFor(size_t capacity)
-    {
-        return 2 + 3 * capacity; // FIXME
-    }
-
-    static Ptr<Bindings> allocBindings(size_t capacity);
-
-    friend class GC;
+    friend class EvalState;
 };
 
 

src/libexpr/common-eval-args.cc

@@ -28,21 +28,19 @@ MixEvalArgs::MixEvalArgs()
         .handler([&](std::string s) { searchPath.push_back(s); });
 }
 
-Ptr<Bindings> MixEvalArgs::getAutoArgs(EvalState & state)
+Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
-    if (!bindings) {
-        bindings = Bindings::allocBindings(autoArgs.size());
-        for (auto & i : autoArgs) {
-            Value * v = state.allocValue();
-            if (i.second[0] == 'E')
-                state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
-            else
-                mkString(*v, string(i.second, 1));
-            bindings->push_back(Attr(state.symbols.create(i.first), v));
-        }
-        bindings->sort();
+    Bindings * res = state.allocBindings(autoArgs.size());
+    for (auto & i : autoArgs) {
+        Value * v = state.allocValue();
+        if (i.second[0] == 'E')
+            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
+        else
+            mkString(*v, string(i.second, 1));
+        res->push_back(Attr(state.symbols.create(i.first), v));
     }
-    return bindings;
+    res->sort();
+    return res;
 }
 
 Path lookupFileArg(EvalState & state, string s)

src/libexpr/common-eval-args.hh

@@ -1,28 +1,24 @@
 #pragma once
 
 #include "args.hh"
-#include "eval.hh"
 
 namespace nix {
 
 class Store;
-
-template<class T>
-struct Ptr;
+class EvalState;
+class Bindings;
 
 struct MixEvalArgs : virtual Args
 {
     MixEvalArgs();
 
-    Ptr<Bindings> getAutoArgs(EvalState & state);
+    Bindings * getAutoArgs(EvalState & state);
 
     Strings searchPath;
 
 private:
 
     std::map<std::string, std::string> autoArgs;
-
-    Ptr<Bindings> bindings;
 };
 
 Path lookupFileArg(EvalState & state, string s);

src/libexpr/eval-inline.hh

@@ -27,27 +27,21 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const
 void EvalState::forceValue(Value & v, const Pos & pos)
 {
     if (v.type == tThunk) {
-        // FIXME: this is necessary because some values (like vList2)
-        // are created non-atomically.
-        Ptr<Env> env(v.thunk.env);
+        Env * env = v.thunk.env;
         Expr * expr = v.thunk.expr;
         try {
             v.type = tBlackhole;
             //checkInterrupt();
             expr->eval(*this, *env, v);
         } catch (...) {
+            v.type = tThunk;
             v.thunk.env = env;
             v.thunk.expr = expr;
-            v.type = tThunk;
             throw;
         }
     }
-    else if (v.type == tApp) {
-        // FIXME: idem.
-        Ptr<Value> left(v.app.left);
-        Ptr<Value> right(v.app.right);
+    else if (v.type == tApp)
         callFunction(*v.app.left, *v.app.right, v, noPos);
-    }
     else if (v.type == tBlackhole)
         throwEvalError("infinite recursion encountered, at %1%", pos);
 }
@@ -84,5 +78,18 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError("value is %1% while a list was expected, at %2%", v, pos);
 }
 
+/* Note: Various places expect the allocated memory to be zeroed. */
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_MALLOC(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
 
 }

src/libexpr/eval.hh

@@ -6,8 +6,8 @@
 #include "symbol-table.hh"
 #include "hash.hh"
 #include "config.hh"
-#include "gc.hh"
 
+#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
@@ -18,7 +18,6 @@ namespace nix {
 
 class Store;
 class EvalState;
-struct Derivation; // FIXME: remove
 struct StorePath;
 enum RepairFlag : bool;
 
@@ -36,56 +35,18 @@ struct PrimOp
 };
 
 
-struct Env : Object
+struct Env
 {
     Env * up;
+    unsigned short prevWith:14; // nr of levels up to next `with' environment
+    enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2;
     Value * values[0];
-
-private:
-
-    constexpr static size_t maxSize = 1 << 16;
-    constexpr static size_t maxPrevWith = 1 << 10;
-
-    Env(Tag type, size_t size, size_t prevWith)
-        : Object(type, size | (prevWith << 16))
-    {
-        if (size >= maxSize)
-            throw Error("environment size %d is too big", size);
-
-        if (prevWith >= maxPrevWith)
-            throw Error("too many nesting levels");
-
-        for (size_t i = 0; i < size; i++)
-            values[i] = nullptr;
-    }
-
-    friend class GC;
-
-public:
-
-    unsigned short getPrevWith() const
-    {
-        return getMisc() >> 16;
-    }
-
-    unsigned short getSize() const
-    {
-        return getMisc() & 0xffff;
-    }
-
-    size_t words() const
-    {
-        return wordsFor(getSize());
-    }
-
-    static size_t wordsFor(unsigned short size)
-    {
-        return 2 + size;
-    }
 };
 
 
-Value & mkString(Value & v, std::string_view s, const PathSet & context);
+Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
+
+void copyContext(const Value & v, PathSet & context);
 
 
 /* Cache for calls to addToStore(); maps source paths to the store
@@ -100,10 +61,14 @@ typedef std::pair<std::string, std::string> SearchPathElem;
 typedef std::list<SearchPathElem> SearchPath;
 
 
+/* Initialise the Boehm GC, if applicable. */
+void initGC();
+
+
 class EvalState
 {
 public:
-    SymbolTable & symbols{nix::symbols}; // FIXME: remove
+    SymbolTable symbols;
 
     const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue,
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
@@ -120,7 +85,7 @@ public:
        mode. */
     std::optional<PathSet> allowedPaths;
 
-    Ptr<Bindings> emptyBindings;
+    Value vEmptySet;
 
     const ref<Store> store;
 
@@ -128,11 +93,19 @@ private:
     SrcToStore srcToStore;
 
     /* A cache from path names to parse trees. */
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *> > > FileParseCache;
+#else
     typedef std::map<Path, Expr *> FileParseCache;
+#endif
     FileParseCache fileParseCache;
 
     /* A cache from path names to values. */
-    typedef std::map<Path, Ptr<Value>> FileEvalCache;
+#if HAVE_BOEHMGC
+    typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value> > > FileEvalCache;
+#else
+    typedef std::map<Path, Value> FileEvalCache;
+#endif
     FileEvalCache fileEvalCache;
 
     SearchPath searchPath;
@@ -142,6 +115,9 @@ private:
     /* Cache used by checkSourcePath(). */
     std::unordered_map<Path, Path> resolvedPaths;
 
+    /* Cache used by prim_match(). */
+    std::unordered_map<std::string, std::regex> regexCache;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -169,8 +145,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(const string & s, const Path & basePath);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath);
 
     Expr * parseStdin();
 
@@ -245,7 +221,7 @@ public:
 
     /* The base environment, containing the builtin functions and
        values. */
-    Ptr<Env> baseEnv;
+    Env & baseEnv;
 
     /* The same, but used during parsing to resolve variables. */
     StaticEnv staticBaseEnv; // !!! should be private
@@ -292,17 +268,13 @@ public:
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
 
     /* Allocation primitives. */
-    Ptr<Value> allocValue()
-    {
-        nrValues++;
-        return gc.alloc<Value>(Value::words());
-    }
+    Value * allocValue();
+    Env & allocEnv(size_t size);
 
-    Ptr<Env> allocEnv(size_t size, size_t prevWith = 0, Tag type = tEnv);
-
-    // Note: the resulting Value is only reachable as long as vAttrs
-    // is reachable.
     Value * allocAttr(Value & vAttrs, const Symbol & name);
+    Value * allocAttr(Value & vAttrs, const std::string & name);
+
+    Bindings * allocBindings(size_t capacity);
 
     void mkList(Value & v, size_t length);
     void mkAttrs(Value & v, size_t capacity);
@@ -347,10 +319,7 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
-
-public:
-
-    std::function<void(const StorePath & drvPath, const Derivation & drv)> derivationHook;
+    friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
 };
 
 
@@ -399,7 +368,7 @@ struct EvalSettings : Config
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/function-trace.cc

@@ -1,4 +1,5 @@
 #include "function-trace.hh"
+#include "logging.hh"
 
 namespace nix {
 

src/libexpr/gc.cc

@@ -1,537 +0,0 @@
-#include "gc.hh"
-#include "value.hh"
-#include "attr-set.hh"
-#include "eval.hh"
-
-#include <chrono>
-
-namespace nix {
-
-GC gc;
-
-GC::GC()
-{
-    nextSize = std::max((size_t) 2, parseSize<size_t>(getEnv("GC_INITIAL_HEAP_SIZE").value_or("131072")) / WORD_SIZE);
-
-    // FIXME: placement new
-    frontPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
-    backPtrSentinel = (Ptr<Object> *) malloc(sizeof(Ptr<Object>));
-
-    frontPtrSentinel->prev = nullptr;
-    frontPtrSentinel->next = backPtrSentinel;
-
-    backPtrSentinel->prev = frontPtrSentinel;
-    backPtrSentinel->next = nullptr;
-
-    frontRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
-    backRootSentinel = (Root<Object> *) malloc(sizeof(Root<Object>));
-
-    frontRootSentinel->prev = nullptr;
-    frontRootSentinel->next = backRootSentinel;
-
-    backRootSentinel->prev = frontRootSentinel;
-    backRootSentinel->next = nullptr;
-
-    freeLists[0].minSize = 2;
-    freeLists[1].minSize = 3;
-    freeLists[2].minSize = 4;
-    freeLists[3].minSize = 8;
-    freeLists[4].minSize = 16;
-    freeLists[5].minSize = 32;
-    freeLists[6].minSize = 64;
-    freeLists[7].minSize = 128;
-
-    addArena(nextSize);
-}
-
-GC::~GC()
-{
-    debug("%d bytes in arenas, %d bytes allocated, %d bytes reclaimed, in %d ms",
-        totalSize * WORD_SIZE,
-        allTimeWordsAllocated * WORD_SIZE,
-        allTimeWordsFreed * WORD_SIZE,
-        totalDurationMs);
-
-    size_t n = 0;
-    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next)
-        n++;
-    if (n)
-        warn("%d GC root pointers still exist on exit", n);
-
-    n = 0;
-    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next)
-        n++;
-    if (n)
-        warn("%d GC root objects still exist on exit", n);
-
-    assert(!frontPtrSentinel->prev);
-    assert(!backPtrSentinel->next);
-    assert(!frontRootSentinel->prev);
-    assert(!backRootSentinel->next);
-}
-
-void GC::addArena(size_t arenaSize)
-{
-    debug("allocating arena of %d bytes", arenaSize * WORD_SIZE);
-
-    auto arena = Arena(arenaSize);
-
-    // Add this arena to a freelist as a single block.
-    addToFreeList(new (arena.start) Free(arenaSize));
-
-    arenas.emplace_back(std::move(arena));
-
-    totalSize += arenaSize;
-
-    nextSize = arenaSize * 1.5; // FIXME: overflow, clamp
-}
-
-void GC::addToFreeList(Free * obj)
-{
-    auto size = obj->words();
-    for (auto i = freeLists.rbegin(); i != freeLists.rend(); ++i)
-        if (size >= i->minSize) {
-            obj->next = i->front;
-            i->front = obj;
-            return;
-        }
-    abort();
-}
-
-void GC::gc()
-{
-    typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
-
-    auto before = steady_time_point::clock::now();
-
-    size_t marked = 0;
-
-    std::stack<Object *> stack;
-
-    // FIXME: ensure this gets inlined.
-    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
-
-    auto pushPointers = [&](Object * obj) {
-        switch (obj->type) {
-
-        case tFree:
-            printError("reached a freed object at %x", obj);
-            abort();
-
-        case tBindings: {
-            auto obj2 = (Bindings *) obj;
-            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
-                push(i->value);
-            break;
-        }
-
-        case tValueList: {
-            auto obj2 = (PtrList<Object> *) obj;
-            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tWithExprEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            break;
-        }
-
-        case tWithAttrsEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            push(obj2->values[0]);
-            break;
-        }
-
-        case tString:
-        case tContext:
-        case tInt:
-        case tBool:
-        case tNull:
-        case tList0:
-        case tFloat:
-        case tShortString:
-        case tStaticString:
-            break;
-
-        case tLongString: {
-            auto obj2 = (Value *) obj;
-            push(obj2->string.s);
-            // See setContext().
-            if (!(((ptrdiff_t) obj2->string.context) & 1))
-                push(obj2->string.context);
-            break;
-        }
-
-        case tPath:
-            push(((Value *) obj)->path);
-            break;
-
-        case tAttrs:
-            push(((Value *) obj)->attrs);
-            break;
-
-        case tList1:
-            push(((Value *) obj)->smallList[0]);
-            break;
-
-        case tList2:
-            push(((Value *) obj)->smallList[0]);
-            push(((Value *) obj)->smallList[1]);
-            break;
-
-        case tListN:
-            push(((Value *) obj)->bigList);
-            break;
-
-        case tThunk:
-        case tBlackhole:
-            push(((Value *) obj)->thunk.env);
-            break;
-
-        case tApp:
-        case tPrimOpApp:
-            push(((Value *) obj)->app.left);
-            push(((Value *) obj)->app.right);
-            break;
-
-        case tLambda:
-            push(((Value *) obj)->lambda.env);
-            break;
-
-        case tPrimOp:
-            // FIXME: GC primops?
-            break;
-
-        default:
-            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
-            abort();
-        }
-    };
-
-    auto processStack = [&]() {
-        while (!stack.empty()) {
-            auto obj = stack.top();
-            stack.pop();
-
-            //printError("MARK %x", obj);
-
-            if (!obj->isMarked()) {
-                marked++;
-                obj->mark();
-                pushPointers(obj);
-            }
-        }
-    };
-
-    for (Root<Object> * p = frontRootSentinel->next; p != backRootSentinel; p = p->next) {
-        pushPointers(&p->value);
-        processStack();
-    }
-
-    for (Ptr<Object> * p = frontPtrSentinel->next; p != backPtrSentinel; p = p->next) {
-        if (!p->value) continue;
-        stack.push(p->value);
-        processStack();
-    }
-
-    auto afterMark = steady_time_point::clock::now();
-
-    // Reset all the freelists.
-    for (auto & freeList : freeLists)
-        freeList.front = nullptr;
-
-    // Go through all the arenas and add free objects to the
-    // appropriate freelists.
-    size_t totalObjectsFreed = 0;
-    size_t totalWordsFreed = 0;
-    size_t totalObjectsKept = 0;
-    size_t totalWordsKept = 0;
-
-    for (auto & arena : arenas) {
-        auto [objectsFreed, wordsFreed, objectsKept, wordsKept] = freeUnmarked(arena);
-        totalObjectsFreed += objectsFreed;
-        totalWordsFreed += wordsFreed;
-        totalObjectsKept += objectsKept;
-        totalWordsKept += wordsKept;
-    }
-
-    auto after = steady_time_point::clock::now();
-
-    auto markDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(afterMark - before).count();
-    auto sweepDurationMs = std::chrono::duration_cast<std::chrono::milliseconds>(after - afterMark).count();
-
-    debug("freed %d dead objects (%d bytes), keeping %d/%d objects (%d bytes), marked in %d ms, swept in %d ms",
-        totalObjectsFreed, totalWordsFreed * WORD_SIZE,
-        marked, totalObjectsKept, totalWordsKept * WORD_SIZE,
-        markDurationMs, sweepDurationMs);
-
-    allTimeWordsFreed += totalWordsFreed;
-    totalDurationMs += markDurationMs + sweepDurationMs;
-}
-
-size_t GC::getObjectSize(Object * obj)
-{
-    auto tag = obj->type;
-    if (tag >= tInt && tag <= tFloat) {
-        return ((Value *) obj)->words();
-    } else {
-        switch (tag) {
-        case tFree:
-            return ((Free *) obj)->words();
-            break;
-        case tString:
-            return ((String *) obj)->words();
-            break;
-        case tBindings:
-            return ((Bindings *) obj)->words();
-            break;
-        case tValueList:
-            return ((PtrList<Value> *) obj)->words();
-            break;
-        case tEnv:
-        case tWithExprEnv:
-        case tWithAttrsEnv:
-            return ((Env *) obj)->words();
-            break;
-        case tContext:
-            return ((Context *) obj)->getSize() + 1;
-            break;
-        default:
-            printError("GC encountered invalid object with tag %d", tag);
-            abort();
-        }
-    }
-}
-
-std::tuple<size_t, size_t, size_t, size_t> GC::freeUnmarked(Arena & arena)
-{
-    size_t objectsFreed = 0;
-    size_t wordsFreed = 0;
-    size_t objectsKept = 0;
-    size_t wordsKept = 0;
-
-    auto end = arena.start + arena.size;
-    auto pos = arena.start;
-
-    Free * curFree = nullptr;
-
-    auto linkCurFree = [&]() {
-        if (curFree && curFree->words() > 1)
-            addToFreeList(curFree);
-        curFree = nullptr;
-    };
-
-    while (pos < end) {
-        auto obj = (Object *) pos;
-
-        auto objSize = getObjectSize(obj);
-
-        // Merge current object into the previous free object.
-        auto mergeFree = [&]() {
-            //printError("MERGE %x %x %d", curFree, obj, curFree->size() + objSize);
-            assert(curFree->words() >= 1);
-            curFree->setSize(curFree->words() + objSize);
-        };
-
-        if (obj->type == tFree) {
-            //debug("KEEP FREE %x %d", obj, obj->getMisc());
-            if (curFree) {
-                // Merge this object into the previous free
-                // object.
-                mergeFree();
-            } else {
-                curFree = (Free *) obj;
-            }
-        } else {
-
-            if (obj->isMarked()) {
-                // Unmark to prepare for the next GC run.
-                //debug("KEEP OBJECT %x %d %d", obj, obj->type, objSize);
-                linkCurFree();
-                obj->unmark();
-                objectsKept += 1;
-                wordsKept += objSize;
-            } else {
-                //debug("FREE OBJECT %x %d %d", obj, obj->type, objSize);
-                #if GC_DEBUG
-                for (size_t i = 0; i < objSize; ++i)
-                    ((Word *) obj)[i] = 0xdeadc0dedeadbeefULL;
-                #endif
-                objectsFreed += 1;
-                wordsFreed += objSize;
-                if (curFree) {
-                    mergeFree();
-                } else {
-                    // Convert to a free object.
-                    curFree = (Free *) obj;
-                    curFree->type = tFree;
-                    curFree->setSize(objSize);
-                }
-            }
-        }
-
-        pos += objSize;
-    }
-
-    linkCurFree();
-
-    assert(pos == end);
-
-    return {objectsFreed, wordsFreed, objectsKept, wordsKept};
-}
-
-bool GC::isObject(void * p)
-{
-    for (auto & arena : arenas)
-        if (p >= arena.start && p < arena.start + arena.size)
-            return true;
-    return false;
-}
-
-std::tuple<size_t, size_t> GC::getObjectClosureSize(Object * p)
-{
-    std::unordered_set<Object *> seen;
-
-    // FIXME: cut&paste.
-
-    std::stack<Object *> stack;
-
-    // FIXME: ensure this gets inlined.
-    auto push = [&](Object * p) { if (p) { assertObject(p); stack.push(p); } };
-
-    auto pushPointers = [&](Object * obj) {
-        switch (obj->type) {
-
-        case tFree:
-            printError("reached a freed object at %x", obj);
-            abort();
-
-        case tBindings: {
-            auto obj2 = (Bindings *) obj;
-            for (auto i = obj2->attrs; i < obj2->attrs + obj2->size_; ++i)
-                push(i->value);
-            break;
-        }
-
-        case tValueList: {
-            auto obj2 = (PtrList<Object> *) obj;
-            for (auto i = obj2->elems; i < obj2->elems + obj2->size(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            for (auto i = obj2->values; i < obj2->values + obj2->getSize(); ++i)
-                push(*i);
-            break;
-        }
-
-        case tWithExprEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            break;
-        }
-
-        case tWithAttrsEnv: {
-            auto obj2 = (Env *) obj;
-            push(obj2->up);
-            push(obj2->values[0]);
-            break;
-        }
-
-        case tString:
-        case tContext:
-        case tInt:
-        case tBool:
-        case tNull:
-        case tList0:
-        case tFloat:
-        case tShortString:
-        case tStaticString:
-            break;
-
-        case tLongString: {
-            auto obj2 = (Value *) obj;
-            push(obj2->string.s);
-            // See setContext().
-            if (!(((ptrdiff_t) obj2->string.context) & 1))
-                push(obj2->string.context);
-            break;
-        }
-
-        case tPath:
-            push(((Value *) obj)->path);
-            break;
-
-        case tAttrs:
-            push(((Value *) obj)->attrs);
-            break;
-
-        case tList1:
-            push(((Value *) obj)->smallList[0]);
-            break;
-
-        case tList2:
-            push(((Value *) obj)->smallList[0]);
-            push(((Value *) obj)->smallList[1]);
-            break;
-
-        case tListN:
-            push(((Value *) obj)->bigList);
-            break;
-
-        case tThunk:
-        case tBlackhole:
-            push(((Value *) obj)->thunk.env);
-            break;
-
-        case tApp:
-        case tPrimOpApp:
-            push(((Value *) obj)->app.left);
-            push(((Value *) obj)->app.right);
-            break;
-
-        case tLambda:
-            push(((Value *) obj)->lambda.env);
-            break;
-
-        case tPrimOp:
-            // FIXME: GC primops?
-            break;
-
-        default:
-            printError("don't know how to traverse object at %x (tag %d)", obj, obj->type);
-            abort();
-        }
-    };
-
-    stack.push(p);
-
-    size_t totalSize = 0;
-
-    while (!stack.empty()) {
-        auto obj = stack.top();
-        stack.pop();
-        if (seen.insert(obj).second) {
-            pushPointers(obj);
-            totalSize += getObjectSize(obj);
-        }
-    }
-
-    return {seen.size(), totalSize};
-}
-
-}

src/libexpr/gc.hh

@@ -1,482 +0,0 @@
-#pragma once
-
-#include "logging.hh"
-
-#include <stack>
-#include <limits>
-#include <cassert>
-#include <cstring>
-
-//#define GC_DEBUG 1
-
-namespace nix {
-
-typedef unsigned long Word;
-
-enum Tag {
-    tFree = 3,
-
-    // Misc types
-    tString,
-    tBindings,
-    tValueList,
-    tEnv,
-    tWithExprEnv,
-    tWithAttrsEnv,
-    tContext,
-
-    // Value tags
-    tInt,
-    tBool,
-    tShortString,
-    tStaticString,
-    tLongString,
-    tPath,
-    tNull,
-    tAttrs,
-    tList0,
-    tList1,
-    tList2,
-    tListN,
-    tThunk,
-    tApp,
-    tLambda,
-    tBlackhole,
-    tPrimOp,
-    tPrimOpApp,
-    tExternal,
-    tFloat
-};
-
-constexpr size_t WORD_SIZE = 8;
-
-struct Object
-{
-    friend class GC;
-
-public:
-    constexpr static size_t miscBytes = 7;
-
-public: // FIXME
-    Tag type:7;
-
-private:
-    bool marked:1;
-
-    unsigned long misc:56;
-
-    void unmark()
-    {
-        marked = false;
-    }
-
-protected:
-
-    Object(Tag type, unsigned long misc) : type(type), marked(false), misc(misc) { }
-
-    bool isMarked()
-    {
-        return marked;
-    }
-
-    void mark()
-    {
-        marked = true;
-    }
-
-    void setMisc(unsigned int misc)
-    {
-        this->misc = misc;
-    }
-
-    unsigned int getMisc() const
-    {
-        return misc;
-    }
-
-    char * getMiscData() const
-    {
-        return ((char *) this) + 1;
-    }
-};
-
-template<class T>
-struct PtrList : Object
-{
-    T * elems[0];
-
-    PtrList(Tag type, size_t size) : Object(type, size)
-    {
-        for (size_t i = 0; i < size; i++)
-            elems[i] = nullptr;
-    }
-
-    size_t size() const { return getMisc(); }
-    size_t words() const { return wordsFor(size()); }
-    static size_t wordsFor(size_t size) { return 1 + size; }
-};
-
-struct Free : Object
-{
-    Free * next;
-
-    Free(size_t size) : Object(tFree, size), next(nullptr) { }
-
-    // return size in words
-    size_t words() const { return getMisc(); }
-
-    void setSize(size_t size) { assert(size >= 1); setMisc(size); }
-};
-
-template<class T>
-struct Ptr;
-
-template<class T>
-struct Root;
-
-struct GC
-{
-
-private:
-
-    Ptr<Object> * frontPtrSentinel;
-    Ptr<Object> * backPtrSentinel;
-
-    Root<Object> * frontRootSentinel;
-    Root<Object> * backRootSentinel;
-
-    template<class T>
-    friend class Ptr;
-
-    template<class T>
-    friend class Root;
-
-    struct Arena
-    {
-        size_t size; // in words
-        Word * start;
-
-        Arena(size_t size)
-            : size(size)
-            , start(new Word[size])
-        {
-            assert(size >= 2);
-        }
-
-        Arena(const Arena & arena) = delete;
-
-        Arena(Arena && arena)
-        {
-            size = arena.size;
-            start = arena.start;
-            arena.start = nullptr;
-        }
-
-        ~Arena()
-        {
-            delete[] start;
-        }
-    };
-
-    size_t totalSize = 0;
-    size_t nextSize;
-
-    std::vector<Arena> arenas;
-
-    struct FreeList
-    {
-        size_t minSize;
-        Free * front = nullptr;
-    };
-
-    std::array<FreeList, 8> freeLists;
-
-public:
-
-    size_t getHeapSize() { return totalSize * WORD_SIZE; }
-
-    size_t allTimeWordsAllocated = 0;
-    size_t allTimeWordsFreed = 0;
-    uint64_t totalDurationMs = 0;
-
-private:
-
-    Object * allocObject(size_t size)
-    {
-        assert(size >= 2);
-
-        for (int attempt = 0; attempt < 3; attempt++) {
-
-            for (size_t i = 0; i < freeLists.size(); ++i) {
-                auto & freeList = freeLists[i];
-
-                if ((size <= freeList.minSize || i == freeLists.size() - 1) && freeList.front) {
-                    //printError("TRY %d %d %d", size, i, freeList.minSize);
-
-                    Free * * prev = &freeList.front;
-
-                    while (Free * freeObj = *prev) {
-                        //printError("LOOK %x %d %x", freeObj, freeObj->words(), freeObj->next);
-                        assert(freeObj->words() >= freeList.minSize);
-                        if (freeObj->words() == size) {
-                            // Convert the free object.
-                            *prev = freeObj->next;
-                            return (Object *) freeObj;
-                        } else if (freeObj->words() >= size + 2) {
-                            // Split the free object.
-                            auto newSize = freeObj->words() - size;
-                            freeObj->setSize(newSize);
-                            if (newSize < freeList.minSize) {
-                                /* The free object is now smaller than
-                                   the minimum size for this freelist,
-                                   so move it to another one. */
-                                //printError("MOVE %x %d -> %d", freeObj, newSize + size, newSize);
-                                *prev = freeObj->next;
-                                addToFreeList(freeObj);
-                            }
-                            return (Object *) (((Word *) freeObj) + newSize);
-                        } else if (freeObj->words() == size + 1) {
-                            // Return the free object and add a padding word.
-                            *prev = freeObj->next;
-                            freeObj->setSize(1);
-                            return (Object *) (((Word *) freeObj) + 1);
-                        } else {
-                            assert(freeObj->words() < size);
-                            prev = &freeObj->next;
-                        }
-                    }
-                }
-            }
-
-            if (attempt == 0) {
-                debug("allocation of %d bytes failed, GCing...", size * WORD_SIZE);
-                gc();
-            } else if (attempt == 1) {
-                addArena(std::max(nextSize, size));
-            }
-        }
-
-        throw Error("allocation of %d bytes failed", size);
-    }
-
-public:
-
-    GC();
-    ~GC();
-
-    template<typename T, typename... Args>
-    Ptr<T> alloc(size_t size, const Args & ... args)
-    {
-        auto raw = allocObject(size);
-        allTimeWordsAllocated += size;
-        return new (raw) T(args...);
-    }
-
-    void gc();
-
-    bool isObject(void * p);
-
-    void assertObject(void * p)
-    {
-        #if GC_DEBUG
-        if (!isObject(p)) {
-            printError("object %p is not an object", p);
-            abort();
-        }
-        #endif
-    }
-
-    /* Return the size in words of object 'obj'. */
-    static size_t getObjectSize(Object * obj);
-
-    /* Return the number and size in words of the objects reachable
-       from object 'obj'. */
-    std::tuple<size_t, size_t> getObjectClosureSize(Object * obj);
-
-private:
-
-    void addArena(size_t arenaSize);
-
-    void addToFreeList(Free * obj);
-
-    std::tuple<size_t, size_t, size_t, size_t> freeUnmarked(Arena & arena);
-};
-
-extern GC gc;
-
-template<class T>
-struct Ptr
-{
-private:
-
-    friend class GC;
-
-    Ptr * prev = nullptr, * next = nullptr;
-    T * value = nullptr;
-
-    void link()
-    {
-        prev = (Ptr *) gc.frontPtrSentinel;
-        next = prev->next;
-        next->prev = this;
-        prev->next = this;
-    }
-
-public:
-
-    Ptr() {
-        link();
-    }
-
-    Ptr(T * value) : value(value)
-    {
-        link();
-    }
-
-    Ptr(const Ptr & p)
-    {
-        auto & p2 = const_cast<Ptr &>(p);
-        value = p2.value;
-        next = &p2;
-        prev = p2.prev;
-        prev->next = this;
-        p2.prev = this;
-    }
-
-    Ptr(Ptr && p)
-    {
-        link();
-        value = p.value;
-        p.value = nullptr;
-    }
-
-    Ptr & operator =(const Ptr & p)
-    {
-        value = p.value;
-        return *this;
-    }
-
-    Ptr & operator =(Ptr && p)
-    {
-        value = p.value;
-        p.value = nullptr;
-        return *this;
-    }
-
-    Ptr & operator =(T * v)
-    {
-        value = v;
-        return *this;
-    }
-
-    ~Ptr()
-    {
-        assert(next);
-        assert(prev);
-        assert(next->prev == this);
-        next->prev = prev;
-        assert(prev->next == this);
-        prev->next = next;
-    }
-
-    T * operator ->()
-    {
-        return value;
-    }
-
-    T * operator ->() const // FIXME
-    {
-        return value;
-    }
-
-    operator T * ()
-    {
-        return value;
-    }
-
-    operator T & ()
-    {
-        assert(value);
-        return *value;
-    }
-
-    operator bool() const
-    {
-        return value != nullptr;
-    }
-};
-
-template<class T>
-struct Root
-{
-    Root * prev = nullptr, * next = nullptr;
-    T value;
-
-    template<typename... Args>
-    Root(const Args & ... args)
-        : value{args... }
-    {
-        prev = (Root *) gc.frontRootSentinel;
-        next = prev->next;
-        next->prev = this;
-        prev->next = this;
-    }
-
-    Root(const Root & p) = delete;
-    Root(Root && p) = delete;
-
-    Root & operator =(const T & v) { value = v; return *this; }
-
-    ~Root()
-    {
-        assert(next);
-        assert(prev);
-        assert(next->prev == this);
-        next->prev = prev;
-        assert(prev->next == this);
-        prev->next = next;
-    }
-
-    T * operator ->()
-    {
-        return &value;
-    }
-
-    operator T * ()
-    {
-        return &value;
-    }
-
-    operator T & ()
-    {
-        return value;
-    }
-};
-
-struct String : Object
-{
-    char s[0];
-
-    String(size_t len, const char * src)
-        : Object(tString, len)
-    {
-        std::memcpy(s, src, len + 1);
-    }
-
-    size_t words() const { return wordsFor(getMisc()); }
-
-    /* Return the number of words needed to store a string of 'len'
-       characters (where 'len' excludes the terminator). */
-    static size_t wordsFor(size_t len)
-    {
-        return len / WORD_SIZE + 2;
-    }
-
-    static String * alloc(const char * src)
-    {
-        auto len = strlen(src);
-        return gc.alloc<String>(wordsFor(len), len, src);
-    }
-};
-
-}

src/libexpr/get-drvs.cc

@@ -115,15 +115,15 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
         return outputs;
 
     /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */
-    const auto outTI = queryMeta("outputsToInstall");
+    const Value * outTI = queryMeta("outputsToInstall");
     if (!outTI) return outputs;
     const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'");
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
     for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if (!(*i)->isString()) throw errMsg;
-        auto out = outputs.find((*i)->getString());
+        if ((*i)->type != tString) throw errMsg;
+        auto out = outputs.find((*i)->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
     }
@@ -178,7 +178,8 @@ bool DrvInfo::checkMeta(Value & v)
             if (!checkMeta(*i.value)) return false;
         return true;
     }
-    else return v.type == tInt || v.type == tBool || v.isString() || v.type == tFloat;
+    else return v.type == tInt || v.type == tBool || v.type == tString ||
+                v.type == tFloat;
 }
 
 
@@ -193,36 +194,36 @@ Value * DrvInfo::queryMeta(const string & name)
 
 string DrvInfo::queryMetaString(const string & name)
 {
-    auto v = queryMeta(name);
-    if (!v || !v->isString()) return "";
-    return v->getString();
+    Value * v = queryMeta(name);
+    if (!v || v->type != tString) return "";
+    return v->string.s;
 }
 
 
 NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
 {
-    auto v = queryMeta(name);
+    Value * v = queryMeta(name);
     if (!v) return def;
     if (v->type == tInt) return v->integer;
-    if (v->isString()) {
+    if (v->type == tString) {
         /* Backwards compatibility with before we had support for
            integer meta fields. */
         NixInt n;
-        if (string2Int(v->getString(), n)) return n;
+        if (string2Int(v->string.s, n)) return n;
     }
     return def;
 }
 
 NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 {
-    auto v = queryMeta(name);
+    Value * v = queryMeta(name);
     if (!v) return def;
     if (v->type == tFloat) return v->fpoint;
-    if (v->isString()) {
+    if (v->type == tString) {
         /* Backwards compatibility with before we had support for
            float meta fields. */
         NixFloat n;
-        if (string2Float(v->getString(), n)) return n;
+        if (string2Float(v->string.s, n)) return n;
     }
     return def;
 }
@@ -230,15 +231,14 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 
 bool DrvInfo::queryMetaBool(const string & name, bool def)
 {
-    auto v = queryMeta(name);
+    Value * v = queryMeta(name);
     if (!v) return def;
     if (v->type == tBool) return v->boolean;
-    if (v->isString()) {
+    if (v->type == tString) {
         /* Backwards compatibility with before we had support for
            Boolean meta fields. */
-        auto s = v->getString();
-        if (strcmp(s, "true") == 0) return true;
-        if (strcmp(s, "false") == 0) return false;
+        if (strcmp(v->string.s, "true") == 0) return true;
+        if (strcmp(v->string.s, "false") == 0) return false;
     }
     return def;
 }
@@ -247,8 +247,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 void DrvInfo::setMeta(const string & name, Value * v)
 {
     getMeta();
-    Ptr<Bindings> old = meta;
-    meta = Bindings::allocBindings(1 + (old ? old->size() : 0));
+    Bindings * old = meta;
+    meta = state->allocBindings(1 + (old ? old->size() : 0));
     Symbol sym = state->symbols.create(name);
     if (old)
         for (auto i : *old)
@@ -260,7 +260,6 @@ void DrvInfo::setMeta(const string & name, Value * v)
 
 
 /* Cache for already considered attrsets. */
-// FIXME: Use Ptr?
 typedef set<Bindings *> Done;
 
 
@@ -320,24 +319,24 @@ static void getDerivations(EvalState & state, Value & vIn,
     DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
-    Root<Value> v;
+    Value v;
     state.autoCallFunction(autoArgs, vIn, v);
 
     /* Process the expression. */
     if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
 
-    else if (v->type == tAttrs) {
+    else if (v.type == tAttrs) {
 
         /* !!! undocumented hackery to support combining channels in
            nix-env.cc. */
-        bool combineChannels = v->attrs->find(state.symbols.create("_combineChannels")) != v->attrs->end();
+        bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end();
 
         /* Consider the attributes in sorted order to get more
            deterministic behaviour in nix-env operations (e.g. when
            there are names clashes between derivations, the derivation
            bound to the attribute with the "lower" name should take
            precedence). */
-        for (auto & i : v->attrs->lexicographicOrder()) {
+        for (auto & i : v.attrs->lexicographicOrder()) {
             debug("evaluating attribute '%1%'", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
@@ -357,11 +356,11 @@ static void getDerivations(EvalState & state, Value & vIn,
         }
     }
 
-    else if (v->isList()) {
-        for (unsigned int n = 0; n < v->listSize(); ++n) {
+    else if (v.isList()) {
+        for (unsigned int n = 0; n < v.listSize(); ++n) {
             string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *v->listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
-                getDerivations(state, *v->listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+            if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
+                getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
     }
 

src/libexpr/get-drvs.hh

@@ -26,8 +26,7 @@ private:
 
     bool failed = false; // set if we get an AssertionError
 
-    Ptr<Bindings> attrs;
-    Ptr<Bindings> meta;
+    Bindings * attrs = nullptr, * meta = nullptr;
 
     Bindings * getMeta();
 
@@ -70,7 +69,11 @@ public:
 };
 
 
-typedef std::list<DrvInfo> DrvInfos;
+#if HAVE_BOEHMGC
+typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
+#else
+typedef list<DrvInfo> DrvInfos;
+#endif
 
 
 /* If value `v' denotes a derivation, return a DrvInfo object

src/libexpr/json-to-value.cc

@@ -11,22 +11,21 @@ namespace nix {
 // for more information, refer to
 // https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp
 class JSONSax : nlohmann::json_sax<json> {
-
     class JSONState {
     protected:
         unique_ptr<JSONState> parent;
-        Ptr<Value> v;
+        Value * v;
     public:
         virtual unique_ptr<JSONState> resolve(EvalState &)
         {
             throw std::logic_error("tried to close toplevel json parser state");
         };
-        explicit JSONState(unique_ptr<JSONState> && p) : parent(std::move(p)), v(nullptr) {};
-        explicit JSONState(Value * v) : v(v) {};
-        JSONState(JSONState & p) = delete;
-        Value & value(EvalState & state)
+        explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {};
+        explicit JSONState(Value* v) : v(v) {};
+        JSONState(JSONState& p) = delete;
+        Value& value(EvalState & state)
         {
-            if (!v)
+            if (v == nullptr)
                 v = state.allocValue();
             return *v;
         };
@@ -39,7 +38,7 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueMap attrs = ValueMap();
         virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
+            Value& v = parent->value(state);
             state.mkAttrs(v, attrs.size());
             for (auto & i : attrs)
                 v.attrs->push_back(Attr(i.first, i.second));
@@ -47,7 +46,7 @@ class JSONSax : nlohmann::json_sax<json> {
         }
         virtual void add() override { v = nullptr; };
     public:
-        void key(string_t & name, EvalState & state)
+        void key(string_t& name, EvalState & state)
         {
             attrs[state.symbols.create(name)] = &value(state);
         }
@@ -57,7 +56,7 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueVector values = ValueVector();
         virtual unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
+            Value& v = parent->value(state);
             state.mkList(v, values.size());
             for (size_t n = 0; n < values.size(); ++n) {
                 v.listElems()[n] = values[n];
@@ -69,7 +68,7 @@ class JSONSax : nlohmann::json_sax<json> {
             v = nullptr;
         };
     public:
-        JSONListState(unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p))
+        JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p))
         {
             values.reserve(reserve);
         }
@@ -115,7 +114,7 @@ public:
 
     bool string(string_t& val)
     {
-        return handle_value<Value & (Value &, std::string_view)>(mkString, (std::string_view) val);
+        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
     }
 
     bool start_object(std::size_t len)

src/libexpr/local.mk

@@ -6,6 +6,8 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+
 libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
@@ -13,6 +15,11 @@ ifneq ($(OS), FreeBSD)
  libexpr_LDFLAGS += -ldl
 endif
 
+# The dependency on libgc must be propagated (i.e. meaning that
+# programs/libraries that use libexpr must explicitly pass -lgc),
+# because inline functions in libexpr's header files call libgc.
+libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS)
+
 libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y
@@ -26,3 +33,5 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh

src/libexpr/nixexpr.cc

@@ -80,7 +80,7 @@ void ExprString::show(std::ostream & str) const
 
 void ExprPath::show(std::ostream & str) const
 {
-    str << v->path->s;
+    str << s;
 }
 
 void ExprVar::show(std::ostream & str) const

src/libexpr/nixexpr.hh

@@ -78,7 +78,7 @@ struct Expr
     virtual void show(std::ostream & str) const;
     virtual void bindVars(const StaticEnv & env);
     virtual void eval(EvalState & state, Env & env, Value & v);
-    virtual Ptr<Value> maybeThunk(EvalState & state, Env & env);
+    virtual Value * maybeThunk(EvalState & state, Env & env);
     virtual void setName(Symbol & name);
 };
 
@@ -92,37 +92,28 @@ std::ostream & operator << (std::ostream & str, const Expr & e);
 struct ExprInt : Expr
 {
     NixInt n;
-    Ptr<Value> v;
-    ExprInt(NixInt n) : n(n) {
-        v = gc.alloc<Value>(Value::words());
-        mkInt(v, n);
-    };
+    Value v;
+    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprFloat : Expr
 {
     NixFloat nf;
-    Ptr<Value> v;
-    ExprFloat(NixFloat nf) : nf(nf) {
-        v = gc.alloc<Value>(Value::words());
-        mkFloat(v, nf);
-    };
+    Value v;
+    ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprString : Expr
 {
     Symbol s;
-    Ptr<Value> v;
-    ExprString(const Symbol & s) : s(s) {
-        v = gc.alloc<Value>(Value::words());
-        mkString(v, s);
-    };
+    Value v;
+    ExprString(const Symbol & s) : s(s) { mkString(v, s); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 /* Temporary class used during parsing of indented strings. */
@@ -134,13 +125,11 @@ struct ExprIndStr : Expr
 
 struct ExprPath : Expr
 {
-    Ptr<Value> v;
-    ExprPath(const string & s) {
-        v = gc.alloc<Value>(Value::words());
-        mkPath(v, s);
-    };
+    string s;
+    Value v;
+    ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprVar : Expr
@@ -164,7 +153,7 @@ struct ExprVar : Expr
     ExprVar(const Symbol & name) : name(name) { };
     ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
     COMMON_METHODS
-    Ptr<Value> maybeThunk(EvalState & state, Env & env) override;
+    Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprSelect : Expr

src/libexpr/parser.y

@@ -611,13 +611,13 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.c_str(), "(string)", basePath, staticEnv);
+    return parse(s.data(), "(string)", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 {
     return parseExprFromString(s, basePath, staticBaseEnv);
 }

src/libexpr/primops.cc

@@ -103,13 +103,14 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     // FIXME
     if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
         Derivation drv = readDerivation(*state.store, realPath);
-        auto w = state.allocValue();
+        Value & w = *state.allocValue();
         state.mkAttrs(w, 3 + drv.outputs.size());
-        auto v2 = state.allocAttr(w, state.sDrvPath);
+        Value * v2 = state.allocAttr(w, state.sDrvPath);
         mkString(*v2, path, {"=" + path});
         v2 = state.allocAttr(w, state.sName);
         mkString(*v2, drv.env["name"]);
-        auto outputsVal = state.allocAttr(w, state.symbols.create("outputs"));
+        Value * outputsVal =
+            state.allocAttr(w, state.symbols.create("outputs"));
         state.mkList(*outputsVal, drv.outputs.size());
         unsigned int outputs_index = 0;
 
@@ -119,19 +120,26 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             outputsVal->listElems()[outputs_index] = state.allocValue();
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
-        w->attrs->sort();
-        auto fun = state.allocValue();
-        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
-        state.forceFunction(fun, pos);
-        mkApp(v, fun, w);
+        w.attrs->sort();
+
+        static Value * fun = nullptr;
+        if (!fun) {
+            fun = state.allocValue();
+            state.eval(state.parseExprFromString(
+                #include "imported-drv-to-derivation.nix.gen.hh"
+                , "/"), *fun);
+        }
+
+        state.forceFunction(*fun, pos);
+        mkApp(v, *fun, w);
         state.forceAttrs(v, pos);
     } else {
         state.forceAttrs(*args[0]);
         if (args[0]->attrs->empty())
             state.evalFile(realPath, v);
         else {
-            auto env = state.allocEnv(args[0]->attrs->size());
-            env->up = state.baseEnv;
+            Env * env = &state.allocEnv(args[0]->attrs->size());
+            env->up = &state.baseEnv;
 
             StaticEnv staticEnv(false, &state.staticBaseEnv);
 
@@ -239,24 +247,19 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
     switch (args[0]->type) {
         case tInt: t = "int"; break;
         case tBool: t = "bool"; break;
-        case tShortString:
-        case tStaticString:
-        case tLongString:
-            t = "string"; break;
+        case tString: t = "string"; break;
         case tPath: t = "path"; break;
         case tNull: t = "null"; break;
         case tAttrs: t = "set"; break;
-        case tList0: case tList1: case tList2: case tListN: t = "list"; break;
+        case tList1: case tList2: case tListN: t = "list"; break;
         case tLambda:
         case tPrimOp:
         case tPrimOpApp:
             t = "lambda";
             break;
-#if 0
         case tExternal:
             t = args[0]->external->typeOf();
             break;
-#endif
         case tFloat: t = "float"; break;
         default: abort();
     }
@@ -309,7 +312,7 @@ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    mkBool(v, args[0]->isString());
+    mkBool(v, args[0]->type == tString);
 }
 
 
@@ -335,8 +338,6 @@ struct CompareValues
             return v1->fpoint < v2->integer;
         if (v1->type == tInt && v2->type == tFloat)
             return v1->integer < v2->fpoint;
-        if (v1->isString() && v2->isString())
-            return strcmp(v1->getString(), v2->getString()) < 0;
         if (v1->type != v2->type)
             throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         switch (v1->type) {
@@ -344,8 +345,10 @@ struct CompareValues
                 return v1->integer < v2->integer;
             case tFloat:
                 return v1->fpoint < v2->fpoint;
+            case tString:
+                return strcmp(v1->string.s, v2->string.s) < 0;
             case tPath:
-                return strcmp(v1->path->s, v2->path->s) < 0;
+                return strcmp(v1->path, v2->path) < 0;
             default:
                 throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2));
         }
@@ -353,7 +356,11 @@ struct CompareValues
 };
 
 
-typedef list<Ptr<Value>> ValueList;
+#if HAVE_BOEHMGC
+typedef list<Value *, gc_allocator<Value *> > ValueList;
+#else
+typedef list<Value *> ValueList;
+#endif
 
 
 static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -383,10 +390,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
        no new elements are found. */
     ValueList res;
     // `doneKeys' doesn't need to be a GC root, because its values are
-    // reachable from res. FIXME: dubious.
+    // reachable from res.
     set<Value *, CompareValues> doneKeys;
     while (!workSet.empty()) {
-        auto e = *(workSet.begin());
+        Value * e = *(workSet.begin());
         workSet.pop_front();
 
         state.forceAttrs(*e, pos);
@@ -401,14 +408,14 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         res.push_back(e);
 
         /* Call the `operator' function with `e' as argument. */
-        Root<Value> call;
+        Value call;
         mkApp(call, *op->value, *e);
         state.forceList(call, pos);
 
         /* Add the values returned by the operator to the work set. */
-        for (unsigned int n = 0; n < call->listSize(); ++n) {
-            state.forceValue(*call->listElems()[n]);
-            workSet.push_back(call->listElems()[n]);
+        for (unsigned int n = 0; n < call.listSize(); ++n) {
+            state.forceValue(*call.listElems()[n]);
+            workSet.push_back(call.listElems()[n]);
         }
     }
 
@@ -498,10 +505,10 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val
 static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0]);
-    if (args[0]->isString())
-        printError("trace: %s", args[0]->getString());
+    if (args[0]->type == tString)
+        printError(format("trace: %1%") % args[0]->string.s);
     else
-        printError("trace: %s", *args[0]);
+        printError(format("trace: %1%") % *args[0]);
     state.forceValue(*args[1]);
     v = *args[1];
 }
@@ -749,8 +756,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
     auto drvPathS = state.store->printStorePath(drvPath);
 
-    if (state.derivationHook) state.derivationHook(drvPath, drv);
-
     printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
 
     /* Optimisation, but required in read-only mode! because in that
@@ -952,20 +957,15 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     DirEntries entries = readDirectory(state.checkSourcePath(path));
     state.mkAttrs(v, entries.size());
 
-    auto sRegular = state.symbols.create("regular");
-    auto sDirectory = state.symbols.create("directory");
-    auto sSymlink = state.symbols.create("symlink");
-    auto sUnknown = state.symbols.create("unknown");
-
     for (auto & ent : entries) {
-        auto ent_val = state.allocAttr(v, state.symbols.create(ent.name));
+        Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name));
         if (ent.type == DT_UNKNOWN)
             ent.type = getFileType(path + "/" + ent.name);
-        mkString(*ent_val,
-            ent.type == DT_REG ? sRegular :
-            ent.type == DT_DIR ? sDirectory :
-            ent.type == DT_LNK ? sSymlink :
-            sUnknown);
+        mkStringNoCopy(*ent_val,
+            ent.type == DT_REG ? "regular" :
+            ent.type == DT_DIR ? "directory" :
+            ent.type == DT_LNK ? "symlink" :
+            "unknown");
     }
 
     v.attrs->sort();
@@ -1048,20 +1048,20 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
 
         /* Call the filter function.  The first argument is the path,
            the second is a string indicating the type of the file. */
-        auto arg1 = state.allocValue();
+        Value arg1;
         mkString(arg1, path);
 
-        Root<Value> fun2;
+        Value fun2;
         state.callFunction(*filterFun, arg1, fun2, noPos);
 
-        auto arg2 = state.allocValue();
+        Value arg2;
         mkString(arg2,
             S_ISREG(st.st_mode) ? "regular" :
             S_ISDIR(st.st_mode) ? "directory" :
             S_ISLNK(st.st_mode) ? "symlink" :
             "unknown" /* not supported, will fail! */);
 
-        Root<Value> res;
+        Value res;
         state.callFunction(fun2, arg2, res, noPos);
 
         return state.forceBool(res, pos);
@@ -1153,7 +1153,7 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V
         mkString(*(v.listElems()[n++] = state.allocValue()), i.name);
 
     std::sort(v.listElems(), v.listElems() + n,
-        [](Value * v1, Value * v2) { return strcmp(v1->getString(), v2->getString()) < 0; });
+              [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; });
 }
 
 
@@ -1230,9 +1230,10 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
 
     /* Get the attribute names to be removed. */
     std::set<Symbol> names;
-    for (unsigned int i = 0; i < args[1]->listSize(); ++i)
-        names.insert(state.symbols.create(
-                state.forceStringNoCtx(*args[1]->listElems()[i], pos)));
+    for (unsigned int i = 0; i < args[1]->listSize(); ++i) {
+        state.forceStringNoCtx(*args[1]->listElems()[i], pos);
+        names.insert(state.symbols.create(args[1]->listElems()[i]->string.s));
+    }
 
     /* Copy all attributes not in that set.  Note that we don't need
        to sort v.attrs because it's a subset of an already sorted
@@ -1368,8 +1369,8 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va
     state.mkAttrs(v, args[1]->attrs->size());
 
     for (auto & i : *args[1]->attrs) {
-        auto vName = state.allocValue();
-        auto vFun2 = state.allocValue();
+        Value * vName = state.allocValue();
+        Value * vFun2 = state.allocValue();
         mkString(*vName, i.name);
         mkApp(*vFun2, *args[0], *vName);
         mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value);
@@ -1456,7 +1457,7 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu
 
     bool same = true;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
-        Root<Value> res;
+        Value res;
         state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos);
         if (state.forceBool(res, pos))
             vs[k++] = args[1]->listElems()[n];
@@ -1511,12 +1512,12 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     state.forceList(*args[2], pos);
 
     if (args[2]->listSize()) {
-        Ptr<Value> vCur = args[1];
+        Value * vCur = args[1];
 
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
-            Root<Value> vTmp; // FIXME: correct?
+            Value vTmp;
             state.callFunction(*args[0], *vCur, vTmp, pos);
-            vCur = n == args[2]->listSize() - 1 ? Ptr(&v) : state.allocValue();
+            vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
             state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
         }
         state.forceValue(v);
@@ -1532,7 +1533,7 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg
     state.forceFunction(*args[0], pos);
     state.forceList(*args[1], pos);
 
-    Root<Value> vTmp;
+    Value vTmp;
     for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
         state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos);
         bool res = state.forceBool(vTmp, pos);
@@ -1568,7 +1569,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     state.mkList(v, len);
 
     for (unsigned int n = 0; n < (unsigned int) len; ++n) {
-        auto arg = state.allocValue();
+        Value * arg = state.allocValue();
         mkInt(*arg, n);
         mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg);
     }
@@ -1597,7 +1598,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
         if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan)
             return CompareValues()(a, b);
 
-        Root<Value> vTmp1, vTmp2; // FIXME
+        Value vTmp1, vTmp2;
         state.callFunction(*args[0], *a, vTmp1, pos);
         state.callFunction(vTmp1, *b, vTmp2, pos);
         return state.forceBool(vTmp2, pos);
@@ -1617,13 +1618,12 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     auto len = args[1]->listSize();
 
-    // Note: these Values are reachable via args[0].
-    std::vector<Value *> right, wrong;
+    ValueVector right, wrong;
 
     for (unsigned int n = 0; n < len; ++n) {
         auto vElem = args[1]->listElems()[n];
         state.forceValue(*vElem);
-        Root<Value> res;
+        Value res;
         state.callFunction(*args[0], *vElem, res, pos);
         if (state.forceBool(res, pos))
             right.push_back(vElem);
@@ -1633,13 +1633,13 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V
 
     state.mkAttrs(v, 2);
 
-    auto vRight = state.allocAttr(v, state.sRight);
+    Value * vRight = state.allocAttr(v, state.sRight);
     auto rsize = right.size();
     state.mkList(*vRight, rsize);
     if (rsize)
         memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize);
 
-    auto vWrong = state.allocAttr(v, state.sWrong);
+    Value * vWrong = state.allocAttr(v, state.sWrong);
     auto wsize = wrong.size();
     state.mkList(*vWrong, wsize);
     if (wsize)
@@ -1657,23 +1657,22 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
     state.forceList(*args[1], pos);
     auto nrLists = args[1]->listSize();
 
-    // FIXME: Root<>[] is inefficient
-    Root<Value> lists[nrLists];
+    Value lists[nrLists];
     size_t len = 0;
 
     for (unsigned int n = 0; n < nrLists; ++n) {
         Value * vElem = args[1]->listElems()[n];
         state.callFunction(*args[0], *vElem, lists[n], pos);
         state.forceList(lists[n], pos);
-        len += lists[n]->listSize();
+        len += lists[n].listSize();
     }
 
     state.mkList(v, len);
     auto out = v.listElems();
     for (unsigned int n = 0, pos = 0; n < nrLists; ++n) {
-        auto l = lists[n]->listSize();
+        auto l = lists[n].listSize();
         if (l)
-            memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
+            memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *));
         pos += l;
     }
 }
@@ -1819,19 +1818,21 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
 
 /* Match a regular expression against a string and return either
    ‘null’ or a list containing substring matches. */
-static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto re = state.forceStringNoCtx(*args[0], pos);
 
     try {
 
-        std::regex regex(re, std::regex::extended);
+        auto regex = state.regexCache.find(re);
+        if (regex == state.regexCache.end())
+            regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
 
         PathSet context;
         const std::string str = state.forceString(*args[1], context, pos);
 
         std::smatch match;
-        if (!std::regex_match(str, match, regex)) {
+        if (!std::regex_match(str, match, regex->second)) {
             mkNull(v);
             return;
         }
@@ -2123,10 +2124,10 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
 
 void EvalState::createBaseEnv()
 {
-    baseEnv->up = 0;
+    baseEnv.up = 0;
 
     /* Add global constants such as `true' to the base environment. */
-    auto v = allocValue();
+    Value v;
 
     /* `builtins' must be first! */
     mkAttrs(v, 128);
@@ -2144,7 +2145,7 @@ void EvalState::createBaseEnv()
     auto vThrow = addPrimOp("throw", 1, prim_throw);
 
     auto addPurityError = [&](const std::string & name) {
-        auto v2 = allocValue();
+        Value * v2 = allocValue();
         mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name));
         mkApp(v, *vThrow, *v2);
         addConstant(name, v);
@@ -2175,7 +2176,7 @@ void EvalState::createBaseEnv()
 
     // Miscellaneous
     auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport);
-    auto v2 = allocValue();
+    Value * v2 = allocValue();
     mkAttrs(*v2, 0);
     mkApp(v, *vScopedImport, *v2);
     forceValue(v);
@@ -2303,7 +2304,7 @@ void EvalState::createBaseEnv()
     mkList(v, searchPath.size());
     int n = 0;
     for (auto & i : searchPath) {
-        v2 = v->listElems()[n++] = allocValue();
+        v2 = v.listElems()[n++] = allocValue();
         mkAttrs(*v2, 2);
         mkString(*allocAttr(*v2, symbols.create("path")), i.second);
         mkString(*allocAttr(*v2, symbols.create("prefix")), i.first);
@@ -2317,7 +2318,7 @@ void EvalState::createBaseEnv()
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
-    baseEnv->values[0]->attrs->sort();
+    baseEnv.values[0]->attrs->sort();
 }
 
 

src/libexpr/primops/context.cc

@@ -160,7 +160,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             if (state.forceBool(*iter->value, *iter->pos)) {
                 if (!isDerivation(i.name)) {
-                    throw EvalError("tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
+                    throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
                 }
                 context.insert("=" + string(i.name));
             }
@@ -170,7 +170,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (iter != i.value->attrs->end()) {
             state.forceList(*iter->value, *iter->pos);
             if (iter->value->listSize() && !isDerivation(i.name)) {
-                throw EvalError("tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, *i.pos);
+                throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos);
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
                 auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);

src/libexpr/primops/fromTOML.cc

@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "cpptoml/cpptoml.h"
+#include "../../cpptoml/cpptoml.h"
 
 namespace nix {
 

src/libexpr/symbol-table.hh

@@ -19,7 +19,6 @@ private:
     const string * s; // pointer into SymbolTable
     Symbol(const string * s) : s(s) { };
     friend class SymbolTable;
-    friend class Value;
 
 public:
     Symbol() : s(0) { };
@@ -90,6 +89,4 @@ public:
     }
 };
 
-extern SymbolTable symbols;
-
 }

src/libexpr/value-to-json.cc

@@ -26,15 +26,13 @@ void printValueAsJSON(EvalState & state, bool strict,
             out.write(v.boolean);
             break;
 
-        case tShortString:
-        case tStaticString:
-        case tLongString:
-            v.getContext(context);
-            out.write(v.getString());
+        case tString:
+            copyContext(v, context);
+            out.write(v.string.s);
             break;
 
         case tPath:
-            out.write(state.copyPathToStore(context, v.path->s));
+            out.write(state.copyPathToStore(context, v.path));
             break;
 
         case tNull:
@@ -63,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-        case tList0: case tList1: case tList2: case tListN: {
+        case tList1: case tList2: case tListN: {
             auto list(out.list());
             for (unsigned int n = 0; n < v.listSize(); ++n) {
                 auto placeholder(list.placeholder());
@@ -72,11 +70,9 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
         }
 
-#if 0
         case tExternal:
             v.external->printValueAsJSON(state, strict, out, context);
             break;
-#endif
 
         case tFloat:
             out.write(v.fpoint);
@@ -94,13 +90,11 @@ void printValueAsJSON(EvalState & state, bool strict,
     printValueAsJSON(state, strict, v, out, context);
 }
 
-#if 0
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,
     JSONPlaceholder & out, PathSet & context) const
 {
     throw TypeError(format("cannot convert %1% to JSON") % showType());
 }
-#endif
 
 
 }

src/libexpr/value-to-xml.cc

@@ -68,16 +68,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
             break;
 
-        case tShortString:
-        case tStaticString:
-        case tLongString:
+        case tString:
             /* !!! show the context? */
-            v.getContext(context);
-            doc.writeEmptyElement("string", singletonAttrs("value", v.getString()));
+            copyContext(v, context);
+            doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
             break;
 
         case tPath:
-            doc.writeEmptyElement("path", singletonAttrs("value", v.path->s));
+            doc.writeEmptyElement("path", singletonAttrs("value", v.path));
             break;
 
         case tNull:
@@ -94,15 +92,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->isString())
-                        xmlAttrs["drvPath"] = drvPath = a->value->getString();
+                    if (a->value->type == tString)
+                        xmlAttrs["drvPath"] = drvPath = a->value->string.s;
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
                     if (strict) state.forceValue(*a->value);
-                    if (a->value->isString())
-                        xmlAttrs["outPath"] = a->value->getString();
+                    if (a->value->type == tString)
+                        xmlAttrs["outPath"] = a->value->string.s;
                 }
 
                 XMLOpenElement _(doc, "derivation", xmlAttrs);
@@ -120,7 +118,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
             break;
 
-        case tList0: case tList1: case tList2: case tListN: {
+        case tList1: case tList2: case tListN: {
             XMLOpenElement _(doc, "list");
             for (unsigned int n = 0; n < v.listSize(); ++n)
                 printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
@@ -145,11 +143,9 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             break;
         }
 
-#if 0
         case tExternal:
             v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
             break;
-#endif
 
         case tFloat:
             doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
@@ -161,13 +157,11 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 }
 
 
-#if 0
 void ExternalValueBase::printValueAsXML(EvalState & state, bool strict,
     bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const
 {
     doc.writeEmptyElement("unevaluated");
 }
-#endif
 
 
 void printValueAsXML(EvalState & state, bool strict, bool location,

src/libexpr/value.hh

@@ -1,12 +1,35 @@
 #pragma once
 
 #include "symbol-table.hh"
-#include "gc.hh"
 
-#include <cstring>
+#if HAVE_BOEHMGC
+#include <gc/gc_allocator.h>
+#endif
 
 namespace nix {
 
+
+typedef enum {
+    tInt = 1,
+    tBool,
+    tString,
+    tPath,
+    tNull,
+    tAttrs,
+    tList1,
+    tList2,
+    tListN,
+    tThunk,
+    tApp,
+    tLambda,
+    tBlackhole,
+    tPrimOp,
+    tPrimOpApp,
+    tExternal,
+    tFloat
+} ValueType;
+
+
 class Bindings;
 struct Env;
 struct Expr;
@@ -22,15 +45,13 @@ class JSONPlaceholder;
 typedef int64_t NixInt;
 typedef double NixFloat;
 
-#if 0
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented
  */
 class ExternalValueBase
 {
     friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
-
-protected:
+    protected:
     /* Print out the value */
     virtual std::ostream & print(std::ostream & str) const = 0;
 
@@ -65,29 +86,11 @@ protected:
 };
 
 std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
-#endif
 
 
-class Context : Object
-{
-    friend class Value;
-    friend class GC;
-
-    Symbol members[0];
-
-    Context(const PathSet & context) : Object(tContext, context.size())
-    {
-        size_t n = 0;
-        for (auto & i : context)
-            members[n++] = symbols.create(i);
-    }
-
-    size_t getSize() { return getMisc(); }
-};
-
-
-struct Value : Object
+struct Value
 {
+    ValueType type;
     union
     {
         NixInt integer;
@@ -110,15 +113,20 @@ struct Value : Object
            with context C is used as a derivation attribute, then the
            derivations in C will be added to the inputDrvs of the
            derivation, and the other store paths in C will be added to
-           the inputSrcs of the derivations. */
+           the inputSrcs of the derivations.
+
+           For canonicity, the store paths should be in sorted order. */
         struct {
-            String * s;
-            Context * context;
+            const char * s;
+            const char * * context; // must be in sorted order
         } string;
-        const char * staticString;
-        String * path;
+
+        const char * path;
         Bindings * attrs;
-        PtrList<Value> * bigList;
+        struct {
+            size_t size;
+            Value * * elems;
+        } bigList;
         Value * smallList[2];
         struct {
             Env * env;
@@ -132,99 +140,46 @@ struct Value : Object
             ExprLambda * fun;
         } lambda;
         PrimOp * primOp;
-        //ExternalValueBase * external;
+        struct {
+            Value * left, * right;
+        } primOpApp;
+        ExternalValueBase * external;
         NixFloat fpoint;
     };
 
-private:
-
-    Value() : Object(tNull, 0) {}
-
-    friend class GC;
-    template<typename T> friend class Root;
-
-public:
-
     bool isList() const
     {
-        return type >= tList0 && type <= tListN;
+        return type == tList1 || type == tList2 || type == tListN;
     }
 
     Value * * listElems()
     {
-        return type == tList0 || type == tList1 || type == tList2 ? smallList : bigList->elems;
+        return type == tList1 || type == tList2 ? smallList : bigList.elems;
     }
 
     const Value * const * listElems() const
     {
-        return type == tList0 || type == tList1 || type == tList2 ? smallList : bigList->elems;
+        return type == tList1 || type == tList2 ? smallList : bigList.elems;
     }
 
     size_t listSize() const
     {
-        return type == tList0 ? 0 : type == tList1 ? 1 : type == tList2 ? 2 : bigList->size();
-    }
-
-    constexpr static size_t words() { return 3; } // FIXME
-
-    void setContext(const PathSet & context)
-    {
-        if (context.size() == 0)
-            string.context = nullptr;
-        else if (context.size() == 1) {
-            // If we have a single context, then store it
-            // directly. This saves allocating a Context object (16
-            // bytes).
-            auto symbol = symbols.create(*context.begin());
-            string.context = (Context *) (((ptrdiff_t) symbol.s) | 1);
-        } else {
-            string.context = gc.alloc<Context>(1 + context.size(), context);
-        }
-    }
-
-    void getContext(PathSet & context)
-    {
-        if (type == tLongString && string.context) {
-            if (((ptrdiff_t) string.context) & 1) {
-                auto s = (const std::string *) (((ptrdiff_t) string.context) & ~1UL);
-                context.insert(*s);
-            } else {
-                auto size = string.context->getSize();
-                for (size_t i = 0; i < size; ++i)
-                    context.insert(string.context->members[i]);
-            }
-        }
-    }
-
-    bool isString() const
-    {
-        return type == tShortString || type == tStaticString || type == tLongString;
-    }
-
-    void setShortString(std::string_view s)
-    {
-        // FIXME: can't use strcpy here because gcc flags it as a
-        // buffer overflow on 'misc'.
-        auto p = getMiscData();
-        memcpy(p, s.data(), s.size());
-        p[s.size()] = 0;
-        type = tShortString;
-    }
-
-    const char * getString() const
-    {
-        if (type == tShortString)
-            return getMiscData();
-        else if (type == tStaticString)
-            return staticString;
-        else
-            return string.s->s;
+        return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
     }
 };
 
 
+/* After overwriting an app node, be sure to clear pointers in the
+   Value to ensure that the target isn't kept alive unnecessarily. */
+static inline void clearValue(Value & v)
+{
+    v.app.left = v.app.right = 0;
+}
+
+
 static inline void mkInt(Value & v, NixInt n)
 {
+    clearValue(v);
     v.type = tInt;
     v.integer = n;
 }
@@ -232,6 +187,7 @@ static inline void mkInt(Value & v, NixInt n)
 
 static inline void mkFloat(Value & v, NixFloat n)
 {
+    clearValue(v);
     v.type = tFloat;
     v.fpoint = n;
 }
@@ -239,6 +195,7 @@ static inline void mkFloat(Value & v, NixFloat n)
 
 static inline void mkBool(Value & v, bool b)
 {
+    clearValue(v);
     v.type = tBool;
     v.boolean = b;
 }
@@ -246,6 +203,7 @@ static inline void mkBool(Value & v, bool b)
 
 static inline void mkNull(Value & v)
 {
+    clearValue(v);
     v.type = tNull;
 }
 
@@ -266,35 +224,41 @@ static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
 }
 
 
-static inline Value & mkString(Value & v, std::string_view s)
+static inline void mkStringNoCopy(Value & v, const char * s)
 {
-    if (s.size() < WORD_SIZE * 2 + Object::miscBytes)
-        v.setShortString(s);
-    else {
-        v.string.s = gc.alloc<String>(String::wordsFor(s.size()), s.size(), s.data());
-        v.string.context = 0;
-        v.type = tLongString;
-    }
-    return v;
+    v.type = tString;
+    v.string.s = s;
+    v.string.context = 0;
 }
 
 
 static inline void mkString(Value & v, const Symbol & s)
 {
-    v.staticString = ((const string &) s).c_str();
-    v.type = tStaticString;
+    mkStringNoCopy(v, ((const string &) s).c_str());
 }
 
 
-static inline void mkPath(Value & v, const std::string & s)
+void mkString(Value & v, const char * s);
+
+
+static inline void mkPathNoCopy(Value & v, const char * s)
 {
-    v.path = String::alloc(s.c_str());
+    clearValue(v);
     v.type = tPath;
+    v.path = s;
 }
 
 
-typedef std::vector<Ptr<Value>> ValueVector; // FIXME: make more efficient
-typedef std::map<Symbol, Ptr<Value>> ValueMap; // FIXME: use Bindings?
+void mkPath(Value & v, const char * s);
+
+
+#if HAVE_BOEHMGC
+typedef std::vector<Value *, gc_allocator<Value *> > ValueVector;
+typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+#else
+typedef std::vector<Value *> ValueVector;
+typedef std::map<Symbol, Value *> ValueMap;
+#endif
 
 
 }

src/libmain/local.mk

@@ -6,6 +6,8 @@ libmain_DIR := $(d)
 
 libmain_SOURCES := $(wildcard $(d)/*.cc)
 
+libmain_CXXFLAGS += -I src/libutil -I src/libstore
+
 libmain_LDFLAGS = $(OPENSSL_LIBS)
 
 libmain_LIBS = libstore libutil

src/libmain/shared.cc

@@ -253,6 +253,9 @@ void printVersion(const string & programName)
     std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl;
     if (verbosity > lvlInfo) {
         Strings cfg;
+#if HAVE_BOEHMGC
+        cfg.push_back("gc");
+#endif
 #if HAVE_SODIUM
         cfg.push_back("signed-caches");
 #endif

src/libmain/shared.hh

@@ -57,7 +57,23 @@ template<class N> N getIntArg(const string & opt,
 {
     ++i;
     if (i == end) throw UsageError(format("'%1%' requires an argument") % opt);
-    return parseSize<N>(*i, allowUnit);
+    string s = *i;
+    N multiplier = 1;
+    if (allowUnit && !s.empty()) {
+        char u = std::toupper(*s.rbegin());
+        if (std::isalpha(u)) {
+            if (u == 'K') multiplier = 1ULL << 10;
+            else if (u == 'M') multiplier = 1ULL << 20;
+            else if (u == 'G') multiplier = 1ULL << 30;
+            else if (u == 'T') multiplier = 1ULL << 40;
+            else throw UsageError(format("invalid unit specifier '%1%'") % u);
+            s.resize(s.size() - 1);
+        }
+    }
+    N n;
+    if (!string2Int(s, n))
+        throw UsageError(format("'%1%' requires an integer argument") % opt);
+    return n * multiplier;
 }
 
 

src/libstore/binary-cache-store.cc

@@ -106,7 +106,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
+        state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
     }
 
     if (diskCache)

src/libstore/build.cc

@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "builtins/buildenv.hh"
 #include "download.hh"
 #include "finally.hh"
 #include "compression.hh"
@@ -1397,7 +1398,7 @@ void DerivationGoal::tryToBuild()
        few seconds and then retry this goal. */
     PathSet lockFiles;
     for (auto & outPath : drv->outputPaths())
-        lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath)));
+        lockFiles.insert(worker.store.Store::toRealPath(outPath));
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         worker.waitForAWhile(shared_from_this());
@@ -1428,7 +1429,7 @@ void DerivationGoal::tryToBuild()
     for (auto & i : drv->outputs) {
         if (worker.store.isValidPath(i.second.path)) continue;
         debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
-        deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path)));
+        deletePath(worker.store.Store::toRealPath(i.second.path));
     }
 
     /* Don't do a remote build if the derivation has the attribute
@@ -1685,7 +1686,7 @@ void DerivationGoal::buildDone()
 
         /* Delete unused redirected outputs (when doing hash rewriting). */
         for (auto & i : redirectedOutputs)
-            deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second)));
+            deletePath(worker.store.Store::toRealPath(i.second));
 
         /* Delete the chroot (if we were using one). */
         autoDelChroot.reset(); /* this runs the destructor */
@@ -1904,7 +1905,7 @@ void DerivationGoal::startBuilder()
             concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
             worker.store.printStorePath(drvPath),
             settings.thisSystem,
-            concatStringsSep(", ", settings.systemFeatures));
+            concatStringsSep<StringSet>(", ", settings.systemFeatures));
 
     if (drv->isBuiltin())
         preloadNSS();
@@ -2071,7 +2072,7 @@ void DerivationGoal::startBuilder()
            environment using bind-mounts.  We put it in the Nix store
            to ensure that we can create hard-links to non-directory
            inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot";
+        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
         deletePath(chrootRootDir);
 
         /* Clean up the chroot directory automatically. */
@@ -2550,7 +2551,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto & structuredAttrs = parsedDrv->getStructuredAttrs();
+    auto structuredAttrs = parsedDrv->getStructuredAttrs();
     if (!structuredAttrs) return;
 
     auto json = *structuredAttrs;
@@ -2916,7 +2917,7 @@ void DerivationGoal::addDependency(const StorePath & path)
 
         #if __linux__
 
-            Path source = worker.store.toRealPath(worker.store.printStorePath(path));
+            Path source = worker.store.Store::toRealPath(path);
             Path target = chrootRootDir + worker.store.printStorePath(path);
             debug("bind-mounting %s -> %s", target, source);
 
@@ -3578,7 +3579,7 @@ void DerivationGoal::registerOutputs()
         if (needsHashRewrite()) {
             auto r = redirectedOutputs.find(i.second.path);
             if (r != redirectedOutputs.end()) {
-                auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second));
+                auto redirected = worker.store.Store::toRealPath(r->second);
                 if (buildMode == bmRepair
                     && redirectedBadOutputs.count(i.second.path)
                     && pathExists(redirected))
@@ -3646,7 +3647,8 @@ void DerivationGoal::registerOutputs()
 
         if (fixedOutput) {
 
-            auto [recursive, h] = i.second.parseHashInfo();
+            bool recursive; Hash h;
+            i.second.parseHashInfo(recursive, h);
 
             if (!recursive) {
                 /* The output path should be a regular file without execute permission. */
@@ -3670,7 +3672,7 @@ void DerivationGoal::registerOutputs()
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
                         worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI)));
 
-                Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest));
+                Path actualDest = worker.store.Store::toRealPath(dest);
 
                 if (worker.store.isValidPath(dest))
                     std::rethrow_exception(delayedException);

src/libstore/builtins.hh

@@ -6,7 +6,6 @@ namespace nix {
 
 // TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
-void builtinBuildenv(const BasicDerivation & drv);
 void builtinUnpackChannel(const BasicDerivation & drv);
 
 }

src/libstore/builtins/buildenv.cc

@@ -1,4 +1,4 @@
-#include "builtins.hh"
+#include "buildenv.hh"
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -7,16 +7,14 @@
 
 namespace nix {
 
-typedef std::map<Path,int> Priorities;
-
-// FIXME: change into local variables.
-
-static Priorities priorities;
-
-static unsigned long symlinks;
+struct State
+{
+    std::map<Path, int> priorities;
+    unsigned long symlinks = 0;
+};
 
 /* For each activated package, create symlinks */
-static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
+static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
 {
     DirEntries srcFiles;
 
@@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISDIR(dstSt.st_mode)) {
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 } else if (S_ISLNK(dstSt.st_mode)) {
                     auto target = canonPath(dstFile, true);
@@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                         throw SysError(format("unlinking '%1%'") % dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
                         throw SysError(format("creating directory '%1%'"));
-                    createLinks(target, dstFile, priorities[dstFile]);
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, target, dstFile, state.priorities[dstFile]);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
@@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISLNK(dstSt.st_mode)) {
-                    auto prevPriority = priorities[dstFile];
+                    auto prevPriority = state.priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(
                                 "packages '%1%' and '%2%' have the same priority %3%; "
@@ -109,67 +107,30 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
         }
 
         createSymlink(srcFile, dstFile);
-        priorities[dstFile] = priority;
-        symlinks++;
+        state.priorities[dstFile] = priority;
+        state.symlinks++;
     }
 }
 
-typedef std::set<Path> FileProp;
-
-static FileProp done;
-static FileProp postponed = FileProp{};
-
-static Path out;
-
-static void addPkg(const Path & pkgDir, int priority)
+void buildProfile(const Path & out, Packages && pkgs)
 {
-    if (!done.insert(pkgDir).second) return;
-    createLinks(pkgDir, out, priority);
+    State state;
 
-    try {
-        for (const auto & p : tokenizeString<std::vector<string>>(
-                readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
-            if (!done.count(p))
-                postponed.insert(p);
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
-    }
-}
+    std::set<Path> done, postponed;
 
-struct Package {
-    Path path;
-    bool active;
-    int priority;
-    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
-};
+    auto addPkg = [&](const Path & pkgDir, int priority) {
+        if (!done.insert(pkgDir).second) return;
+        createLinks(state, pkgDir, out, priority);
 
-typedef std::vector<Package> Packages;
-
-void builtinBuildenv(const BasicDerivation & drv)
-{
-    auto getAttr = [&](const string & name) {
-        auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
-        return i->second;
-    };
-
-    out = getAttr("out");
-    createDirs(out);
-
-    /* Convert the stuff we get from the environment back into a
-     * coherent data type. */
-    Packages pkgs;
-    auto derivations = tokenizeString<Strings>(getAttr("derivations"));
-    while (!derivations.empty()) {
-        /* !!! We're trusting the caller to structure derivations env var correctly */
-        auto active = derivations.front(); derivations.pop_front();
-        auto priority = stoi(derivations.front()); derivations.pop_front();
-        auto outputs = stoi(derivations.front()); derivations.pop_front();
-        for (auto n = 0; n < outputs; n++) {
-            auto path = derivations.front(); derivations.pop_front();
-            pkgs.emplace_back(path, active != "false", priority);
+        try {
+            for (const auto & p : tokenizeString<std::vector<string>>(
+                    readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
+                if (!done.count(p))
+                    postponed.insert(p);
+        } catch (SysError & e) {
+            if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
         }
-    }
+    };
 
     /* Symlink to the packages that have been installed explicitly by the
      * user. Process in priority order to reduce unnecessary
@@ -189,13 +150,42 @@ void builtinBuildenv(const BasicDerivation & drv)
      */
     auto priorityCounter = 1000;
     while (!postponed.empty()) {
-        auto pkgDirs = postponed;
-        postponed = FileProp{};
+        std::set<Path> pkgDirs;
+        postponed.swap(pkgDirs);
         for (const auto & pkgDir : pkgDirs)
             addPkg(pkgDir, priorityCounter++);
     }
 
-    printError("created %d symlinks in user environment", symlinks);
+    debug("created %d symlinks in user environment", state.symlinks);
+}
+
+void builtinBuildenv(const BasicDerivation & drv)
+{
+    auto getAttr = [&](const string & name) {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        return i->second;
+    };
+
+    Path out = getAttr("out");
+    createDirs(out);
+
+    /* Convert the stuff we get from the environment back into a
+     * coherent data type. */
+    Packages pkgs;
+    auto derivations = tokenizeString<Strings>(getAttr("derivations"));
+    while (!derivations.empty()) {
+        /* !!! We're trusting the caller to structure derivations env var correctly */
+        auto active = derivations.front(); derivations.pop_front();
+        auto priority = stoi(derivations.front()); derivations.pop_front();
+        auto outputs = stoi(derivations.front()); derivations.pop_front();
+        for (auto n = 0; n < outputs; n++) {
+            auto path = derivations.front(); derivations.pop_front();
+            pkgs.emplace_back(path, active != "false", priority);
+        }
+    }
+
+    buildProfile(out, std::move(pkgs));
 
     createSymlink(getAttr("manifest"), out + "/manifest.nix");
 }

src/libstore/builtins/buildenv.hh

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "derivations.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+struct Package {
+    Path path;
+    bool active;
+    int priority;
+    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
+};
+
+typedef std::vector<Package> Packages;
+
+void buildProfile(const Path & out, Packages && pkgs);
+
+void builtinBuildenv(const BasicDerivation & drv);
+
+}

src/libstore/derivations.cc

@@ -9,19 +9,21 @@
 namespace nix {
 
 
-std::pair<bool, Hash> DerivationOutput::parseHashInfo() const
+void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const
 {
-    bool recursive = false;
+    recursive = false;
     string algo = hashAlgo;
 
-    if (string(algo, 0, 2) == "r:")
+    if (string(algo, 0, 2) == "r:") {
         recursive = true;
+        algo = string(algo, 2);
+    }
 
-    HashType hashType = parseHashType(recursive ? string(algo, 2) : algo);
+    HashType hashType = parseHashType(algo);
     if (hashType == htUnknown)
         throw Error("unknown hash algorithm '%s'", algo);
 
-    return {recursive, Hash(this->hash, hashType)};
+    hash = Hash(this->hash, hashType);
 }
 
 
@@ -63,7 +65,7 @@ bool BasicDerivation::isBuiltin() const
 
 
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair)
+    const Derivation & drv, std::string_view name, RepairFlag repair)
 {
     auto references = cloneStorePathSet(drv.inputSrcs);
     for (auto & i : drv.inputDrvs)
@@ -71,8 +73,8 @@ StorePath writeDerivation(ref<Store> store,
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-    string suffix = name + drvExtension;
-    string contents = drv.unparse(*store, false);
+    auto suffix = std::string(name) + drvExtension;
+    auto contents = drv.unparse(*store, false);
     return settings.readOnlyMode
         ? store->computeStorePathForText(suffix, contents, references)
         : store->addTextToStore(suffix, contents, references, repair);
@@ -211,15 +213,26 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
 }
 
 
-static void printString(string & res, const string & s)
+static void printString(string & res, std::string_view s)
+{
+    char buf[s.size() * 2 + 2];
+    char * p = buf;
+    *p++ = '"';
+    for (auto c : s)
+        if (c == '\"' || c == '\\') { *p++ = '\\'; *p++ = c; }
+        else if (c == '\n') { *p++ = '\\'; *p++ = 'n'; }
+        else if (c == '\r') { *p++ = '\\'; *p++ = 'r'; }
+        else if (c == '\t') { *p++ = '\\'; *p++ = 't'; }
+        else *p++ = c;
+    *p++ = '"';
+    res.append(buf, p - buf);
+}
+
+
+static void printUnquotedString(string & res, std::string_view s)
 {
     res += '"';
-    for (const char * i = s.c_str(); *i; i++)
-        if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
-        else if (*i == '\n') res += "\\n";
-        else if (*i == '\r') res += "\\r";
-        else if (*i == '\t') res += "\\t";
-        else res += *i;
+    res.append(s);
     res += '"';
 }
 
@@ -237,6 +250,19 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
 }
 
 
+template<class ForwardIterator>
+static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterator j)
+{
+    res += '[';
+    bool first = true;
+    for ( ; i != j; ++i) {
+        if (first) first = false; else res += ',';
+        printUnquotedString(res, *i);
+    }
+    res += ']';
+}
+
+
 string Derivation::unparse(const Store & store, bool maskOutputs,
     std::map<std::string, StringSet> * actualInputs) const
 {
@@ -247,10 +273,10 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     bool first = true;
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
-        s += '('; printString(s, i.first);
-        s += ','; printString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
-        s += ','; printString(s, i.second.hashAlgo);
-        s += ','; printString(s, i.second.hash);
+        s += '('; printUnquotedString(s, i.first);
+        s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
+        s += ','; printUnquotedString(s, i.second.hashAlgo);
+        s += ','; printUnquotedString(s, i.second.hash);
         s += ')';
     }
 
@@ -259,24 +285,24 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     if (actualInputs) {
         for (auto & i : *actualInputs) {
             if (first) first = false; else s += ',';
-            s += '('; printString(s, i.first);
-            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, i.first);
+            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
             s += ')';
         }
     } else {
         for (auto & i : inputDrvs) {
             if (first) first = false; else s += ',';
-            s += '('; printString(s, store.printStorePath(i.first));
-            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, store.printStorePath(i.first));
+            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
             s += ')';
         }
     }
 
     s += "],";
     auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow
-    printStrings(s, paths.begin(), paths.end());
+    printUnquotedStrings(s, paths.begin(), paths.end());
 
-    s += ','; printString(s, platform);
+    s += ','; printUnquotedString(s, platform);
     s += ','; printString(s, builder);
     s += ','; printStrings(s, args.begin(), args.end());
 

src/libstore/derivations.hh

@@ -22,7 +22,7 @@ struct DerivationOutput
         , hashAlgo(std::move(hashAlgo))
         , hash(std::move(hash))
     { }
-    std::pair<bool, Hash> parseHashInfo() const;
+    void parseHashInfo(bool & recursive, Hash & hash) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -79,7 +79,7 @@ class Store;
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
+    const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
 Derivation readDerivation(const Store & store, const Path & drvPath);

src/libstore/download.cc

@@ -308,7 +308,7 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
-            if (writtenToSink)
+            if (writtenToSink && acceptRanges && encoding.empty())
                 curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
 
             result.data = std::make_shared<std::string>();
@@ -414,16 +414,14 @@ struct CurlDownloader : public Downloader
 
                 /* If this is a transient error, then maybe retry the
                    download after a while. If we're writing to a
-                   sink, we can only retry if the server supports
-                   ranged requests. */
-                if (err == Transient
-                    && attempt < request.tries
-                    && (!this->request.dataCallback
-                        || writtenToSink == 0
-                        || (acceptRanges && encoding.empty())))
+                   sink, we can only resume if the server supports
+                   ranged requests, otherwise fallback to a plain
+                   retry.
+                */
+                if (err == Transient && attempt < request.tries)
                 {
                     int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
-                    if (writtenToSink)
+                    if (writtenToSink && acceptRanges && encoding.empty())
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
                         warn("%s; retrying in %d ms", exc.what(), ms);

src/libstore/gc.cc

@@ -255,12 +255,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
-        Path storePath = toStorePath(target);
-        // FIXME
-        if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
-            roots[parseStorePath(storePath)].emplace(path);
+        auto storePath = maybeParseStorePath(toStorePath(target));
+        if (storePath && isValidPath(*storePath))
+            roots[std::move(*storePath)].emplace(path);
         else
-            printInfo("skipping invalid root from '%1%' to '%2%'", path, storePath);
+            printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
     };
 
     try {
@@ -296,10 +295,9 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         }
 
         else if (type == DT_REG) {
-            Path storePath = storeDir + "/" + std::string(baseNameOf(path));
-            // FIXME
-            if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
-                roots[parseStorePath(storePath)].emplace(path);
+            auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path)));
+            if (storePath && isValidPath(*storePath))
+                roots[std::move(*storePath)].emplace(path);
         }
 
     }
@@ -523,14 +521,14 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 
     unsigned long long size = 0;
 
-    // FIXME
-    if (isStorePath(path) && isValidPath(parseStorePath(path))) {
+    auto storePath = maybeParseStorePath(path);
+    if (storePath && isValidPath(*storePath)) {
         StorePathSet referrers;
-        queryReferrers(parseStorePath(path), referrers);
+        queryReferrers(*storePath, referrers);
         for (auto & i : referrers)
             if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
-        size = queryPathInfo(parseStorePath(path))->narSize;
-        invalidatePathChecked(parseStorePath(path));
+        size = queryPathInfo(*storePath)->narSize;
+        invalidatePathChecked(*storePath);
     }
 
     Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
@@ -593,8 +591,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
 
     visited.insert(path.clone());
 
-    //FIXME
-    if (!isStorePath(printStorePath(path)) || !isValidPath(path)) return false;
+    if (!isValidPath(path)) return false;
 
     StorePathSet incoming;
 
@@ -637,8 +634,9 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 
     //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
 
-    // FIXME
-    if (!isStorePath(path) || !isValidPath(parseStorePath(path))) {
+    auto storePath = maybeParseStorePath(path);
+
+    if (!storePath || !isValidPath(*storePath)) {
         /* A lock file belonging to a path that we're building right
            now isn't garbage. */
         if (isActiveTempFile(state, path, ".lock")) return;
@@ -655,7 +653,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 
     StorePathSet visited;
 
-    if (canReachRoot(state, visited, parseStorePath(path))) {
+    if (storePath && canReachRoot(state, visited, *storePath)) {
         debug("cannot delete '%s' because it's still reachable", path);
     } else {
         /* No path we visited was a root, so everything is garbage.
@@ -818,8 +816,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 string name = dirent->d_name;
                 if (name == "." || name == "..") continue;
                 Path path = storeDir + "/" + name;
-                // FIXME
-                if (isStorePath(path) && isValidPath(parseStorePath(path)))
+                auto storePath = maybeParseStorePath(path);
+                if (storePath && isValidPath(*storePath))
                     entries.push_back(path);
                 else
                     tryToDelete(state, path);

src/libstore/globals.cc

@@ -20,13 +20,6 @@ namespace nix {
    must be deleted and recreated on startup.) */
 #define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
 
-/* chroot-like behavior from Apple's sandbox */
-#if __APPLE__
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
-#else
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
-#endif
-
 Settings settings;
 
 static GlobalConfig::Register r1(&settings);
@@ -68,7 +61,12 @@ Settings::Settings()
     sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
 #endif
 
-    allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
+
+/* chroot-like behavior from Apple's sandbox */
+#if __APPLE__
+    sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib");
+    allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
+#endif
 }
 
 void loadConfFile()

src/libstore/globals.hh

@@ -311,12 +311,7 @@ public:
     Setting<bool> printMissing{this, true, "print-missing",
         "Whether to print what paths need to be built or downloaded."};
 
-    Setting<std::string> preBuildHook{this,
-#if __APPLE__
-        nixLibexecDir + "/nix/resolve-system-dependencies",
-#else
-        "",
-#endif
+    Setting<std::string> preBuildHook{this, "",
         "pre-build-hook",
         "A program to run just before a build to set derivation-specific build settings."};
 

src/libstore/http-binary-cache-store.cc

@@ -163,10 +163,11 @@ static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
+    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
     if (std::string(uri, 0, 7) != "http://" &&
         std::string(uri, 0, 8) != "https://" &&
-        (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
-        ) return 0;
+        (!forceHttp || std::string(uri, 0, 7) != "file://"))
+        return 0;
     auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
     store->init();
     return store;

src/libstore/local-store.cc

@@ -298,9 +298,7 @@ void LocalStore::openDB(State & state, bool create)
     /* Open the Nix database. */
     string dbPath = dbDir + "/db.sqlite";
     auto & db(state.db);
-    if (sqlite3_open_v2(dbPath.c_str(), &db.db,
-            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
-        throw Error(format("cannot open Nix database '%1%'") % dbPath);
+    state.db = SQLite(dbPath, create);
 
 #ifdef __CYGWIN__
     /* The cygwin version of sqlite3 has a patch which calls
@@ -312,11 +310,6 @@ void LocalStore::openDB(State & state, bool create)
     SetDllDirectoryW(L"");
 #endif
 
-    if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
-        throwSQLiteError(db, "setting timeout");
-
-    db.exec("pragma foreign_keys = 1");
-
     /* !!! check whether sqlite has been built with foreign key
        support */
 
@@ -350,7 +343,7 @@ void LocalStore::openDB(State & state, bool create)
 
     /* Initialise the database schema, if necessary. */
     if (create) {
-        const char * schema =
+        static const char schema[] =
 #include "schema.sql.gen.hh"
             ;
         db.exec(schema);
@@ -547,30 +540,33 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     std::string drvName(drvPath.name());
     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 
+    auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName)
+    {
+        if (actual != expected)
+            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                printStorePath(drvPath), printStorePath(actual), printStorePath(expected));
+        auto j = drv.env.find(varName);
+        if (j == drv.env.end() || parseStorePath(j->second) != actual)
+            throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
+                printStorePath(drvPath), varName, printStorePath(actual));
+    };
+
+
     if (drv.isFixedOutput()) {
         DerivationOutputs::const_iterator out = drv.outputs.find("out");
         if (out == drv.outputs.end())
             throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
-        auto [recursive, h] = out->second.parseHashInfo();
-        auto outPath = makeFixedOutputPath(recursive, h, drvName);
+        bool recursive; Hash h;
+        out->second.parseHashInfo(recursive, h);
 
-        StringPairs::const_iterator j = drv.env.find("out");
-        if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
-            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath));
+        check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
     }
 
     else {
         Hash h = hashDerivationModulo(*this, drv, true);
-
-        for (auto & i : drv.outputs) {
-            auto outPath = makeOutputPath(i.first, h, drvName);
-            StringPairs::const_iterator j = drv.env.find(i.first);
-            if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
-                throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                    printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath));
-        }
+        for (auto & i : drv.outputs)
+            check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
     }
 }
 
@@ -619,7 +615,8 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), std::make_shared<ValidPathInfo>(info));
+        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)),
+            PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
     return id;
@@ -1000,16 +997,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             deletePath(realPath);
 
+            if (info.ca != "" &&
+                !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
+                    || info.references.empty()))
+                settings.requireExperimentalFeature("ca-references");
+
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
             std::unique_ptr<AbstractHashSink> hashSink;
-            if (info.ca == "")
+            if (info.ca == "" || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(htSHA256);
-            else {
-                if (!info.references.empty())
-                    settings.requireExperimentalFeature("ca-references");
+            else
                 hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
-            }
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
@@ -1264,12 +1263,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
                 std::unique_ptr<AbstractHashSink> hashSink;
-                if (info->ca == "")
+                if (info->ca == "" || !info->references.count(info->path))
                     hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
                     hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
 
-                dumpPath(toRealPath(printStorePath(i)), *hashSink);
+                dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {

src/libstore/local.mk

@@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1)
 	libstore_LDFLAGS += -lseccomp
 endif
 
-libstore_CXXFLAGS = \
+libstore_CXXFLAGS += \
+ -I src/libutil -I src/libstore \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \

src/libstore/nar-info-disk-cache.cc

@@ -78,12 +78,7 @@ public:
 
         state->db = SQLite(dbPath);
 
-        if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
-            throwSQLiteError(state->db, "setting timeout");
-
-        // We can always reproduce the cache.
-        state->db.exec("pragma synchronous = off");
-        state->db.exec("pragma main.journal_mode = truncate");
+        state->db.isCache();
 
         state->db.exec(schema);
 

src/libstore/nar-info-disk-cache.hh

@@ -10,7 +10,7 @@ class NarInfoDiskCache
 public:
     typedef enum { oValid, oInvalid, oUnknown } Outcome;
 
-    virtual ~NarInfoDiskCache() { };
+    virtual ~NarInfoDiskCache() { }
 
     virtual void createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;

src/libstore/parsed-derivations.cc

@@ -1,5 +1,7 @@
 #include "parsed-derivations.hh"
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
@@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
     auto jsonAttr = drv.env.find("__json");
     if (jsonAttr != drv.env.end()) {
         try {
-            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
+            structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second));
         } catch (std::exception & e) {
             throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what());
         }
     }
 }
 
+ParsedDerivation::~ParsedDerivation() { }
+
 std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
 {
     if (structuredAttrs) {

src/libstore/parsed-derivations.hh

@@ -1,6 +1,6 @@
 #include "derivations.hh"
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix {
 
@@ -8,15 +8,17 @@ class ParsedDerivation
 {
     StorePath drvPath;
     BasicDerivation & drv;
-    std::optional<nlohmann::json> structuredAttrs;
+    std::unique_ptr<nlohmann::json> structuredAttrs;
 
 public:
 
     ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
 
-    const std::optional<nlohmann::json> & getStructuredAttrs() const
+    ~ParsedDerivation();
+
+    const nlohmann::json * getStructuredAttrs() const
     {
-        return structuredAttrs;
+        return structuredAttrs.get();
     }
 
     std::optional<std::string> getStringAttr(const std::string & name) const;

src/libstore/path.cc

@@ -55,6 +55,20 @@ StorePath Store::parseStorePath(std::string_view path) const
     return StorePath::make(path, storeDir);
 }
 
+std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
+{
+    try {
+        return parseStorePath(path);
+    } catch (Error &) {
+        return {};
+    }
+}
+
+bool Store::isStorePath(std::string_view path) const
+{
+    return (bool) maybeParseStorePath(path);
+}
+
 StorePathSet Store::parseStorePathSet(const PathSet & paths) const
 {
     StorePathSet res;

src/libstore/profiles.cc

@@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile)
 }
 
 
+Path getDefaultProfile()
+{
+    Path profileLink = getHome() + "/.nix-profile";
+    try {
+        if (!pathExists(profileLink)) {
+            replaceSymlink(
+                getuid() == 0
+                ? settings.nixStateDir + "/profiles/default"
+                : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
+                profileLink);
+        }
+        return absPath(readLink(profileLink), dirOf(profileLink));
+    } catch (Error &) {
+        return profileLink;
+    }
+}
+
+
 }

src/libstore/profiles.hh

@@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile);
    rebuilt. */
 string optimisticLockProfile(const Path & profile);
 
+/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
+   it. */
+Path getDefaultProfile();
+
 }

src/libstore/sandbox-defaults.sb

@@ -71,6 +71,12 @@
        (literal "/dev/zero")
        (subpath "/dev/fd"))
 
+; Allow pseudo-terminals.
+(allow file*
+       (literal "/dev/ptmx")
+       (regex #"^/dev/pty[a-z]+")
+       (regex #"^/dev/ttys[0-9]+"))
+
 ; Does nothing, but reduces build noise.
 (allow file* (literal "/dev/dtracehelper"))
 
@@ -85,3 +91,7 @@
        (literal "/etc")
        (literal "/var")
        (literal "/private/var/tmp"))
+
+; This is used by /bin/sh on macOS 10.15 and later.
+(allow file*
+       (literal "/private/var/select/sh"))

src/libstore/sqlite.cc

@@ -25,11 +25,16 @@ namespace nix {
         throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
 }
 
-SQLite::SQLite(const Path & path)
+SQLite::SQLite(const Path & path, bool create)
 {
     if (sqlite3_open_v2(path.c_str(), &db,
-            SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
+            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
         throw Error(format("cannot open SQLite database '%s'") % path);
+
+    if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
+        throwSQLiteError(db, "setting timeout");
+
+    exec("pragma foreign_keys = 1");
 }
 
 SQLite::~SQLite()
@@ -42,6 +47,12 @@ SQLite::~SQLite()
     }
 }
 
+void SQLite::isCache()
+{
+    exec("pragma synchronous = off");
+    exec("pragma main.journal_mode = truncate");
+}
+
 void SQLite::exec(const std::string & stmt)
 {
     retrySQLite<void>([&]() {
@@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
     return *this;
 }
 
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
+{
+    if (notNull) {
+        if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
+            throwSQLiteError(stmt.db, "binding argument");
+    } else
+        bind();
+    return *this;
+}
+
 SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
 {
     if (notNull) {

src/libstore/sqlite.hh

@@ -5,8 +5,8 @@
 
 #include "types.hh"
 
-class sqlite3;
-class sqlite3_stmt;
+struct sqlite3;
+struct sqlite3_stmt;
 
 namespace nix {
 
@@ -15,13 +15,16 @@ struct SQLite
 {
     sqlite3 * db = 0;
     SQLite() { }
-    SQLite(const Path & path);
+    SQLite(const Path & path, bool create = true);
     SQLite(const SQLite & from) = delete;
     SQLite& operator = (const SQLite & from) = delete;
     SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
     ~SQLite();
     operator sqlite3 * () { return db; }
 
+    /* Disable synchronous mode, set truncate journal mode. */
+    void isCache();
+
     void exec(const std::string & stmt);
 };
 
@@ -52,6 +55,7 @@ struct SQLiteStmt
 
         /* Bind the next parameter. */
         Use & operator () (const std::string & value, bool notNull = true);
+        Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
         Use & operator () (int64_t value, bool notNull = true);
         Use & bind(); // null
 

src/libstore/ssh.cc

@@ -33,6 +33,9 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
     out.create();
 
     auto conn = std::make_unique<Connection>();
+    ProcessOptions options;
+    options.dieWithParent = false;
+
     conn->sshPid = startProcess([&]() {
         restoreSignals();
 
@@ -64,7 +67,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
         // could not exec ssh/bash
         throw SysError("unable to execute '%s'", args.front());
-    });
+    }, options);
 
 
     in.readSide = -1;
@@ -91,6 +94,9 @@ Path SSHMaster::startMaster()
     Pipe out;
     out.create();
 
+    ProcessOptions options;
+    options.dieWithParent = false;
+
     state->sshMaster = startProcess([&]() {
         restoreSignals();
 
@@ -110,7 +116,7 @@ Path SSHMaster::startMaster()
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("unable to execute '%s'", args.front());
-    });
+    }, options);
 
     out.writeSide = -1;
 

src/libstore/store-api.cc

@@ -19,14 +19,6 @@ bool Store::isInStore(const Path & path) const
 }
 
 
-bool Store::isStorePath(const Path & path) const
-{
-    return isInStore(path)
-        && path.size() >= storeDir.size() + 1 + storePathHashLen
-        && path.find('/', storeDir.size() + 1) == Path::npos;
-}
-
-
 Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
@@ -234,6 +226,14 @@ std::string Store::getUri()
     return "";
 }
 
+bool Store::PathInfoCacheValue::isKnownNow()
+{
+    std::chrono::duration ttl = didExist()
+        ? std::chrono::seconds(settings.ttlPositiveNarInfoCache)
+        : std::chrono::seconds(settings.ttlNegativeNarInfoCache);
+
+    return std::chrono::steady_clock::now() < time_point + ttl;
+}
 
 bool Store::isValidPath(const StorePath & storePath)
 {
@@ -242,9 +242,9 @@ bool Store::isValidPath(const StorePath & storePath)
     {
         auto state_(state.lock());
         auto res = state_->pathInfoCache.get(hashPart);
-        if (res) {
+        if (res && res->isKnownNow()) {
             stats.narInfoReadAverted++;
-            return *res != 0;
+            return res->didExist();
         }
     }
 
@@ -254,7 +254,7 @@ bool Store::isValidPath(const StorePath & storePath)
             stats.narInfoReadAverted++;
             auto state_(state.lock());
             state_->pathInfoCache.upsert(hashPart,
-                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+                res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second });
             return res.first == NarInfoDiskCache::oValid;
         }
     }
@@ -309,11 +309,11 @@ void Store::queryPathInfo(const StorePath & storePath,
 
         {
             auto res = state.lock()->pathInfoCache.get(hashPart);
-            if (res) {
+            if (res && res->isKnownNow()) {
                 stats.narInfoReadAverted++;
-                if (!*res)
+                if (!res->didExist())
                     throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-                return callback(ref<const ValidPathInfo>(*res));
+                return callback(ref<const ValidPathInfo>(res->value));
             }
         }
 
@@ -324,7 +324,7 @@ void Store::queryPathInfo(const StorePath & storePath,
                 {
                     auto state_(state.lock());
                     state_->pathInfoCache.upsert(hashPart,
-                        res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+                        res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
                     if (res.first == NarInfoDiskCache::oInvalid ||
                         res.second->path != storePath)
                         throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
@@ -348,7 +348,7 @@ void Store::queryPathInfo(const StorePath & storePath,
 
                 {
                     auto state_(state.lock());
-                    state_->pathInfoCache.upsert(hashPart, info);
+                    state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
                 }
 
                 if (!info || info->path != parseStorePath(storePath)) {
@@ -441,7 +441,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
 
 
 void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
-    bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
+    bool includeImpureInfo, bool showClosureSize,
+    Base hashBase,
+    AllowInvalidFlag allowInvalid)
 {
     auto jsonList = jsonOut.list();
 
@@ -453,7 +455,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
             auto info = queryPathInfo(storePath);
 
             jsonPath
-                .attr("narHash", info->narHash.to_string())
+                .attr("narHash", info->narHash.to_string(hashBase))
                 .attr("narSize", info->narSize);
 
             {
@@ -741,12 +743,7 @@ std::string Store::showPaths(const StorePathSet & paths)
 
 string showPaths(const PathSet & paths)
 {
-    string s;
-    for (auto & i : paths) {
-        if (s.size() != 0) s += ", ";
-        s += "'" + i + "'";
-    }
-    return s;
+    return concatStringsSep(", ", quoteStrings(paths));
 }
 
 

src/libstore/store-api.hh

@@ -16,6 +16,7 @@
 #include <unordered_set>
 #include <memory>
 #include <string>
+#include <chrono>
 
 
 namespace nix {
@@ -261,10 +262,28 @@ public:
 
 protected:
 
+    struct PathInfoCacheValue {
+
+        // Time of cache entry creation or update
+        std::chrono::time_point<std::chrono::steady_clock> time_point = std::chrono::steady_clock::now();
+
+        // Null if missing
+        std::shared_ptr<const ValidPathInfo> value;
+
+        // Whether the value is valid as a cache entry. The path may not exist.
+        bool isKnownNow();
+
+        // Past tense, because a path can only be assumed to exists when
+        // isKnownNow() && didExist()
+        inline bool didExist() {
+          return value != nullptr;
+        }
+    };
+
     struct State
     {
         // FIXME: fix key
-        LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache;
+        LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
     };
 
     Sync<State> state;
@@ -281,6 +300,8 @@ public:
 
     StorePath parseStorePath(std::string_view path) const;
 
+    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
+
     std::string printStorePath(const StorePath & path) const;
 
     // FIXME: remove
@@ -303,7 +324,7 @@ public:
 
     /* Return true if ‘path’ is a store path, i.e. a direct child of
        the Nix store. */
-    bool isStorePath(const Path & path) const;
+    bool isStorePath(std::string_view path) const;
 
     /* Chop off the parts after the top-level store name, e.g.,
        /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
@@ -540,6 +561,7 @@ public:
        each path is included. */
     void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
         bool includeImpureInfo, bool showClosureSize,
+        Base hashBase = Base32,
         AllowInvalidFlag allowInvalid = DisallowInvalid);
 
     /* Return the size of the closure of the specified path, that is,
@@ -655,6 +677,11 @@ public:
         return storePath;
     }
 
+    Path toRealPath(const StorePath & storePath)
+    {
+        return toRealPath(printStorePath(storePath));
+    }
+
     virtual void createUser(const std::string & userName, uid_t userId)
     { }
 

src/libutil/hash.hh

@@ -80,6 +80,18 @@ struct Hash
        or base-64. By default, this is prefixed by the hash type
        (e.g. "sha256:"). */
     std::string to_string(Base base = Base32, bool includeType = true) const;
+
+    std::string gitRev() const
+    {
+        assert(type == htSHA1);
+        return to_string(Base16, false);
+    }
+
+    std::string gitShortRev() const
+    {
+        assert(type == htSHA1);
+        return std::string(to_string(Base16, false), 0, 7);
+    }
 };
 
 

src/libutil/util.cc

@@ -97,10 +97,10 @@ void replaceEnv(std::map<std::string, std::string> newEnv)
 }
 
 
-Path absPath(Path path, Path dir)
+Path absPath(Path path, std::optional<Path> dir)
 {
     if (path[0] != '/') {
-        if (dir == "") {
+        if (!dir) {
 #ifdef __GNU__
             /* GNU (aka. GNU/Hurd) doesn't have any limitation on path
                lengths and doesn't define `PATH_MAX'.  */
@@ -116,7 +116,7 @@ Path absPath(Path path, Path dir)
             free(buf);
 #endif
         }
-        path = dir + "/" + path;
+        path = *dir + "/" + path;
     }
     return canonPath(path);
 }
@@ -1205,28 +1205,6 @@ template StringSet tokenizeString(std::string_view s, const string & separators)
 template vector<string> tokenizeString(std::string_view s, const string & separators);
 
 
-string concatStringsSep(const string & sep, const Strings & ss)
-{
-    string s;
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-
-string concatStringsSep(const string & sep, const StringSet & ss)
-{
-    string s;
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-
 string chomp(const string & s)
 {
     size_t i = s.find_last_not_of(" \n\r\t");