downloader: when retry but can't resume, retry the whole file

(cherry picked from commit 2c692a3b14)

.dir-locals.el

@@ -13,4 +13,5 @@
   (eval . (c-set-offset 'arglist-cont-nonempty '+))
   (eval . (c-set-offset 'substatement-open 0))
   (eval . (c-set-offset 'access-label '-))
+  (eval . (c-set-offset 'inlambda 0))
   )))

.github/workflows/test.yml

@@ -0,0 +1,14 @@
+name: "Test"
+on:
+  pull_request:
+  push:
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v8
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

.gitignore

@@ -47,7 +47,7 @@ perl/Makefile.config
 /src/libexpr/nix.tbl
 
 # /src/libstore/
-/src/libstore/*.gen.hh
+*.gen.*
 
 /src/nix/nix
 

Makefile.config.in

@@ -37,6 +37,7 @@ prefix = @prefix@
 sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
+system = @system@
 doc_generate = @doc_generate@
 xmllint = @xmllint@
 xsltproc = @xsltproc@

README.md

@@ -9,16 +9,13 @@ appear with Nix.
 
 To find out more about the tool, usage and installation instructions, please
 read the manual, which is available on the Nix website at
-<http://nixos.org/nix/manual>.
+<https://nixos.org/nix/manual>.
 
 ## Contributing
 
-Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking)
+Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking)
 of the manual. It helps you to get started with building Nix from source.
 
 ## License
 
 Nix is released under the LGPL v2.1
-
-This product includes software developed by the OpenSSL Project for
-use in the [OpenSSL Toolkit](http://www.OpenSSL.org/).

corepkgs/local.mk

@@ -1,4 +1,7 @@
-corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix
+corepkgs_FILES = \
+  unpack-channel.nix \
+  derivation.nix \
+  fetchurl.nix
 
 $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs)))
 

doc/manual/command-ref/nix-env.xml

@@ -1066,7 +1066,8 @@ user environment elements, etc. -->
     the derivation, which can be used to unambiguously select it using
     the <link linkend="opt-attr"><option>--attr</option> option</link>
     available in commands that install derivations like
-    <literal>nix-env --install</literal>.</para></listitem>
+    <literal>nix-env --install</literal>. This option only works
+    together with <option>--available</option></para></listitem>
 
   </varlistentry>
 

doc/manual/command-ref/nix-store.xml

@@ -360,7 +360,6 @@ EOF
     <arg choice='plain'><option>--print-roots</option></arg>
     <arg choice='plain'><option>--print-live</option></arg>
     <arg choice='plain'><option>--print-dead</option></arg>
-    <arg choice='plain'><option>--delete</option></arg>
   </group>
   <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg>
 </cmdsynopsis>
@@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of
 
   </varlistentry>
 
-  <varlistentry><term><option>--delete</option></term>
-
-    <listitem><para>This operation performs an actual garbage
-    collection.  All dead paths are removed from the
-    store.  This is the default.</para></listitem>
-
-  </varlistentry>
-
 </variablelist>
 
 <para>By default, all unreachable paths are deleted.  The following
@@ -444,10 +435,10 @@ and <link
 linkend="conf-keep-derivations"><literal>keep-derivations</literal></link>
 variables in the Nix configuration file.</para>
 
-<para>With <option>--delete</option>, the collector prints the total
-number of freed bytes when it finishes (or when it is interrupted).
-With <option>--print-dead</option>, it prints the number of bytes that
-would be freed.</para>
+<para>By default, the collector prints the total number of freed bytes
+when it finishes (or when it is interrupted). With
+<option>--print-dead</option>, it prints the number of bytes that would
+be freed.</para>
 
 </refsection>
 
@@ -1148,7 +1139,7 @@ the information that Nix considers important.  For instance,
 timestamps are elided because all files in the Nix store have their
 timestamp set to 0 anyway.  Likewise, all permissions are left out
 except for the execute bit, because all files in the Nix store have
-644 or 755 permission.</para>
+444 or 555 permission.</para>
 
 <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning
 that “equal” paths always produce the same NAR archive.  For instance,

doc/manual/installation/installing-binary.xml

@@ -142,7 +142,7 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
   <para>
     NixOS.org hosts version-specific installation URLs for all Nix
     versions since 1.11.16, at
-    <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>.
+    <literal>https://releases.nixos.org/nix/nix-<replaceable>version</replaceable>/install</literal>.
   </para>
 
   <para>

doc/manual/installation/prerequisites-source.xml

@@ -8,6 +8,14 @@
 
 <itemizedlist>
 
+  <listitem><para>GNU Autoconf
+  (<link xlink:href="https://www.gnu.org/software/autoconf/"/>)
+  and the autoconf-archive macro collection
+  (<link xlink:href="https://www.gnu.org/software/autoconf-archive/"/>).
+  These are only needed to run the bootstrap script, and are not necessary
+  if your source distribution came with a pre-built
+  <literal>./configure</literal> script.</para></listitem>
+
   <listitem><para>GNU Make.</para></listitem>
   
   <listitem><para>Bash Shell. The <literal>./configure</literal> script

doc/manual/local.mk

@@ -4,11 +4,10 @@ ifeq ($(doc_generate),yes)
 XSLTPROC = $(xsltproc) --nonet $(xmlflags) \
   --param section.autolabel 1 \
   --param section.label.includes.component.label 1 \
-  --param html.stylesheet \'style.css\' \
   --param xref.with.number.and.title 1 \
   --param toc.section.depth 3 \
   --param admon.style \'\' \
-  --param callout.graphics.extension \'.gif\' \
+  --param callout.graphics 0 \
   --param contrib.inline.enabled 0 \
   --stringparam generate.toc "book toc" \
   --param keep.relative.image.uris 0
@@ -66,12 +65,10 @@ $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid
 	  $(docbookxsl)/profiling/profile.xsl $< | \
 	  $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl -
 
-$(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
+$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual)))
 
 $(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures)))
 
-$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts)))
-
 $(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html))
 
 

doc/manual/release-notes/rl-2.0.xml

@@ -503,14 +503,14 @@
   </listitem>
 
   <listitem>
-    <para><emphasis>Pure evaluation mode</emphasis>. This is a variant
-    of the existing restricted evaluation mode. In pure mode, the Nix
-    evaluator forbids access to anything that could cause different
-    evaluations of the same command line arguments to produce a
+    <para><emphasis>Pure evaluation mode</emphasis>. With the
+    <literal>--pure-eval</literal> flag, Nix enables a variant of the existing
+    restricted evaluation mode that forbids access to anything that could cause
+    different evaluations of the same command line arguments to produce a
     different result. This includes builtin functions such as
     <function>builtins.getEnv</function>, but more importantly,
-    <emphasis>all</emphasis> filesystem or network access unless a
-    content hash or commit hash is specified. For example, calls to
+    <emphasis>all</emphasis> filesystem or network access unless a content hash
+    or commit hash is specified. For example, calls to
     <function>builtins.fetchGit</function> are only allowed if a
     <varname>rev</varname> attribute is specified.</para>
 

doc/manual/style.css

@@ -1,263 +0,0 @@
-/* Copied from http://bakefile.sourceforge.net/, which appears
-   licensed under the GNU GPL. */
-
-
-/***************************************************************************
-                             Basic headers and text:
- ***************************************************************************/
-
-body
-{
-    font-family: "Nimbus Sans L", sans-serif;
-    background: white;
-    margin: 2em 1em 2em 1em;
-}
-
-h1, h2, h3, h4
-{
-    color: #005aa0;
-}
-
-h1 /* title */
-{
-    font-size: 200%;
-}
-
-div.part h1
-{
-    font-size: 240%;
-}
-
-h2 /* chapters, appendices, subtitle */
-{
-    font-size: 180%;
-}
-
-div.part
-{
-    margin-top: 4em;
-}
-
-/* Extra space between chapters, appendices. */
-div.chapter > div.titlepage h2, div.appendix > div.titlepage h2 
-{ 
-    margin-top: 1.5em;
-}
-
-div.section > div.titlepage h2 /* sections */
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-h3 /* subsections */
-{
-    font-size: 125%;
-}
-
-div.simplesect h2
-{
-    font-size: 110%;
-}
-
-div.appendix h3
-{
-    font-size: 150%;
-    margin-top: 1.5em;
-}
-
-div.refentry\.separator
-{
-    margin-top: 2.5em;
-    margin-bottom: 2em;
-}
-
-div.refnamediv h2, div.refsynopsisdiv h2, div.refsection h2 /* refentry parts */
-{
-    margin-top: 1.4em;
-    font-size: 125%;
-}
-
-div.refsection h3
-{
-    font-size: 110%;
-}
-
-
-/***************************************************************************
-                               Examples:
- ***************************************************************************/
-
-div.example
-{
-    border: 1px solid #b0b0b0;
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    background: #f4f4f8;
-    border-radius: 0.4em;
-}
-
-div.example p.title
-{
-    margin-top: 0em;
-}
-
-div.example pre
-{
-}
-
-
-/***************************************************************************
-                            Screen dumps:
- ***************************************************************************/
-
-pre.screen, pre.programlisting
-{
-    padding: 6px 6px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    color: #600000;
-    background: #f4f4f8;
-    font-family: monospace;
-}
-
-div.example pre.programlisting
-{
-    border: 0px;
-    padding: 0 0;
-    margin: 0 0 0 0;
-}
-
-
-/***************************************************************************
-                               Notes, warnings etc:
- ***************************************************************************/
-
-.note, .warning
-{
-    border: 1px solid #b0b0b0;
-    padding: 3px 3px;
-    margin-left: 1.5em;
-    margin-right: 1.5em;
-    margin-bottom: 1em;
-    padding: 0.3em 0.3em 0.3em 0.3em;
-    background: #fffff5;
-    border-radius: 0.4em;
-}
-
-div.note, div.warning
-{
-    font-style: italic;
-}
-
-div.note h3, div.warning h3
-{
-    color: red;
-    font-size: 100%;
-    padding-right: 0.5em;
-    display: inline;
-}
-
-div.note p, div.warning p
-{
-    margin-bottom: 0em;
-}
-
-div.note h3 + p, div.warning h3 + p
-{
-    display: inline;
-}
-
-div.note h3
-{
-    color: blue;
-    font-size: 100%;
-}
-
-div.navfooter *
-{
-    font-size: 90%;
-}
-
-
-/***************************************************************************
-                        Links colors and highlighting: 
- ***************************************************************************/
-
-a { text-decoration: none; }
-a:hover { text-decoration: underline; }
-a:link { color: #0048b3; }
-a:visited { color: #002a6a; }
-
-
-/***************************************************************************
-                              Table of contents:
- ***************************************************************************/
-
-div.toc
-{
-    font-size: 90%;
-}
-
-div.toc dl
-{
-    margin-top: 0em;
-    margin-bottom: 0em;
-}
-
-
-/***************************************************************************
-                               Special elements:
- ***************************************************************************/
-
-tt, code
-{
-    color: #400000;
-}
-
-.term
-{
-    font-weight: bold;
-    
-}
-
-div.variablelist dd p, div.glosslist dd p
-{
-    margin-top: 0em;
-}
-
-div.variablelist dd, div.glosslist dd
-{
-    margin-left: 1.5em;
-}
-
-div.glosslist dt
-{
-    font-style: italic;
-}
-
-.varname
-{
-    color: #400000;
-}
-
-span.command strong
-{
-    font-weight: normal;
-    color: #400000;
-}
-
-div.calloutlist table
-{
-}
-
-table
-{
-    border-collapse: collapse;
-}
-
-div.affiliation
-{
-    font-style: italic;
-}

local.mk

@@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure
 
 clean-files += Makefile.config
 
-GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations
+GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
 
 $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
 $(GCH) $(PCH): src/libutil/util.hh config.h
+
+GCH_CXXFLAGS = -I src/libutil

mk/README.md

@@ -1,6 +0,0 @@
-This is a set of helper Makefiles for doing non-recursive builds with
-GNU Make.  The canonical source can be found at
-https://github.com/edolstra/make-rules.  You should copy the files
-into the `mk` subdirectory of your project.
-
-TODO: write more documentation.

mk/precompiled-headers.mk

@@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch
 $(GCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 PCH = $(buildprefix)precompiled-headers.h.pch
 
 $(PCH): precompiled-headers.h
 	@rm -f $@
 	@mkdir -p "$(dir $@)"
-	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS)
+	$(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS)
 
 clean-files += $(GCH) $(PCH)
 

perl/lib/Nix/Store.xs

@@ -102,7 +102,7 @@ SV * queryPathInfo(char * path, int base32)
     PPCODE:
         try {
             auto info = store()->queryPathInfo(store()->parseStorePath(path));
-            if (info->deriver)
+            if (!info->deriver)
                 XPUSHs(&PL_sv_undef);
             else
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));

release-common.nix

@@ -34,7 +34,7 @@ rec {
       "--with-sandbox-shell=${sh}/bin/busybox"
     ];
 
-  tarballDeps =
+  buildDeps =
     [ bison
       flex
       libxml2
@@ -43,12 +43,10 @@ rec {
       docbook_xsl_ns
       autoconf-archive
       autoreconfHook
-    ];
 
-  buildDeps =
-    [ curl
+      curl
       bzip2 xz brotli zlib editline
-      openssl pkgconfig sqlite boehmgc
+      openssl pkgconfig sqlite
       libarchive
       boost
       nlohmann_json
@@ -73,6 +71,10 @@ rec {
         */
       }));
 
+  propagatedDeps =
+    [ (boehmgc.override { enableLargeConfig = true; })
+    ];
+
   perlDeps =
     [ perl
       perlPackages.DBDSQLite

release.nix

@@ -1,5 +1,5 @@
 { nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz
+, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
 , officialRelease ? false
 , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
 }:
@@ -8,11 +8,14 @@ let
 
   pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
 
+  version =
+    builtins.readFile ./.version
+    + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
+
   jobs = rec {
 
     # Create a "vendor" directory that contains the crates listed in
-    # Cargo.lock, and include it in the Nix tarball. This allows Nix
-    # to be built without network access.
+    # Cargo.lock. This allows Nix to be built without network access.
     vendoredCrates =
       let
         lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock);
@@ -55,48 +58,6 @@ let
         '';
 
 
-    tarball =
-      with pkgs;
-
-      with import ./release-common.nix { inherit pkgs; };
-
-      releaseTools.sourceTarball {
-        name = "nix-tarball";
-        version = builtins.readFile ./.version;
-        versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}";
-        src = nix;
-        inherit officialRelease;
-
-        buildInputs = tarballDeps ++ buildDeps;
-
-        postUnpack = ''
-          (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files
-          cat $sourceRoot/.dist-files
-        '';
-
-        preConfigure = ''
-          (cd perl ; autoreconf --install --force --verbose)
-          # TeX needs a writable font cache.
-          export VARTEXFONTS=$TMPDIR/texfonts
-        '';
-
-        distPhase =
-          ''
-            cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/
-
-            runHook preDist
-            make dist
-            mkdir -p $out/tarballs
-            cp *.tar.* $out/tarballs
-          '';
-
-        preDist = ''
-          make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk
-          echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products
-        '';
-      };
-
-
     build = pkgs.lib.genAttrs systems (system:
 
       let pkgs = import nixpkgs { inherit system; }; in
@@ -105,16 +66,21 @@ let
 
       with import ./release-common.nix { inherit pkgs; };
 
-      releaseTools.nixBuild {
-        name = "nix";
-        src = tarball;
+      stdenv.mkDerivation {
+        name = "nix-${version}";
+
+        src = nix;
+
+        outputs = [ "out" "dev" "doc" ];
 
         buildInputs = buildDeps;
 
+        propagatedBuildInputs = propagatedDeps;
+
         preConfigure =
-          # Copy libboost_context so we don't get all of Boost in our closure.
-          # https://github.com/NixOS/nixpkgs/issues/45462
           ''
+            # Copy libboost_context so we don't get all of Boost in our closure.
+            # https://github.com/NixOS/nixpkgs/issues/45462
             mkdir -p $out/lib
             cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
             rm -f $out/lib/*.a
@@ -122,6 +88,10 @@ let
               chmod u+w $out/lib/*.so.*
               patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
             ''}
+
+            ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor
+
+            (cd perl; autoreconf --install --force --verbose)
           '';
 
         configureFlags = configureFlags ++
@@ -133,8 +103,17 @@ let
 
         installFlags = "sysconfdir=$(out)/etc";
 
+        doCheck = true;
+
         doInstallCheck = true;
         installCheckFlags = "sysconfdir=$(out)/etc";
+
+        separateDebugInfo = true;
+
+        preDist = ''
+          mkdir -p $doc/nix-support
+          echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+        '';
       });
 
 
@@ -143,11 +122,21 @@ let
       let pkgs = import nixpkgs { inherit system; }; in with pkgs;
 
       releaseTools.nixBuild {
-        name = "nix-perl";
-        src = tarball;
+        name = "nix-perl-${version}";
+
+        src = nix;
 
         buildInputs =
-          [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ]
+          [ autoconf-archive
+            autoreconfHook
+            jobs.build.${system}
+            curl
+            bzip2
+            xz
+            pkgconfig
+            pkgs.perl
+            boost
+          ]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
 
         configureFlags = ''
@@ -167,7 +156,6 @@ let
 
       let
         toplevel = builtins.getAttr system jobs.build;
-        version = toplevel.src.version;
         installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
       in
 
@@ -237,12 +225,13 @@ let
       with import ./release-common.nix { inherit pkgs; };
 
       releaseTools.coverageAnalysis {
-        name = "nix-build";
-        src = tarball;
+        name = "nix-coverage-${version}";
+
+        src = nix;
 
         enableParallelBuilding = true;
 
-        buildInputs = buildDeps;
+        buildInputs = buildDeps ++ propagatedDeps;
 
         dontInstall = false;
 
@@ -334,45 +323,20 @@ let
 
     installerScript =
       pkgs.runCommand "installer-script"
-        { buildInputs = [ build.x86_64-linux ];
-        }
+        { buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
         ''
           mkdir -p $out/nix-support
 
           substitute ${./scripts/install.in} $out/install \
             ${pkgs.lib.concatMapStrings
               (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
-              [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+              systems
             } \
-            --replace '@nixVersion@' ${build.x86_64-linux.src.version}
+            --replace '@nixVersion@' ${version}
 
           echo "file installer $out/install" >> $out/nix-support/hydra-build-products
         '';
 
-
-    # Aggregate job containing the release-critical jobs.
-    release = pkgs.releaseTools.aggregate {
-      name = "nix-${tarball.version}";
-      meta.description = "Release-critical builds";
-      constituents =
-        [ tarball
-          build.i686-linux
-          build.x86_64-darwin
-          build.x86_64-linux
-          build.aarch64-linux
-          binaryTarball.i686-linux
-          binaryTarball.x86_64-darwin
-          binaryTarball.x86_64-linux
-          binaryTarball.aarch64-linux
-          tests.remoteBuilds
-          tests.nix-copy-closure
-          tests.binaryTarball
-          #tests.evalNixpkgs
-          #tests.evalNixOS
-          installerScript
-        ];
-    };
-
   };
 
 

scripts/install-multi-user.sh

@@ -13,12 +13,12 @@ set -o pipefail
 # however tracking which bits came from which would be impossible.
 
 readonly ESC='\033[0m'
-readonly BOLD='\033[38;1m'
-readonly BLUE='\033[38;34m'
-readonly BLUE_UL='\033[38;4;34m'
-readonly GREEN='\033[38;32m'
-readonly GREEN_UL='\033[38;4;32m'
-readonly RED='\033[38;31m'
+readonly BOLD='\033[1m'
+readonly BLUE='\033[34m'
+readonly BLUE_UL='\033[4;34m'
+readonly GREEN='\033[32m'
+readonly GREEN_UL='\033[4;32m'
+readonly RED='\033[31m'
 
 readonly NIX_USER_COUNT="32"
 readonly NIX_BUILD_GROUP_ID="30000"
@@ -567,7 +567,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt ./store/* "$NIX_ROOT/store/"
+              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

scripts/install-nix-from-closure.sh

@@ -87,7 +87,7 @@ if ! [ -e $dest ]; then
 fi
 
 if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

scripts/install-systemd-multi-user.sh

@@ -88,7 +88,7 @@ poly_configure_nix_daemon_service() {
           systemctl start nix-daemon.socket
 
     _sudo "to start the nix-daemon.service" \
-          systemctl start nix-daemon.service
+          systemctl restart nix-daemon.service
 
 }
 

scripts/install.in

@@ -30,12 +30,13 @@ case "$(uname -s).$(uname -m)" in
     *) oops "sorry, there is no binary distribution of Nix for your platform";;
 esac
 
-url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
+url="https://releases.nixos.org/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz"
 
 tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")"
 
 require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
+require_util xz "unpack the binary tarball"
 
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
 curl -L "$url" -o "$tarball" || oops "failed to download '$url'"

shell.nix

@@ -1,13 +1,13 @@
 { useClang ? false }:
 
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-19.09.tar.gz) {};
+with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
 
 with import ./release-common.nix { inherit pkgs; };
 
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
 
   inherit configureFlags;
 

src/build-remote/build-remote.cc

@@ -17,7 +17,7 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using std::cin;

src/libexpr/attr-path.cc

@@ -32,7 +32,7 @@ static Strings parseAttrPath(const string & s)
 }
 
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
@@ -41,6 +41,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
         Error(format("attribute selection path '%1%' does not match expression") % attrPath);
 
     Value * v = &vIn;
+    Pos pos = noPos;
 
     for (auto & attr : tokens) {
 
@@ -70,8 +71,9 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
             if (a == v->attrs->end())
-                throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath);
+                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
             v = &*a->value;
+            pos = *a->pos;
         }
 
         else if (apType == apIndex) {
@@ -82,14 +84,15 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath,
                     % attrPath % showType(*v));
 
             if (attrIndex >= v->listSize())
-                throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath);
+                throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
 
             v = v->listElems()[attrIndex];
+            pos = noPos;
         }
 
     }
 
-    return v;
+    return {v, pos};
 }
 
 
@@ -98,9 +101,9 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
     Value * v2;
     try {
         auto dummyArgs = state.allocBindings(0);
-        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v);
+        v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first;
     } catch (Error &) {
-        throw Error("package '%s' has no source location information", what);
+        throw NoPositionInfo("package '%s' has no source location information", what);
     }
 
     // FIXME: is it possible to extract the Pos object instead of doing this

src/libexpr/attr-path.hh

@@ -7,7 +7,10 @@
 
 namespace nix {
 
-Value * findAlongAttrPath(EvalState & state, const string & attrPath,
+MakeError(AttrPathNotFound, Error);
+MakeError(NoPositionInfo, Error);
+
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */

src/libexpr/attr-set.cc

@@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 }
 
 
+Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+{
+    return allocAttr(vAttrs, symbols.create(name));
+}
+
+
 void Bindings::sort()
 {
     std::sort(begin(), end());

src/libexpr/eval.hh

@@ -7,6 +7,7 @@
 #include "hash.hh"
 #include "config.hh"
 
+#include <regex>
 #include <map>
 #include <optional>
 #include <unordered_map>
@@ -114,6 +115,9 @@ private:
     /* Cache used by checkSourcePath(). */
     std::unordered_map<Path, Path> resolvedPaths;
 
+    /* Cache used by prim_match(). */
+    std::unordered_map<std::string, std::regex> regexCache;
+
 public:
 
     EvalState(const Strings & _searchPath, ref<Store> store);
@@ -141,8 +145,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(const string & s, const Path & basePath);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string_view s, const Path & basePath);
 
     Expr * parseStdin();
 
@@ -268,6 +272,7 @@ public:
     Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
+    Value * allocAttr(Value & vAttrs, const std::string & name);
 
     Bindings * allocBindings(size_t capacity);
 
@@ -314,6 +319,7 @@ private:
     friend struct ExprOpConcatLists;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
 };
 
 
@@ -362,7 +368,7 @@ struct EvalSettings : Config
         "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."};
 
     Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"};
+        "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/function-trace.cc

@@ -1,4 +1,5 @@
 #include "function-trace.hh"
+#include "logging.hh"
 
 namespace nix {
 

src/libexpr/local.mk

@@ -6,6 +6,8 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
 
+libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr
+
 libexpr_LIBS = libutil libstore libnixrust
 
 libexpr_LDFLAGS =
@@ -31,3 +33,5 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh

src/libexpr/parser.y

@@ -611,13 +611,13 @@ Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.c_str(), "(string)", basePath, staticEnv);
+    return parse(s.data(), "(string)", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 {
     return parseExprFromString(s, basePath, staticBaseEnv);
 }

src/libexpr/primops.cc

@@ -121,10 +121,17 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
             mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
         }
         w.attrs->sort();
-        Value fun;
-        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
-        state.forceFunction(fun, pos);
-        mkApp(v, fun, w);
+
+        static Value * fun = nullptr;
+        if (!fun) {
+            fun = state.allocValue();
+            state.eval(state.parseExprFromString(
+                #include "imported-drv-to-derivation.nix.gen.hh"
+                , "/"), *fun);
+        }
+
+        state.forceFunction(*fun, pos);
+        mkApp(v, *fun, w);
         state.forceAttrs(v, pos);
     } else {
         state.forceAttrs(*args[0]);
@@ -1811,19 +1818,21 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
 
 /* Match a regular expression against a string and return either
    ‘null’ or a list containing substring matches. */
-static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
+void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     auto re = state.forceStringNoCtx(*args[0], pos);
 
     try {
 
-        std::regex regex(re, std::regex::extended);
+        auto regex = state.regexCache.find(re);
+        if (regex == state.regexCache.end())
+            regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first;
 
         PathSet context;
         const std::string str = state.forceString(*args[1], context, pos);
 
         std::smatch match;
-        if (!std::regex_match(str, match, regex)) {
+        if (!std::regex_match(str, match, regex->second)) {
             mkNull(v);
             return;
         }

src/libexpr/primops/fromTOML.cc

@@ -1,7 +1,7 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "cpptoml/cpptoml.h"
+#include "../../cpptoml/cpptoml.h"
 
 namespace nix {
 

src/libmain/local.mk

@@ -6,6 +6,8 @@ libmain_DIR := $(d)
 
 libmain_SOURCES := $(wildcard $(d)/*.cc)
 
+libmain_CXXFLAGS += -I src/libutil -I src/libstore
+
 libmain_LDFLAGS = $(OPENSSL_LIBS)
 
 libmain_LIBS = libstore libutil

src/libstore/binary-cache-store.cc

@@ -106,7 +106,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
+        state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
     }
 
     if (diskCache)

src/libstore/build.cc

@@ -6,6 +6,7 @@
 #include "archive.hh"
 #include "affinity.hh"
 #include "builtins.hh"
+#include "builtins/buildenv.hh"
 #include "download.hh"
 #include "finally.hh"
 #include "compression.hh"
@@ -1397,7 +1398,7 @@ void DerivationGoal::tryToBuild()
        few seconds and then retry this goal. */
     PathSet lockFiles;
     for (auto & outPath : drv->outputPaths())
-        lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath)));
+        lockFiles.insert(worker.store.Store::toRealPath(outPath));
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         worker.waitForAWhile(shared_from_this());
@@ -1428,7 +1429,7 @@ void DerivationGoal::tryToBuild()
     for (auto & i : drv->outputs) {
         if (worker.store.isValidPath(i.second.path)) continue;
         debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
-        deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path)));
+        deletePath(worker.store.Store::toRealPath(i.second.path));
     }
 
     /* Don't do a remote build if the derivation has the attribute
@@ -1685,7 +1686,7 @@ void DerivationGoal::buildDone()
 
         /* Delete unused redirected outputs (when doing hash rewriting). */
         for (auto & i : redirectedOutputs)
-            deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second)));
+            deletePath(worker.store.Store::toRealPath(i.second));
 
         /* Delete the chroot (if we were using one). */
         autoDelChroot.reset(); /* this runs the destructor */
@@ -1904,7 +1905,7 @@ void DerivationGoal::startBuilder()
             concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
             worker.store.printStorePath(drvPath),
             settings.thisSystem,
-            concatStringsSep(", ", settings.systemFeatures));
+            concatStringsSep<StringSet>(", ", settings.systemFeatures));
 
     if (drv->isBuiltin())
         preloadNSS();
@@ -2071,7 +2072,7 @@ void DerivationGoal::startBuilder()
            environment using bind-mounts.  We put it in the Nix store
            to ensure that we can create hard-links to non-directory
            inputs in the fake Nix store in the chroot (see below). */
-        chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot";
+        chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
         deletePath(chrootRootDir);
 
         /* Clean up the chroot directory automatically. */
@@ -2550,7 +2551,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
 
 void DerivationGoal::writeStructuredAttrs()
 {
-    auto & structuredAttrs = parsedDrv->getStructuredAttrs();
+    auto structuredAttrs = parsedDrv->getStructuredAttrs();
     if (!structuredAttrs) return;
 
     auto json = *structuredAttrs;
@@ -2916,7 +2917,7 @@ void DerivationGoal::addDependency(const StorePath & path)
 
         #if __linux__
 
-            Path source = worker.store.toRealPath(worker.store.printStorePath(path));
+            Path source = worker.store.Store::toRealPath(path);
             Path target = chrootRootDir + worker.store.printStorePath(path);
             debug("bind-mounting %s -> %s", target, source);
 
@@ -3578,7 +3579,7 @@ void DerivationGoal::registerOutputs()
         if (needsHashRewrite()) {
             auto r = redirectedOutputs.find(i.second.path);
             if (r != redirectedOutputs.end()) {
-                auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second));
+                auto redirected = worker.store.Store::toRealPath(r->second);
                 if (buildMode == bmRepair
                     && redirectedBadOutputs.count(i.second.path)
                     && pathExists(redirected))
@@ -3671,7 +3672,7 @@ void DerivationGoal::registerOutputs()
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
                         worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI)));
 
-                Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest));
+                Path actualDest = worker.store.Store::toRealPath(dest);
 
                 if (worker.store.isValidPath(dest))
                     std::rethrow_exception(delayedException);

src/libstore/builtins.hh

@@ -6,7 +6,6 @@ namespace nix {
 
 // TODO: make pluggable.
 void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData);
-void builtinBuildenv(const BasicDerivation & drv);
 void builtinUnpackChannel(const BasicDerivation & drv);
 
 }

src/libstore/builtins/buildenv.cc

@@ -1,4 +1,4 @@
-#include "builtins.hh"
+#include "buildenv.hh"
 
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -7,16 +7,14 @@
 
 namespace nix {
 
-typedef std::map<Path,int> Priorities;
-
-// FIXME: change into local variables.
-
-static Priorities priorities;
-
-static unsigned long symlinks;
+struct State
+{
+    std::map<Path, int> priorities;
+    unsigned long symlinks = 0;
+};
 
 /* For each activated package, create symlinks */
-static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
+static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority)
 {
     DirEntries srcFiles;
 
@@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISDIR(dstSt.st_mode)) {
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 } else if (S_ISLNK(dstSt.st_mode)) {
                     auto target = canonPath(dstFile, true);
@@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
                         throw SysError(format("unlinking '%1%'") % dstFile);
                     if (mkdir(dstFile.c_str(), 0755) == -1)
                         throw SysError(format("creating directory '%1%'"));
-                    createLinks(target, dstFile, priorities[dstFile]);
-                    createLinks(srcFile, dstFile, priority);
+                    createLinks(state, target, dstFile, state.priorities[dstFile]);
+                    createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
             } else if (errno != ENOENT)
@@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
             auto res = lstat(dstFile.c_str(), &dstSt);
             if (res == 0) {
                 if (S_ISLNK(dstSt.st_mode)) {
-                    auto prevPriority = priorities[dstFile];
+                    auto prevPriority = state.priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(
                                 "packages '%1%' and '%2%' have the same priority %3%; "
@@ -109,67 +107,30 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority)
         }
 
         createSymlink(srcFile, dstFile);
-        priorities[dstFile] = priority;
-        symlinks++;
+        state.priorities[dstFile] = priority;
+        state.symlinks++;
     }
 }
 
-typedef std::set<Path> FileProp;
-
-static FileProp done;
-static FileProp postponed = FileProp{};
-
-static Path out;
-
-static void addPkg(const Path & pkgDir, int priority)
+void buildProfile(const Path & out, Packages && pkgs)
 {
-    if (!done.insert(pkgDir).second) return;
-    createLinks(pkgDir, out, priority);
+    State state;
 
-    try {
-        for (const auto & p : tokenizeString<std::vector<string>>(
-                readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
-            if (!done.count(p))
-                postponed.insert(p);
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
-    }
-}
+    std::set<Path> done, postponed;
 
-struct Package {
-    Path path;
-    bool active;
-    int priority;
-    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
-};
+    auto addPkg = [&](const Path & pkgDir, int priority) {
+        if (!done.insert(pkgDir).second) return;
+        createLinks(state, pkgDir, out, priority);
 
-typedef std::vector<Package> Packages;
-
-void builtinBuildenv(const BasicDerivation & drv)
-{
-    auto getAttr = [&](const string & name) {
-        auto i = drv.env.find(name);
-        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
-        return i->second;
-    };
-
-    out = getAttr("out");
-    createDirs(out);
-
-    /* Convert the stuff we get from the environment back into a
-     * coherent data type. */
-    Packages pkgs;
-    auto derivations = tokenizeString<Strings>(getAttr("derivations"));
-    while (!derivations.empty()) {
-        /* !!! We're trusting the caller to structure derivations env var correctly */
-        auto active = derivations.front(); derivations.pop_front();
-        auto priority = stoi(derivations.front()); derivations.pop_front();
-        auto outputs = stoi(derivations.front()); derivations.pop_front();
-        for (auto n = 0; n < outputs; n++) {
-            auto path = derivations.front(); derivations.pop_front();
-            pkgs.emplace_back(path, active != "false", priority);
+        try {
+            for (const auto & p : tokenizeString<std::vector<string>>(
+                    readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n"))
+                if (!done.count(p))
+                    postponed.insert(p);
+        } catch (SysError & e) {
+            if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw;
         }
-    }
+    };
 
     /* Symlink to the packages that have been installed explicitly by the
      * user. Process in priority order to reduce unnecessary
@@ -189,13 +150,42 @@ void builtinBuildenv(const BasicDerivation & drv)
      */
     auto priorityCounter = 1000;
     while (!postponed.empty()) {
-        auto pkgDirs = postponed;
-        postponed = FileProp{};
+        std::set<Path> pkgDirs;
+        postponed.swap(pkgDirs);
         for (const auto & pkgDir : pkgDirs)
             addPkg(pkgDir, priorityCounter++);
     }
 
-    printError("created %d symlinks in user environment", symlinks);
+    debug("created %d symlinks in user environment", state.symlinks);
+}
+
+void builtinBuildenv(const BasicDerivation & drv)
+{
+    auto getAttr = [&](const string & name) {
+        auto i = drv.env.find(name);
+        if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
+        return i->second;
+    };
+
+    Path out = getAttr("out");
+    createDirs(out);
+
+    /* Convert the stuff we get from the environment back into a
+     * coherent data type. */
+    Packages pkgs;
+    auto derivations = tokenizeString<Strings>(getAttr("derivations"));
+    while (!derivations.empty()) {
+        /* !!! We're trusting the caller to structure derivations env var correctly */
+        auto active = derivations.front(); derivations.pop_front();
+        auto priority = stoi(derivations.front()); derivations.pop_front();
+        auto outputs = stoi(derivations.front()); derivations.pop_front();
+        for (auto n = 0; n < outputs; n++) {
+            auto path = derivations.front(); derivations.pop_front();
+            pkgs.emplace_back(path, active != "false", priority);
+        }
+    }
+
+    buildProfile(out, std::move(pkgs));
 
     createSymlink(getAttr("manifest"), out + "/manifest.nix");
 }

src/libstore/builtins/buildenv.hh

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "derivations.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+struct Package {
+    Path path;
+    bool active;
+    int priority;
+    Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
+};
+
+typedef std::vector<Package> Packages;
+
+void buildProfile(const Path & out, Packages && pkgs);
+
+void builtinBuildenv(const BasicDerivation & drv);
+
+}

src/libstore/derivations.cc

@@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const
 
 
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair)
+    const Derivation & drv, std::string_view name, RepairFlag repair)
 {
     auto references = cloneStorePathSet(drv.inputSrcs);
     for (auto & i : drv.inputDrvs)
@@ -73,8 +73,8 @@ StorePath writeDerivation(ref<Store> store,
     /* Note that the outputs of a derivation are *not* references
        (that can be missing (of course) and should not necessarily be
        held during a garbage collection). */
-    string suffix = name + drvExtension;
-    string contents = drv.unparse(*store, false);
+    auto suffix = std::string(name) + drvExtension;
+    auto contents = drv.unparse(*store, false);
     return settings.readOnlyMode
         ? store->computeStorePathForText(suffix, contents, references)
         : store->addTextToStore(suffix, contents, references, repair);
@@ -213,15 +213,26 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
 }
 
 
-static void printString(string & res, const string & s)
+static void printString(string & res, std::string_view s)
+{
+    char buf[s.size() * 2 + 2];
+    char * p = buf;
+    *p++ = '"';
+    for (auto c : s)
+        if (c == '\"' || c == '\\') { *p++ = '\\'; *p++ = c; }
+        else if (c == '\n') { *p++ = '\\'; *p++ = 'n'; }
+        else if (c == '\r') { *p++ = '\\'; *p++ = 'r'; }
+        else if (c == '\t') { *p++ = '\\'; *p++ = 't'; }
+        else *p++ = c;
+    *p++ = '"';
+    res.append(buf, p - buf);
+}
+
+
+static void printUnquotedString(string & res, std::string_view s)
 {
     res += '"';
-    for (const char * i = s.c_str(); *i; i++)
-        if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; }
-        else if (*i == '\n') res += "\\n";
-        else if (*i == '\r') res += "\\r";
-        else if (*i == '\t') res += "\\t";
-        else res += *i;
+    res.append(s);
     res += '"';
 }
 
@@ -239,6 +250,19 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j)
 }
 
 
+template<class ForwardIterator>
+static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterator j)
+{
+    res += '[';
+    bool first = true;
+    for ( ; i != j; ++i) {
+        if (first) first = false; else res += ',';
+        printUnquotedString(res, *i);
+    }
+    res += ']';
+}
+
+
 string Derivation::unparse(const Store & store, bool maskOutputs,
     std::map<std::string, StringSet> * actualInputs) const
 {
@@ -249,10 +273,10 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     bool first = true;
     for (auto & i : outputs) {
         if (first) first = false; else s += ',';
-        s += '('; printString(s, i.first);
-        s += ','; printString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
-        s += ','; printString(s, i.second.hashAlgo);
-        s += ','; printString(s, i.second.hash);
+        s += '('; printUnquotedString(s, i.first);
+        s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
+        s += ','; printUnquotedString(s, i.second.hashAlgo);
+        s += ','; printUnquotedString(s, i.second.hash);
         s += ')';
     }
 
@@ -261,24 +285,24 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
     if (actualInputs) {
         for (auto & i : *actualInputs) {
             if (first) first = false; else s += ',';
-            s += '('; printString(s, i.first);
-            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, i.first);
+            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
             s += ')';
         }
     } else {
         for (auto & i : inputDrvs) {
             if (first) first = false; else s += ',';
-            s += '('; printString(s, store.printStorePath(i.first));
-            s += ','; printStrings(s, i.second.begin(), i.second.end());
+            s += '('; printUnquotedString(s, store.printStorePath(i.first));
+            s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
             s += ')';
         }
     }
 
     s += "],";
     auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow
-    printStrings(s, paths.begin(), paths.end());
+    printUnquotedStrings(s, paths.begin(), paths.end());
 
-    s += ','; printString(s, platform);
+    s += ','; printUnquotedString(s, platform);
     s += ','; printString(s, builder);
     s += ','; printStrings(s, args.begin(), args.end());
 

src/libstore/derivations.hh

@@ -79,7 +79,7 @@ class Store;
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,
-    const Derivation & drv, const string & name, RepairFlag repair = NoRepair);
+    const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
 
 /* Read a derivation from a file. */
 Derivation readDerivation(const Store & store, const Path & drvPath);

src/libstore/download.cc

@@ -308,7 +308,7 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
             curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 
-            if (writtenToSink)
+            if (writtenToSink && acceptRanges && encoding.empty())
                 curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
 
             result.data = std::make_shared<std::string>();
@@ -414,16 +414,14 @@ struct CurlDownloader : public Downloader
 
                 /* If this is a transient error, then maybe retry the
                    download after a while. If we're writing to a
-                   sink, we can only retry if the server supports
-                   ranged requests. */
-                if (err == Transient
-                    && attempt < request.tries
-                    && (!this->request.dataCallback
-                        || writtenToSink == 0
-                        || (acceptRanges && encoding.empty())))
+                   sink, we can only resume if the server supports
+                   ranged requests, otherwise fallback to a plain
+                   retry.
+                */
+                if (err == Transient && attempt < request.tries)
                 {
                     int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
-                    if (writtenToSink)
+                    if (writtenToSink && acceptRanges && encoding.empty())
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
                         warn("%s; retrying in %d ms", exc.what(), ms);

src/libstore/gc.cc

@@ -255,12 +255,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
 void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
     auto foundRoot = [&](const Path & path, const Path & target) {
-        Path storePath = toStorePath(target);
-        // FIXME
-        if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
-            roots[parseStorePath(storePath)].emplace(path);
+        auto storePath = maybeParseStorePath(toStorePath(target));
+        if (storePath && isValidPath(*storePath))
+            roots[std::move(*storePath)].emplace(path);
         else
-            printInfo("skipping invalid root from '%1%' to '%2%'", path, storePath);
+            printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
     };
 
     try {
@@ -296,10 +295,9 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
         }
 
         else if (type == DT_REG) {
-            Path storePath = storeDir + "/" + std::string(baseNameOf(path));
-            // FIXME
-            if (isStorePath(storePath) && isValidPath(parseStorePath(storePath)))
-                roots[parseStorePath(storePath)].emplace(path);
+            auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path)));
+            if (storePath && isValidPath(*storePath))
+                roots[std::move(*storePath)].emplace(path);
         }
 
     }
@@ -523,14 +521,14 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
 
     unsigned long long size = 0;
 
-    // FIXME
-    if (isStorePath(path) && isValidPath(parseStorePath(path))) {
+    auto storePath = maybeParseStorePath(path);
+    if (storePath && isValidPath(*storePath)) {
         StorePathSet referrers;
-        queryReferrers(parseStorePath(path), referrers);
+        queryReferrers(*storePath, referrers);
         for (auto & i : referrers)
             if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
-        size = queryPathInfo(parseStorePath(path))->narSize;
-        invalidatePathChecked(parseStorePath(path));
+        size = queryPathInfo(*storePath)->narSize;
+        invalidatePathChecked(*storePath);
     }
 
     Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
@@ -593,8 +591,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto
 
     visited.insert(path.clone());
 
-    //FIXME
-    if (!isStorePath(printStorePath(path)) || !isValidPath(path)) return false;
+    if (!isValidPath(path)) return false;
 
     StorePathSet incoming;
 
@@ -637,8 +634,9 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 
     //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
 
-    // FIXME
-    if (!isStorePath(path) || !isValidPath(parseStorePath(path))) {
+    auto storePath = maybeParseStorePath(path);
+
+    if (!storePath || !isValidPath(*storePath)) {
         /* A lock file belonging to a path that we're building right
            now isn't garbage. */
         if (isActiveTempFile(state, path, ".lock")) return;
@@ -655,7 +653,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path)
 
     StorePathSet visited;
 
-    if (canReachRoot(state, visited, parseStorePath(path))) {
+    if (storePath && canReachRoot(state, visited, *storePath)) {
         debug("cannot delete '%s' because it's still reachable", path);
     } else {
         /* No path we visited was a root, so everything is garbage.
@@ -818,8 +816,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 string name = dirent->d_name;
                 if (name == "." || name == "..") continue;
                 Path path = storeDir + "/" + name;
-                // FIXME
-                if (isStorePath(path) && isValidPath(parseStorePath(path)))
+                auto storePath = maybeParseStorePath(path);
+                if (storePath && isValidPath(*storePath))
                     entries.push_back(path);
                 else
                     tryToDelete(state, path);

src/libstore/globals.cc

@@ -20,13 +20,6 @@ namespace nix {
    must be deleted and recreated on startup.) */
 #define DEFAULT_SOCKET_PATH "/daemon-socket/socket"
 
-/* chroot-like behavior from Apple's sandbox */
-#if __APPLE__
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh"
-#else
-    #define DEFAULT_ALLOWED_IMPURE_PREFIXES ""
-#endif
-
 Settings settings;
 
 static GlobalConfig::Register r1(&settings);
@@ -68,7 +61,12 @@ Settings::Settings()
     sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL);
 #endif
 
-    allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES);
+
+/* chroot-like behavior from Apple's sandbox */
+#if __APPLE__
+    sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib");
+    allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh");
+#endif
 }
 
 void loadConfFile()

src/libstore/globals.hh

@@ -311,12 +311,7 @@ public:
     Setting<bool> printMissing{this, true, "print-missing",
         "Whether to print what paths need to be built or downloaded."};
 
-    Setting<std::string> preBuildHook{this,
-#if __APPLE__
-        nixLibexecDir + "/nix/resolve-system-dependencies",
-#else
-        "",
-#endif
+    Setting<std::string> preBuildHook{this, "",
         "pre-build-hook",
         "A program to run just before a build to set derivation-specific build settings."};
 

src/libstore/http-binary-cache-store.cc

@@ -163,10 +163,11 @@ static RegisterStoreImplementation regStore([](
     const std::string & uri, const Store::Params & params)
     -> std::shared_ptr<Store>
 {
+    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1";
     if (std::string(uri, 0, 7) != "http://" &&
         std::string(uri, 0, 8) != "https://" &&
-        (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://")
-        ) return 0;
+        (!forceHttp || std::string(uri, 0, 7) != "file://"))
+        return 0;
     auto store = std::make_shared<HttpBinaryCacheStore>(params, uri);
     store->init();
     return store;

src/libstore/local-store.cc

@@ -298,9 +298,7 @@ void LocalStore::openDB(State & state, bool create)
     /* Open the Nix database. */
     string dbPath = dbDir + "/db.sqlite";
     auto & db(state.db);
-    if (sqlite3_open_v2(dbPath.c_str(), &db.db,
-            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
-        throw Error(format("cannot open Nix database '%1%'") % dbPath);
+    state.db = SQLite(dbPath, create);
 
 #ifdef __CYGWIN__
     /* The cygwin version of sqlite3 has a patch which calls
@@ -312,11 +310,6 @@ void LocalStore::openDB(State & state, bool create)
     SetDllDirectoryW(L"");
 #endif
 
-    if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
-        throwSQLiteError(db, "setting timeout");
-
-    db.exec("pragma foreign_keys = 1");
-
     /* !!! check whether sqlite has been built with foreign key
        support */
 
@@ -350,7 +343,7 @@ void LocalStore::openDB(State & state, bool create)
 
     /* Initialise the database schema, if necessary. */
     if (create) {
-        const char * schema =
+        static const char schema[] =
 #include "schema.sql.gen.hh"
             ;
         db.exec(schema);
@@ -547,6 +540,18 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     std::string drvName(drvPath.name());
     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 
+    auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName)
+    {
+        if (actual != expected)
+            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                printStorePath(drvPath), printStorePath(actual), printStorePath(expected));
+        auto j = drv.env.find(varName);
+        if (j == drv.env.end() || parseStorePath(j->second) != actual)
+            throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
+                printStorePath(drvPath), varName, printStorePath(actual));
+    };
+
+
     if (drv.isFixedOutput()) {
         DerivationOutputs::const_iterator out = drv.outputs.find("out");
         if (out == drv.outputs.end())
@@ -554,24 +559,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 
         bool recursive; Hash h;
         out->second.parseHashInfo(recursive, h);
-        auto outPath = makeFixedOutputPath(recursive, h, drvName);
 
-        StringPairs::const_iterator j = drv.env.find("out");
-        if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
-            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath));
+        check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out");
     }
 
     else {
         Hash h = hashDerivationModulo(*this, drv, true);
-
-        for (auto & i : drv.outputs) {
-            auto outPath = makeOutputPath(i.first, h, drvName);
-            StringPairs::const_iterator j = drv.env.find(i.first);
-            if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath)
-                throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                    printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath));
-        }
+        for (auto & i : drv.outputs)
+            check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
     }
 }
 
@@ -620,7 +615,8 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), std::make_shared<ValidPathInfo>(info));
+        state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)),
+            PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
     return id;
@@ -1001,16 +997,18 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             deletePath(realPath);
 
+            if (info.ca != "" &&
+                !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
+                    || info.references.empty()))
+                settings.requireExperimentalFeature("ca-references");
+
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
             std::unique_ptr<AbstractHashSink> hashSink;
-            if (info.ca == "")
+            if (info.ca == "" || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(htSHA256);
-            else {
-                if (!info.references.empty())
-                    settings.requireExperimentalFeature("ca-references");
+            else
                 hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path)));
-            }
 
             LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t {
                 size_t n = source.read(data, len);
@@ -1265,12 +1263,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
                 std::unique_ptr<AbstractHashSink> hashSink;
-                if (info->ca == "")
+                if (info->ca == "" || !info->references.count(info->path))
                     hashSink = std::make_unique<HashSink>(info->narHash.type);
                 else
                     hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path)));
 
-                dumpPath(toRealPath(printStorePath(i)), *hashSink);
+                dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();
 
                 if (info->narHash != nullHash && info->narHash != current.first) {

src/libstore/local.mk

@@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1)
 	libstore_LDFLAGS += -lseccomp
 endif
 
-libstore_CXXFLAGS = \
+libstore_CXXFLAGS += \
+ -I src/libutil -I src/libstore \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \

src/libstore/nar-info-disk-cache.cc

@@ -78,12 +78,7 @@ public:
 
         state->db = SQLite(dbPath);
 
-        if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
-            throwSQLiteError(state->db, "setting timeout");
-
-        // We can always reproduce the cache.
-        state->db.exec("pragma synchronous = off");
-        state->db.exec("pragma main.journal_mode = truncate");
+        state->db.isCache();
 
         state->db.exec(schema);
 

src/libstore/nar-info-disk-cache.hh

@@ -10,7 +10,7 @@ class NarInfoDiskCache
 public:
     typedef enum { oValid, oInvalid, oUnknown } Outcome;
 
-    virtual ~NarInfoDiskCache() { };
+    virtual ~NarInfoDiskCache() { }
 
     virtual void createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;

src/libstore/parsed-derivations.cc

@@ -1,5 +1,7 @@
 #include "parsed-derivations.hh"
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
@@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv)
     auto jsonAttr = drv.env.find("__json");
     if (jsonAttr != drv.env.end()) {
         try {
-            structuredAttrs = nlohmann::json::parse(jsonAttr->second);
+            structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second));
         } catch (std::exception & e) {
             throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what());
         }
     }
 }
 
+ParsedDerivation::~ParsedDerivation() { }
+
 std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
 {
     if (structuredAttrs) {

src/libstore/parsed-derivations.hh

@@ -1,6 +1,6 @@
 #include "derivations.hh"
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json_fwd.hpp>
 
 namespace nix {
 
@@ -8,15 +8,17 @@ class ParsedDerivation
 {
     StorePath drvPath;
     BasicDerivation & drv;
-    std::optional<nlohmann::json> structuredAttrs;
+    std::unique_ptr<nlohmann::json> structuredAttrs;
 
 public:
 
     ParsedDerivation(StorePath && drvPath, BasicDerivation & drv);
 
-    const std::optional<nlohmann::json> & getStructuredAttrs() const
+    ~ParsedDerivation();
+
+    const nlohmann::json * getStructuredAttrs() const
     {
-        return structuredAttrs;
+        return structuredAttrs.get();
     }
 
     std::optional<std::string> getStringAttr(const std::string & name) const;

src/libstore/path.cc

@@ -55,6 +55,20 @@ StorePath Store::parseStorePath(std::string_view path) const
     return StorePath::make(path, storeDir);
 }
 
+std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const
+{
+    try {
+        return parseStorePath(path);
+    } catch (Error &) {
+        return {};
+    }
+}
+
+bool Store::isStorePath(std::string_view path) const
+{
+    return (bool) maybeParseStorePath(path);
+}
+
 StorePathSet Store::parseStorePathSet(const PathSet & paths) const
 {
     StorePathSet res;

src/libstore/profiles.cc

@@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile)
 }
 
 
+Path getDefaultProfile()
+{
+    Path profileLink = getHome() + "/.nix-profile";
+    try {
+        if (!pathExists(profileLink)) {
+            replaceSymlink(
+                getuid() == 0
+                ? settings.nixStateDir + "/profiles/default"
+                : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
+                profileLink);
+        }
+        return absPath(readLink(profileLink), dirOf(profileLink));
+    } catch (Error &) {
+        return profileLink;
+    }
+}
+
+
 }

src/libstore/profiles.hh

@@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile);
    rebuilt. */
 string optimisticLockProfile(const Path & profile);
 
+/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create
+   it. */
+Path getDefaultProfile();
+
 }

src/libstore/sandbox-defaults.sb

@@ -71,6 +71,12 @@
        (literal "/dev/zero")
        (subpath "/dev/fd"))
 
+; Allow pseudo-terminals.
+(allow file*
+       (literal "/dev/ptmx")
+       (regex #"^/dev/pty[a-z]+")
+       (regex #"^/dev/ttys[0-9]+"))
+
 ; Does nothing, but reduces build noise.
 (allow file* (literal "/dev/dtracehelper"))
 
@@ -85,3 +91,7 @@
        (literal "/etc")
        (literal "/var")
        (literal "/private/var/tmp"))
+
+; This is used by /bin/sh on macOS 10.15 and later.
+(allow file*
+       (literal "/private/var/select/sh"))

src/libstore/sqlite.cc

@@ -25,11 +25,16 @@ namespace nix {
         throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path);
 }
 
-SQLite::SQLite(const Path & path)
+SQLite::SQLite(const Path & path, bool create)
 {
     if (sqlite3_open_v2(path.c_str(), &db,
-            SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
+            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
         throw Error(format("cannot open SQLite database '%s'") % path);
+
+    if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
+        throwSQLiteError(db, "setting timeout");
+
+    exec("pragma foreign_keys = 1");
 }
 
 SQLite::~SQLite()
@@ -42,6 +47,12 @@ SQLite::~SQLite()
     }
 }
 
+void SQLite::isCache()
+{
+    exec("pragma synchronous = off");
+    exec("pragma main.journal_mode = truncate");
+}
+
 void SQLite::exec(const std::string & stmt)
 {
     retrySQLite<void>([&]() {
@@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool
     return *this;
 }
 
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull)
+{
+    if (notNull) {
+        if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK)
+            throwSQLiteError(stmt.db, "binding argument");
+    } else
+        bind();
+    return *this;
+}
+
 SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
 {
     if (notNull) {

src/libstore/sqlite.hh

@@ -5,8 +5,8 @@
 
 #include "types.hh"
 
-class sqlite3;
-class sqlite3_stmt;
+struct sqlite3;
+struct sqlite3_stmt;
 
 namespace nix {
 
@@ -15,13 +15,16 @@ struct SQLite
 {
     sqlite3 * db = 0;
     SQLite() { }
-    SQLite(const Path & path);
+    SQLite(const Path & path, bool create = true);
     SQLite(const SQLite & from) = delete;
     SQLite& operator = (const SQLite & from) = delete;
     SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
     ~SQLite();
     operator sqlite3 * () { return db; }
 
+    /* Disable synchronous mode, set truncate journal mode. */
+    void isCache();
+
     void exec(const std::string & stmt);
 };
 
@@ -52,6 +55,7 @@ struct SQLiteStmt
 
         /* Bind the next parameter. */
         Use & operator () (const std::string & value, bool notNull = true);
+        Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
         Use & operator () (int64_t value, bool notNull = true);
         Use & bind(); // null
 

src/libstore/ssh.cc

@@ -33,6 +33,9 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
     out.create();
 
     auto conn = std::make_unique<Connection>();
+    ProcessOptions options;
+    options.dieWithParent = false;
+
     conn->sshPid = startProcess([&]() {
         restoreSignals();
 
@@ -64,7 +67,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
 
         // could not exec ssh/bash
         throw SysError("unable to execute '%s'", args.front());
-    });
+    }, options);
 
 
     in.readSide = -1;
@@ -91,6 +94,9 @@ Path SSHMaster::startMaster()
     Pipe out;
     out.create();
 
+    ProcessOptions options;
+    options.dieWithParent = false;
+
     state->sshMaster = startProcess([&]() {
         restoreSignals();
 
@@ -110,7 +116,7 @@ Path SSHMaster::startMaster()
         execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());
 
         throw SysError("unable to execute '%s'", args.front());
-    });
+    }, options);
 
     out.writeSide = -1;
 

src/libstore/store-api.cc

@@ -19,14 +19,6 @@ bool Store::isInStore(const Path & path) const
 }
 
 
-bool Store::isStorePath(const Path & path) const
-{
-    return isInStore(path)
-        && path.size() >= storeDir.size() + 1 + storePathHashLen
-        && path.find('/', storeDir.size() + 1) == Path::npos;
-}
-
-
 Path Store::toStorePath(const Path & path) const
 {
     if (!isInStore(path))
@@ -234,6 +226,14 @@ std::string Store::getUri()
     return "";
 }
 
+bool Store::PathInfoCacheValue::isKnownNow()
+{
+    std::chrono::duration ttl = didExist()
+        ? std::chrono::seconds(settings.ttlPositiveNarInfoCache)
+        : std::chrono::seconds(settings.ttlNegativeNarInfoCache);
+
+    return std::chrono::steady_clock::now() < time_point + ttl;
+}
 
 bool Store::isValidPath(const StorePath & storePath)
 {
@@ -242,9 +242,9 @@ bool Store::isValidPath(const StorePath & storePath)
     {
         auto state_(state.lock());
         auto res = state_->pathInfoCache.get(hashPart);
-        if (res) {
+        if (res && res->isKnownNow()) {
             stats.narInfoReadAverted++;
-            return *res != 0;
+            return res->didExist();
         }
     }
 
@@ -254,7 +254,7 @@ bool Store::isValidPath(const StorePath & storePath)
             stats.narInfoReadAverted++;
             auto state_(state.lock());
             state_->pathInfoCache.upsert(hashPart,
-                res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+                res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second });
             return res.first == NarInfoDiskCache::oValid;
         }
     }
@@ -309,11 +309,11 @@ void Store::queryPathInfo(const StorePath & storePath,
 
         {
             auto res = state.lock()->pathInfoCache.get(hashPart);
-            if (res) {
+            if (res && res->isKnownNow()) {
                 stats.narInfoReadAverted++;
-                if (!*res)
+                if (!res->didExist())
                     throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
-                return callback(ref<const ValidPathInfo>(*res));
+                return callback(ref<const ValidPathInfo>(res->value));
             }
         }
 
@@ -324,7 +324,7 @@ void Store::queryPathInfo(const StorePath & storePath,
                 {
                     auto state_(state.lock());
                     state_->pathInfoCache.upsert(hashPart,
-                        res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
+                        res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
                     if (res.first == NarInfoDiskCache::oInvalid ||
                         res.second->path != storePath)
                         throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
@@ -348,7 +348,7 @@ void Store::queryPathInfo(const StorePath & storePath,
 
                 {
                     auto state_(state.lock());
-                    state_->pathInfoCache.upsert(hashPart, info);
+                    state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
                 }
 
                 if (!info || info->path != parseStorePath(storePath)) {
@@ -441,7 +441,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
 
 
 void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
-    bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid)
+    bool includeImpureInfo, bool showClosureSize,
+    Base hashBase,
+    AllowInvalidFlag allowInvalid)
 {
     auto jsonList = jsonOut.list();
 
@@ -453,7 +455,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
             auto info = queryPathInfo(storePath);
 
             jsonPath
-                .attr("narHash", info->narHash.to_string())
+                .attr("narHash", info->narHash.to_string(hashBase))
                 .attr("narSize", info->narSize);
 
             {
@@ -741,12 +743,7 @@ std::string Store::showPaths(const StorePathSet & paths)
 
 string showPaths(const PathSet & paths)
 {
-    string s;
-    for (auto & i : paths) {
-        if (s.size() != 0) s += ", ";
-        s += "'" + i + "'";
-    }
-    return s;
+    return concatStringsSep(", ", quoteStrings(paths));
 }
 
 

src/libstore/store-api.hh

@@ -16,6 +16,7 @@
 #include <unordered_set>
 #include <memory>
 #include <string>
+#include <chrono>
 
 
 namespace nix {
@@ -261,10 +262,28 @@ public:
 
 protected:
 
+    struct PathInfoCacheValue {
+
+        // Time of cache entry creation or update
+        std::chrono::time_point<std::chrono::steady_clock> time_point = std::chrono::steady_clock::now();
+
+        // Null if missing
+        std::shared_ptr<const ValidPathInfo> value;
+
+        // Whether the value is valid as a cache entry. The path may not exist.
+        bool isKnownNow();
+
+        // Past tense, because a path can only be assumed to exists when
+        // isKnownNow() && didExist()
+        inline bool didExist() {
+          return value != nullptr;
+        }
+    };
+
     struct State
     {
         // FIXME: fix key
-        LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache;
+        LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
     };
 
     Sync<State> state;
@@ -281,6 +300,8 @@ public:
 
     StorePath parseStorePath(std::string_view path) const;
 
+    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
+
     std::string printStorePath(const StorePath & path) const;
 
     // FIXME: remove
@@ -303,7 +324,7 @@ public:
 
     /* Return true if ‘path’ is a store path, i.e. a direct child of
        the Nix store. */
-    bool isStorePath(const Path & path) const;
+    bool isStorePath(std::string_view path) const;
 
     /* Chop off the parts after the top-level store name, e.g.,
        /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
@@ -540,6 +561,7 @@ public:
        each path is included. */
     void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
         bool includeImpureInfo, bool showClosureSize,
+        Base hashBase = Base32,
         AllowInvalidFlag allowInvalid = DisallowInvalid);
 
     /* Return the size of the closure of the specified path, that is,
@@ -655,6 +677,11 @@ public:
         return storePath;
     }
 
+    Path toRealPath(const StorePath & storePath)
+    {
+        return toRealPath(printStorePath(storePath));
+    }
+
     virtual void createUser(const std::string & userName, uid_t userId)
     { }
 

src/libutil/hash.hh

@@ -80,6 +80,18 @@ struct Hash
        or base-64. By default, this is prefixed by the hash type
        (e.g. "sha256:"). */
     std::string to_string(Base base = Base32, bool includeType = true) const;
+
+    std::string gitRev() const
+    {
+        assert(type == htSHA1);
+        return to_string(Base16, false);
+    }
+
+    std::string gitShortRev() const
+    {
+        assert(type == htSHA1);
+        return std::string(to_string(Base16, false), 0, 7);
+    }
 };
 
 

src/libutil/util.cc

@@ -97,10 +97,10 @@ void replaceEnv(std::map<std::string, std::string> newEnv)
 }
 
 
-Path absPath(Path path, Path dir)
+Path absPath(Path path, std::optional<Path> dir)
 {
     if (path[0] != '/') {
-        if (dir == "") {
+        if (!dir) {
 #ifdef __GNU__
             /* GNU (aka. GNU/Hurd) doesn't have any limitation on path
                lengths and doesn't define `PATH_MAX'.  */
@@ -116,7 +116,7 @@ Path absPath(Path path, Path dir)
             free(buf);
 #endif
         }
-        path = dir + "/" + path;
+        path = *dir + "/" + path;
     }
     return canonPath(path);
 }
@@ -1205,28 +1205,6 @@ template StringSet tokenizeString(std::string_view s, const string & separators)
 template vector<string> tokenizeString(std::string_view s, const string & separators);
 
 
-string concatStringsSep(const string & sep, const Strings & ss)
-{
-    string s;
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-
-string concatStringsSep(const string & sep, const StringSet & ss)
-{
-    string s;
-    for (auto & i : ss) {
-        if (s.size() != 0) s += sep;
-        s += i;
-    }
-    return s;
-}
-
-
 string chomp(const string & s)
 {
     size_t i = s.find_last_not_of(" \n\r\t");

src/libutil/util.hh

@@ -46,7 +46,7 @@ void clearEnv();
 /* Return an absolutized path, resolving paths relative to the
    specified directory, or the current directory otherwise.  The path
    is also canonicalised. */
-Path absPath(Path path, Path dir = "");
+Path absPath(Path path, std::optional<Path> dir = {});
 
 /* Canonicalise a path by removing all `.' or `..' components and
    double or trailing slashes.  Optionally resolves all symlink
@@ -345,8 +345,26 @@ template<class C> C tokenizeString(std::string_view s, const string & separators
 
 /* Concatenate the given strings with a separator between the
    elements. */
-string concatStringsSep(const string & sep, const Strings & ss);
-string concatStringsSep(const string & sep, const StringSet & ss);
+template<class C>
+string concatStringsSep(const string & sep, const C & ss)
+{
+    string s;
+    for (auto & i : ss) {
+        if (s.size() != 0) s += sep;
+        s += i;
+    }
+    return s;
+}
+
+
+/* Add quotes around a collection of strings. */
+template<class C> Strings quoteStrings(const C & c)
+{
+    Strings res;
+    for (auto & s : c)
+        res.push_back("'" + s + "'");
+    return res;
+}
 
 
 /* Remove trailing whitespace from a string. */
@@ -433,6 +451,13 @@ void ignoreException();
 #define ANSI_BLUE "\e[34;1m"
 
 
+/* Tree formatting. */
+constexpr char treeConn[] = "├───";
+constexpr char treeLast[] = "└───";
+constexpr char treeLine[] = "│   ";
+constexpr char treeNull[] = "    ";
+
+
 /* Truncate a string to 'width' printable characters. If 'filterAll'
    is true, all ANSI escape sequences are filtered out. Otherwise,
    some escape sequences (such as colour setting) are copied but not
@@ -451,10 +476,11 @@ string base64Decode(const string & s);
 /* Get a value for the specified key from an associate container, or a
    default value if the key doesn't exist. */
 template <class T>
-std::optional<std::string> get(const T & map, const std::string & key)
+std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
 {
     auto i = map.find(key);
-    return i == map.end() ? std::optional<std::string>() : i->second;
+    if (i == map.end()) return {};
+    return std::optional<typename T::mapped_type>(i->second);
 }
 
 
@@ -558,4 +584,31 @@ extern PathFilter defaultPathFilter;
 AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
 
 
+// A Rust/Python-like enumerate() iterator adapter.
+// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.
+template <typename T,
+          typename TIter = decltype(std::begin(std::declval<T>())),
+          typename = decltype(std::end(std::declval<T>()))>
+constexpr auto enumerate(T && iterable)
+{
+    struct iterator
+    {
+        size_t i;
+        TIter iter;
+        bool operator != (const iterator & other) const { return iter != other.iter; }
+        void operator ++ () { ++i; ++iter; }
+        auto operator * () const { return std::tie(i, *iter); }
+    };
+
+    struct iterable_wrapper
+    {
+        T iterable;
+        auto begin() { return iterator{ 0, std::begin(iterable) }; }
+        auto end() { return iterator{ 0, std::end(iterable) }; }
+    };
+
+    return iterable_wrapper{ std::forward<T>(iterable) };
+}
+
+
 }

src/nix-build/nix-build.cc

@@ -16,7 +16,7 @@
 #include "get-drvs.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 using namespace std::string_literals;
@@ -97,7 +97,11 @@ static void _main(int argc, char * * argv)
     std::string outLink = "./result";
 
     // List of environment variables kept for --pure
-    std::set<string> keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"};
+    std::set<string> keepVars{
+        "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM",
+        "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL",
+        "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy"
+    };
 
     Strings args;
     for (int i = 1; i < argc; ++i)
@@ -291,7 +295,8 @@ static void _main(int argc, char * * argv)
                 try {
                     absolute = canonPath(absPath(i), true);
                 } catch (Error & e) {};
-                if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
+                auto [path, outputNames] = parsePathWithOutputs(absolute);
+                if (store->isStorePath(path) && hasSuffix(path, ".drv"))
                     drvs.push_back(DrvInfo(*state, store, absolute));
                 else
                     /* If we're in a #! script, interpret filenames
@@ -309,7 +314,7 @@ static void _main(int argc, char * * argv)
         state->eval(e, vRoot);
 
         for (auto & i : attrPaths) {
-            Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot));
+            Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first);
             state->forceValue(v);
             getDerivations(*state, v, "", *autoArgs, drvs, false);
         }
@@ -423,13 +428,18 @@ static void _main(int argc, char * * argv)
            lose the current $PATH directories. */
         auto rcfile = (Path) tmpDir + "/rc";
         writeFile(rcfile, fmt(
-                (keepTmp ? "" : "rm -rf '%1%'; "s) +
+                R"(_nix_shell_clean_tmpdir() { rm -rf %1%; }; )"s +
+                (keepTmp ?
+                    "trap _nix_shell_clean_tmpdir EXIT; "
+                    "exitHooks+=(_nix_shell_clean_tmpdir); "
+                    "failureHooks+=(_nix_shell_clean_tmpdir); ":
+                    "_nix_shell_clean_tmpdir; ") +
                 (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") +
                 "%2%"
                 "dontAddDisableDepTrack=1; "
                 "[ -e $stdenv/setup ] && source $stdenv/setup; "
                 "%3%"
-                "PATH=\"%4%:$PATH\"; "
+                "PATH=%4%:\"$PATH\"; "
                 "SHELL=%5%; "
                 "set +e; "
                 R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s"
@@ -438,12 +448,12 @@ static void _main(int argc, char * * argv)
                 "shopt -u nullglob; "
                 "unset TZ; %6%"
                 "%7%",
-                (Path) tmpDir,
+                shellEscape(tmpDir),
                 (pure ? "" : "p=$PATH; "),
                 (pure ? "" : "PATH=$PATH:$p; unset p; "),
-                dirOf(*shell),
-                *shell,
-                (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""),
+                shellEscape(dirOf(*shell)),
+                shellEscape(*shell),
+                (getenv("TZ") ? (string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""),
                 envCommand));
 
         Strings envStrs;

src/nix-channel/nix-channel.cc

@@ -2,7 +2,7 @@
 #include "globals.hh"
 #include "download.hh"
 #include "store-api.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <fcntl.h>
 #include <regex>

src/nix-collect-garbage/nix-collect-garbage.cc

@@ -2,7 +2,7 @@
 #include "profiles.hh"
 #include "shared.hh"
 #include "globals.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <iostream>
 #include <cerrno>

src/nix-copy-closure/nix-copy-closure.cc

@@ -1,6 +1,6 @@
 #include "shared.hh"
 #include "store-api.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 using namespace nix;
 

src/nix-daemon/nix-daemon.cc

@@ -6,7 +6,7 @@
 #include "globals.hh"
 #include "derivations.hh"
 #include "finally.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 #include "daemon.hh"
 
 #include <algorithm>

src/nix-env/nix-env.cc

@@ -13,7 +13,7 @@
 #include "json.hh"
 #include "value-to-json.hh"
 #include "xml-writer.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <cerrno>
 #include <ctime>
@@ -178,7 +178,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath,
     Value vRoot;
     loadSourceExpr(state, nixExprPath, vRoot);
 
-    Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot));
+    Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first);
 
     getDerivations(state, v, pathPrefix, autoArgs, elems, true);
 
@@ -408,7 +408,7 @@ static void queryInstSources(EvalState & state,
             Value vRoot;
             loadSourceExpr(state, instSource.nixExprPath, vRoot);
             for (auto & i : args) {
-                Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot));
+                Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first);
                 getDerivations(state, v, "", *instSource.autoArgs, elems, true);
             }
             break;
@@ -914,6 +914,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
             throw UsageError(format("unknown flag '%1%'") % arg);
     }
 
+    if (printAttrPath && source != sAvailable)
+        throw UsageError("--attr-path(-P) only works with --available");
 
     /* Obtain derivation information from the specified source. */
     DrvInfos availElems, installedElems;
@@ -1426,21 +1428,8 @@ static int _main(int argc, char * * argv)
         if (globals.profile == "")
             globals.profile = getEnv("NIX_PROFILE").value_or("");
 
-        if (globals.profile == "") {
-            Path profileLink = getHome() + "/.nix-profile";
-            try {
-                if (!pathExists(profileLink)) {
-                    replaceSymlink(
-                        getuid() == 0
-                        ? settings.nixStateDir + "/profiles/default"
-                        : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()),
-                        profileLink);
-                }
-                globals.profile = absPath(readLink(profileLink), dirOf(profileLink));
-            } catch (Error &) {
-                globals.profile = profileLink;
-            }
-        }
+        if (globals.profile == "")
+            globals.profile = getDefaultProfile();
 
         op(globals, opFlags, opArgs);
 

src/nix-env/user-env.cc

@@ -106,7 +106,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
 
     /* Get the environment builder expression. */
     Value envBuilder;
-    state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder);
+    state.eval(state.parseExprFromString(
+        #include "buildenv.nix.gen.hh"
+            , "/"), envBuilder);
 
     /* Construct a Nix expression that calls the user environment
        builder with the manifest as argument. */

src/nix-instantiate/nix-instantiate.cc

@@ -9,7 +9,7 @@
 #include "util.hh"
 #include "store-api.hh"
 #include "common-eval-args.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <map>
 #include <iostream>
@@ -39,7 +39,7 @@ void processExpr(EvalState & state, const Strings & attrPaths,
     state.eval(e, vRoot);
 
     for (auto & i : attrPaths) {
-        Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot));
+        Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first);
         state.forceValue(v);
 
         PathSet context;

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -6,9 +6,9 @@
 #include "eval-inline.hh"
 #include "common-eval-args.hh"
 #include "attr-path.hh"
-#include "legacy.hh"
 #include "finally.hh"
-#include "progress-bar.hh"
+#include "../nix/legacy.hh"
+#include "../nix/progress-bar.hh"
 #include "tarfile.hh"
 
 #include <iostream>
@@ -120,7 +120,7 @@ static int _main(int argc, char * * argv)
             Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
             Value vRoot;
             state->evalFile(path, vRoot);
-            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot));
+            Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first);
             state->forceAttrs(v);
 
             /* Extract the URI. */

src/nix-store/dotgraph.cc

@@ -44,46 +44,6 @@ static string makeNode(const string & id, std::string_view label,
 }
 
 
-#if 0
-string pathLabel(const Path & nePath, const string & elemPath)
-{
-    return (string) nePath + "-" + elemPath;
-}
-
-
-void printClosure(const Path & nePath, const StoreExpr & fs)
-{
-    PathSet workList(fs.closure.roots);
-    PathSet doneSet;
-
-    for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) {
-        cout << makeEdge(pathLabel(nePath, *i), nePath);
-    }
-
-    while (!workList.empty()) {
-        Path path = *(workList.begin());
-        workList.erase(path);
-
-        if (doneSet.insert(path).second) {
-            ClosureElems::const_iterator elem = fs.closure.elems.find(path);
-            if (elem == fs.closure.elems.end())
-                throw Error(format("bad closure, missing path '%1%'") % path);
-
-            for (StringSet::const_iterator i = elem->second.refs.begin();
-                 i != elem->second.refs.end(); ++i)
-            {
-                workList.insert(*i);
-                cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path));
-            }
-
-            cout << makeNode(pathLabel(nePath, path),
-                symbolicName(path), "#ff0000");
-        }
-    }
-}
-#endif
-
-
 void printDotGraph(ref<Store> store, StorePathSet && roots)
 {
     StorePathSet workList(std::move(roots));
@@ -101,40 +61,9 @@ void printDotGraph(ref<Store> store, StorePathSet && roots)
         for (auto & p : store->queryPathInfo(path)->references) {
             if (p != path) {
                 workList.insert(p.clone());
-                cout << makeEdge(std::string(p.to_string()), std::string(p.to_string()));
+                cout << makeEdge(std::string(p.to_string()), std::string(path.to_string()));
             }
         }
-
-#if 0
-        StoreExpr ne = storeExprFromPath(path);
-
-        string label, colour;
-
-        if (ne.type == StoreExpr::neDerivation) {
-            for (PathSet::iterator i = ne.derivation.inputs.begin();
-                 i != ne.derivation.inputs.end(); ++i)
-            {
-                workList.insert(*i);
-                cout << makeEdge(*i, path);
-            }
-
-            label = "derivation";
-            colour = "#00ff00";
-            for (StringPairs::iterator i = ne.derivation.env.begin();
-                 i != ne.derivation.env.end(); ++i)
-                if (i->first == "name") label = i->second;
-        }
-
-        else if (ne.type == StoreExpr::neClosure) {
-            label = "<closure>";
-            colour = "#00ffff";
-            printClosure(path, ne);
-        }
-
-        else abort();
-
-        cout << makeNode(path, label, colour);
-#endif
     }
 
     cout << "}\n";

src/nix-store/nix-store.cc

@@ -9,7 +9,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "graphml.hh"
-#include "legacy.hh"
+#include "../nix/legacy.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -229,12 +229,6 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
 /* Some code to print a tree representation of a derivation dependency
    graph.  Topological sorting is used to keep the tree relatively
    flat. */
-
-const string treeConn = "+---";
-const string treeLine = "|   ";
-const string treeNull = "    ";
-
-
 static void printTree(const StorePath & path,
     const string & firstPad, const string & tailPad, StorePathSet & done)
 {
@@ -254,10 +248,11 @@ static void printTree(const StorePath & path,
     auto sorted = store->topoSortPaths(info->references);
     reverse(sorted.begin(), sorted.end());
 
-    for (auto i = sorted.begin(); i != sorted.end(); ++i) {
-        auto j = i; ++j;
-        printTree(*i, tailPad + treeConn,
-            j == sorted.end() ? tailPad + treeNull : tailPad + treeLine,
+    for (const auto &[n, i] : enumerate(sorted)) {
+        bool last = n + 1 == sorted.size();
+        printTree(i,
+            tailPad + (last ? treeLast : treeConn),
+            tailPad + (last ? treeNull : treeLine),
             done);
     }
 }
@@ -577,7 +572,6 @@ static void opGC(Strings opFlags, Strings opArgs)
         if (*i == "--print-roots") printRoots = true;
         else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
         else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
-        else if (*i == "--delete") options.action = GCOptions::gcDeleteDead;
         else if (*i == "--max-freed") {
             long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true);
             options.maxFreed = maxFreed >= 0 ? maxFreed : 0;

src/nix/command.hh

@@ -3,17 +3,12 @@
 #include "args.hh"
 #include "common-eval-args.hh"
 #include "path.hh"
+#include "eval.hh"
 
 namespace nix {
 
 extern std::string programPath;
 
-struct Value;
-class Bindings;
-class EvalState;
-struct Pos;
-class Store;
-
 /* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {
@@ -48,7 +43,7 @@ struct Installable
 
     Buildable toBuildable();
 
-    virtual Value * toValue(EvalState & state)
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
     }

src/nix/edit.cc

@@ -29,9 +29,15 @@ struct CmdEdit : InstallableCommand
     {
         auto state = getEvalState();
 
-        auto v = installable->toValue(*state);
+        auto [v, pos] = installable->toValue(*state);
 
-        Pos pos = findDerivationFilename(*state, *v, installable->what());
+        try {
+            pos = findDerivationFilename(*state, *v, installable->what());
+        } catch (NoPositionInfo &) {
+        }
+
+        if (pos == noPos)
+            throw Error("cannot find position information for '%s", installable->what());
 
         stopProgressBar();
 

src/nix/eval.cc

@@ -52,7 +52,7 @@ struct CmdEval : MixJSON, InstallableCommand
 
         auto state = getEvalState();
 
-        auto v = installable->toValue(*state);
+        auto v = installable->toValue(*state).first;
         PathSet context;
 
         stopProgressBar();

src/nix/installables.cc

@@ -121,7 +121,7 @@ struct InstallableValue : Installable
     {
         auto state = cmd.getEvalState();
 
-        auto v = toValue(*state);
+        auto v = toValue(*state).first;
 
         Bindings & autoArgs = *cmd.getAutoArgs(*state);
 
@@ -169,11 +169,11 @@ struct InstallableExpr : InstallableValue
 
     std::string what() override { return text; }
 
-    Value * toValue(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto v = state.allocValue();
         state.eval(state.parseExprFromString(text, absPath(".")), *v);
-        return v;
+        return {v, noPos};
     }
 };
 
@@ -187,16 +187,16 @@ struct InstallableAttrPath : InstallableValue
 
     std::string what() override { return attrPath; }
 
-    Value * toValue(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto source = cmd.getSourceExpr(state);
 
         Bindings & autoArgs = *cmd.getAutoArgs(state);
 
-        Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source);
+        auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first;
         state.forceValue(*v);
 
-        return v;
+        return {v, noPos};
     }
 };
 

src/nix/local.mk

@@ -15,6 +15,8 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain
+
 nix_LIBS = libexpr libmain libstore libutil libnixrust
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
@@ -23,3 +25,5 @@ $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
   $(eval $(call install-symlink, nix, $(bindir)/$(name))))
 $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
+
+src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh

src/nix/main.cc

@@ -55,6 +55,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 {
     bool printBuildLogs = false;
     bool useNet = true;
+    bool refresh = false;
 
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
@@ -92,6 +93,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
             .longName("no-net")
             .description("disable substituters and consider all previously downloaded files up-to-date")
             .handler([&]() { useNet = false; });
+
+        mkFlag()
+            .longName("refresh")
+            .description("consider all previously downloaded files out-of-date")
+            .handler([&]() { refresh = true; });
     }
 
     void printFlags(std::ostream & out) override
@@ -176,6 +182,9 @@ void mainWrapped(int argc, char * * argv)
             downloadSettings.connectTimeout = 1;
     }
 
+    if (args.refresh)
+        settings.tarballTtl = 0;
+
     args.command->prepare();
     args.command->run();
 }

src/nix/path-info.cc

@@ -89,7 +89,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             store->pathInfoToJSON(jsonRoot,
                 // FIXME: preserve order?
                 storePathsToSet(storePaths),
-                true, showClosureSize, AllowInvalid);
+                true, showClosureSize, SRI, AllowInvalid);
         }
 
         else {