Mark official release

Add regression / performance tests against a fixed revision of Nixpkgs

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,7 @@
+**Release Notes**
+Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed.
+
+
+**Testing**
+
+If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master).

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: "Test"
+name: "CI"
 
 on:
   pull_request:
@@ -74,3 +74,35 @@ jobs:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
     - run: nix-instantiate -E 'builtins.currentTime' --eval
+
+  docker_push_image:
+    needs: [check_cachix, tests]
+    if: >-
+      github.event_name == 'push' &&
+      github.ref_name == 'master' &&
+      needs.check_cachix.outputs.secret == 'true'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v16
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - run: echo NIX_VERSION="$(nix-instantiate --eval -E '(import ./default.nix).defaultPackage.${builtins.currentSystem}.version' | tr -d \")" >> $GITHUB_ENV
+    - uses: cachix/cachix-action@v10
+      if: needs.check_cachix.outputs.secret == 'true'
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - run: nix-build -A checks.$(nix-instantiate --eval -E 'builtins.currentSystem' --json).dockerImage
+    - run: docker load -i ./result/image.tar.gz
+    - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
+    - run: docker tag nix:$NIX_VERSION nixos/nix:master
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - run: docker push nixos/nix:$NIX_VERSION
+    - run: docker push nixos/nix:master

.github/workflows/hydra_status.yml

@@ -0,0 +1,16 @@
+name: Hydra status
+on:
+  schedule:
+    - cron: "12,42 * * * *"
+  workflow_dispatch:
+jobs:
+  check_hydra_status:
+    name: Check Hydra status
+    if: github.repository_owner == 'NixOS'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        fetch-depth: 0
+    - run: bash scripts/check-hydra-status.sh
+

.version

@@ -1 +1 @@
-2.5
+2.6.0

Makefile

@@ -10,6 +10,7 @@ makefiles = \
   src/libexpr/local.mk \
   src/libcmd/local.mk \
   src/nix/local.mk \
+  src/nlohmann/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \

configure.ac

@@ -188,17 +188,24 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
     [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
 ])
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
 # Look for libcpuid.
+have_libcpuid=
 if test "$machine_name" = "x86_64"; then
-  PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"])
-  have_libcpuid=1
-  AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])
+    AC_ARG_ENABLE([cpuid],
+                  AS_HELP_STRING([--disable-cpuid], [Do not determine microarchitecture levels with libcpuid (relevant to x86_64 only)]))
+    if test "x$enable_cpuid" != "xno"; then
+      PKG_CHECK_MODULES([LIBCPUID], [libcpuid],
+        [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"
+         have_libcpuid=1
+         AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])]
+      )
+    fi
 fi
 AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
 

doc/manual/generate-options.nix

@@ -8,17 +8,19 @@ concatStrings (map
     let option = options.${name}; in
     "  - `${name}`  \n\n"
     + concatStrings (map (s: "    ${s}\n") (splitLines option.description)) + "\n\n"
-    + "    **Default:** " + (
-      if option.value == "" || option.value == []
-      then "*empty*"
-      else if isBool option.value
-      then (if option.value then "`true`" else "`false`")
-      else
-        # n.b. a StringMap value type is specified as a string, but
-        # this shows the value type.  The empty stringmap is "null" in
-        # JSON, but that converts to "{ }" here.
-        (if isAttrs option.value then "`\"\"`"
-         else "`" + toString option.value + "`")) + "\n\n"
+    + (if option.documentDefault
+       then "    **Default:** " + (
+       if option.value == "" || option.value == []
+       then "*empty*"
+       else if isBool option.value
+       then (if option.value then "`true`" else "`false`")
+       else
+         # n.b. a StringMap value type is specified as a string, but
+         # this shows the value type.  The empty stringmap is "null" in
+         # JSON, but that converts to "{ }" here.
+         (if isAttrs option.value then "`\"\"`"
+          else "`" + toString option.value + "`")) + "\n\n"
+       else "    **Default:** *machine-specific*")
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
        else "")

doc/manual/local.mk

@@ -12,11 +12,13 @@ man-pages := $(foreach n, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
+# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
 	NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
-	NIX_STATE_DIR=/dummy
+	NIX_STATE_DIR=/dummy \
+	NIX_CONFIG='cores = 0'
 
 nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 

doc/manual/src/SUMMARY.md.in

@@ -71,7 +71,8 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md)
+  - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md)
   - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)

doc/manual/src/advanced-topics/distributed-builds.md

@@ -53,8 +53,8 @@ example, the following command allows you to build a derivation for
 $ uname
 Linux
 
-$ nix build \
-  '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+$ nix build --impure \
+  --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
 [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
 

doc/manual/src/command-ref/nix-shell.md

@@ -101,7 +101,8 @@ The following common options are supported:
 
   - `NIX_BUILD_SHELL`\
     Shell used to start the interactive environment. Defaults to the
-    `bash` found in `PATH`.
+    `bash` found in `<nixpkgs>`, falling back to the `bash` found in
+    `PATH` if not found.
 
 # Examples
 

doc/manual/src/contributing/hacking.md

@@ -35,6 +35,25 @@ variables are set up so that those dependencies can be found:
 $ nix-shell
 ```
 
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop
+```
+
+To get a shell with a different compilation environment (e.g. stdenv,
+gccStdenv, clangStdenv, clang11Stdenv):
+
+```console
+$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages
+```
+
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop .#clang11StdenvPackages
+```
+
 To build Nix itself in this shell:
 
 ```console

doc/manual/src/expressions/language-constructs.md

@@ -284,6 +284,10 @@ The points of interest are:
     function is called with the `localServer` argument set to `true` but
     the `db4` argument set to `null`, then the evaluation fails.
 
+    Note that `->` is the [logical
+    implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication)
+    Boolean operation.
+
 2.  This is a more subtle condition: if Subversion is built with Apache
     (`httpServer`) support, then the Expat library (an XML library) used
     by Subversion should be same as the one used by Apache. This is

doc/manual/src/glossary.md

@@ -47,7 +47,7 @@
     the store object at `P` contains the path `Q` somewhere. The
     *references* of a store path are the set of store paths to which it
     has a reference.
-    
+
     A derivation can reference other derivations and sources (but not
     output paths), whereas an output path only references other output
     paths.
@@ -66,7 +66,7 @@
     is necessary to deploy whole closures, since otherwise at runtime
     files could be missing. The command `nix-store -qR` prints out
     closures of store paths.
-    
+
     As an example, if the store object at path `P` contains a reference
     to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
     references `R` then `R` is also in the closure of `P`.
@@ -98,3 +98,7 @@
     store. It can contain regular files, directories and symbolic
     links.  NARs are generated and unpacked using `nix-store --dump`
     and `nix-store --restore`.
+  - `∅` \
+    The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.
+  - `ε` \
+    The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.

doc/manual/src/installation/installing-binary.md

@@ -119,6 +119,30 @@ this to run the installer, but it may help if you run into trouble:
 - update `/etc/synthetic.conf` to direct macOS to create a "synthetic"
   empty root directory to mount your volume
 - specify mount options for the volume in `/etc/fstab`
+  - `rw`: read-write
+  - `noauto`: prevent the system from auto-mounting the volume (so the
+    LaunchDaemon mentioned below can control mounting it, and to avoid
+    masking problems with that mounting service).
+  - `nobrowse`: prevent the Nix Store volume from showing up on your
+    desktop; also keeps Spotlight from spending resources to index
+    this volume
+  <!-- TODO:
+  - `suid`: honor setuid? surely not? ...
+  - `owners`: honor file ownership on the volume
+
+    For now I'll avoid pretending to understand suid/owners more
+    than I do. There've been some vague reports of file-ownership
+    and permission issues, particularly in cloud/VM/headless setups.
+    My pet theory is that this has something to do with these setups
+    not having a token that gets delegated to initial/admin accounts
+    on macOS. See scripts/create-darwin-volume.sh for a little more.
+
+    In any case, by Dec 4 2021, it _seems_ like some combination of
+    suid, owners, and calling diskutil enableOwnership have stopped
+    new reports from coming in. But I hesitate to celebrate because we
+    haven't really named and catalogued the behavior, understood what
+    we're fixing, and validated that all 3 components are essential.
+  -->
 - if you have FileVault enabled
     - generate an encryption password
     - put it in your system Keychain

doc/manual/src/installation/installing-docker.md

@@ -3,7 +3,7 @@
 To run the latest stable release of Nix with Docker run the following command:
 
 ```console
-$ docker -ti run nixos/nix
+$ docker run -ti nixos/nix
 Unable to find image 'nixos/nix:latest' locally
 latest: Pulling from nixos/nix
 5843afab3874: Pull complete
@@ -16,7 +16,7 @@ nix (Nix) 2.3.12
 35ca4ada6e96:/# exit
 ```
 
-# What is included in Nix' Docker image?
+# What is included in Nix's Docker image?
 
 The official Docker image is created using `pkgs.dockerTools.buildLayeredImage`
 (and not with `Dockerfile` as it is usual with Docker images). You can still
@@ -54,6 +54,6 @@ You can also build a Docker image from source yourself:
 
 ```console
 $ nix build ./\#hydraJobs.dockerImage.x86_64-linux
-$ docker load -i ./result
+$ docker load -i ./result/image.tar.gz
 $ docker run -ti nix:2.5pre20211105
 ```

doc/manual/src/installation/prerequisites-source.md

@@ -44,6 +44,11 @@
     obtained from the its repository
     <https://github.com/troglobit/editline>.
 
+  - The `libsodium` library for verifying cryptographic signatures
+    of contents fetched from binary caches.
+    It can be obtained from the official web site
+    <https://libsodium.org>.
+
   - Recent versions of Bison and Flex to build the parser. (This is
     because Nix needs GLR support in Bison and reentrancy support in
     Flex.) For Bison, you need version 2.6, which can be obtained from
@@ -58,3 +63,11 @@
     `--disable-seccomp-sandboxing` option to the `configure` script (Not
     recommended unless your system doesn't support `libseccomp`). To get
     the library, visit <https://github.com/seccomp/libseccomp>.
+
+  - On 64-bit x86 machines only, `libcpuid` library
+    is used to determine which microarchitecture levels are supported
+    (e.g., as whether to have `x86_64-v2-linux` among additional system types).
+    The library is available from its homepage
+    <http://libcpuid.sourceforge.net>.
+    This is an optional dependency and can be disabled
+    by providing a `--disable-cpuid` to the `configure` script.

doc/manual/src/installation/supported-platforms.md

@@ -4,4 +4,4 @@ Nix is currently supported on the following platforms:
 
   - Linux (i686, x86\_64, aarch64).
 
-  - macOS (x86\_64).
+  - macOS (x86\_64, aarch64).

doc/manual/src/package-management/basic-package-mgmt.md

@@ -40,7 +40,7 @@ $ nix-channel --update
 > 
 > On NixOS, you’re automatically subscribed to a NixOS channel
 > corresponding to your NixOS major release (e.g.
-> <http://nixos.org/channels/nixos-14.12>). A NixOS channel is identical
+> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical
 > to the Nixpkgs channel, except that it contains only Linux binaries
 > and is updated only if a set of regression tests succeed.
 

doc/manual/src/release-notes/rl-2.4.md

@@ -276,6 +276,9 @@ more than 2800 commits from 195 contributors since release 2.3.
 
 * Plugins can now register `nix` subcommands.
 
+* The `--indirect` flag to `nix-store --add-root` has become a no-op.
+  `--add-root` will always generate indirect GC roots from now on.
+
 ## Incompatible changes
 
 * The `nix` command is now marked as an experimental feature. This

doc/manual/src/release-notes/rl-2.5.md

@@ -0,0 +1,16 @@
+# Release 2.5 (2021-12-13)
+
+* The garbage collector no longer blocks new builds, so the message
+  `waiting for the big garbage collector lock...` is a thing of the
+  past.
+
+* Binary cache stores now have a setting `compression-level`.
+
+* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
+
+* Lists can now be compared lexicographically using the `<` operator.
+
+* New built-in function: `builtins.groupBy`, with the same functionality as
+  Nixpkgs' `lib.groupBy`, but faster.
+
+* `nix repl` now has a `:log` command.

doc/manual/src/release-notes/rl-2.6.md

@@ -0,0 +1,21 @@
+# Release 2.6 (2022-01-24)
+
+* The Nix CLI now searches for a `flake.nix` up until the root of the current
+  Git repository or a filesystem boundary rather than just in the current
+  directory.
+* The TOML parser used by `builtins.fromTOML` has been replaced by [a
+  more compliant one](https://github.com/ToruNiina/toml11).
+* Added `:st`/`:show-trace` commands to `nix repl`, which are used to
+  set or toggle display of error traces.
+* New builtin function `builtins.zipAttrsWith` with the same
+  functionality as `lib.zipAttrsWith` from Nixpkgs, but much more
+  efficient.
+* New command `nix store copy-log` to copy build logs from one store
+  to another.
+* The `commit-lockfile-summary` option can be set to a non-empty
+  string to override the commit summary used when commiting an updated
+  lockfile.  This may be used in conjunction with the `nixConfig`
+  attribute in `flake.nix` to better conform to repository
+  conventions.
+* `docker run -ti nixos/nix:master` will place you in the Docker
+  container with the latest version of Nix from the `master` branch.

doc/manual/src/release-notes/rl-next.md

@@ -1,7 +1,2 @@
-# Release 2.5 (2021-XX-XX)
+# Release X.Y (202?-??-??)
 
-* Binary cache stores now have a setting `compression-level`.
-
-* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
-
-* Lists can now be compared lexicographically using the `<` operator.

docker.nix

@@ -20,6 +20,8 @@ let
     man
     cacert.out
     findutils
+    iana-etc
+    git
   ];
 
   users = {
@@ -137,11 +139,8 @@ let
         name = "root-profile-env";
         paths = defaultPkgs;
       };
-      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
-        mkdir $out
-        cp -a ${rootEnv}/* $out/
-
-        cat > $out/manifest.nix <<EOF
+      manifest = pkgs.buildPackages.runCommand "manifest.nix" { } ''
+        cat > $out <<EOF
         [
         ${lib.concatStringsSep "\n" (builtins.map (drv: let
           outputs = drv.outputsToInstall or [ "out" ];
@@ -161,6 +160,11 @@ let
         ]
         EOF
       '';
+      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+        mkdir $out
+        cp -a ${rootEnv}/* $out/
+        ln -s ${manifest} $out/manifest.nix
+      '';
     in
     pkgs.runCommand "base-system"
       {
@@ -178,6 +182,9 @@ let
       set -x
       mkdir -p $out/etc
 
+      mkdir -p $out/etc/ssl/certs
+      ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs
+
       cat $passwdContentsPath > $out/etc/passwd
       echo "" >> $out/etc/passwd
 
@@ -194,6 +201,8 @@ let
 
       mkdir $out/tmp
 
+      mkdir -p $out/var/tmp
+
       mkdir -p $out/etc/nix
       cat $nixConfContentsPath > $out/etc/nix/nix.conf
 
@@ -227,6 +236,10 @@ pkgs.dockerTools.buildLayeredImageWithNixDb {
     rm -rf nix-support
     ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles
   '';
+  fakeRootCommands = ''
+    chmod 1777 tmp
+    chmod 1777 var/tmp
+  '';
 
   config = {
     Cmd = [ "/root/.nix-profile/bin/bash" ];

flake.lock

@@ -31,10 +31,26 @@
         "type": "indirect"
       }
     },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "indirect"
+      }
+    },
     "root": {
       "inputs": {
         "lowdown-src": "lowdown-src",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-regression": "nixpkgs-regression"
       }
     }
   },

flake.nix

@@ -2,9 +2,10 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small";
+  inputs.nixpkgs-regression.url = "nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
-  outputs = { self, nixpkgs, lowdown-src }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }:
 
     let
 
@@ -14,7 +15,7 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
@@ -22,15 +23,36 @@
 
       crossSystems = [ "armv6l-linux" "armv7l-linux" ];
 
+      stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ];
+
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+      forAllSystemsAndStdenvs = f: forAllSystems (system:
+        nixpkgs.lib.listToAttrs
+          (map
+            (n:
+            nixpkgs.lib.nameValuePair "${n}Packages" (
+              f system n
+            )) stdenvs
+          )
+      );
+
+      forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv);
 
       # Memoize nixpkgs for different platforms for efficiency.
-      nixpkgsFor = forAllSystems (system:
-        import nixpkgs {
-          inherit system;
-          overlays = [ self.overlay ];
-        }
-      );
+      nixpkgsFor =
+        let stdenvsPackages = forAllSystemsAndStdenvs
+          (system: stdenv:
+            import nixpkgs {
+              inherit system;
+              overlays = [
+                (overlayFor (p: p.${stdenv}))
+              ];
+            }
+          );
+        in
+        # Add the `stdenvPackages` at toplevel, both because these are the ones
+        # we want most of the time and for backwards compatibility
+        forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages);
 
       commonDeps = pkgs: with pkgs; rec {
         # Use "busybox-sandbox-shell" if present,
@@ -75,7 +97,7 @@
             buildPackages.mdbook
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
-            buildPackages.pkgconfig
+            buildPackages.pkg-config
 
             # Tests
             buildPackages.git
@@ -255,18 +277,15 @@
             $(cat ${installerClosureInfo}/store-paths)
         '';
 
-    in {
-
-      # A Nixpkgs overlay that overrides the 'nix' and
-      # 'nix.perl-bindings' packages.
-      overlay = final: prev: {
-
+      overlayFor = getStdenv: final: prev:
+      let currentStdenv = getStdenv final; in
+      {
         nixStable = prev.nix;
 
         # Forward from the previous stage as we don’t want it to pick the lowdown override
         nixUnstable = prev.nixUnstable;
 
-        nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
+        nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
 
@@ -281,6 +300,8 @@
 
           propagatedBuildInputs = propagatedDeps;
 
+          disallowedReferences = [ boost ];
+
           preConfigure =
             ''
               # Copy libboost_context so we don't get all of Boost in our closure.
@@ -288,9 +309,16 @@
               mkdir -p $out/lib
               cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
               rm -f $out/lib/*.a
-              ${lib.optionalString stdenv.isLinux ''
+              ${lib.optionalString currentStdenv.isLinux ''
                 chmod u+w $out/lib/*.so.*
-                patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+                patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+              ''}
+              ${lib.optionalString currentStdenv.isDarwin ''
+                for LIB in $out/lib/*.dylib; do
+                  chmod u+w $LIB
+                  install_name_tool -id $LIB $LIB
+                done
+                install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
               ''}
             '';
 
@@ -308,6 +336,12 @@
           postInstall = ''
             mkdir -p $doc/nix-support
             echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+            ${lib.optionalString currentStdenv.isDarwin ''
+            install_name_tool \
+              -change ${boost}/lib/libboost_context.dylib \
+              $out/lib/libboost_context.dylib \
+              $out/lib/libnixutil.dylib
+            ''}
           '';
 
           doInstallCheck = true;
@@ -317,7 +351,7 @@
 
           strictDeps = true;
 
-          passthru.perl-bindings = with final; stdenv.mkDerivation {
+          passthru.perl-bindings = with final; currentStdenv.mkDerivation {
             name = "nix-perl-${version}";
 
             src = self;
@@ -325,7 +359,7 @@
             nativeBuildInputs =
               [ buildPackages.autoconf-archive
                 buildPackages.autoreconfHook
-                buildPackages.pkgconfig
+                buildPackages.pkg-config
               ];
 
             buildInputs =
@@ -336,8 +370,8 @@
                 pkgs.perl
                 boost
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
+              ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium
+              ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -351,7 +385,7 @@
 
         };
 
-        lowdown-nix = with final; stdenv.mkDerivation rec {
+        lowdown-nix = with final; currentStdenv.mkDerivation rec {
           name = "lowdown-0.9.0";
 
           src = lowdown-src;
@@ -361,15 +395,20 @@
           nativeBuildInputs = [ buildPackages.which ];
 
           configurePhase = ''
-              ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
+              ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
               ./configure \
                 PREFIX=${placeholder "dev"} \
                 BINDIR=${placeholder "bin"}/bin
-            '';
+          '';
         };
-
       };
 
+    in {
+
+      # A Nixpkgs overlay that overrides the 'nix' and
+      # 'nix.perl-bindings' packages.
+      overlay = overlayFor (p: p.stdenv);
+
       hydraJobs = {
 
         # Binary package for various platforms.
@@ -480,29 +519,23 @@
               inherit (self) overlay;
             });
 
-        /*
-        # Check whether we can still evaluate all of Nixpkgs.
+        # Make sure that nix-env still produces the exact same result
+        # on a particular version of Nixpkgs.
         tests.evalNixpkgs =
-          import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
-            # FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
-            inherit nixpkgs;
-            pkgs = nixpkgsFor.x86_64-linux;
-            officialRelease = false;
-          };
-
-        # Check whether we can still evaluate NixOS.
-        tests.evalNixOS =
           with nixpkgsFor.x86_64-linux;
           runCommand "eval-nixos" { buildInputs = [ nix ]; }
             ''
-              export NIX_STATE_DIR=$TMPDIR
-
-              nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
-                --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
-
-              touch $out
+              type -p nix-env
+              # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
+              time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+              [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]]
+              mkdir $out
             '';
-        */
+
+        metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
+          pkgs = nixpkgsFor.x86_64-linux;
+          nixpkgs = nixpkgs-regression;
+        };
 
         installTests = forAllSystems (system:
           let pkgs = nixpkgsFor.${system}; in
@@ -524,9 +557,9 @@
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
-      } // (if system == "x86_64-linux" then {
+      } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
-      } else {}));
+      });
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;
@@ -610,15 +643,22 @@
           doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
         };
-      }) crossSystems)));
+      }) crossSystems)) // (builtins.listToAttrs (map (stdenvName:
+      nixpkgsFor.${system}.lib.nameValuePair
+        "nix-${stdenvName}"
+        nixpkgsFor.${system}."${stdenvName}Packages".nix
+      ) stdenvs))
+      );
 
       defaultPackage = forAllSystems (system: self.packages.${system}.nix);
 
-      devShell = forAllSystems (system:
+      devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages);
+
+      devShells = forAllSystemsAndStdenvs (system: stdenv:
         with nixpkgsFor.${system};
         with commonDeps pkgs;
 
-        stdenv.mkDerivation {
+        nixpkgsFor.${system}.${stdenv}.mkDerivation {
           name = "nix";
 
           outputs = [ "out" "dev" "doc" ];
@@ -637,6 +677,9 @@
               PATH=$prefix/bin:$PATH
               unset PYTHONPATH
               export MANPATH=$out/share/man:$MANPATH
+
+              # Make bash completion work.
+              XDG_DATA_DIRS+=:$out/share
             '';
         });
 

misc/bash/completion.sh

@@ -7,13 +7,15 @@ function _complete_nix {
         local completion=${line%%	*}
         if [[ -z $have_type ]]; then
             have_type=1
-            if [[ $completion = filenames ]]; then
+            if [[ $completion == filenames ]]; then
                 compopt -o filenames
+            elif [[ $completion == attrs ]]; then
+                compopt -o nospace
             fi
         else
             COMPREPLY+=("$completion")
         fi
-    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
+    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}")
     __ltrim_colon_completions "$cur"
 }
 

misc/fish/completion.fish

@@ -19,7 +19,6 @@ end
 
 function _nix_accepts_files
   set -l response (_nix_complete)
-  # First line is either filenames or no-filenames.
   test $response[1] = 'filenames'
 end
 

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -25,5 +25,10 @@
     <string>/var/log/nix-daemon.log</string>
     <key>StandardOutPath</key>
     <string>/dev/null</string>
+    <key>SoftResourceLimits</key>
+    <dict>
+      <key>NumberOfFiles</key>
+      <integer>4096</integer>
+    </dict>
   </dict>
 </plist>

perl/configure.ac

@@ -41,7 +41,7 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};')
 AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
 AC_MSG_RESULT($perllibdir)
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Check for the required Perl dependencies (DBI and DBD::SQLite).

scripts/check-hydra-status.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+# set -x
+
+
+# mapfile BUILDS_FOR_LATEST_EVAL < <(
+# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+#   jq -r '.evals[0].builds[] | @sh')
+BUILDS_FOR_LATEST_EVAL=$(
+curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+  jq -r '.evals[0].builds[]')
+
+someBuildFailed=0
+
+for buildId in $BUILDS_FOR_LATEST_EVAL; do
+  buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
+
+  buildStatus=$(echo "$buildInfo" | \
+    jq -r '.buildstatus')
+
+  if [[ "$buildStatus" -ne 0 ]]; then
+    someBuildFailed=1
+    echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra"
+  fi
+done
+
+exit "$someBuildFailed"

scripts/create-darwin-volume.sh

@@ -440,7 +440,22 @@ add_nix_vol_fstab_line() {
     # shellcheck disable=SC1003,SC2026
     local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}"
     shift
-    EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <<EOF
+
+    # wrap `ex` to work around a problem with vim plugins breaking exit codes;
+    # (see https://github.com/NixOS/nix/issues/5468)
+    # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split
+    # the EDITOR env.
+    #
+    # TODO: at some point we should switch to `--clean`, but it wasn't added
+    # until https://github.com/vim/vim/releases/tag/v8.0.1554 while the macOS
+    # minver 10.12.6 seems to have released with vim 7.4
+    cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF
+#!/bin/sh
+/usr/bin/ex --noplugin "\$@"
+EOF
+    chmod 755 "$SCRATCH/ex_cleanroom_wrapper"
+
+    EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" <<EOF
 :a
 UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners
 .
@@ -631,7 +646,7 @@ EOF
         # technically /etc/synthetic.d/nix is supported in Big Sur+
         # but handling both takes even more code...
         _sudo "to add Nix to /etc/synthetic.conf" \
-            /usr/bin/ex /etc/synthetic.conf <<EOF
+            /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF
 :a
 ${NIX_ROOT:1}
 .
@@ -794,7 +809,7 @@ setup_volume_daemon() {
     local volume_uuid="$2"
     if ! test_voldaemon; then
         task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2
-        _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <<EOF
+        _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF
 :a
 $(generate_mount_daemon "$cmd_type" "$volume_uuid")
 .

scripts/install-darwin-multi-user.sh

@@ -218,7 +218,7 @@ EOF
         setup_darwin_volume
     fi
 
-    if [ "$(diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
-        failure "This script needs a /nix volume with global permissions! This may require running sudo diskutil enableOwnership /nix."
+    if [ "$(/usr/sbin/diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        failure "This script needs a /nix volume with global permissions! This may require running sudo /usr/sbin/diskutil enableOwnership /nix."
     fi
 }

scripts/install-multi-user.sh

@@ -377,6 +377,11 @@ cure_artifacts() {
 }
 
 validate_starting_assumptions() {
+    task "Checking for artifacts of previous installs"
+    cat <<EOF
+Before I try to install, I'll check for signs Nix already is or has
+been installed on this system.
+EOF
     if type nix-env 2> /dev/null >&2; then
         warning <<EOF
 Nix already appears to be installed. This installer may run into issues.
@@ -386,6 +391,11 @@ $(uninstall_directions)
 EOF
     fi
 
+    # TODO: I think it would be good for this step to accumulate more
+    #       knowledge of older obsolete artifacts, if there are any.
+    #       We could issue a "reminder" here that the user might want
+    #       to clean them up?
+
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         # TODO: I think it would be good to accumulate a list of all
         #       of the copies so that people don't hit this 2 or 3x in

scripts/install-nix-from-closure.sh

@@ -38,7 +38,7 @@ fi
 
 # Determine if we could use the multi-user installer or not
 if [ "$(uname -s)" = "Linux" ]; then
-    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
+    echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2
 fi
 
 case "$(uname -s)" in
@@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do
                 echo "              providing multi-user support and better isolation for local builds."
                 echo "              Both for security and reproducibility, this method is recommended if"
                 echo "              supported on your platform."
-                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo "              See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation"
                 echo ""
                 echo " --no-daemon: Simple, single-user installation that does not require root and is"
                 echo "              trivial to uninstall."
@@ -144,7 +144,7 @@ if ! [ -e "$dest" ]; then
 fi
 
 if ! [ -w "$dest" ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

src/cpptoml/LICENSE

@@ -1,18 +0,0 @@
-Copyright (c) 2014 Chase Geigle
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

src/libcmd/command.cc

@@ -54,6 +54,36 @@ void StoreCommand::run()
     run(getStore());
 }
 
+CopyCommand::CopyCommand()
+{
+    addFlag({
+        .longName = "from",
+        .description = "URL of the source Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&srcUri},
+    });
+
+    addFlag({
+        .longName = "to",
+        .description = "URL of the destination Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&dstUri},
+    });
+}
+
+ref<Store> CopyCommand::createStore()
+{
+    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+}
+
+ref<Store> CopyCommand::getDstStore()
+{
+    if (srcUri.empty() && dstUri.empty())
+        throw UsageError("you must pass '--from' and/or '--to'");
+
+    return dstUri.empty() ? openStore() : openStore(dstUri);
+}
+
 EvalCommand::EvalCommand()
 {
 }
@@ -74,7 +104,15 @@ ref<Store> EvalCommand::getEvalStore()
 ref<EvalState> EvalCommand::getEvalState()
 {
     if (!evalState)
-        evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
+        evalState =
+            #if HAVE_BOEHMGC
+            std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
+                searchPath, getEvalStore(), getStore())
+            #else
+            std::make_shared<EvalState>(
+                searchPath, getEvalStore(), getStore())
+            #endif
+            ;
     return ref<EvalState>(evalState);
 }
 

src/libcmd/command.hh

@@ -43,6 +43,19 @@ private:
     std::shared_ptr<Store> _store;
 };
 
+/* A command that copies something between `--from` and `--to`
+   stores. */
+struct CopyCommand : virtual StoreCommand
+{
+    std::string srcUri, dstUri;
+
+    CopyCommand();
+
+    ref<Store> createStore() override;
+
+    ref<Store> getDstStore();
+};
+
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     EvalCommand();

src/libcmd/installables.cc

@@ -191,7 +191,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
         auto sep = prefix_.rfind('.');
         std::string searchWord;
         if (sep != std::string::npos) {
-            searchWord = prefix_.substr(sep, std::string::npos);
+            searchWord = prefix_.substr(sep + 1, std::string::npos);
             prefix_ = prefix_.substr(0, sep);
         } else {
             searchWord = prefix_;
@@ -203,6 +203,8 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
         Value v2;
         state->autoCallFunction(*autoArgs, v1, v2);
 
+        completionType = ctAttrs;
+
         if (v2.type() == nAttrs) {
             for (auto & i : *v2.attrs) {
                 std::string name = i.name;
@@ -232,7 +234,9 @@ void completeFlakeRefWithFragment(
        prefix. */
     try {
         auto hash = prefix.find('#');
-        if (hash != std::string::npos) {
+        if (hash == std::string::npos) {
+            completeFlakeRef(evalState->store, prefix);
+        } else {
             auto fragment = prefix.substr(hash + 1);
             auto flakeRefS = std::string(prefix.substr(0, hash));
             // FIXME: do tilde expansion.
@@ -248,6 +252,8 @@ void completeFlakeRefWithFragment(
                flake. */
             attrPathPrefixes.push_back("");
 
+            completionType = ctAttrs;
+
             for (auto & attrPathPrefixS : attrPathPrefixes) {
                 auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS);
                 auto attrPathS = attrPathPrefixS + std::string(fragment);
@@ -285,12 +291,13 @@ void completeFlakeRefWithFragment(
     } catch (Error & e) {
         warn(e.msg());
     }
-
-    completeFlakeRef(evalState->store, prefix);
 }
 
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
+    if (!settings.isExperimentalFeatureEnabled(Xp::Flakes))
+        return;
+
     if (prefix == "")
         completions->add(".");
 
@@ -338,6 +345,18 @@ Installable::getCursor(EvalState & state)
     return cursors[0];
 }
 
+static StorePath getDeriver(
+    ref<Store> store,
+    const Installable & i,
+    const StorePath & drvPath)
+{
+    auto derivers = store->queryValidDerivers(drvPath);
+    if (derivers.empty())
+        throw Error("'%s' does not have a known deriver", i.what());
+    // FIXME: use all derivers?
+    return *derivers.begin();
+}
+
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
@@ -346,7 +365,7 @@ struct InstallableStorePath : Installable
     InstallableStorePath(ref<Store> store, StorePath && storePath)
         : store(store), storePath(std::move(storePath)) { }
 
-    std::string what() override { return store->printStorePath(storePath); }
+    std::string what() const override { return store->printStorePath(storePath); }
 
     DerivedPaths toDerivedPaths() override
     {
@@ -367,6 +386,15 @@ struct InstallableStorePath : Installable
         }
     }
 
+    StorePathSet toDrvPaths(ref<Store> store) override
+    {
+        if (storePath.isDerivation()) {
+            return {storePath};
+        } else {
+            return {getDeriver(store, *this, storePath)};
+        }
+    }
+
     std::optional<StorePath> getStorePath() override
     {
         return storePath;
@@ -395,6 +423,14 @@ DerivedPaths InstallableValue::toDerivedPaths()
     return res;
 }
 
+StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
+{
+    StorePathSet res;
+    for (auto & drv : toDerivations())
+        res.insert(drv.drvPath);
+    return res;
+}
+
 struct InstallableAttrPath : InstallableValue
 {
     SourceExprCommand & cmd;
@@ -405,7 +441,7 @@ struct InstallableAttrPath : InstallableValue
         : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
     { }
 
-    std::string what() override { return attrPath; }
+    std::string what() const override { return attrPath; }
 
     std::pair<Value *, Pos> toValue(EvalState & state) override
     {
@@ -829,11 +865,7 @@ StorePathSet toDerivations(
                 [&](const DerivedPath::Opaque & bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
-                    auto derivers = store->queryValidDerivers(bo.path);
-                    if (derivers.empty())
-                        throw Error("'%s' does not have a known deriver", i->what());
-                    // FIXME: use all derivers?
-                    drvPaths.insert(*derivers.begin());
+                    drvPaths.insert(getDeriver(store, *i, bo.path));
                 },
                 [&](const DerivedPath::Built & bfd) {
                     drvPaths.insert(bfd.drvPath);

src/libcmd/installables.hh

@@ -33,10 +33,15 @@ struct Installable
 {
     virtual ~Installable() { }
 
-    virtual std::string what() = 0;
+    virtual std::string what() const = 0;
 
     virtual DerivedPaths toDerivedPaths() = 0;
 
+    virtual StorePathSet toDrvPaths(ref<Store> store)
+    {
+        throw Error("'%s' cannot be converted to a derivation path", what());
+    }
+
     DerivedPath toDerivedPath();
 
     UnresolvedApp toApp(EvalState & state);
@@ -81,6 +86,8 @@ struct InstallableValue : Installable
     virtual std::vector<DerivationInfo> toDerivations() = 0;
 
     DerivedPaths toDerivedPaths() override;
+
+    StorePathSet toDrvPaths(ref<Store> store) override;
 };
 
 struct InstallableFlake : InstallableValue
@@ -99,7 +106,7 @@ struct InstallableFlake : InstallableValue
         Strings && prefixes,
         const flake::LockFlags & lockFlags);
 
-    std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
+    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
 
     std::vector<std::string> getActualAttrPaths();
 

src/libexpr/attr-set.cc

@@ -7,26 +7,19 @@
 namespace nix {
 
 
+
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
 Bindings * EvalState::allocBindings(size_t capacity)
 {
+    if (capacity == 0)
+        return &emptyBindings;
     if (capacity > std::numeric_limits<Bindings::size_t>::max())
         throw Error("attribute set of size %d is too big", capacity);
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
-}
-
-
-void EvalState::mkAttrs(Value & v, size_t capacity)
-{
-    if (capacity == 0) {
-        v = vEmptySet;
-        return;
-    }
-    v.mkAttrs(allocBindings(capacity));
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
@@ -41,15 +34,36 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 }
 
 
-Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+Value * EvalState::allocAttr(Value & vAttrs, std::string_view name)
 {
     return allocAttr(vAttrs, symbols.create(name));
 }
 
 
+Value & BindingsBuilder::alloc(const Symbol & name, ptr<Pos> pos)
+{
+    auto value = state.allocValue();
+    bindings->push_back(Attr(name, value, pos));
+    return *value;
+}
+
+
+Value & BindingsBuilder::alloc(std::string_view name, ptr<Pos> pos)
+{
+    return alloc(state.symbols.create(name), pos);
+}
+
+
 void Bindings::sort()
 {
-    std::sort(begin(), end());
+    if (size_) std::sort(begin(), end());
+}
+
+
+Value & Value::mkAttrs(BindingsBuilder & bindings)
+{
+    mkAttrs(bindings.finish());
+    return *this;
 }
 
 

src/libexpr/attr-set.hh

@@ -113,5 +113,52 @@ public:
     friend class EvalState;
 };
 
+/* A wrapper around Bindings that ensures that its always in sorted
+   order at the end. The only way to consume a BindingsBuilder is to
+   call finish(), which sorts the bindings. */
+class BindingsBuilder
+{
+    Bindings * bindings;
+
+public:
+    // needed by std::back_inserter
+    using value_type = Attr;
+
+    EvalState & state;
+
+    BindingsBuilder(EvalState & state, Bindings * bindings)
+        : bindings(bindings), state(state)
+    { }
+
+    void insert(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos))
+    {
+        insert(Attr(name, value, pos));
+    }
+
+    void insert(const Attr & attr)
+    {
+        push_back(attr);
+    }
+
+    void push_back(const Attr & attr)
+    {
+        bindings->push_back(attr);
+    }
+
+    Value & alloc(const Symbol & name, ptr<Pos> pos = ptr(&noPos));
+
+    Value & alloc(std::string_view name, ptr<Pos> pos = ptr(&noPos));
+
+    Bindings * finish()
+    {
+        bindings->sort();
+        return bindings;
+    }
+
+    Bindings * alreadySorted()
+    {
+        return bindings;
+    }
+};
 
 }

src/libexpr/common-eval-args.cc

@@ -73,17 +73,16 @@ MixEvalArgs::MixEvalArgs()
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
-    Bindings * res = state.allocBindings(autoArgs.size());
+    auto res = state.buildBindings(autoArgs.size());
     for (auto & i : autoArgs) {
-        Value * v = state.allocValue();
+        auto v = state.allocValue();
         if (i.second[0] == 'E')
             state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
         else
-            mkString(*v, string(i.second, 1));
-        res->push_back(Attr(state.symbols.create(i.first), v));
+            v->mkString(((std::string_view) i.second).substr(1));
+        res.insert(state.symbols.create(i.first), v);
     }
-    res->sort();
-    return res;
+    return res.finish();
 }
 
 Path lookupFileArg(EvalState & state, string s)

src/libexpr/eval.cc

@@ -36,6 +36,19 @@
 namespace nix {
 
 
+static char * allocString(size_t size)
+{
+    char * t;
+#if HAVE_BOEHMGC
+    t = (char *) GC_MALLOC_ATOMIC(size);
+#else
+    t = malloc(size);
+#endif
+    if (!t) throw std::bad_alloc();
+    return t;
+}
+
+
 static char * dupString(const char * s)
 {
     char * t;
@@ -145,7 +158,7 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         str << v.fpoint;
         break;
     default:
-        throw Error("invalid value");
+        abort();
     }
 
     active.erase(&v);
@@ -412,10 +425,21 @@ EvalState::EvalState(
     , sDescription(symbols.create("description"))
     , sSelf(symbols.create("self"))
     , sEpsilon(symbols.create(""))
+    , sStartSet(symbols.create("startSet"))
+    , sOperator(symbols.create("operator"))
+    , sKey(symbols.create("key"))
+    , sPath(symbols.create("path"))
+    , sPrefix(symbols.create("prefix"))
     , repair(NoRepair)
+    , emptyBindings(0)
     , store(store)
     , buildStore(buildStore ? buildStore : store)
     , regexCache(makeRegexCache())
+#if HAVE_BOEHMGC
+    , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+#else
+    , valueAllocCache(std::make_shared<void *>(nullptr))
+#endif
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
 {
@@ -454,8 +478,6 @@ EvalState::EvalState(
         }
     }
 
-    vEmptySet.mkAttrs(allocBindings(0));
-
     createBaseEnv();
 }
 
@@ -613,7 +635,7 @@ Value * EvalState::addPrimOp(const string & name,
         auto vPrimOp = allocValue();
         vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym });
         Value v;
-        mkApp(v, *vPrimOp, *vPrimOp);
+        v.mkApp(vPrimOp, vPrimOp);
         return addConstant(name, v);
     }
 
@@ -635,7 +657,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
         auto vPrimOp = allocValue();
         vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
         Value v;
-        mkApp(v, *vPrimOp, *vPrimOp);
+        v.mkApp(vPrimOp, vPrimOp);
         return addConstant(primOp.name, v);
     }
 
@@ -766,15 +788,14 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con
 }
 
 
-void mkString(Value & v, const char * s)
+void Value::mkString(std::string_view s)
 {
-    v.mkString(dupString(s));
+    mkString(dupStringWithLen(s.data(), s.size()));
 }
 
 
-Value & mkString(Value & v, std::string_view s, const PathSet & context)
+static void copyContextToValue(Value & v, const PathSet & context)
 {
-    v.mkString(dupStringWithLen(s.data(), s.size()));
     if (!context.empty()) {
         size_t n = 0;
         v.string.context = (const char * *)
@@ -783,13 +804,24 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context)
             v.string.context[n++] = dupString(i.c_str());
         v.string.context[n] = 0;
     }
-    return v;
+}
+
+void Value::mkString(std::string_view s, const PathSet & context)
+{
+    mkString(s);
+    copyContextToValue(*this, context);
+}
+
+void Value::mkStringMove(const char * s, const PathSet & context)
+{
+    mkString(s);
+    copyContextToValue(*this, context);
 }
 
 
-void mkPath(Value & v, const char * s)
+void Value::mkPath(std::string_view s)
 {
-    v.mkPath(dupString(s));
+    mkPath(dupStringWithLen(s.data(), s.size()));
 }
 
 
@@ -821,8 +853,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 
 Value * EvalState::allocValue()
 {
+    /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
+       GC_malloc_many returns a linked list of objects of the given size, where the first word
+       of each object is also the pointer to the next object in the list. This also means that we
+       have to explicitly clear the first word of every object we take. */
+    if (!*valueAllocCache) {
+        *valueAllocCache = GC_malloc_many(sizeof(Value));
+        if (!*valueAllocCache) throw std::bad_alloc();
+    }
+
+    /* GC_NEXT is a convenience macro for accessing the first word of an object.
+       Take the first list item, advance the list to the next item, and clear the next pointer. */
+    void * p = *valueAllocCache;
+    GC_PTR_STORE_AND_DIRTY(&*valueAllocCache, GC_NEXT(p));
+    GC_NEXT(p) = nullptr;
+
     nrValues++;
-    auto v = (Value *) allocBytes(sizeof(Value));
+    auto v = (Value *) p;
     return v;
 }
 
@@ -867,13 +914,13 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 void EvalState::mkPos(Value & v, ptr<Pos> pos)
 {
     if (pos->file.set()) {
-        mkAttrs(v, 3);
-        mkString(*allocAttr(v, sFile), pos->file);
-        mkInt(*allocAttr(v, sLine), pos->line);
-        mkInt(*allocAttr(v, sColumn), pos->column);
-        v.attrs->sort();
+        auto attrs = buildBindings(3);
+        attrs.alloc(sFile).mkString(pos->file);
+        attrs.alloc(sLine).mkInt(pos->line);
+        attrs.alloc(sColumn).mkInt(pos->column);
+        v.mkAttrs(attrs);
     } else
-        mkNull(v);
+        v.mkNull();
 }
 
 
@@ -1052,8 +1099,8 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
 
 void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 {
-    state.mkAttrs(v, attrs.size() + dynamicAttrs.size());
-    Env *dynamicEnv = &env;
+    v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish());
+    auto dynamicEnv = &env;
 
     if (recursive) {
         /* Create a new environment that contains the attributes in
@@ -1244,14 +1291,14 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
         if (vAttrs->type() != nAttrs ||
             (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
         {
-            mkBool(v, false);
+            v.mkBool(false);
             return;
         } else {
             vAttrs = j->value;
         }
     }
 
-    mkBool(v, true);
+    v.mkBool(true);
 }
 
 
@@ -1326,7 +1373,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     /* Nope, so show the first unexpected argument to the
                        user. */
                     for (auto & i : *args[0]->attrs)
-                        if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
+                        if (!lambda.formals->argNames.count(i.name))
                             throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
                     abort(); // can't happen
                 }
@@ -1469,22 +1516,20 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         return;
     }
 
-    Value * actualArgs = allocValue();
-    mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
+    auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
 
     if (fun.lambda.fun->formals->ellipsis) {
         // If the formals have an ellipsis (eg the function accepts extra args) pass
         // all available automatic arguments (which includes arguments specified on
         // the command line via --arg/--argstr)
-        for (auto& v : args) {
-            actualArgs->attrs->push_back(v);
-        }
+        for (auto & v : args)
+            attrs.insert(v);
     } else {
         // Otherwise, only pass the arguments that the function accepts
         for (auto & i : fun.lambda.fun->formals->formals) {
             Bindings::iterator j = args.find(i.name);
             if (j != args.end()) {
-                actualArgs->attrs->push_back(*j);
+                attrs.insert(*j);
             } else if (!i.def) {
                 throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
 
@@ -1497,9 +1542,7 @@ https://nixos.org/manual/nix/stable/#ss-functions.)", i.name);
         }
     }
 
-    actualArgs->attrs->sort();
-
-    callFunction(fun, *actualArgs, res, noPos);
+    callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos);
 }
 
 
@@ -1534,7 +1577,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
 
 void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
 {
-    mkBool(v, !state.evalBool(env, e));
+    v.mkBool(!state.evalBool(env, e));
 }
 
 
@@ -1542,7 +1585,7 @@ void ExprOpEq::eval(EvalState & state, Env & env, Value & v)
 {
     Value v1; e1->eval(state, env, v1);
     Value v2; e2->eval(state, env, v2);
-    mkBool(v, state.eqValues(v1, v2));
+    v.mkBool(state.eqValues(v1, v2));
 }
 
 
@@ -1550,25 +1593,25 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v)
 {
     Value v1; e1->eval(state, env, v1);
     Value v2; e2->eval(state, env, v2);
-    mkBool(v, !state.eqValues(v1, v2));
+    v.mkBool(!state.eqValues(v1, v2));
 }
 
 
 void ExprOpAnd::eval(EvalState & state, Env & env, Value & v)
 {
-    mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos));
+    v.mkBool(state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos));
 }
 
 
 void ExprOpOr::eval(EvalState & state, Env & env, Value & v)
 {
-    mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos));
+    v.mkBool(state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos));
 }
 
 
 void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
 {
-    mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos));
+    v.mkBool(!state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos));
 }
 
 
@@ -1583,7 +1626,7 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
     if (v1.attrs->size() == 0) { v = v2; return; }
     if (v2.attrs->size() == 0) { v = v1; return; }
 
-    state.mkAttrs(v, v1.attrs->size() + v2.attrs->size());
+    auto attrs = state.buildBindings(v1.attrs->size() + v2.attrs->size());
 
     /* Merge the sets, preferring values from the second set.  Make
        sure to keep the resulting vector in sorted order. */
@@ -1592,17 +1635,19 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
 
     while (i != v1.attrs->end() && j != v2.attrs->end()) {
         if (i->name == j->name) {
-            v.attrs->push_back(*j);
+            attrs.insert(*j);
             ++i; ++j;
         }
         else if (i->name < j->name)
-            v.attrs->push_back(*i++);
+            attrs.insert(*i++);
         else
-            v.attrs->push_back(*j++);
+            attrs.insert(*j++);
     }
 
-    while (i != v1.attrs->end()) v.attrs->push_back(*i++);
-    while (j != v2.attrs->end()) v.attrs->push_back(*j++);
+    while (i != v1.attrs->end()) attrs.insert(*i++);
+    while (j != v2.attrs->end()) attrs.insert(*j++);
+
+    v.mkAttrs(attrs.alreadySorted());
 
     state.nrOpUpdateValuesCopied += v.attrs->size();
 }
@@ -1649,14 +1694,35 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po
 void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 {
     PathSet context;
-    std::ostringstream s;
+    std::vector<std::string> s;
+    size_t sSize = 0;
     NixInt n = 0;
     NixFloat nf = 0;
 
     bool first = !forceString;
     ValueType firstType = nString;
 
-    for (auto & i : *es) {
+    const auto str = [&] {
+        std::string result;
+        result.reserve(sSize);
+        for (const auto & part : s) result += part;
+        return result;
+    };
+    /* c_str() is not str().c_str() because we want to create a string
+       Value. allocating a GC'd string directly and moving it into a
+       Value lets us avoid an allocation and copy. */
+    const auto c_str = [&] {
+        char * result = allocString(sSize + 1);
+        char * tmp = result;
+        for (const auto & part : s) {
+            memcpy(tmp, part.c_str(), part.size());
+            tmp += part.size();
+        }
+        *tmp = 0;
+        return result;
+    };
+
+    for (auto & [i_pos, i] : *es) {
         Value vTmp;
         i->eval(state, env, vTmp);
 
@@ -1677,34 +1743,37 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 nf = n;
                 nf += vTmp.fpoint;
             } else
-                throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
+                throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp));
         } else if (firstType == nFloat) {
             if (vTmp.type() == nInt) {
                 nf += vTmp.integer;
             } else if (vTmp.type() == nFloat) {
                 nf += vTmp.fpoint;
             } else
-                throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
-        } else
+                throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp));
+        } else {
+            if (s.empty()) s.reserve(es->size());
             /* skip canonization of first path, which would only be not
             canonized in the first place if it's coming from a ./${foo} type
             path */
-            s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first);
+            s.emplace_back(
+                state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first));
+            sSize += s.back().size();
+        }
 
         first = false;
     }
 
     if (firstType == nInt)
-        mkInt(v, n);
+        v.mkInt(n);
     else if (firstType == nFloat)
-        mkFloat(v, nf);
+        v.mkFloat(nf);
     else if (firstType == nPath) {
         if (!context.empty())
             throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
-        auto path = canonPath(s.str());
-        mkPath(v, path.c_str());
+        v.mkPath(canonPath(str()));
     } else
-        mkString(v, s.str(), context);
+        v.mkStringMove(c_str(), context);
 }
 
 

src/libexpr/eval.hh

@@ -44,8 +44,6 @@ struct Env
 };
 
 
-Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
-
 void copyContext(const Value & v, PathSet & context);
 
 
@@ -82,7 +80,8 @@ public:
         sContentAddressed,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
-        sDescription, sSelf, sEpsilon;
+        sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
+        sPrefix;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -93,7 +92,7 @@ public:
        mode. */
     std::optional<PathSet> allowedPaths;
 
-    Value vEmptySet;
+    Bindings emptyBindings;
 
     /* Store used to materialise .drv files. */
     const ref<Store> store;
@@ -133,6 +132,9 @@ private:
     /* Cache used by prim_match(). */
     std::shared_ptr<RegexCache> regexCache;
 
+    /* Allocation cache for GC'd Value objects. */
+    std::shared_ptr<void *> valueAllocCache;
+
 public:
 
     EvalState(
@@ -178,8 +180,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(std::string_view s, const Path & basePath);
+    Expr * parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string s, const Path & basePath);
 
     Expr * parseStdin();
 
@@ -307,7 +309,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, FileOrigin origin, const Path & path,
+    Expr * parse(char * text, size_t length, FileOrigin origin, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:
@@ -336,12 +338,16 @@ public:
     Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
-    Value * allocAttr(Value & vAttrs, const std::string & name);
+    Value * allocAttr(Value & vAttrs, std::string_view name);
 
     Bindings * allocBindings(size_t capacity);
 
+    BindingsBuilder buildBindings(size_t capacity)
+    {
+        return BindingsBuilder(*this, allocBindings(capacity));
+    }
+
     void mkList(Value & v, size_t length);
-    void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, ptr<Pos> pos);
 
@@ -350,7 +356,10 @@ public:
     /* Print statistics. */
     void printStats();
 
-    void realiseContext(const PathSet & context);
+    /* Realise the given context, and return a mapping from the placeholders
+     * used to construct the associated value to their final store path
+     */
+    [[nodiscard]] StringMap realiseContext(const PathSet & context);
 
 private:
 
@@ -391,6 +400,9 @@ private:
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
     friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
+    friend struct Value;
 };
 
 

src/libexpr/flake/config.cc

@@ -38,11 +38,11 @@ void ConfigFile::apply()
 
         // FIXME: Move into libutil/config.cc.
         std::string valueS;
-        if (auto s = std::get_if<std::string>(&value))
+        if (auto* s = std::get_if<std::string>(&value))
             valueS = *s;
-        else if (auto n = std::get_if<int64_t>(&value))
-            valueS = fmt("%d", n);
-        else if (auto b = std::get_if<Explicit<bool>>(&value))
+        else if (auto* n = std::get_if<int64_t>(&value))
+            valueS = fmt("%d", *n);
+        else if (auto* b = std::get_if<Explicit<bool>>(&value))
             valueS = b->t ? "true" : "false";
         else if (auto ss = std::get_if<std::vector<std::string>>(&value))
             valueS = concatStringsSep(" ", *ss); // FIXME: evil

src/libexpr/flake/flake.cc

@@ -155,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         if (!attrs.empty())
             throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
         if (url)
-            input.ref = parseFlakeRef(*url, baseDir, true);
+            input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake);
     }
 
     if (!input.follows && !input.ref)
@@ -194,8 +194,8 @@ static Flake getFlake(
         state, originalRef, allowLookup, flakeCache);
 
     // Guard against symlink attacks.
-    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir);
-    auto flakeFile = canonPath(flakeDir + "/flake.nix");
+    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true);
+    auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
             lockedRef, state.store->printStorePath(sourceInfo.storePath));
@@ -251,10 +251,14 @@ static Flake getFlake(
             forceTrivialValue(state, *setting.value, *setting.pos);
             if (setting.value->type() == nString)
                 flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
+            else if (setting.value->type() == nPath) {
+                PathSet emptyContext = {};
+                flake.config.settings.insert({setting.name, state.coerceToString(*setting.pos, *setting.value, emptyContext, false, true, true)});
+            }
             else if (setting.value->type() == nInt)
                 flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
             else if (setting.value->type() == nBool)
-                flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
+                flake.config.settings.insert({setting.name, Explicit<bool> { state.forceBool(*setting.value, *setting.pos) }});
             else if (setting.value->type() == nList) {
                 std::vector<std::string> ss;
                 for (auto elem : setting.value->listItems()) {
@@ -344,7 +348,8 @@ LockedFlake lockFlake(
             const InputPath & inputPathPrefix,
             std::shared_ptr<const Node> oldNode,
             const LockParent & parent,
-            const Path & parentPath)>
+            const Path & parentPath,
+            bool trustLock)>
             computeLocks;
 
         computeLocks = [&](
@@ -353,7 +358,8 @@ LockedFlake lockFlake(
             const InputPath & inputPathPrefix,
             std::shared_ptr<const Node> oldNode,
             const LockParent & parent,
-            const Path & parentPath)
+            const Path & parentPath,
+            bool trustLock)
         {
             debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
@@ -464,14 +470,20 @@ LockedFlake lockFlake(
                                         .isFlake = (*lockedNode)->isFlake,
                                     });
                                 } else if (auto follows = std::get_if<1>(&i.second)) {
-                                    auto o = input.overrides.find(i.first);
-                                    // If the override disappeared, we have to refetch the flake,
-                                    // since some of the inputs may not be present in the lockfile.
-                                    if (o == input.overrides.end()) {
-                                        mustRefetch = true;
-                                        // There's no point populating the rest of the fake inputs,
-                                        // since we'll refetch the flake anyways.
-                                        break;
+                                    if (! trustLock) {
+                                        // It is possible that the flake has changed,
+                                        // so we must confirm all the follows that are in the lockfile are also in the flake.
+                                        auto overridePath(inputPath);
+                                        overridePath.push_back(i.first);
+                                        auto o = overrides.find(overridePath);
+                                        // If the override disappeared, we have to refetch the flake,
+                                        // since some of the inputs may not be present in the lockfile.
+                                        if (o == overrides.end()) {
+                                            mustRefetch = true;
+                                            // There's no point populating the rest of the fake inputs,
+                                            // since we'll refetch the flake anyways.
+                                            break;
+                                        }
                                     }
                                     fakeInputs.emplace(i.first, FlakeInput {
                                         .follows = *follows,
@@ -480,11 +492,16 @@ LockedFlake lockFlake(
                             }
                         }
 
+                        LockParent newParent {
+                            .path = inputPath,
+                            .absolute = true
+                        };
+
                         computeLocks(
                             mustRefetch
                             ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs
                             : fakeInputs,
-                            childNode, inputPath, oldLock, parent, parentPath);
+                            childNode, inputPath, oldLock, newParent, parentPath, !mustRefetch);
 
                     } else {
                         /* We need to create a new lock file entry. So fetch
@@ -541,7 +558,7 @@ LockedFlake lockFlake(
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
                                 : LockFile::read(
                                     inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root,
-                                newParent, localPath);
+                                newParent, localPath, false);
                         }
 
                         else {
@@ -565,11 +582,11 @@ LockedFlake lockFlake(
         };
 
         // Bring in the current ref for relative path resolution if we have it
-        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir);
+        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true);
 
         computeLocks(
             flake.inputs, newLockFile.root, {},
-            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath);
+            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath, false);
 
         for (auto & i : lockFlags.inputOverrides)
             if (!overridesUsed.count(i.first))
@@ -616,12 +633,24 @@ LockedFlake lockFlake(
 
                         newLockFile.write(path);
 
+                        std::optional<std::string> commitMessage = std::nullopt;
+                        if (lockFlags.commitLockFile) {
+                            std::string cm;
+
+                            cm = settings.commitLockFileSummary.get();
+
+                            if (cm == "") {
+                                cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
+                            }
+
+                            cm += "\n\nFlake lock file updates:\n\n";
+                            cm += filterANSIEscapes(diff, true);
+                            commitMessage = cm;
+                        }
+
                         topRef.input.markChangedFile(
                             (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
-                            lockFlags.commitLockFile
-                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s",
-                                    relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true)))
-                            : std::nullopt);
+                            commitMessage);
 
                         /* Rewriting the lockfile changed the top-level
                            repo, so we should re-read it. FIXME: we could
@@ -669,7 +698,7 @@ void callFlake(EvalState & state,
     auto vTmp1 = state.allocValue();
     auto vTmp2 = state.allocValue();
 
-    mkString(*vLocks, lockedFlake.lockFile.to_string());
+    vLocks->mkString(lockedFlake.lockFile.to_string());
 
     emitTreeAttrs(
         state,
@@ -679,7 +708,7 @@ void callFlake(EvalState & state,
         false,
         lockedFlake.flake.forceDirty);
 
-    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+    vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
 
     if (!state.vCallFlake) {
         state.vCallFlake = allocRootValue(state.allocValue());

src/libexpr/flake/flakeref.cc

@@ -48,9 +48,12 @@ FlakeRef FlakeRef::resolve(ref<Store> store) const
 }
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
     if (fragment != "")
         throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
     return flakeRef;
@@ -67,7 +70,10 @@ std::optional<FlakeRef> maybeParseFlakeRef(
 }
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
     using namespace fetchers;
 
@@ -112,46 +118,71 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
                to 'baseDir'). If so, search upward to the root of the
                repo (i.e. the directory containing .git). */
 
-            path = absPath(path, baseDir, true);
+            path = absPath(path, baseDir);
 
-            if (!S_ISDIR(lstat(path).st_mode))
-                throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+            if (isFlake) {
 
-            if (!allowMissing && !pathExists(path + "/flake.nix"))
-                throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+                if (!allowMissing && !pathExists(path + "/flake.nix")){
+                    notice("path '%s' does not contain a 'flake.nix', searching up",path);
 
-            auto flakeRoot = path;
-            std::string subdir;
-
-            while (flakeRoot != "/") {
-                if (pathExists(flakeRoot + "/.git")) {
-                    auto base = std::string("git+file://") + flakeRoot;
-
-                    auto parsedURL = ParsedURL{
-                        .url = base, // FIXME
-                        .base = base,
-                        .scheme = "git+file",
-                        .authority = "",
-                        .path = flakeRoot,
-                        .query = decodeQuery(match[2]),
-                    };
-
-                    if (subdir != "") {
-                        if (parsedURL.query.count("dir"))
-                            throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
-                        parsedURL.query.insert_or_assign("dir", subdir);
+                    // Save device to detect filesystem boundary
+                    dev_t device = lstat(path).st_dev;
+                    bool found = false;
+                    while (path != "/") {
+                        if (pathExists(path + "/flake.nix")) {
+                            found = true;
+                            break;
+                        } else if (pathExists(path + "/.git"))
+                            throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
+                        else {
+                            if (lstat(path).st_dev != device)
+                                throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
+                        }
+                        path = dirOf(path);
                     }
-
-                    if (pathExists(flakeRoot + "/.git/shallow"))
-                        parsedURL.query.insert_or_assign("shallow", "1");
-
-                    return std::make_pair(
-                        FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
-                        fragment);
+                    if (!found)
+                        throw BadURL("could not find a flake.nix file");
                 }
 
-                subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-                flakeRoot = dirOf(flakeRoot);
+                if (!S_ISDIR(lstat(path).st_mode))
+                    throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+
+                if (!allowMissing && !pathExists(path + "/flake.nix"))
+                    throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+
+                auto flakeRoot = path;
+                std::string subdir;
+
+                while (flakeRoot != "/") {
+                    if (pathExists(flakeRoot + "/.git")) {
+                        auto base = std::string("git+file://") + flakeRoot;
+
+                        auto parsedURL = ParsedURL{
+                            .url = base, // FIXME
+                            .base = base,
+                            .scheme = "git+file",
+                            .authority = "",
+                            .path = flakeRoot,
+                            .query = decodeQuery(match[2]),
+                        };
+
+                        if (subdir != "") {
+                            if (parsedURL.query.count("dir"))
+                                throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                            parsedURL.query.insert_or_assign("dir", subdir);
+                        }
+
+                        if (pathExists(flakeRoot + "/.git/shallow"))
+                            parsedURL.query.insert_or_assign("shallow", "1");
+
+                        return std::make_pair(
+                            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                            fragment);
+                    }
+
+                    subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+                    flakeRoot = dirOf(flakeRoot);
+                }
             }
 
         } else {

src/libexpr/flake/flakeref.hh

@@ -62,13 +62,19 @@ struct FlakeRef
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<FlakeRef> maybeParseFlake(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});

src/libexpr/get-drvs.cc

@@ -254,15 +254,14 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 void DrvInfo::setMeta(const string & name, Value * v)
 {
     getMeta();
-    Bindings * old = meta;
-    meta = state->allocBindings(1 + (old ? old->size() : 0));
+    auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0));
     Symbol sym = state->symbols.create(name);
-    if (old)
-        for (auto i : *old)
+    if (meta)
+        for (auto i : *meta)
             if (i.name != sym)
-                meta->push_back(i);
-    if (v) meta->push_back(Attr(sym, v));
-    meta->sort();
+                attrs.insert(i);
+    if (v) attrs.insert(sym, v);
+    meta = attrs.finish();
 }
 
 

src/libexpr/json-to-value.cc

@@ -37,10 +37,10 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueMap attrs;
         std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
-            state.mkAttrs(v, attrs.size());
+            auto attrs2 = state.buildBindings(attrs.size());
             for (auto & i : attrs)
-                v.attrs->push_back(Attr(i.first, i.second));
+                attrs2.insert(i.first, i.second);
+            parent->value(state).mkAttrs(attrs2.alreadySorted());
             return std::move(parent);
         }
         void add() override { v = nullptr; }
@@ -76,45 +76,51 @@ class JSONSax : nlohmann::json_sax<json> {
     EvalState & state;
     std::unique_ptr<JSONState> rs;
 
-    template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
-    {
-        f(rs->value(state), args...);
-        rs->add();
-        return true;
-    }
-
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
 
     bool null()
     {
-        return handle_value(mkNull);
+        rs->value(state).mkNull();
+        rs->add();
+        return true;
     }
 
     bool boolean(bool val)
     {
-        return handle_value(mkBool, val);
+        rs->value(state).mkBool(val);
+        rs->add();
+        return true;
     }
 
     bool number_integer(number_integer_t val)
     {
-        return handle_value(mkInt, val);
+        rs->value(state).mkInt(val);
+        rs->add();
+        return true;
     }
 
     bool number_unsigned(number_unsigned_t val)
     {
-        return handle_value(mkInt, val);
+        rs->value(state).mkInt(val);
+        rs->add();
+        return true;
     }
 
     bool number_float(number_float_t val, const string_t & s)
     {
-        return handle_value(mkFloat, val);
+        rs->value(state).mkFloat(val);
+        rs->add();
+        return true;
     }
 
     bool string(string_t & val)
     {
-        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
+        rs->value(state).mkString(val);
+        rs->add();
+        return true;
     }
+
 #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
     bool binary(binary_t&)
     {

src/libexpr/lexer.l

@@ -64,29 +64,32 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
-// FIXME: optimize
-static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
+// we make use of the fact that the parser receives a private copy of the input
+// string and can munge around in it.
+static Expr * unescapeStr(SymbolTable & symbols, char * s, size_t length)
 {
-    string t;
-    t.reserve(length);
+    char * result = s;
+    char * t = s;
     char c;
+    // the input string is terminated with *two* NULs, so we can safely take
+    // *one* character after the one being checked against.
     while ((c = *s++)) {
         if (c == '\\') {
-            assert(*s);
             c = *s++;
-            if (c == 'n') t += '\n';
-            else if (c == 'r') t += '\r';
-            else if (c == 't') t += '\t';
-            else t += c;
+            if (c == 'n') *t = '\n';
+            else if (c == 'r') *t = '\r';
+            else if (c == 't') *t = '\t';
+            else *t = c;
         }
         else if (c == '\r') {
             /* Normalise CR and CR/LF into LF. */
-            t += '\n';
+            *t = '\n';
             if (*s == '\n') s++; /* cr/lf */
         }
-        else t += c;
+        else *t = c;
+        t++;
     }
-    return new ExprString(symbols.create(t));
+    return new ExprString(symbols.create({result, size_t(t - result)}));
 }
 
 
@@ -139,7 +142,7 @@ or          { return OR_KW; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
 
-{ID}        { yylval->id = strdup(yytext); return ID; }
+{ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
@@ -221,14 +224,14 @@ or          { return OR_KW; }
 <PATH_START>{PATH_SEG} {
   POP_STATE();
   PUSH_STATE(INPATH_SLASH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return PATH;
 }
 
 <PATH_START>{HPATH_START} {
   POP_STATE();
   PUSH_STATE(INPATH_SLASH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return HPATH;
 }
 
@@ -237,7 +240,7 @@ or          { return OR_KW; }
     PUSH_STATE(INPATH_SLASH);
   else
     PUSH_STATE(INPATH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return PATH;
 }
 {HPATH} {
@@ -245,7 +248,7 @@ or          { return OR_KW; }
     PUSH_STATE(INPATH_SLASH);
   else
     PUSH_STATE(INPATH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return HPATH;
 }
 
@@ -280,8 +283,8 @@ or          { return OR_KW; }
   throw ParseError("path has a trailing slash");
 }
 
-{SPATH}     { yylval->path = strdup(yytext); return SPATH; }
-{URI}       { yylval->uri = strdup(yytext); return URI; }
+{SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
+{URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
 
 [ \t\r\n]+    /* eat up whitespace */
 \#[^\r\n]*    /* single-line comments */

src/libexpr/nixexpr.cc

@@ -191,7 +191,7 @@ void ExprConcatStrings::show(std::ostream & str) const
     str << "(";
     for (auto & i : *es) {
         if (first) first = false; else str << " + ";
-        str << *i;
+        str << *i.second;
     }
     str << ")";
 }
@@ -439,7 +439,7 @@ void ExprOpNot::bindVars(const StaticEnv & env)
 void ExprConcatStrings::bindVars(const StaticEnv & env)
 {
     for (auto & i : *es)
-        i->bindVars(env);
+        i.second->bindVars(env);
 }
 
 void ExprPos::bindVars(const StaticEnv & env)
@@ -473,7 +473,7 @@ string ExprLambda::showNamePos() const
 size_t SymbolTable::totalSize() const
 {
     size_t n = 0;
-    for (auto & i : symbols)
+    for (auto & i : store)
         n += i.size();
     return n;
 }

src/libexpr/nixexpr.hh

@@ -94,7 +94,7 @@ struct ExprInt : Expr
 {
     NixInt n;
     Value v;
-    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
+    ExprInt(NixInt n) : n(n) { v.mkInt(n); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -103,7 +103,7 @@ struct ExprFloat : Expr
 {
     NixFloat nf;
     Value v;
-    ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
+    ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -112,7 +112,7 @@ struct ExprString : Expr
 {
     Symbol s;
     Value v;
-    ExprString(const Symbol & s) : s(s) { mkString(v, s); };
+    ExprString(const Symbol & s) : s(s) { v.mkString(s); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -332,8 +332,8 @@ struct ExprConcatStrings : Expr
 {
     Pos pos;
     bool forceString;
-    vector<Expr *> * es;
-    ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es)
+    vector<std::pair<Pos, Expr *> > * es;
+    ExprConcatStrings(const Pos & pos, bool forceString, vector<std::pair<Pos, Expr *> > * es)
         : pos(pos), forceString(forceString), es(es) { };
     COMMON_METHODS
 };
@@ -368,6 +368,13 @@ struct StaticEnv
             [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
     }
 
+    void deduplicate()
+    {
+        const auto last = std::unique(vars.begin(), vars.end(),
+            [] (const Vars::value_type & a, const Vars::value_type & b) { return a.first == b.first; });
+        vars.erase(last, vars.end());
+    }
+
     Vars::const_iterator find(const Symbol & name) const
     {
         Vars::value_type key(name, 0);

src/libexpr/parser.y

@@ -152,7 +152,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 }
 
 
-static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Expr *> & es)
+static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<std::pair<Pos, Expr *> > & es)
 {
     if (es.empty()) return new ExprString(symbols.create(""));
 
@@ -162,7 +162,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
     size_t minIndent = 1000000;
     size_t curIndent = 0;
-    for (auto & i : es) {
+    for (auto & [i_pos, i] : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
             /* Anti-quotations end the current start-of-line whitespace. */
@@ -192,12 +192,12 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     }
 
     /* Strip spaces from each line. */
-    vector<Expr *> * es2 = new vector<Expr *>;
+    vector<std::pair<Pos, Expr *> > * es2 = new vector<std::pair<Pos, Expr *> >;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
-    for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
-        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+    for (vector<std::pair<Pos, Expr *> >::iterator i = es.begin(); i != es.end(); ++i, --n) {
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(i->second);
         if (!e) {
             atStartOfLine = false;
             curDropped = 0;
@@ -234,11 +234,11 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
                 s2 = string(s2, 0, p + 1);
         }
 
-        es2->push_back(new ExprString(symbols.create(s2)));
+        es2->emplace_back(i->first, new ExprString(symbols.create(s2)));
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2);
+    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2);
 }
 
 
@@ -273,11 +273,18 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
   nix::Formal * formal;
   nix::NixInt n;
   nix::NixFloat nf;
-  const char * id; // !!! -> Symbol
-  char * path;
-  char * uri;
+  // using C a struct allows us to avoid having to define the special
+  // members that using string_view here would implicitly delete.
+  struct StringToken {
+    const char * p;
+    size_t l;
+    operator std::string_view() const { return {p, l}; }
+  };
+  StringToken id; // !!! -> Symbol
+  StringToken path;
+  StringToken uri;
   std::vector<nix::AttrName> * attrNames;
-  std::vector<nix::Expr *> * string_parts;
+  std::vector<std::pair<nix::Pos, nix::Expr *> > * string_parts;
 }
 
 %type <e> start expr expr_function expr_if expr_op
@@ -364,7 +371,7 @@ expr_op
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); }
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
+    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
   | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
   | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
   | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
@@ -397,7 +404,8 @@ expr_select
 
 expr_simple
   : ID {
-      if (strcmp($1, "__curPos") == 0)
+      std::string_view s = "__curPos";
+      if (strncmp($1.p, s.data(), s.size()) == 0)
           $$ = new ExprPos(CUR_POS);
       else
           $$ = new ExprVar(CUR_POS, data->symbols.create($1));
@@ -410,11 +418,11 @@ expr_simple
   }
   | path_start PATH_END { $$ = $1; }
   | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), $1);
+      $2->insert($2->begin(), {makeCurPos(@1, data), $1});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
-      string path($1 + 1, strlen($1) - 2);
+      string path($1.p + 1, $1.l - 2);
       $$ = new ExprCall(CUR_POS,
           new ExprVar(data->symbols.create("__findFile")),
           {new ExprVar(data->symbols.create("__nixPath")),
@@ -448,34 +456,34 @@ string_parts
   ;
 
 string_parts_interpolated
-  : string_parts_interpolated STR { $$ = $1; $1->push_back($2); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
-  | DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($2); }
+  : string_parts_interpolated STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
+  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | DOLLAR_CURLY expr '}' { $$ = new vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); }
   | STR DOLLAR_CURLY expr '}' {
-      $$ = new vector<Expr *>;
-      $$->push_back($1);
-      $$->push_back($3);
+      $$ = new vector<std::pair<Pos, Expr *> >;
+      $$->emplace_back(makeCurPos(@1, data), $1);
+      $$->emplace_back(makeCurPos(@2, data), $3);
     }
   ;
 
 path_start
   : PATH {
-    Path path(absPath($1, data->basePath));
+    Path path(absPath({$1.p, $1.l}, data->basePath));
     /* add back in the trailing '/' to the first segment */
-    if ($1[strlen($1)-1] == '/' && strlen($1) > 1)
+    if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
     $$ = new ExprPath(path);
   }
   | HPATH {
-    Path path(getHome() + string($1 + 1));
+    Path path(getHome() + string($1.p + 1, $1.l - 1));
     $$ = new ExprPath(path);
   }
   ;
 
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
-  | { $$ = new vector<Expr *>; }
+  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | { $$ = new vector<std::pair<Pos, Expr *> >; }
   ;
 
 binds
@@ -543,7 +551,7 @@ attrpath
 
 attr
   : ID { $$ = $1; }
-  | OR_KW { $$ = "or"; }
+  | OR_KW { $$ = {"or", 2}; }
   ;
 
 string_attr
@@ -589,7 +597,7 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text, FileOrigin origin,
+Expr * EvalState::parse(char * text, size_t length, FileOrigin origin,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
@@ -609,7 +617,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
     data.basePath = basePath;
 
     yylex_init(&scanner);
-    yy_scan_string(text, scanner);
+    yy_scan_buffer(text, length, scanner);
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
@@ -655,26 +663,33 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
+    auto buffer = readFile(path);
+    // readFile should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), foFile, path, dirOf(path), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), foString, "", basePath, staticEnv);
+    s.append("\0\0", 2);
+    return parse(s.data(), s.size(), foString, "", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string s, const Path & basePath)
 {
-    return parseExprFromString(s, basePath, staticBaseEnv);
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
 }
 
 
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
+    auto buffer = drainFD(0);
+    // drainFD should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 

src/libexpr/primops/context.cc

@@ -7,8 +7,7 @@ namespace nix {
 static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    mkString(v, s, PathSet());
+    v.mkString(state.coerceToString(pos, *args[0], context));
 }
 
 static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
@@ -18,7 +17,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
 {
     PathSet context;
     state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
+    v.mkBool(!context.empty());
 }
 
 static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
@@ -39,7 +38,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po
     for (auto & p : context)
         context2.insert(p.at(0) == '=' ? string(p, 1) : p);
 
-    mkString(v, s, context2);
+    v.mkString(s, context2);
 }
 
 static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
@@ -103,27 +102,26 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
         }
     }
 
-    state.mkAttrs(v, contextInfos.size());
+    auto attrs = state.buildBindings(contextInfos.size());
 
     auto sPath = state.symbols.create("path");
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (const auto & info : contextInfos) {
-        auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
-        state.mkAttrs(infoVal, 3);
+        auto infoAttrs = state.buildBindings(3);
         if (info.second.path)
-            mkBool(*state.allocAttr(infoVal, sPath), true);
+            infoAttrs.alloc(sPath).mkBool(true);
         if (info.second.allOutputs)
-            mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
+            infoAttrs.alloc(sAllOutputs).mkBool(true);
         if (!info.second.outputs.empty()) {
-            auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
+            auto & outputsVal = infoAttrs.alloc(state.sOutputs);
             state.mkList(outputsVal, info.second.outputs.size());
-            size_t i = 0;
-            for (const auto & output : info.second.outputs)
-                mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
+            for (const auto & [i, output] : enumerate(info.second.outputs))
+                (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
         }
-        infoVal.attrs->sort();
+        attrs.alloc(info.first).mkAttrs(infoAttrs);
     }
-    v.attrs->sort();
+
+    v.mkAttrs(attrs);
 }
 
 static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
@@ -187,7 +185,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         }
     }
 
-    mkString(v, orig, context);
+    v.mkString(orig, context);
 }
 
 static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext);

src/libexpr/primops/fetchMercurial.cc

@@ -70,19 +70,19 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     // FIXME: use name
     auto [tree, input2] = input.fetch(state.store);
 
-    state.mkAttrs(v, 8);
+    auto attrs2 = state.buildBindings(8);
     auto storePath = state.store->printStorePath(tree.storePath);
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    attrs2.alloc(state.sOutPath).mkString(storePath, {storePath});
     if (input2.getRef())
-        mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
+        attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
     auto rev2 = input2.getRev().value_or(Hash(htSHA1));
-    mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
-    mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
+    attrs2.alloc("rev").mkString(rev2.gitRev());
+    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
     if (auto revCount = input2.getRevCount())
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
-    v.attrs->sort();
+        attrs2.alloc("revCount").mkInt(*revCount);
+    v.mkAttrs(attrs2);
 
     state.allowPath(tree.storePath);
 }

src/libexpr/primops/fetchTree.cc

@@ -21,49 +21,48 @@ void emitTreeAttrs(
 {
     assert(input.isImmutable());
 
-    state.mkAttrs(v, 8);
+    auto attrs = state.buildBindings(8);
 
     auto storePath = state.store->printStorePath(tree.storePath);
 
-    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
+    attrs.alloc(state.sOutPath).mkString(storePath, {storePath});
 
     // FIXME: support arbitrary input attributes.
 
     auto narHash = input.getNarHash();
     assert(narHash);
-    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
-        narHash->to_string(SRI, true));
+    attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
 
     if (input.getType() == "git")
-        mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
+        attrs.alloc("submodules").mkBool(
             fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
 
     if (!forceDirty) {
 
         if (auto rev = input.getRev()) {
-            mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
-            mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
+            attrs.alloc("rev").mkString(rev->gitRev());
+            attrs.alloc("shortRev").mkString(rev->gitShortRev());
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
             auto emptyHash = Hash(htSHA1);
-            mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
-            mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
+            attrs.alloc("rev").mkString(emptyHash.gitRev());
+            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
         if (auto revCount = input.getRevCount())
-            mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
+            attrs.alloc("revCount").mkInt(*revCount);
         else if (emptyRevFallback)
-            mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
+            attrs.alloc("revCount").mkInt(0);
 
     }
 
     if (auto lastModified = input.getLastModified()) {
-        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
-        mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
+        attrs.alloc("lastModified").mkInt(*lastModified);
+        attrs.alloc("lastModifiedDate").mkString(
             fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
     }
 
-    v.attrs->sort();
+    v.mkAttrs(attrs);
 }
 
 std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
@@ -248,7 +247,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     state.allowPath(storePath);
 
     auto path = state.store->printStorePath(storePath);
-    mkString(v, path, PathSet({path}));
+    v.mkString(path, PathSet({path}));
 }
 
 static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v)

src/libexpr/primops/fromTOML.cc

@@ -1,86 +1,76 @@
 #include "primops.hh"
 #include "eval-inline.hh"
 
-#include "../../cpptoml/cpptoml.h"
+#include "../../toml11/toml.hpp"
 
 namespace nix {
 
-static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & val)
 {
-    using namespace cpptoml;
-
     auto toml = state.forceStringNoCtx(*args[0], pos);
 
     std::istringstream tomlStream(toml);
 
-    std::function<void(Value &, std::shared_ptr<base>)> visit;
+    std::function<void(Value &, toml::value)> visit;
 
-    visit = [&](Value & v, std::shared_ptr<base> t) {
+    visit = [&](Value & v, toml::value t) {
 
-        if (auto t2 = t->as_table()) {
+        switch(t.type())
+        {
+            case toml::value_t::table:
+                {
+                    auto table = toml::get<toml::table>(t);
 
-            size_t size = 0;
-            for (auto & i : *t2) { (void) i; size++; }
+                    size_t size = 0;
+                    for (auto & i : table) { (void) i; size++; }
 
-            state.mkAttrs(v, size);
+                    auto attrs = state.buildBindings(size);
 
-            for (auto & i : *t2) {
-                auto & v2 = *state.allocAttr(v, state.symbols.create(i.first));
+                    for(auto & elem : table)
+                        visit(attrs.alloc(elem.first), elem.second);
 
-                if (auto i2 = i.second->as_table_array()) {
-                    size_t size2 = i2->get().size();
-                    state.mkList(v2, size2);
-                    for (size_t j = 0; j < size2; ++j)
-                        visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]);
+                    v.mkAttrs(attrs);
                 }
-                else
-                    visit(v2, i.second);
-            }
+                break;;
+            case toml::value_t::array:
+                {
+                    auto array = toml::get<std::vector<toml::value>>(t);
+
+                    size_t size = array.size();
+                    state.mkList(v, size);
+                    for (size_t i = 0; i < size; ++i)
+                        visit(*(v.listElems()[i] = state.allocValue()), array[i]);
+                }
+                break;;
+            case toml::value_t::boolean:
+                v.mkBool(toml::get<bool>(t));
+                break;;
+            case toml::value_t::integer:
+                v.mkInt(toml::get<int64_t>(t));
+                break;;
+            case toml::value_t::floating:
+                v.mkFloat(toml::get<NixFloat>(t));
+                break;;
+            case toml::value_t::string:
+                v.mkString(toml::get<std::string>(t));
+                break;;
+            case toml::value_t::local_datetime:
+            case toml::value_t::offset_datetime:
+            case toml::value_t::local_date:
+            case toml::value_t::local_time:
+                // We fail since Nix doesn't have date and time types
+                throw std::runtime_error("Dates and times are not supported");
+                break;;
+            case toml::value_t::empty:
+                v.mkNull();
+                break;;
 
-            v.attrs->sort();
         }
-
-        else if (auto t2 = t->as_array()) {
-            size_t size = t2->get().size();
-
-            state.mkList(v, size);
-
-            for (size_t i = 0; i < size; ++i)
-                visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]);
-        }
-
-        // Handle cases like 'a = [[{ a = true }]]', which IMHO should be
-        // parsed as a array containing an array containing a table,
-        // but instead are parsed as an array containing a table array
-        // containing a table.
-        else if (auto t2 = t->as_table_array()) {
-            size_t size = t2->get().size();
-
-            state.mkList(v, size);
-
-            for (size_t j = 0; j < size; ++j)
-                visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]);
-        }
-
-        else if (t->is_value()) {
-            if (auto val = t->as<int64_t>())
-                mkInt(v, val->get());
-            else if (auto val = t->as<NixFloat>())
-                mkFloat(v, val->get());
-            else if (auto val = t->as<bool>())
-                mkBool(v, val->get());
-            else if (auto val = t->as<std::string>())
-                mkString(v, val->get());
-            else
-                throw EvalError("unsupported value type in TOML");
-        }
-
-        else abort();
     };
 
     try {
-        visit(v, parser(tomlStream).parse());
-    } catch (std::runtime_error & e) {
+        visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */));
+    } catch (std::exception & e) { // TODO: toml::syntax_error
         throw EvalError({
             .msg = hintfmt("while parsing a TOML string: %s", e.what()),
             .errPos = pos

src/libexpr/symbol-table.hh

@@ -1,7 +1,8 @@
 #pragma once
 
+#include <list>
 #include <map>
-#include <unordered_set>
+#include <unordered_map>
 
 #include "types.hh"
 
@@ -70,15 +71,21 @@ public:
 class SymbolTable
 {
 private:
-    typedef std::unordered_set<string> Symbols;
-    Symbols symbols;
+    std::unordered_map<std::string_view, Symbol> symbols;
+    std::list<string> store;
 
 public:
     Symbol create(std::string_view s)
     {
-        // FIXME: avoid allocation if 's' already exists in the symbol table.
-        std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s));
-        return Symbol(&*res.first);
+        // Most symbols are looked up more than once, so we trade off insertion performance
+        // for lookup performance.
+        // TODO: could probably be done more efficiently with transparent Hash and Equals
+        // on the original implementation using unordered_set
+        auto it = symbols.find(s);
+        if (it != symbols.end()) return it->second;
+
+        const string & rawSym = store.emplace_back(s);
+        return symbols.emplace(rawSym, Symbol(&rawSym)).first->second;
     }
 
     size_t size() const
@@ -91,7 +98,7 @@ public:
     template<typename T>
     void dump(T callback)
     {
-        for (auto & s : symbols)
+        for (auto & s : store)
             callback(s);
     }
 };

src/libexpr/value.hh

@@ -10,6 +10,8 @@
 
 namespace nix {
 
+class BindingsBuilder;
+
 
 typedef enum {
     tInt = 1,
@@ -235,6 +237,17 @@ public:
         string.context = context;
     }
 
+    void mkString(std::string_view s);
+
+    void mkString(std::string_view s, const PathSet & context);
+
+    void mkStringMove(const char * s, const PathSet & context);
+
+    inline void mkString(const Symbol & s)
+    {
+        mkString(((const std::string &) s).c_str());
+    }
+
     inline void mkPath(const char * s)
     {
         clearValue();
@@ -242,6 +255,8 @@ public:
         path = s;
     }
 
+    void mkPath(std::string_view s);
+
     inline void mkNull()
     {
         clearValue();
@@ -255,6 +270,8 @@ public:
         attrs = a;
     }
 
+    Value & mkAttrs(BindingsBuilder & bindings);
+
     inline void mkList(size_t size)
     {
         clearValue();
@@ -383,51 +400,14 @@ public:
 };
 
 
-
-// TODO: Remove these static functions, replace call sites with v.mk* instead
-static inline void mkInt(Value & v, NixInt n)
-{
-    v.mkInt(n);
-}
-
-static inline void mkFloat(Value & v, NixFloat n)
-{
-    v.mkFloat(n);
-}
-
-static inline void mkBool(Value & v, bool b)
-{
-    v.mkBool(b);
-}
-
-static inline void mkNull(Value & v)
-{
-    v.mkNull();
-}
-
-static inline void mkApp(Value & v, Value & left, Value & right)
-{
-    v.mkApp(&left, &right);
-}
-
-static inline void mkString(Value & v, const Symbol & s)
-{
-    v.mkString(((const string &) s).c_str());
-}
-
-
-void mkString(Value & v, const char * s);
-
-
-void mkPath(Value & v, const char * s);
-
-
 #if HAVE_BOEHMGC
 typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector;
 typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap;
+typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector> > > ValueVectorMap;
 #else
 typedef std::vector<Value *> ValueVector;
 typedef std::map<Symbol, Value *> ValueMap;
+typedef std::map<Symbol, ValueVector> ValueVectorMap;
 #endif
 
 

src/libfetchers/path.cc

@@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme
             // for security, ensure that if the parent is a store path, it's inside it
             if (store->isInStore(parent)) {
                 auto storePath = store->printStorePath(store->toStorePath(parent).first);
-                if (!isInDir(absPath, storePath))
+                if (!isDirOrInDir(absPath, storePath))
                     throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
             }
         } else

src/libfetchers/tarball.cc

@@ -67,18 +67,18 @@ DownloadFileResult downloadFile(
         storePath = std::move(cached->storePath);
     } else {
         StringSink sink;
-        dumpString(*res.data, sink);
-        auto hash = hashString(htSHA256, *res.data);
+        dumpString(res.data, sink);
+        auto hash = hashString(htSHA256, res.data);
         ValidPathInfo info {
             store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
-            hashString(htSHA256, *sink.s),
+            hashString(htSHA256, sink.s),
         };
-        info.narSize = sink.s->size();
+        info.narSize = sink.s.size();
         info.ca = FixedOutputHash {
             .method = FileIngestionMethod::Flat,
             .hash = hash,
         };
-        auto source = StringSource { *sink.s };
+        auto source = StringSource(sink.s);
         store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
     }
@@ -176,6 +176,7 @@ struct TarballInputScheme : InputScheme
 
         if (!hasSuffix(url.path, ".zip")
             && !hasSuffix(url.path, ".tar")
+            && !hasSuffix(url.path, ".tgz")
             && !hasSuffix(url.path, ".tar.gz")
             && !hasSuffix(url.path, ".tar.xz")
             && !hasSuffix(url.path, ".tar.bz2")

src/libmain/progress-bar.cc

@@ -11,7 +11,7 @@
 
 namespace nix {
 
-static std::string getS(const std::vector<Logger::Field> & fields, size_t n)
+static std::string_view getS(const std::vector<Logger::Field> & fields, size_t n)
 {
     assert(n < fields.size());
     assert(fields[n].type == Logger::Field::tString);

src/libmain/shared.cc

@@ -145,10 +145,10 @@ static void preloadNSS() {
          *
          * All other platforms are unaffected.
          */
-        if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) {
-            printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend"));
-        }
-        __nss_configure_lookup ("hosts", "dns");
+        if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW))
+            warn("unable to load nss_dns backend");
+        // FIXME: get hosts entry from nsswitch.conf.
+        __nss_configure_lookup("hosts", "files dns");
 #endif
     });
 }
@@ -427,7 +427,7 @@ RunPager::RunPager()
     });
 
     pid.setKillSignal(SIGINT);
-
+    stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0);
     if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1)
         throw SysError("dupping stdout");
 }
@@ -438,7 +438,7 @@ RunPager::~RunPager()
     try {
         if (pid != -1) {
             std::cout.flush();
-            close(STDOUT_FILENO);
+            dup2(stdout, STDOUT_FILENO);
             pid.wait();
         }
     } catch (...) {

src/libmain/shared.hh

@@ -88,6 +88,7 @@ public:
 
 private:
     Pid pid;
+    int stdout;
 };
 
 extern volatile ::sig_atomic_t blockInt;

src/libstore/binary-cache-store.cc

@@ -31,7 +31,7 @@ BinaryCacheStore::BinaryCacheStore(const Params & params)
 
     StringSink sink;
     sink << narVersionMagic1;
-    narMagic = *sink.s;
+    narMagic = sink.s;
 }
 
 void BinaryCacheStore::init()
@@ -68,7 +68,7 @@ void BinaryCacheStore::upsertFile(const std::string & path,
 }
 
 void BinaryCacheStore::getFile(const std::string & path,
-    Callback<std::shared_ptr<std::string>> callback) noexcept
+    Callback<std::optional<std::string>> callback) noexcept
 {
     try {
         callback(getFile(path));
@@ -77,9 +77,9 @@ void BinaryCacheStore::getFile(const std::string & path,
 
 void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
 {
-    std::promise<std::shared_ptr<std::string>> promise;
+    std::promise<std::optional<std::string>> promise;
     getFile(path,
-        {[&](std::future<std::shared_ptr<std::string>> result) {
+        {[&](std::future<std::optional<std::string>> result) {
             try {
                 promise.set_value(result.get());
             } catch (...) {
@@ -89,15 +89,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
     sink(*promise.get_future().get());
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
+std::optional<std::string> BinaryCacheStore::getFile(const std::string & path)
 {
     StringSink sink;
     try {
         getFile(path, sink);
     } catch (NoSuchBinaryCacheFile &) {
-        return nullptr;
+        return std::nullopt;
     }
-    return sink.s;
+    return std::move(sink.s);
 }
 
 std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath)
@@ -367,11 +367,11 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     getFile(narInfoFile,
-        {[=](std::future<std::shared_ptr<std::string>> fut) {
+        {[=](std::future<std::optional<std::string>> fut) {
             try {
                 auto data = fut.get();
 
-                if (!data) return (*callbackPtr)(nullptr);
+                if (!data) return (*callbackPtr)({});
 
                 stats.narInfoRead++;
 
@@ -429,7 +429,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
 
     StringSink sink;
     dumpString(s, sink);
-    auto source = StringSource { *sink.s };
+    StringSource source(sink.s);
     return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info { path, nar.first };
         info.narSize = nar.second;
@@ -446,11 +446,11 @@ void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
 
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
-    Callback<std::shared_ptr<std::string>> newCallback = {
-        [=](std::future<std::shared_ptr<std::string>> fut) {
+    Callback<std::optional<std::string>> newCallback = {
+        [=](std::future<std::optional<std::string>> fut) {
             try {
                 auto data = fut.get();
-                if (!data) return (*callbackPtr)(nullptr);
+                if (!data) return (*callbackPtr)({});
 
                 auto realisation = Realisation::fromJSON(
                     nlohmann::json::parse(*data), outputInfoFilePath);
@@ -490,7 +490,7 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
     writeNarInfo(narInfo);
 }
 
-std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
+std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path)
 {
     auto drvPath = path;
 
@@ -498,10 +498,10 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
         try {
             auto info = queryPathInfo(path);
             // FIXME: add a "Log" field to .narinfo
-            if (!info->deriver) return nullptr;
+            if (!info->deriver) return std::nullopt;
             drvPath = *info->deriver;
         } catch (InvalidPath &) {
-            return nullptr;
+            return std::nullopt;
         }
     }
 
@@ -512,4 +512,14 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat
     return getFile(logPath);
 }
 
+void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    upsertFile(
+        "log/" + std::string(drvPath.to_string()),
+        (std::string) log, // FIXME: don't copy
+        "text/plain; charset=utf-8");
+}
+
 }

src/libstore/binary-cache-store.hh

@@ -51,6 +51,7 @@ public:
         const std::string & mimeType) = 0;
 
     void upsertFile(const std::string & path,
+        // FIXME: use std::string_view
         std::string && data,
         const std::string & mimeType);
 
@@ -62,10 +63,11 @@ public:
 
     /* Fetch the specified file and call the specified callback with
        the result. A subclass may implement this asynchronously. */
-    virtual void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) noexcept;
+    virtual void getFile(
+        const std::string & path,
+        Callback<std::optional<std::string>> callback) noexcept;
 
-    std::shared_ptr<std::string> getFile(const std::string & path);
+    std::optional<std::string> getFile(const std::string & path);
 
 public:
 
@@ -117,7 +119,9 @@ public:
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 
-    std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
+    std::optional<std::string> getBuildLog(const StorePath & path) override;
+
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
 
 };
 

src/libstore/build/derivation-goal.cc

@@ -17,6 +17,7 @@
 #include <regex>
 #include <queue>
 
+#include <fstream>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -193,7 +194,7 @@ void DerivationGoal::loadDerivation()
     assert(worker.evalStore.isValidPath(drvPath));
 
     /* Get the derivation. */
-    drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
+    drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
 
     haveDerivation();
 }
@@ -277,7 +278,7 @@ void DerivationGoal::outputsSubstitutionTried()
 
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
         done(BuildResult::TransientFailure,
-            fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
+            Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
                 worker.store.printStorePath(drvPath)));
         return;
     }
@@ -464,7 +465,6 @@ void DerivationGoal::inputsRealised()
             Derivation drvResolved { *std::move(attempt) };
 
             auto pathResolved = writeDerivation(worker.store, drvResolved);
-            resolvedDrv = drvResolved;
 
             auto msg = fmt("Resolved derivation: '%s' -> '%s'",
                 worker.store.printStorePath(drvPath),
@@ -475,9 +475,9 @@ void DerivationGoal::inputsRealised()
                        worker.store.printStorePath(pathResolved),
                    });
 
-            auto resolvedGoal = worker.makeDerivationGoal(
+            resolvedDrvGoal = worker.makeDerivationGoal(
                 pathResolved, wantedOutputs, buildMode);
-            addWaitee(resolvedGoal);
+            addWaitee(resolvedDrvGoal);
 
             state = &DerivationGoal::resolvedFinished;
             return;
@@ -655,7 +655,7 @@ void DerivationGoal::tryLocalBuild() {
     throw Error(
         "unable to build with a primary store that isn't a local store; "
         "either pass a different '--store' or enable remote builds."
-        "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+        "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 }
 
 
@@ -938,16 +938,17 @@ void DerivationGoal::buildDone()
 }
 
 void DerivationGoal::resolvedFinished() {
-    assert(resolvedDrv);
+    assert(resolvedDrvGoal);
+    auto resolvedDrv = *resolvedDrvGoal->drv;
 
-    auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
+    auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
 
     StorePathSet outputPaths;
 
     // `wantedOutputs` might be empty, which means “all the outputs”
     auto realWantedOutputs = wantedOutputs;
     if (realWantedOutputs.empty())
-        realWantedOutputs = resolvedDrv->outputNames();
+        realWantedOutputs = resolvedDrv.outputNames();
 
     for (auto & wantedOutput : realWantedOutputs) {
         assert(initialOutputs.count(wantedOutput) != 0);
@@ -979,9 +980,17 @@ void DerivationGoal::resolvedFinished() {
         outputPaths
     );
 
-    // This is potentially a bit fishy in terms of error reporting. Not sure
-    // how to do it in a cleaner way
-    amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
+    auto status = [&]() {
+        auto resolvedResult = resolvedDrvGoal->getResult();
+        switch (resolvedResult.status) {
+            case BuildResult::AlreadyValid:
+                return BuildResult::ResolvesToAlreadyValid;
+            default:
+                return resolvedResult.status;
+        }
+    }();
+
+    done(status);
 }
 
 HookReply DerivationGoal::tryBuildHook()
@@ -1329,6 +1338,13 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex)
     }
 
     worker.updateProgress();
+
+    auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or("");
+    if (traceBuiltOutputsFile != "") {
+        std::fstream fs;
+        fs.open(traceBuiltOutputsFile, std::fstream::out);
+        fs << worker.store.printStorePath(drvPath) << "\t" << result.toString() << std::endl;
+    }
 }
 
 

src/libstore/build/derivation-goal.hh

@@ -50,8 +50,8 @@ struct DerivationGoal : public Goal
     /* The path of the derivation. */
     StorePath drvPath;
 
-    /* The path of the corresponding resolved derivation */
-    std::optional<BasicDerivation> resolvedDrv;
+    /* The goal for the corresponding resolved derivation */
+    std::shared_ptr<DerivationGoal> resolvedDrvGoal;
 
     /* The specific outputs that we need to build.  Empty means all of
        them. */

src/libstore/build/local-derivation-goal.cc

@@ -260,6 +260,7 @@ void LocalDerivationGoal::cleanupHookFinally()
 void LocalDerivationGoal::cleanupPreChildKill()
 {
     sandboxMountNamespace = -1;
+    sandboxUserNamespace = -1;
 }
 
 
@@ -906,11 +907,14 @@ void LocalDerivationGoal::startBuilder()
                 "nobody:x:65534:65534:Nobody:/:/noshell\n",
                 sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
 
-        /* Save the mount namespace of the child. We have to do this
+        /* Save the mount- and user namespace of the child. We have to do this
            *before* the child does a chroot. */
         sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
         if (sandboxMountNamespace.get() == -1)
             throw SysError("getting sandbox mount namespace");
+        sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY);
+        if (sandboxUserNamespace.get() == -1)
+            throw SysError("getting sandbox user namespace");
 
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
@@ -1438,6 +1442,9 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
                    child process.*/
                 Pid child(startProcess([&]() {
 
+                    if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1))
+                        throw SysError("entering sandbox user namespace");
+
                     if (setns(sandboxMountNamespace.get(), 0) == -1)
                         throw SysError("entering sandbox mount namespace");
 
@@ -1779,11 +1786,14 @@ void LocalDerivationGoal::runChild()
            i686-linux build on an x86_64-linux machine. */
         struct utsname utsbuf;
         uname(&utsbuf);
-        if (drv->platform == "i686-linux" &&
-            (settings.thisSystem == "x86_64-linux" ||
-             (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
+        if ((drv->platform == "i686-linux"
+                && (settings.thisSystem == "x86_64-linux"
+                    || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
+            || drv->platform == "armv7l-linux"
+            || drv->platform == "armv6l-linux")
+        {
             if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set i686-linux personality");
+                throw SysError("cannot set 32-bit personality");
         }
 
         /* Impersonate a Linux 2.6 machine to get some determinism in
@@ -2216,8 +2226,8 @@ void LocalDerivationGoal::registerOutputs()
                 StringSink sink;
                 dumpPath(actualPath, sink);
                 deletePath(actualPath);
-                sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
-                StringSource source(*sink.s);
+                sink.s = rewriteStrings(sink.s, outputRewrites);
+                StringSource source(sink.s);
                 restorePath(actualPath, source);
             }
         };
@@ -2285,7 +2295,7 @@ void LocalDerivationGoal::registerOutputs()
                     StringSink sink;
                     dumpPath(actualPath, sink);
                     RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink);
-                    rsink2(*sink.s);
+                    rsink2(sink.s);
                     rsink2.flush();
                 });
                 Path tmpPath = actualPath + ".tmp";
@@ -2457,7 +2467,7 @@ void LocalDerivationGoal::registerOutputs()
         }
 
         if (curRound == nrRounds) {
-            localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
+            localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences()
             worker.markContentsGood(newInfo.path);
         }
 

src/libstore/build/local-derivation-goal.hh

@@ -27,9 +27,10 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Pipe for synchronising updates to the builder namespaces. */
     Pipe userNamespaceSync;
 
-    /* The mount namespace of the builder, used to add additional
+    /* The mount namespace and user namespace of the builder, used to add additional
        paths to the sandbox as a result of recursive Nix calls. */
     AutoCloseFD sandboxMountNamespace;
+    AutoCloseFD sandboxUserNamespace;
 
     /* On Linux, whether we're doing the build in its own user
        namespace. */

src/libstore/build/substitution-goal.cc

@@ -138,8 +138,8 @@ void PathSubstitutionGoal::tryNext()
        only after we've downloaded the path. */
     if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
     {
-        warn("substituter '%s' does not have a valid signature for path '%s'",
-            sub->getUri(), worker.store.printStorePath(storePath));
+        warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'",
+            worker.store.printStorePath(storePath), sub->getUri());
         tryNext();
         return;
     }

src/libstore/build/worker.cc

@@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals)
                 if (getMachines().empty())
                    throw Error("unable to start any build; either increase '--max-jobs' "
                             "or enable remote builds."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
                 else
                    throw Error("unable to start any build; remote machines may not have "
                             "all required system features."
-                            "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+                            "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html");
 
             }
             assert(!awake.empty());

src/libstore/ca-specific-schema.sql

@@ -19,3 +19,8 @@ create table if not exists RealisationsRefs (
     foreign key (referrer) references Realisations(id) on delete cascade,
     foreign key (realisationReference) references Realisations(id) on delete restrict
 );
+
+-- used by QueryRealisationReferences
+create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
+-- used by cascade deletion when ValidPaths is deleted
+create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);

src/libstore/daemon.cc

@@ -4,7 +4,6 @@
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 #include "finally.hh"
-#include "affinity.hh"
 #include "archive.hh"
 #include "derivations.hh"
 #include "args.hh"
@@ -70,7 +69,7 @@ struct TunnelLogger : public Logger
 
         StringSink buf;
         buf << STDERR_NEXT << (fs.s + "\n");
-        enqueueMsg(*buf.s);
+        enqueueMsg(buf.s);
     }
 
     void logEI(const ErrorInfo & ei) override
@@ -82,7 +81,7 @@ struct TunnelLogger : public Logger
 
         StringSink buf;
         buf << STDERR_NEXT << oss.str();
-        enqueueMsg(*buf.s);
+        enqueueMsg(buf.s);
     }
 
     /* startWork() means that we're starting an operation for which we
@@ -130,7 +129,7 @@ struct TunnelLogger : public Logger
 
         StringSink buf;
         buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent;
-        enqueueMsg(*buf.s);
+        enqueueMsg(buf.s);
     }
 
     void stopActivity(ActivityId act) override
@@ -138,7 +137,7 @@ struct TunnelLogger : public Logger
         if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
         StringSink buf;
         buf << STDERR_STOP_ACTIVITY << act;
-        enqueueMsg(*buf.s);
+        enqueueMsg(buf.s);
     }
 
     void result(ActivityId act, ResultType type, const Fields & fields) override
@@ -146,7 +145,7 @@ struct TunnelLogger : public Logger
         if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
         StringSink buf;
         buf << STDERR_RESULT << act << type << fields;
-        enqueueMsg(*buf.s);
+        enqueueMsg(buf.s);
     }
 };
 
@@ -431,25 +430,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 hashAlgo = parseHashType(hashAlgoRaw);
             }
 
-            StringSink saved;
-            TeeSource savedNARSource(from, saved);
-            RetrieveRegularNARSink savedRegular { saved };
-
-            if (method == FileIngestionMethod::Recursive) {
-                /* Get the entire NAR dump from the client and save it to
-                  a string so that we can pass it to
-                  addToStoreFromDump(). */
-                ParseSink sink; /* null sink; just parse the NAR */
-                parseDump(sink, savedNARSource);
-            } else
-                parseDump(savedRegular, from);
-
+            auto dumpSource = sinkToSource([&](Sink & saved) {
+                if (method == FileIngestionMethod::Recursive) {
+                    /* We parse the NAR dump through into `saved` unmodified,
+                       so why all this extra work? We still parse the NAR so
+                       that we aren't sending arbitrary data to `saved`
+                       unwittingly`, and we know when the NAR ends so we don't
+                       consume the rest of `from` and can't parse another
+                       command. (We don't trust `addToStoreFromDump` to not
+                       eagerly consume the entire stream it's given, past the
+                       length of the Nar. */
+                    TeeSource savedNARSource(from, saved);
+                    ParseSink sink; /* null sink; just parse the NAR */
+                    parseDump(sink, savedNARSource);
+                } else {
+                    /* Incrementally parse the NAR file, stripping the
+                       metadata, and streaming the sole file we expect into
+                       `saved`. */
+                    RetrieveRegularNARSink savedRegular { saved };
+                    parseDump(savedRegular, from);
+                    if (!savedRegular.regular) throw Error("regular file expected");
+                }
+            });
             logger->startWork();
-            if (!savedRegular.regular) throw Error("regular file expected");
-
-            // FIXME: try to stream directly from `from`.
-            StringSource dumpSource { *saved.s };
-            auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
+            auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
             logger->stopWork();
 
             to << store->printStorePath(path);
@@ -464,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             dontCheckSigs = false;
 
         logger->startWork();
-        FramedSource source(from);
-        store->addMultipleToStore(source,
-            RepairFlag{repair},
-            dontCheckSigs ? NoCheckSigs : CheckSigs);
+        {
+            FramedSource source(from);
+            store->addMultipleToStore(source,
+                RepairFlag{repair},
+                dontCheckSigs ? NoCheckSigs : CheckSigs);
+        }
         logger->stopWork();
         break;
     }
@@ -848,14 +854,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
 
         else {
             std::unique_ptr<Source> source;
+            StringSink saved;
             if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
                 source = std::make_unique<TunnelSource>(from, to);
             else {
-                StringSink saved;
                 TeeSource tee { from, saved };
                 ParseSink ether;
                 parseDump(ether, tee);
-                source = std::make_unique<StringSource>(std::move(*saved.s));
+                source = std::make_unique<StringSource>(saved.s);
             }
 
             logger->startWork();
@@ -916,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case wopAddBuildLog: {
+        StorePath path{readString(from)};
+        logger->startWork();
+        if (!trusted)
+            throw Error("you are not privileged to add logs");
+        {
+            FramedSource source(from);
+            StringSink sink;
+            source.drainInto(sink);
+            store->addBuildLog(path, sink.s);
+        }
+        logger->stopWork();
+        to << 1;
+        break;
+    }
+
     default:
         throw Error("invalid operation %1%", op);
     }
@@ -951,12 +973,12 @@ void processConnection(
 
     Finally finally([&]() {
         _isInterrupted = false;
-        prevLogger->log(lvlDebug, fmt("%d operations", opCount));
+        printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
     });
 
     if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
-        auto affinity = readInt(from);
-        setAffinityTo(affinity);
+        // Obsolete CPU affinity.
+        readInt(from);
     }
 
     readInt(from); // obsolete reserveSpace
@@ -984,6 +1006,8 @@ void processConnection(
                 break;
             }
 
+            printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op);
+
             opCount++;
 
             try {

src/libstore/derivations.cc

@@ -4,6 +4,7 @@
 #include "util.hh"
 #include "worker-protocol.hh"
 #include "fs-accessor.hh"
+#include <boost/container/small_vector.hpp>
 
 namespace nix {
 
@@ -272,7 +273,9 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi
 
 static void printString(string & res, std::string_view s)
 {
-    char buf[s.size() * 2 + 2];
+    boost::container::small_vector<char, 64 * 1024> buffer;
+    buffer.reserve(s.size() * 2 + 2);
+    char * buf = buffer.data();
     char * p = buf;
     *p++ = '"';
     for (auto c : s)

src/libstore/export-import.cc

@@ -75,20 +75,20 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {});
         auto deriver = readString(source);
-        auto narHash = hashString(htSHA256, *saved.s);
+        auto narHash = hashString(htSHA256, saved.s);
 
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
         info.references = references;
-        info.narSize = saved.s->size();
+        info.narSize = saved.s.size();
 
         // Ignore optional legacy signature.
         if (readInt(source) == 1)
             readString(source);
 
         // Can't use underlying source, which would have been exhausted
-        auto source = StringSource { *saved.s };
+        auto source = StringSource(saved.s);
         addToStore(info, source, NoRepair, checkSigs);
 
         res.push_back(info.path);

src/libstore/filetransfer.cc

@@ -106,7 +106,7 @@ struct curlFileTransfer : public FileTransfer
                         this->request.dataCallback(data);
                     }
                 } else
-                    this->result.data->append(data);
+                    this->result.data.append(data);
               })
         {
             if (!request.expectedETag.empty())
@@ -195,7 +195,7 @@ struct curlFileTransfer : public FileTransfer
             std::smatch match;
             if (std::regex_match(line, match, statusLine)) {
                 result.etag = "";
-                result.data = std::make_shared<std::string>();
+                result.data.clear();
                 result.bodySize = 0;
                 statusMsg = trim(match[1]);
                 acceptRanges = false;
@@ -340,7 +340,7 @@ struct curlFileTransfer : public FileTransfer
             if (writtenToSink)
                 curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
 
-            result.data = std::make_shared<std::string>();
+            result.data.clear();
             result.bodySize = 0;
         }
 
@@ -434,21 +434,21 @@ struct curlFileTransfer : public FileTransfer
 
                 attempt++;
 
-                std::shared_ptr<std::string> response;
+                std::optional<std::string> response;
                 if (errorSink)
-                    response = errorSink->s;
+                    response = std::move(errorSink->s);
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
+                    ? FileTransferError(Interrupted, std::move(response), "%s of '%s' was interrupted", request.verb(), request.uri)
                     : httpStatus != 0
                     ? FileTransferError(err,
-                        response,
+                        std::move(response),
                         fmt("unable to %s '%s': HTTP error %d ('%s')",
                             request.verb(), request.uri, httpStatus, statusMsg)
                         + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
                         )
                     : FileTransferError(err,
-                        response,
+                        std::move(response),
                         fmt("unable to %s '%s': %s (%d)",
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
@@ -544,13 +544,7 @@ struct curlFileTransfer : public FileTransfer
             stopWorkerThread();
         });
 
-#ifdef __linux__
-        /* Cause this thread to not share any FS attributes with the main thread,
-           because this causes setns() in restoreMountNamespace() to fail.
-           Ideally, this would happen in the std::thread() constructor. */
-        if (unshare(CLONE_FS) != 0)
-            throw SysError("unsharing filesystem state in download thread");
-#endif
+        unshareFilesystem();
 
         std::map<CURL *, std::shared_ptr<TransferItem>> items;
 
@@ -711,7 +705,7 @@ struct curlFileTransfer : public FileTransfer
                 FileTransferResult res;
                 if (!s3Res.data)
                     throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
-                res.data = s3Res.data;
+                res.data = std::move(*s3Res.data);
                 callback(std::move(res));
 #else
                 throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);
@@ -865,7 +859,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
 }
 
 template<typename... Args>
-FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
+FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args)
     : Error(args...), error(error), response(response)
 {
     const auto hf = hintfmt(args...);

src/libstore/filetransfer.hh

@@ -59,7 +59,7 @@ struct FileTransferRequest
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
-    std::shared_ptr<std::string> data;
+    std::optional<std::string> data;
     std::string mimeType;
     std::function<void(std::string_view data)> dataCallback;
 
@@ -77,7 +77,7 @@ struct FileTransferResult
     bool cached = false;
     std::string etag;
     std::string effectiveUri;
-    std::shared_ptr<std::string> data;
+    std::string data;
     uint64_t bodySize = 0;
 };
 
@@ -119,10 +119,10 @@ class FileTransferError : public Error
 {
 public:
     FileTransfer::Error error;
-    std::shared_ptr<string> response; // intentionally optional
+    std::optional<string> response; // intentionally optional
 
     template<typename... Args>
-    FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
+    FileTransferError(FileTransfer::Error error, std::optional<string> response, const Args & ... args);
 
     virtual const char* sname() const override { return "FileTransferError"; }
 };

src/libstore/gc.cc

@@ -126,7 +126,17 @@ void LocalStore::addTempRoot(const StorePath & path)
             auto socketPath = stateDir.get() + gcSocketPath;
             debug("connecting to '%s'", socketPath);
             state->fdRootsSocket = createUnixDomainSocket();
-            nix::connect(state->fdRootsSocket.get(), socketPath);
+            try {
+                nix::connect(state->fdRootsSocket.get(), socketPath);
+            } catch (SysError & e) {
+                /* The garbage collector may have exited, so we need to
+                   restart. */
+                if (e.errNo == ECONNREFUSED) {
+                    debug("GC socket connection refused");
+                    state->fdRootsSocket.close();
+                    goto restart;
+                }
+            }
         }
 
         try {
@@ -523,6 +533,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
                 if (!fdClient) continue;
 
+                debug("GC roots server accepted new client");
+
                 /* Process the connection in a separate thread. */
                 auto fdClient_ = fdClient.get();
                 std::thread clientThread([&, fdClient = std::move(fdClient)]() {
@@ -535,6 +547,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                         }
                     });
 
+                    /* On macOS, accepted sockets inherit the
+                       non-blocking flag from the server socket, so
+                       explicitly make it blocking. */
+                    if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1)
+                        abort();
+
                     while (true) {
                         try {
                             auto path = readLine(fdClient.get());
@@ -559,7 +577,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                             } else
                                 printError("received garbage instead of a root from client");
                             writeFull(fdClient.get(), "1", false);
-                        } catch (Error &) { break; }
+                        } catch (Error & e) {
+                            debug("reading GC root from client: %s", e.msg());
+                            break;
+                        }
                     }
                 });
 

src/libstore/globals.hh

@@ -21,7 +21,7 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
-        : BaseSetting<unsigned int>(def, name, description, aliases)
+        : BaseSetting<unsigned int>(def, true, name, description, aliases)
     {
         options->addSetting(this);
     }
@@ -38,7 +38,7 @@ struct PluginFilesSetting : public BaseSetting<Paths>
         const std::string & name,
         const std::string & description,
         const std::set<std::string> & aliases = {})
-        : BaseSetting<Paths>(def, name, description, aliases)
+        : BaseSetting<Paths>(def, true, name, description, aliases)
     {
         options->addSetting(this);
     }
@@ -130,7 +130,9 @@ public:
         {"build-max-jobs"}};
 
     Setting<unsigned int> buildCores{
-        this, getDefaultCores(), "cores",
+        this,
+        getDefaultCores(),
+        "cores",
         R"(
           Sets the value of the `NIX_BUILD_CORES` environment variable in the
           invocation of builders. Builders can use this variable at their
@@ -141,7 +143,7 @@ public:
           command line switch and defaults to `1`. The value `0` means that
           the builder should use all available CPU cores in the system.
         )",
-        {"build-cores"}};
+        {"build-cores"}, false};
 
     /* Read-only mode.  Don't copy stuff to the store, don't change
        the database. */
@@ -583,10 +585,11 @@ public:
           platform and generate incompatible code, so you may wish to
           cross-check the results of using this option against proper
           natively-built versions of your derivations.
-        )"};
+        )", {}, false};
 
     Setting<StringSet> systemFeatures{
-        this, getDefaultSystemFeatures(),
+        this,
+        getDefaultSystemFeatures(),
         "system-features",
         R"(
           A set of system “features” supported by this machine, e.g. `kvm`.
@@ -602,7 +605,7 @@ public:
           This setting by default includes `kvm` if `/dev/kvm` is accessible,
           and the pseudo-features `nixos-test`, `benchmark` and `big-parallel`
           that are used in Nixpkgs to route builds to specific machines.
-        )"};
+        )", {}, false};
 
     Setting<Strings> substituters{
         this,
@@ -797,6 +800,15 @@ public:
           may be useful in certain scenarios (e.g. to spin up containers or
           set up userspace network interfaces in tests).
         )"};
+
+    Setting<StringSet> ignoredAcls{
+        this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls",
+        R"(
+          A list of ACLs that should be ignored, normally Nix attempts to
+          remove all ACLs from files and directories in the Nix store, but
+          some ACLs like `security.selinux` or `system.nfs4_acl` can't be
+          removed even by root. Therefore it's best to just ignore them.
+        )"};
 #endif
 
     Setting<Strings> hashedMirrors{
@@ -954,6 +966,13 @@ public:
 
     Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config",
         "Whether to accept nix configuration from a flake without prompting."};
+
+    Setting<std::string> commitLockFileSummary{
+        this, "", "commit-lockfile-summary",
+        R"(
+          The commit summary to use when commiting changed flake lock files. If
+          empty, the summary is generated based on the action performed.
+        )"};
 };
 
 

src/libstore/http-binary-cache-store.cc

@@ -126,7 +126,7 @@ protected:
         const std::string & mimeType) override
     {
         auto req = makeRequest(path);
-        req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
+        req.data = StreamToSourceAdapter(istream).drain();
         req.mimeType = mimeType;
         try {
             getFileTransfer()->upload(req);
@@ -159,7 +159,7 @@ protected:
     }
 
     void getFile(const std::string & path,
-        Callback<std::shared_ptr<std::string>> callback) noexcept override
+        Callback<std::optional<std::string>> callback) noexcept override
     {
         checkEnabled();
 
@@ -170,10 +170,10 @@ protected:
         getFileTransfer()->enqueueFileTransfer(request,
             {[callbackPtr, this](std::future<FileTransferResult> result) {
                 try {
-                    (*callbackPtr)(result.get().data);
+                    (*callbackPtr)(std::move(result.get().data));
                 } catch (FileTransferError & e) {
                     if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
-                        return (*callbackPtr)(std::shared_ptr<std::string>());
+                        return (*callbackPtr)({});
                     maybeDisable();
                     callbackPtr->rethrow();
                 } catch (...) {

src/libstore/legacy-ssh-store.cc

@@ -94,7 +94,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 conn->sshConn->in.close();
                 auto msg = conn->from.drain();
                 throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
-                    host, chomp(*saved.s + msg));
+                    host, chomp(saved.s + msg));
             }
             conn->remoteVersion = readInt(conn->from);
             if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)

src/libstore/local-binary-cache-store.cc

@@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init()
     createDirs(binaryCacheDir + "/" + realisationsPrefix);
     if (writeDebugInfo)
         createDirs(binaryCacheDir + "/debuginfo");
+    createDirs(binaryCacheDir + "/log");
     BinaryCacheStore::init();
 }
 

src/libstore/local-fs-store.cc

@@ -87,34 +87,32 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
 
 const string LocalFSStore::drvsLogDir = "drvs";
 
-
-
-std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
+std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
 {
     auto path = path_;
 
     if (!path.isDerivation()) {
         try {
             auto info = queryPathInfo(path);
-            if (!info->deriver) return nullptr;
+            if (!info->deriver) return std::nullopt;
             path = *info->deriver;
         } catch (InvalidPath &) {
-            return nullptr;
+            return std::nullopt;
         }
     }
 
-    auto baseName = std::string(baseNameOf(printStorePath(path)));
+    auto baseName = path.to_string();
 
     for (int j = 0; j < 2; j++) {
 
         Path logPath =
             j == 0
-            ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2))
+            ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2))
             : fmt("%s/%s/%s", logDir, drvsLogDir, baseName);
         Path logBz2Path = logPath + ".bz2";
 
         if (pathExists(logPath))
-            return std::make_shared<std::string>(readFile(logPath));
+            return readFile(logPath);
 
         else if (pathExists(logBz2Path)) {
             try {
@@ -124,7 +122,7 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_)
 
     }
 
-    return nullptr;
+    return std::nullopt;
 }
 
 }

src/libstore/local-fs-store.hh

@@ -45,7 +45,8 @@ public:
         return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
     }
 
-    std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
+    std::optional<std::string> getBuildLog(const StorePath & path) override;
+
 };
 
 }

src/libstore/local-store.cc

@@ -8,6 +8,8 @@
 #include "references.hh"
 #include "callback.hh"
 #include "topo-sort.hh"
+#include "finally.hh"
+#include "compression.hh"
 
 #include <iostream>
 #include <algorithm>
@@ -79,7 +81,7 @@ int getSchema(Path schemaPath)
 
 void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
 {
-    const int nixCASchemaVersion = 2;
+    const int nixCASchemaVersion = 3;
     int curCASchema = getSchema(schemaPath);
     if (curCASchema != nixCASchemaVersion) {
         if (curCASchema > nixCASchemaVersion) {
@@ -130,6 +132,17 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
             txn.commit();
         }
 
+        if (curCASchema < 3) {
+            SQLiteTxn txn(db);
+            // Apply new indices added in this schema update.
+            db.exec(R"(
+                -- used by QueryRealisationReferences
+                create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer);
+                -- used by cascade deletion when ValidPaths is deleted
+                create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath);
+            )");
+            txn.commit();
+        }
         writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
         lockFile(lockFd.get(), ltRead, true);
     }
@@ -589,9 +602,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe
             throw SysError("querying extended attributes of '%s'", path);
 
         for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) {
-            /* Ignore SELinux security labels since these cannot be
-               removed even by root. */
-            if (eaName == "security.selinux") continue;
+            if (settings.ignoredAcls.get().count(eaName)) continue;
             if (lremovexattr(path.c_str(), eaName.c_str()) == -1)
                 throw SysError("removing extended attribute '%s' from '%s'", eaName, path);
         }
@@ -1297,7 +1308,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             canonicalisePathMetaData(realPath, -1);
 
-            optimisePath(realPath); // FIXME: combine with hashPath()
+            optimisePath(realPath, repair); // FIXME: combine with hashPath()
 
             registerValidPath(info);
         }
@@ -1333,13 +1344,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
         auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
         dump.resize(oldSize + want);
         auto got = 0;
+        Finally cleanup([&]() {
+            dump.resize(oldSize + got);
+        });
         try {
             got = source.read(dump.data() + oldSize, want);
         } catch (EndOfFile &) {
             inMemory = true;
             break;
         }
-        dump.resize(oldSize + got);
     }
 
     std::unique_ptr<AutoDelete> delTempDir;
@@ -1407,7 +1420,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
 
             canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
 
-            optimisePath(realPath);
+            optimisePath(realPath, repair);
 
             ValidPathInfo info { dstPath, narHash.first };
             info.narSize = narHash.second;
@@ -1449,12 +1462,12 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
             StringSink sink;
             dumpString(s, sink);
-            auto narHash = hashString(htSHA256, *sink.s);
+            auto narHash = hashString(htSHA256, sink.s);
 
-            optimisePath(realPath);
+            optimisePath(realPath, repair);
 
             ValidPathInfo info { dstPath, narHash };
-            info.narSize = sink.s->size();
+            info.narSize = sink.s.size();
             info.references = references;
             info.ca = TextHash { .hash = hash };
             registerValidPath(info);
@@ -1886,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath(
     };
 }
 
+void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
+{
+    assert(drvPath.isDerivation());
+
+    auto baseName = drvPath.to_string();
+
+    auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
+
+    if (pathExists(logPath)) return;
+
+    createDirs(dirOf(logPath));
+
+    auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
+
+    writeFile(tmpFile, compress("bzip2", log));
+
+    if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
+        throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
+}
+
 }  // namespace nix

src/libstore/local-store.hh

@@ -172,8 +172,9 @@ public:
 
     void optimiseStore() override;
 
-    /* Optimise a single store path. */
-    void optimisePath(const Path & path);
+    /* Optimise a single store path. Optionally, test the encountered
+       symlinks for corruption. */
+    void optimisePath(const Path & path, RepairFlag repair);
 
     bool verifyStore(bool checkContents, RepairFlag repair) override;
 
@@ -253,7 +254,7 @@ private:
 
     InodeHash loadInodeHash();
     Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
-    void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
+    void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair);
 
     // Internal versions that are not wrapped in retry_sqlite.
     bool isValidPath_(State & state, const StorePath & path);
@@ -279,6 +280,8 @@ private:
         const std::string_view pathHash
     );
 
+    void addBuildLog(const StorePath & drvPath, std::string_view log) override;
+
     friend struct LocalDerivationGoal;
     friend struct PathSubstitutionGoal;
     friend struct SubstitutionGoal;

src/libstore/misc.cc

@@ -7,6 +7,7 @@
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "closure.hh"
+#include "filetransfer.hh"
 
 namespace nix {
 
@@ -100,7 +101,8 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
 
     downloadSize_ = narSize_ = 0;
 
-    ThreadPool pool;
+    // FIXME: make async.
+    ThreadPool pool(fileTransferSettings.httpConnections);
 
     struct State
     {

src/libstore/names.cc

@@ -42,7 +42,7 @@ DrvName::~DrvName()
 { }
 
 
-bool DrvName::matches(DrvName & n)
+bool DrvName::matches(const DrvName & n)
 {
     if (name != "*") {
         if (!regex) {

src/libstore/names.hh

@@ -19,7 +19,7 @@ struct DrvName
     DrvName(std::string_view s);
     ~DrvName();
 
-    bool matches(DrvName & n);
+    bool matches(const DrvName & n);
 
 private:
     std::unique_ptr<Regex> regex;

src/libstore/nar-accessor.cc

@@ -28,7 +28,7 @@ struct NarMember
 
 struct NarAccessor : public FSAccessor
 {
-    std::shared_ptr<const std::string> nar;
+    std::optional<const std::string> nar;
 
     GetNarBytes getNarBytes;
 
@@ -104,7 +104,7 @@ struct NarAccessor : public FSAccessor
         }
     };
 
-    NarAccessor(ref<const std::string> nar) : nar(nar)
+    NarAccessor(std::string && _nar) : nar(_nar)
     {
         StringSource source(*nar);
         NarIndexer indexer(*this, source);
@@ -224,9 +224,9 @@ struct NarAccessor : public FSAccessor
     }
 };
 
-ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
+ref<FSAccessor> makeNarAccessor(std::string && nar)
 {
-    return make_ref<NarAccessor>(nar);
+    return make_ref<NarAccessor>(std::move(nar));
 }
 
 ref<FSAccessor> makeNarAccessor(Source & source)

src/libstore/nar-accessor.hh

@@ -10,7 +10,7 @@ struct Source;
 
 /* Return an object that provides access to the contents of a NAR
    file. */
-ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
+ref<FSAccessor> makeNarAccessor(std::string && nar);
 
 ref<FSAccessor> makeNarAccessor(Source & source);
 

src/libstore/optimise-store.cc

@@ -88,7 +88,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa
 
 
 void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
-    const Path & path, InodeHash & inodeHash)
+    const Path & path, InodeHash & inodeHash, RepairFlag repair)
 {
     checkInterrupt();
 
@@ -110,7 +110,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     if (S_ISDIR(st.st_mode)) {
         Strings names = readDirectoryIgnoringInodes(path, inodeHash);
         for (auto & i : names)
-            optimisePath_(act, stats, path + "/" + i, inodeHash);
+            optimisePath_(act, stats, path + "/" + i, inodeHash, repair);
         return;
     }
 
@@ -151,7 +151,20 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
     /* Check if this is a known hash. */
     Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
 
- retry:
+    /* Maybe delete the link, if it has been corrupted. */
+    if (pathExists(linkPath)) {
+        auto stLink = lstat(linkPath);
+        if (st.st_size != stLink.st_size
+            || (repair && hash != hashPath(htSHA256, linkPath).first))
+        {
+            // XXX: Consider overwriting linkPath with our valid version.
+            warn("removing corrupted link '%s'", linkPath);
+            warn("There may be more corrupted paths."
+                 "\nYou should run `nix-store --verify --check-contents --repair` to fix them all");
+            unlink(linkPath.c_str());
+        }
+    }
+
     if (!pathExists(linkPath)) {
         /* Nope, create a hard link in the links directory. */
         if (link(path.c_str(), linkPath.c_str()) == 0) {
@@ -187,12 +200,6 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
         return;
     }
 
-    if (st.st_size != stLink.st_size) {
-        warn("removing corrupted link '%s'", linkPath);
-        unlink(linkPath.c_str());
-        goto retry;
-    }
-
     printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath);
 
     /* Make the containing directory writable, but only if it's not
@@ -260,7 +267,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
         if (!isValidPath(i)) continue; /* path was GC'ed, probably */
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i)));
-            optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash);
+            optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair);
         }
         done++;
         act.progress(done, paths.size());
@@ -278,12 +285,12 @@ void LocalStore::optimiseStore()
         stats.filesLinked);
 }
 
-void LocalStore::optimisePath(const Path & path)
+void LocalStore::optimisePath(const Path & path, RepairFlag repair)
 {
     OptimiseStats stats;
     InodeHash inodeHash;
 
-    if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash);
+    if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash, repair);
 }
 
 

src/libstore/realisation.cc

@@ -78,7 +78,7 @@ Realisation Realisation::fromJSON(
         auto fieldIterator = json.find(fieldName);
         if (fieldIterator == json.end())
             return std::nullopt;
-        return *fieldIterator;
+        return {*fieldIterator};
     };
     auto getField = [&](std::string fieldName) -> std::string {
         if (auto field = getOptionalField(fieldName))

src/libstore/remote-fs-accessor.cc

@@ -22,9 +22,18 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin
     return fmt("%s/%s.%s", cacheDir, hashPart, ext);
 }
 
-void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
-    ref<FSAccessor> narAccessor)
+ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)
 {
+    if (cacheDir != "") {
+        try {
+            /* FIXME: do this asynchronously. */
+            writeFile(makeCacheFile(hashPart, "nar"), nar);
+        } catch (...) {
+            ignoreException();
+        }
+    }
+
+    auto narAccessor = makeNarAccessor(std::move(nar));
     nars.emplace(hashPart, narAccessor);
 
     if (cacheDir != "") {
@@ -33,14 +42,12 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string &
             JSONPlaceholder jsonRoot(str);
             listNar(jsonRoot, narAccessor, "", true);
             writeFile(makeCacheFile(hashPart, "ls"), str.str());
-
-            /* FIXME: do this asynchronously. */
-            writeFile(makeCacheFile(hashPart, "nar"), nar);
-
         } catch (...) {
             ignoreException();
         }
     }
+
+    return narAccessor;
 }
 
 std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
@@ -55,7 +62,6 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
     auto i = nars.find(std::string(storePath.hashPart()));
     if (i != nars.end()) return {i->second, restPath};
 
-    StringSink sink;
     std::string listing;
     Path cacheFile;
 
@@ -86,19 +92,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo
         } catch (SysError &) { }
 
         try {
-            *sink.s = nix::readFile(cacheFile);
-
-            auto narAccessor = makeNarAccessor(sink.s);
+            auto narAccessor = makeNarAccessor(nix::readFile(cacheFile));
             nars.emplace(storePath.hashPart(), narAccessor);
             return {narAccessor, restPath};
-
         } catch (SysError &) { }
     }
 
+    StringSink sink;
     store->narFromPath(storePath, sink);
-    auto narAccessor = makeNarAccessor(sink.s);
-    addToCache(storePath.hashPart(), *sink.s, narAccessor);
-    return {narAccessor, restPath};
+    return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath};
 }
 
 FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)

src/libstore/remote-fs-accessor.hh

@@ -20,8 +20,7 @@ class RemoteFSAccessor : public FSAccessor
 
     Path makeCacheFile(std::string_view hashPart, const std::string & ext);
 
-    void addToCache(std::string_view hashPart, const std::string & nar,
-        ref<FSAccessor> narAccessor);
+    ref<FSAccessor> addToCache(std::string_view hashPart, std::string && nar);
 
 public: