Set 2.4 release date

(cherry picked from commit bc4b7521f4)

.github/workflows/test.yml

@@ -4,6 +4,7 @@ on:
   push:
 jobs:
   tests:
+    needs: [check_cachix]
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
@@ -13,15 +14,15 @@ jobs:
     - uses: actions/checkout@v2.3.4
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v14
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v10
+      if: needs.check_cachix.outputs.secret == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    #- run: nix flake check
-    - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
+    - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest
@@ -44,7 +45,7 @@ jobs:
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v14
     - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -62,7 +63,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2.3.4
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v14
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"

.gitignore

@@ -15,6 +15,7 @@ perl/Makefile.config
 /doc/manual/*.1
 /doc/manual/*.5
 /doc/manual/*.8
+/doc/manual/generated/*
 /doc/manual/nix.json
 /doc/manual/conf-file.json
 /doc/manual/builtins.json
@@ -56,9 +57,6 @@ perl/Makefile.config
 
 /src/nix-prefetch-url/nix-prefetch-url
 
-# /src/nix-daemon/
-/src/nix-daemon/nix-daemon
-
 /src/nix-collect-garbage/nix-collect-garbage
 
 # /src/nix-channel/
@@ -76,7 +74,6 @@ perl/Makefile.config
 # /tests/
 /tests/test-tmp
 /tests/common.sh
-/tests/dummy
 /tests/result*
 /tests/restricted-innocent
 /tests/shell

Makefile

@@ -4,6 +4,7 @@ makefiles = \
   src/libutil/local.mk \
   src/libutil/tests/local.mk \
   src/libstore/local.mk \
+  src/libstore/tests/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
@@ -12,6 +13,7 @@ makefiles = \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \
+  misc/fish/local.mk \
   misc/zsh/local.mk \
   misc/systemd/local.mk \
   misc/launchd/local.mk \
@@ -32,4 +34,4 @@ endif
 
 include mk/lib.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17
+GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++17 -I src

Makefile.config.in

@@ -1,3 +1,4 @@
+HOST_OS = @host_os@
 AR = @AR@
 BDW_GC_LIBS = @BDW_GC_LIBS@
 BOOST_LDFLAGS = @BOOST_LDFLAGS@

benchmark.sh

@@ -1,53 +0,0 @@
-#!/bin/sh
-
-set -euo pipefail
-set -x
-
-callNix () {
-    nix \
-        --experimental-features "nix-command flakes" \
-        --store /tmp/nix \
-        "$@"
-}
-
-callBuild () {
-    callNix \
-        eval --impure --file "$THINGTOBENCH" drvPath \
-        "$@"
-}
-getCompletions () {
-    NIX_GET_COMPLETIONS=6 callNix build "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31#firef" "$@"
-}
-runSearch () {
-    callNix search "github:NixOS/nixpkgs?rev=ad0d20345219790533ebe06571f82ed6b034db31" firefox "$@"
-}
-
-noCache () {
-    "$@" --option eval-cache false
-}
-coldCache () {
-    if [[ -e ~/.cache/nix/eval-cache-v2 || -e ~/.cache/nix/eval-cache-v3 ]]; then
-        echo "Error: The cache should be clean"
-        exit 1
-    fi
-    "$@"
-}
-
-run_all () {
-
-    mkdir -p "$out"
-
-    NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$out/eval-stats.json bash $0 noCache callBuild
-
-    hyperfine \
-        --warmup 2 \
-        --export-csv "$out/result.csv" \
-        --export-json "$out/result.json" \
-        --export-markdown "$out/result.md" \
-        --style basic \
-        --prepare '' "bash $0 noCache callBuild" \
-        --prepare 'rm -rf ~/.cache/nix/' "bash $0 coldCache callBuild" \
-        --prepare '' "bash $0 callBuild"
-}
-
-"$@"

boehmgc-coroutine-sp-fallback.diff

@@ -0,0 +1,42 @@
+diff --git a/pthread_stop_world.c b/pthread_stop_world.c
+index 1cee6a0b..46c3acd9 100644
+--- a/pthread_stop_world.c
++++ b/pthread_stop_world.c
+@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
+     struct GC_traced_stack_sect_s *traced_stack_sect;
+     pthread_t self = pthread_self();
+     word total_size = 0;
++    size_t stack_limit;
++    pthread_attr_t pattr;
+ 
+     if (!EXPECT(GC_thr_initialized, TRUE))
+       GC_thr_init();
+@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void)
+           hi = p->altstack + p->altstack_size;
+           /* FIXME: Need to scan the normal stack too, but how ? */
+           /* FIXME: Assume stack grows down */
++        } else {
++          if (pthread_getattr_np(p->id, &pattr)) {
++            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
++          }
++          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
++            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
++          }
++          // When a thread goes into a coroutine, we lose its original sp until
++          // control flow returns to the thread.
++          // While in the coroutine, the sp points outside the thread stack,
++          // so we can detect this and push the entire thread stack instead,
++          // as an approximation.
++          // We assume that the coroutine has similarly added its entire stack.
++          // This could be made accurate by cooperating with the application
++          // via new functions and/or callbacks.
++          #ifndef STACK_GROWS_UP
++            if (lo >= hi || lo < hi - stack_limit) { // sp outside stack
++              lo = hi - stack_limit;
++            }
++          #else
++          #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
++          #endif
+         }
+         GC_push_all_stack_sections(lo, hi, traced_stack_sect);
+ #       ifdef STACK_GROWS_UP

configure.ac

@@ -32,14 +32,6 @@ AC_ARG_WITH(system, AS_HELP_STRING([--with-system=SYSTEM],[Platform identifier (
         system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";;
    esac])
 
-sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
-
-case $sys_name in
-    cygwin*)
-        sys_name=cygwin
-        ;;
-esac
-
 AC_MSG_RESULT($system)
 AC_SUBST(system)
 AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
@@ -63,10 +55,12 @@ AC_SYS_LARGEFILE
 
 # Solaris-specific stuff.
 AC_STRUCT_DIRENT_D_TYPE
-if test "$sys_name" = sunos; then
+case "$host_os" in
+  solaris*)
     # Solaris requires -lsocket -lnsl for network functions
-    LIBS="-lsocket -lnsl $LIBS"
-fi
+    LDFLAGS="-lsocket -lnsl $LDFLAGS"
+    ;;
+esac
 
 
 # Check for pubsetbuf.
@@ -150,7 +144,7 @@ int main() {
 }]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes)
 AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC)
 if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then
-    LIBS="-latomic $LIBS"
+    LDFLAGS="-latomic $LDFLAGS"
 fi
 
 PKG_PROG_PKG_CONFIG
@@ -210,28 +204,31 @@ AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
 
 
 # Look for libseccomp, required for Linux sandboxing.
-if test "$sys_name" = linux; then
-  AC_ARG_ENABLE([seccomp-sandboxing],
-                AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)
-                              ]))
-  if test "x$enable_seccomp_sandboxing" != "xno"; then
-    PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
-                      [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
-    have_seccomp=1
-    AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
-  else
+case "$host_os" in
+  linux*)
+    AC_ARG_ENABLE([seccomp-sandboxing],
+                  AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)
+                                ]))
+    if test "x$enable_seccomp_sandboxing" != "xno"; then
+      PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
+                        [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
+      have_seccomp=1
+      AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
+    else
+      have_seccomp=
+    fi
+    ;;
+  *)
     have_seccomp=
-  fi
-else
-  have_seccomp=
-fi
+    ;;
+esac
 AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 
 # Look for aws-cpp-sdk-s3.
 AC_LANG_PUSH(C++)
 AC_CHECK_HEADERS([aws/s3/S3Client.h],
-  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1], 
+  [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=1],
   [AC_DEFINE([ENABLE_S3], [0], [Whether to enable S3 support via aws-sdk-cpp.]) enable_s3=])
 AC_SUBST(ENABLE_S3, [$enable_s3])
 AC_LANG_POP(C++)
@@ -263,6 +260,8 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
+# Look for lowdown library.
+PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.8.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])
@@ -274,9 +273,11 @@ AC_CHECK_FUNCS([strsignal posix_fallocate sysconf])
 
 # This is needed if bzip2 is a static library, and the Nix libraries
 # are dynamic.
-if test "$(uname)" = "Darwin"; then
+case "${host_os}" in
+  darwin*)
     LDFLAGS="-all_load $LDFLAGS"
-fi
+    ;;
+esac
 
 
 AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]),

doc/manual/generate-builtins.nix

@@ -6,9 +6,11 @@ builtins:
 concatStrings (map
   (name:
     let builtin = builtins.${name}; in
-    "  - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args)
-    + "  \n\n"
-    + concatStrings (map (s: "    ${s}\n") (splitLines builtin.doc)) + "\n\n"
+    "<dt><code>${name} "
+    + concatStringsSep " " (map (s: "<var>${s}</var>") builtin.args)
+    + "</code></dt>"
+    + "<dd>\n\n"
+    + builtin.doc
+    + "\n\n</dd>"
   )
   (attrNames builtins))
-

doc/manual/generate-manpage.nix

@@ -1,4 +1,4 @@
-command:
+{ command, renderLinks ? false }:
 
 with builtins;
 with import ./utils.nix;
@@ -20,7 +20,11 @@ let
            categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
            listCommands = cmds:
              concatStrings (map (name:
-               "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n")
+               "* "
+               + (if renderLinks
+                  then "[`${command} ${name}`](./${appendName filename name}.md)"
+                  else "`${command} ${name}`")
+               + " - ${cmds.${name}.description}\n")
                (attrNames cmds));
          in
          "where *subcommand* is one of the following:\n\n"
@@ -89,7 +93,7 @@ let
 in
 
 let
-  manpages = processCommand { filename = "nix"; command = "nix"; def = command; };
+  manpages = processCommand { filename = "nix"; command = "nix"; def = builtins.fromJSON command; };
   summary = concatStrings (map (manpage: "    - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages);
 in
 (listToAttrs manpages) // { "SUMMARY.md" = summary; }

doc/manual/local.mk

@@ -1,7 +1,5 @@
 ifeq ($(doc_generate),yes)
 
-MANUAL_SRCS := $(call rwildcard, $(d)/src, *.md)
-
 # Generate man pages.
 man-pages := $(foreach n, \
   nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
@@ -46,7 +44,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@
-	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))'
+	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }'
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
@@ -64,6 +62,7 @@ $(d)/conf-file.json: $(bindir)/nix
 $(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix
 	@cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp
 	$(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp
+	@cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp
 	@mv $@.tmp $@
 
 $(d)/builtins.json: $(bindir)/nix
@@ -74,17 +73,28 @@ $(d)/builtins.json: $(bindir)/nix
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
-install: $(d)/src/command-ref/new-cli
+install: $(mandir)/man1/nix3-manpages
+man: doc/manual/generated/man1/nix3-manpages
+all: doc/manual/generated/man1/nix3-manpages
+
+$(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages
+	@mkdir -p $(DESTDIR)$$(dirname $@)
+	$(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@)
+
+doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
+	@mkdir -p $(DESTDIR)$$(dirname $@)
 	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
 	  name=$$(basename $$i .md); \
+	  tmpFile=$$(mktemp); \
 	  if [[ $$name = SUMMARY ]]; then continue; fi; \
-	  printf "Title: %s\n\n" "$$name" > $$i.tmp; \
-	  cat $$i >> $$i.tmp; \
-	  lowdown -sT man -M section=1 $$i.tmp -o $(mandir)/man1/$$name.1; \
+	  printf "Title: %s\n\n" "$$name" > $$tmpFile; \
+	  cat $$i >> $$tmpFile; \
+	  lowdown -sT man -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \
+	  rm $$tmpFile; \
 	done
+	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
-	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual
-	@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md)
+	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual
 
 endif

doc/manual/src/SUMMARY.md.in

@@ -70,6 +70,7 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
+  - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)
   - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md)

doc/manual/src/command-ref/env-common.md

@@ -10,35 +10,39 @@ Most Nix commands interpret the following environment variables:
     A colon-separated list of directories used to look up Nix
     expressions enclosed in angle brackets (i.e., `<path>`). For
     instance, the value
-    
+
         /home/eelco/Dev:/etc/nixos
-    
+
     will cause Nix to look for paths relative to `/home/eelco/Dev` and
     `/etc/nixos`, in this order. It is also possible to match paths
     against a prefix. For example, the value
-    
+
         nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos
-    
+
     will cause Nix to search for `<nixpkgs/path>` in
     `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`.
-    
+
     If a path in the Nix search path starts with `http://` or
     `https://`, it is interpreted as the URL of a tarball that will be
     downloaded and unpacked to a temporary location. The tarball must
     consist of a single top-level directory. For example, setting
     `NIX_PATH` to
-    
-        nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz
-    
-    tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09
-    channel.
-    
-    A following shorthand can be used to refer to the official channels:
-    
-        nixpkgs=channel:nixos-15.09
-    
-    The search path can be extended using the `-I` option, which takes
-    precedence over `NIX_PATH`.
+
+        nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz
+
+    tells Nix to download and use the current contents of the
+    `master` branch in the `nixpkgs` repository.
+
+    The URLs of the tarballs from the official nixos.org channels (see
+    [the manual for `nix-channel`](nix-channel.md)) can be abbreviated
+    as `channel:<channel-name>`.  For instance, the following two
+    values of `NIX_PATH` are equivalent:
+
+        nixpkgs=channel:nixos-21.05
+        nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz
+
+    The Nix search path can also be extended using the `-I` option to
+    many Nix commands, which takes precedence over `NIX_PATH`.
 
   - `NIX_IGNORE_SYMLINK_STORE`\
     Normally, the Nix store directory (typically `/nix/store`) is not
@@ -50,7 +54,7 @@ Most Nix commands interpret the following environment variables:
     builds are deployed to machines where `/nix/store` resolves
     differently. If you are sure that you’re not going to do that, you
     can set `NIX_IGNORE_SYMLINK_STORE` to `1`.
-    
+
     Note that if you’re symlinking the Nix store so that you can put it
     on another file system than the root file system, on Linux you’re
     better off using `bind` mount points, e.g.,
@@ -59,7 +63,7 @@ Most Nix commands interpret the following environment variables:
     $ mkdir /nix
     $ mount -o bind /mnt/otherdisk/nix /nix
     ```
-    
+
     Consult the mount 8 manual page for details.
 
   - `NIX_STORE_DIR`\

doc/manual/src/expressions/builtins-prefix.md

@@ -9,7 +9,8 @@ scope. Instead, you can access them through the `builtins` built-in
 value, which is a set that contains all built-in functions and values.
 For instance, `derivation` is also available as `builtins.derivation`.
 
-  - `derivation` *attrs*; `builtins.derivation` *attrs*\
-
-    `derivation` is described in [its own section](derivations.md).
-
+<dl>
+  <dt><code>derivation <var>attrs</var></code>;
+      <code>builtins.derivation <var>attrs</var></code></dt>
+  <dd><p><var>derivation</var> in described in
+         <a href="derivations.md">its own section</a>.</p></dd>

doc/manual/src/expressions/builtins-suffix.md

@@ -0,0 +1 @@
+</dl>

doc/manual/src/expressions/language-values.md

@@ -139,6 +139,13 @@ Nix has the following basic data types:
     environment variable `NIX_PATH` will be searched for the given file
     or directory name.
 
+    Antiquotation is supported in any paths except those in angle brackets.
+    `./${foo}-${bar}.nix` is a more convenient way of writing 
+    `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At
+    least one slash must appear *before* any antiquotations for this to be
+    recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division
+    operation. `./a.${foo}/b.${bar}` is a path.
+
   - *Booleans* with values `true` and `false`.
 
   - The null value, denoted as `null`.

doc/manual/src/installation/env-variables.md

@@ -40,7 +40,7 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
 > **Note**
 > 
 > You must not add the export and then do the install, as the Nix
-> installer will detect the presense of Nix configuration, and abort.
+> installer will detect the presence of Nix configuration, and abort.
 
 ## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon
 

doc/manual/src/installation/prerequisites-source.md

@@ -26,15 +26,6 @@
     available for download from the official repository
     <https://github.com/google/brotli>.
 
-  - The bzip2 compressor program and the `libbz2` library. Thus you must
-    have bzip2 installed, including development headers and libraries.
-    If your distribution does not provide these, you can obtain bzip2
-    from
-    <https://sourceware.org/bzip2/>.
-
-  - `liblzma`, which is provided by XZ Utils. If your distribution does
-    not provide this, you can get it from <https://tukaani.org/xz/>.
-
   - cURL and its library. If your distribution does not provide it, you
     can get it from <https://curl.haxx.se/>.
 

doc/manual/src/release-notes/rl-2.4.md

@@ -1,8 +1,525 @@
-# Release 2.4 (202X-XX-XX)
+# Release 2.4 (2021-11-01)
 
-  - It is now an error to modify the `plugin-files` setting via a
-    command-line flag that appears after the first non-flag argument
-    to any command, including a subcommand to `nix`. For example,
-    `nix-instantiate default.nix --plugin-files ""` must now become
-    `nix-instantiate --plugin-files "" default.nix`.
-  - Plugins that add new `nix` subcommands are now actually respected.
+This is the first release in more than two years and is the result of
+more than 2800 commits from 195 contributors since release 2.3.
+
+## Highlights
+
+* Nix's **error messages** have been improved a lot. For instance,
+  evaluation errors now point out the location of the error:
+
+  ```
+  $ nix build
+  error: undefined variable 'bzip3'
+
+         at /nix/store/449lv242z0zsgwv95a8124xi11sp419f-source/flake.nix:88:13:
+
+             87|           [ curl
+             88|             bzip3 xz brotli editline
+               |             ^
+             89|             openssl sqlite
+  ```
+
+* The **`nix` command** has seen a lot of work and is now almost at
+  feature parity with the old command-line interface (the `nix-*`
+  commands). It aims to be [more modern, consistent and pleasant to
+  use](../contributing/cli-guideline.md) than the old CLI. It is still
+  marked as experimental but its interface should not change much
+  anymore in future releases.
+
+* **Flakes** are a new format to package Nix-based projects in a more
+  discoverable, composable, consistent and reproducible way. A flake
+  is just a repository or tarball containing a file named `flake.nix`
+  that specifies dependencies on other flakes and returns any Nix
+  assets such as packages, Nixpkgs overlays, NixOS modules or CI
+  tests. The new `nix` CLI is primarily based around flakes; for
+  example, a command like `nix run nixpkgs#hello` runs the `hello`
+  application from the `nixpkgs` flake.
+
+  Flakes are currently marked as experimental. For an introduction,
+  see [this blog
+  post](https://www.tweag.io/blog/2020-05-25-flakes/). For detailed
+  information about flake syntax and semantics, see the [`nix flake`
+  manual page](../command-ref/new-cli/nix3-flake.md).
+
+* Nix's store can now be **content-addressed**, meaning that the hash
+  component of a store path is the hash of the path's
+  contents. Previously Nix could only build **input-addressed** store
+  paths, where the hash is computed from the derivation dependency
+  graph. Content-addressing allows deduplication, early cutoff in
+  build systems, and unprivileged closure copying. This is still [an
+  experimental
+  feature](https://discourse.nixos.org/t/content-addressed-nix-call-for-testers/12881).
+
+* The Nix manual has been converted into Markdown, making it easier to
+  contribute. In addition, every `nix` subcommand now has a manual
+  page, documenting every option.
+
+* A new setting that allows **experimental features** to be enabled
+  selectively. This allows us to merge unstable features into Nix more
+  quickly and do more frequent releases.
+
+## Other features
+
+* There are many new `nix` subcommands:
+
+  - `nix develop` is intended to replace `nix-shell`. It has a number
+    of new features:
+
+    * It automatically sets the output environment variables (such as
+      `$out`) to writable locations (such as `./outputs/out`).
+
+    * It can store the environment in a profile. This is useful for
+      offline work.
+
+    * It can run specific phases directly. For instance, `nix develop
+      --build` runs `buildPhase`.
+
+    - It allows dependencies in the Nix store to be "redirected" to
+      arbitrary directories using the `--redirect` flag. This is
+      useful if you want to hack on a package *and* some of its
+      dependencies at the same time.
+
+  - `nix print-dev-env` prints the environment variables and bash
+    functions defined by a derivation. This is useful for users of
+    other shells than bash (especially with `--json`).
+
+  - `nix shell` was previously named `nix run` and is intended to
+    replace `nix-shell -p`, but without the `stdenv` overhead. It
+    simply starts a shell where some packages have been added to
+    `$PATH`.
+
+  - `nix run` (not to be confused with the old subcommand that has
+    been renamed to `nix shell`) runs an "app", a flake output that
+    specifies a command to run, or an eponymous program from a
+    package. For example, `nix run nixpkgs#hello` runs the `hello`
+    program from the `hello` package in `nixpkgs`.
+
+  - `nix flake` is the container for flake-related operations, such as
+    creating a new flake, querying the contents of a flake or updating
+    flake lock files.
+
+  - `nix registry` allows you to query and update the flake registry,
+    which maps identifiers such as `nixpkgs` to concrete flake URLs.
+
+  - `nix profile` is intended to replace `nix-env`. Its main advantage
+    is that it keeps track of the provenance of installed packages
+    (e.g. exactly which flake version a package came from). It also
+    has some helpful subcommands:
+
+    * `nix profile history` shows what packages were added, upgraded
+      or removed between each version of a profile.
+
+    * `nix profile diff-closures` shows the changes between the
+      closures of each version of a profile. This allows you to
+      discover the addition or removal of dependencies or size
+      changes.
+
+    **Warning**: after a profile has been updated using `nix profile`,
+    it is no longer usable with `nix-env`.
+
+  - `nix store diff-closures` shows the differences between the
+    closures of two store paths in terms of the versions and sizes of
+    dependencies in the closures.
+
+  - `nix store make-content-addressable` rewrites an arbitrary closure
+    to make it content-addressed. Such paths can be copied into other
+    stores without requiring signatures.
+
+  - `nix bundle` uses the [`nix-bundle`
+    program](https://github.com/matthewbauer/nix-bundle) to convert a
+    closure into a self-extracting executable.
+
+  - Various other replacements for the old CLI, e.g. `nix store gc`,
+    `nix store delete`, `nix store repair`, `nix nar dump-path`, `nix
+    store prefetch-file`, `nix store prefetch-tarball`, `nix key` and
+    `nix daemon`.
+
+* Nix now has an **evaluation cache** for flake outputs. For example,
+  a second invocation of the command `nix run nixpkgs#firefox` will
+  not need to evaluate the `firefox` attribute because it's already in
+  the evaluation cache. This is made possible by the hermetic
+  evaluation model of flakes.
+
+* The new `--offline` flag disables substituters and causes all
+  locally cached tarballs and repositories to be considered
+  up-to-date.
+
+* The new `--refresh` flag causes all locally cached tarballs and
+  repositories to be considered out-of-date.
+
+* Many `nix` subcommands now have a `--json` option to produce
+  machine-readable output.
+
+* `nix repl` has a new `:doc` command to show documentation about
+  builtin functions (e.g. `:doc builtins.map`).
+
+* Binary cache stores now have an option `index-debug-info` to create
+  an index of DWARF debuginfo files for use by
+  [`dwarffs`](https://github.com/edolstra/dwarffs).
+
+* To support flakes, Nix now has an extensible mechanism for fetching
+  source trees. Currently it has the following backends:
+
+  * Git repositories
+
+  * Mercurial repositories
+
+  * GitHub and GitLab repositories (an optimisation for faster
+    fetching than Git)
+
+  * Tarballs
+
+  * Arbitrary directories
+
+  The fetcher infrastructure is exposed via flake input specifications
+  and via the `fetchTree` built-in.
+
+* **Languages changes**: the only new language feature is that you can
+  now have antiquotations in paths, e.g. `./${foo}` instead of `./. +
+  foo`.
+
+* **New built-in functions**:
+
+  - `builtins.fetchTree` allows fetching a source tree using any
+    backends supported by the fetcher infrastructure. It subsumes the
+    functionality of existing built-ins like `fetchGit`,
+    `fetchMercurial` and `fetchTarball`.
+
+  - `builtins.getFlake` fetches a flake and returns its output
+    attributes. This function should not be used inside flakes! Use
+    flake inputs instead.
+
+  - `builtins.floor` and `builtins.ceil` round a floating-point number
+    down and up, respectively.
+
+* Experimental support for recursive Nix. This means that Nix
+  derivations can now call Nix to build other derivations. This is not
+  in a stable state yet and not well
+  [documented](https://github.com/NixOS/nix/commit/c4d7c76b641d82b2696fef73ce0ac160043c18da).
+
+* The new experimental feature `no-url-literals` disables URL
+  literals. This helps to implement [RFC
+  45](https://github.com/NixOS/rfcs/pull/45).
+
+* Nix now uses `libarchive` to decompress and unpack tarballs and zip
+  files, so `tar` is no longer required.
+
+* The priority of substituters can now be overridden using the
+  `priority` substituter setting (e.g. `--substituters
+  'http://cache.nixos.org?priority=100 daemon?priority=10'`).
+
+* `nix edit` now supports non-derivation attributes, e.g. `nix edit
+  .#nixosConfigurations.bla`.
+
+* The `nix` command now provides command line completion for `bash`,
+  `zsh` and `fish`. Since the support for getting completions is built
+  into `nix`, it's easy to add support for other shells.
+
+* The new `--log-format` flag selects what Nix's output looks like. It
+  defaults to a terse progress indicator. There is a new
+  `internal-json` output format for use by other programs.
+
+* `nix eval` has a new `--apply` flag that applies a function to the
+  evaluation result.
+
+* `nix eval` has a new `--write-to` flag that allows it to write a
+  nested attribute set of string leaves to a corresponding directory
+  tree.
+
+* Memory improvements: many operations that add paths to the store or
+  copy paths between stores now run in constant memory.
+
+* Many `nix` commands now support the flag `--derivation` to operate
+  on a `.drv` file itself instead of its outputs.
+
+* There is a new store called `dummy://` that does not support
+  building or adding paths. This is useful if you want to use the Nix
+  evaluator but don't have a Nix store.
+
+* The `ssh-ng://` store now allows substituting paths on the remote,
+  as `ssh://` already did.
+
+* When auto-calling a function with an ellipsis, all arguments are now
+  passed.
+
+* New `nix-shell` features:
+
+  - It preserves the `PS1` environment variable if
+    `NIX_SHELL_PRESERVE_PROMPT` is set.
+
+  - With `-p`, it passes any `--arg`s as Nixpkgs arguments.
+
+  - Support for structured attributes.
+
+* `nix-prefetch-url` has a new `--executable` flag.
+
+* On `x86_64` systems, [`x86_64` microarchitecture
+  levels](https://lwn.net/Articles/844831/) are mapped to additional
+  system types (e.g. `x86_64-v1-linux`).
+
+* The new `--eval-store` flag allows you to use a different store for
+  evaluation than for building or storing the build result. This is
+  primarily useful when you want to query whether something exists in
+  a read-only store, such as a binary cache:
+
+  ```
+  # nix path-info --json --store https://cache.nixos.org \
+    --eval-store auto nixpkgs#hello
+  ```
+
+  (Here `auto` indicates the local store.)
+
+* The Nix daemon has a new low-latency mechanism for copying
+  closures. This is useful when building on remote stores such as
+  `ssh-ng://`.
+
+* Plugins can now register `nix` subcommands.
+
+## Incompatible changes
+
+* The `nix` command is now marked as an experimental feature. This
+  means that you need to add
+
+  > experimental-features = nix-command
+
+  to your `nix.conf` if you want to use it, or pass
+  `--extra-experimental-features nix-command` on the command line.
+
+* The `nix` command no longer has a syntax for referring to packages
+  in a channel. This means that the following no longer works:
+
+  > nix build nixpkgs.hello # Nix 2.3
+
+  Instead, you can either use the `#` syntax to select a package from
+  a flake, e.g.
+
+  > nix build nixpkgs#hello
+
+  Or, if you want to use the `nixpkgs` channel in the `NIX_PATH`
+  environment variable:
+
+  > nix build -f '<nixpkgs>' hello
+
+* The old `nix run` has been renamed to `nix shell`, while there is a
+  new `nix run` that runs a default command. So instead of
+
+  > nix run nixpkgs.hello -c hello # Nix 2.3
+
+  you should use
+
+  > nix shell nixpkgs#hello -c hello
+
+  or just
+
+  > nix run nixpkgs#hello
+
+  if the command you want to run has the same name as the package.
+
+* It is now an error to modify the `plugin-files` setting via a
+  command-line flag that appears after the first non-flag argument to
+  any command, including a subcommand to `nix`. For example,
+  `nix-instantiate default.nix --plugin-files ""` must now become
+  `nix-instantiate --plugin-files "" default.nix`.
+
+* We no longer release source tarballs. If you want to build from
+  source, please build from the tags in the Git repository.
+
+## Contributors
+
+This release has contributions from
+Adam Höse,
+Albert Safin,
+Alex Kovar,
+Alex Zero,
+Alexander Bantyev,
+Alexandre Esteves,
+Alyssa Ross,
+Anatole Lucet,
+Anders Kaseorg,
+Andreas Rammhold,
+Antoine Eiche,
+Antoine Martin,
+Arnout Engelen,
+Arthur Gautier,
+aszlig,
+Ben Burdette,
+Benjamin Hipple,
+Bernardo Meurer,
+Björn Gohla,
+Bjørn Forsman,
+Bob van der Linden,
+Brian Leung,
+Brian McKenna,
+Brian Wignall,
+Bruce Toll,
+Bryan Richter,
+Calle Rosenquist,
+Calvin Loncaric,
+Carlo Nucera,
+Carlos D'Agostino,
+Chaz Schlarp,
+Christian Höppner,
+Christian Kampka,
+Chua Hou,
+Chuck,
+Cole Helbling,
+Daiderd Jordan,
+Dan Callahan,
+Dani,
+Daniel Fitzpatrick,
+Danila Fedorin,
+Daniël de Kok,
+Danny Bautista,
+DavHau,
+David McFarland,
+Dima,
+Domen Kožar,
+Dominik Schrempf,
+Dominique Martinet,
+dramforever,
+Dustin DeWeese,
+edef,
+Eelco Dolstra,
+Emilio Karakey,
+Emily,
+Eric Culp,
+Ersin Akinci,
+Fabian Möller,
+Farid Zakaria,
+Federico Pellegrin,
+Finn Behrens,
+Florian Franzen,
+Félix Baylac-Jacqué,
+Gabriel Gonzalez,
+Geoff Reedy,
+Georges Dubus,
+Graham Christensen,
+Greg Hale,
+Greg Price,
+Gregor Kleen,
+Gregory Hale,
+Griffin Smith,
+Guillaume Bouchard,
+Harald van Dijk,
+illustris,
+Ivan Zvonimir Horvat,
+Jade,
+Jake Waksbaum,
+jakobrs,
+James Ottaway,
+Jan Tojnar,
+Janne Heß,
+Jaroslavas Pocepko,
+Jarrett Keifer,
+Jeremy Schlatter,
+Joachim Breitner,
+Joe Hermaszewski,
+Joe Pea,
+John Ericson,
+Jonathan Ringer,
+Josef Kemetmüller,
+Joseph Lucas,
+Jude Taylor,
+Julian Stecklina,
+Julien Tanguy,
+Jörg Thalheim,
+Kai Wohlfahrt,
+keke,
+Keshav Kini,
+Kevin Quick,
+Kevin Stock,
+Kjetil Orbekk,
+Krzysztof Gogolewski,
+kvtb,
+Lars Mühmel,
+Leonhard Markert,
+Lily Ballard,
+Linus Heckemann,
+Lorenzo Manacorda,
+Lucas Desgouilles,
+Lucas Franceschino,
+Lucas Hoffmann,
+Luke Granger-Brown,
+Madeline Haraj,
+Marwan Aljubeh,
+Mat Marini,
+Mateusz Piotrowski,
+Matthew Bauer,
+Matthew Kenigsberg,
+Mauricio Scheffer,
+Maximilian Bosch,
+Michael Adler,
+Michael Bishop,
+Michael Fellinger,
+Michael Forney,
+Michael Reilly,
+mlatus,
+Mykola Orliuk,
+Nathan van Doorn,
+Naïm Favier,
+ng0,
+Nick Van den Broeck,
+Nicolas Stig124 Formichella,
+Niels Egberts,
+Niklas Hambüchen,
+Nikola Knezevic,
+oxalica,
+p01arst0rm,
+Pamplemousse,
+Patrick Hilhorst,
+Paul Opiyo,
+Pavol Rusnak,
+Peter Kolloch,
+Philipp Bartsch,
+Philipp Middendorf,
+Piotr Szubiakowski,
+Profpatsch,
+Puck Meerburg,
+Ricardo M. Correia,
+Rickard Nilsson,
+Robert Hensing,
+Robin Gloster,
+Rodrigo,
+Rok Garbas,
+Ronnie Ebrin,
+Rovanion Luckey,
+Ryan Burns,
+Ryan Mulligan,
+Ryne Everett,
+Sam Doshi,
+Sam Lidder,
+Samir Talwar,
+Samuel Dionne-Riel,
+Sebastian Ullrich,
+Sergei Trofimovich,
+Sevan Janiyan,
+Shao Cheng,
+Shea Levy,
+Silvan Mosberger,
+Stefan Frijters,
+Stefan Jaax,
+sternenseemann,
+Steven Shaw,
+Stéphan Kochen,
+SuperSandro2000,
+Suraj Barkale,
+Taeer Bar-Yam,
+Thomas Churchman,
+Théophane Hufschmitt,
+Timothy DeHerrera,
+Timothy Klim,
+Tobias Möst,
+Tobias Pflug,
+Tom Bereknyei,
+Travis A. Everett,
+Ujjwal Jain,
+Vladimír Čunát,
+Wil Taylor,
+Will Dietz,
+Yaroslav Bolyukin,
+Yestin L. Harrison,
+YI,
+Yorick van Pelt,
+Yuriy Taraday and
+zimbatm.

flake.lock

@@ -3,27 +3,26 @@
     "lowdown-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1617481909,
-        "narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=",
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
         "owner": "kristapsdz",
         "repo": "lowdown",
-        "rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
         "type": "github"
       },
       "original": {
         "owner": "kristapsdz",
-        "ref": "VERSION_0_8_4",
         "repo": "lowdown",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1622593737,
-        "narHash": "sha256-9loxFJg85AbzJrSkU4pE/divZ1+zOxDy2FSjlrufCB8=",
+        "lastModified": 1632864508,
+        "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "bb8a5e54845012ed1375ffd5f317d2fdf434b20e",
+        "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
         "type": "github"
       },
       "original": {

flake.nix

@@ -2,7 +2,7 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small";
-  inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; };
+  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
   outputs = { self, nixpkgs, lowdown-src }:
 
@@ -14,12 +14,14 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
       systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ];
 
+      crossSystems = [ "armv6l-linux" "armv7l-linux" ];
+
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
 
       # Memoize nixpkgs for different platforms for efficiency.
@@ -68,7 +70,7 @@
           [
             buildPackages.bison
             buildPackages.flex
-            (lib.getBin buildPackages.lowdown)
+            (lib.getBin buildPackages.lowdown-nix)
             buildPackages.mdbook
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
@@ -76,10 +78,10 @@
 
             # Tests
             buildPackages.git
-            buildPackages.mercurial
+            buildPackages.mercurial # FIXME: remove? only needed for tests
             buildPackages.jq
           ]
-          ++ lib.optionals stdenv.isLinux [(pkgs.util-linuxMinimal or pkgs.utillinuxMinimal)];
+          ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
 
         buildDeps =
           [ curl
@@ -87,13 +89,12 @@
             openssl sqlite
             libarchive
             boost
-            nlohmann_json
-            lowdown
+            lowdown-nix
             gmock
           ]
           ++ lib.optionals stdenv.isLinux [libseccomp]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-          ++ lib.optional stdenv.isx86_64 libcpuid;
+          ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
 
         awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
           (aws-sdk-cpp.override {
@@ -102,7 +103,13 @@
           });
 
         propagatedDeps =
-          [ (boehmgc.override { enableLargeConfig = true; })
+          [ ((boehmgc.override {
+              enableLargeConfig = true;
+            }).overrideAttrs(o: {
+              patches = (o.patches or []) ++ [
+                ./boehmgc-coroutine-sp-fallback.diff
+              ];
+            }))
           ];
 
         perlDeps =
@@ -119,8 +126,7 @@
           ''
             mkdir -p $out/nix-support
 
-            # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
-            # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
+            # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
             tarballPath() {
               # Remove the store prefix
               local path=''${1#${builtins.storeDir}/}
@@ -133,10 +139,11 @@
 
             substitute ${./scripts/install.in} $out/install \
               ${pkgs.lib.concatMapStrings
-                (system:
-                  '' \
-                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
-                  --replace '@tarballPath_${system}@' $(tarballPath ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) \
+                (system: let
+                    tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system};
+                  in '' \
+                  --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
+                  --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
                   ''
                 )
                 systems
@@ -145,13 +152,15 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
-      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
+      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
-        # Must keep this name short as OSX has a rather strict limit on the
-        # socket path length, and this name appears in the path of the
-        # nix-daemon socket used in the tests
-        name = "nix-tests";
+        name =
+          "nix-tests"
+          + optionalString
+            (versionAtLeast daemon.version "2.4pre20211005" &&
+             versionAtLeast client.version "2.4pre20211005")
+            "-${client.version}-against-${daemon.version}";
         inherit version;
 
         src = self;
@@ -170,21 +179,92 @@
         installPhase = ''
           mkdir -p $out
         '';
-        installCheckPhase = "make installcheck";
 
+        installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES";
       };
 
+      binaryTarball = buildPackages: nix: pkgs: let
+        inherit (pkgs) cacert;
+        installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
+      in
+
+      buildPackages.runCommand "nix-binary-tarball-${version}"
+        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+          meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
+        }
+        ''
+          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+          substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+
+          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+            --subst-var-by nix ${nix} \
+            --subst-var-by cacert ${cacert}
+
+          if type -p shellcheck; then
+            # SC1090: Don't worry about not being able to find
+            #         $nix/etc/profile.d/nix.sh
+            shellcheck --exclude SC1090 $TMPDIR/install
+            shellcheck $TMPDIR/create-darwin-volume.sh
+            shellcheck $TMPDIR/install-darwin-multi-user.sh
+            shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+            # SC1091: Don't panic about not being able to source
+            #         /etc/profile
+            # SC2002: Ignore "useless cat" "error", when loading
+            #         .reginfo, as the cat is a much cleaner
+            #         implementation, even though it is "useless"
+            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+            #         root's home directory
+            shellcheck --external-sources \
+              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+          fi
+
+          chmod +x $TMPDIR/install
+          chmod +x $TMPDIR/create-darwin-volume.sh
+          chmod +x $TMPDIR/install-darwin-multi-user.sh
+          chmod +x $TMPDIR/install-systemd-multi-user.sh
+          chmod +x $TMPDIR/install-multi-user
+          dir=nix-${version}-${pkgs.system}
+          fn=$out/$dir.tar.xz
+          mkdir -p $out/nix-support
+          echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+          tar cvfJ $fn \
+            --owner=0 --group=0 --mode=u+rw,uga+r \
+            --absolute-names \
+            --hard-dereference \
+            --transform "s,$TMPDIR/install,$dir/install," \
+            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
+            --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+            --transform "s,$NIX_STORE,$dir/store,S" \
+            $TMPDIR/install \
+            $TMPDIR/create-darwin-volume.sh \
+            $TMPDIR/install-darwin-multi-user.sh \
+            $TMPDIR/install-systemd-multi-user.sh \
+            $TMPDIR/install-multi-user \
+            $TMPDIR/reginfo \
+            $(cat ${installerClosureInfo}/store-paths)
+        '';
+
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
       overlay = final: prev: {
 
-        # An older version of Nix to test against when using the daemon.
-        # Currently using `nixUnstable` as the stable one doesn't respect
-        # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests.
         nixStable = prev.nix;
 
+        # Forward from the previous stage as we don’t want it to pick the lowdown override
+        nixUnstable = prev.nixUnstable;
+
         nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
@@ -229,7 +309,7 @@
             echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
           '';
 
-          doInstallCheck = false;
+          doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
 
           separateDebugInfo = true;
@@ -254,9 +334,9 @@
                 xz
                 pkgs.perl
                 boost
-                nlohmann_json
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -270,21 +350,14 @@
 
         };
 
-        lowdown = with final; stdenv.mkDerivation rec {
-          name = "lowdown-0.8.4";
-
-          /*
-          src = fetchurl {
-            url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz";
-            hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw==";
-          };
-          */
+        lowdown-nix = with final; stdenv.mkDerivation rec {
+          name = "lowdown-0.9.0";
 
           src = lowdown-src;
 
           outputs = [ "out" "bin" "dev" ];
 
-          nativeBuildInputs = [ which ];
+          nativeBuildInputs = [ buildPackages.which ];
 
           configurePhase = ''
               ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
@@ -294,13 +367,6 @@
             '';
         };
 
-        nix-benchmarks = prev.runCommandNoCC "nix-benchmarks" {
-          buildInputs = [ final.nix prev.hyperfine ];
-          THINGTOBENCH = ./thingToBench.nix;
-          NIX_PATH="nixpkgs=${prev.path}";
-        } ''
-          bash ${./benchmark.sh} run_all
-        '';
       };
 
       hydraJobs = {
@@ -310,92 +376,33 @@
 
         buildStatic = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static);
 
+        buildCross = nixpkgs.lib.genAttrs crossSystems (crossSystem:
+          nixpkgs.lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}"));
+
         # Perl bindings for various platforms.
         perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
 
         # Binary tarball for various platforms, containing a Nix store
         # with the closure of 'nix' package, and the second half of
         # the installation script.
-        binaryTarball = nixpkgs.lib.genAttrs systems (system:
+        binaryTarball = nixpkgs.lib.genAttrs systems (system: binaryTarball nixpkgsFor.${system} nixpkgsFor.${system}.nix nixpkgsFor.${system});
 
-          with nixpkgsFor.${system};
-
-          let
-            installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
-          in
-
-          runCommand "nix-binary-tarball-${version}"
-            { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-              meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
-            }
-            ''
-              cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-              cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-              substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-              substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-                --subst-var-by nix ${nix} \
-                --subst-var-by cacert ${cacert}
-
-              if type -p shellcheck; then
-                # SC1090: Don't worry about not being able to find
-                #         $nix/etc/profile.d/nix.sh
-                shellcheck --exclude SC1090 $TMPDIR/install
-                shellcheck $TMPDIR/create-darwin-volume.sh
-                shellcheck $TMPDIR/install-darwin-multi-user.sh
-                shellcheck $TMPDIR/install-systemd-multi-user.sh
-
-                # SC1091: Don't panic about not being able to source
-                #         /etc/profile
-                # SC2002: Ignore "useless cat" "error", when loading
-                #         .reginfo, as the cat is a much cleaner
-                #         implementation, even though it is "useless"
-                # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-                #         root's home directory
-                shellcheck --external-sources \
-                  --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-              fi
-
-              chmod +x $TMPDIR/install
-              chmod +x $TMPDIR/create-darwin-volume.sh
-              chmod +x $TMPDIR/install-darwin-multi-user.sh
-              chmod +x $TMPDIR/install-systemd-multi-user.sh
-              chmod +x $TMPDIR/install-multi-user
-              dir=nix-${version}-${system}
-              fn=$out/$dir.tar.xz
-              mkdir -p $out/nix-support
-              echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-              tar cvfJ $fn \
-                --owner=0 --group=0 --mode=u+rw,uga+r \
-                --absolute-names \
-                --hard-dereference \
-                --transform "s,$TMPDIR/install,$dir/install," \
-                --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
-                --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-                --transform "s,$NIX_STORE,$dir/store,S" \
-                $TMPDIR/install \
-                $TMPDIR/create-darwin-volume.sh \
-                $TMPDIR/install-darwin-multi-user.sh \
-                $TMPDIR/install-systemd-multi-user.sh \
-                $TMPDIR/install-multi-user \
-                $TMPDIR/reginfo \
-                $(cat ${installerClosureInfo}/store-paths)
-            '');
+        binaryTarballCross = nixpkgs.lib.genAttrs ["x86_64-linux"] (system: builtins.listToAttrs (map (crossSystem: {
+          name = crossSystem;
+          value = let
+            nixpkgsCross = import nixpkgs {
+              inherit system crossSystem;
+              overlays = [ self.overlay ];
+            };
+          in binaryTarball nixpkgsFor.${system} self.packages.${system}."nix-${crossSystem}" nixpkgsCross;
+        }) crossSystems));
 
         # The first half of the installation script. This is uploaded
         # to https://nixos.org/nix/install. It downloads the binary
         # tarball for the user's system and calls the second half of the
         # installation script.
-        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
-        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" ];
+        installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
+        installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
         # Line coverage analysis.
         coverage =
@@ -484,7 +491,11 @@
           let pkgs = nixpkgsFor.${system}; in
           pkgs.runCommand "install-tests" {
             againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
-            againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
+            againstCurrentUnstable =
+              # FIXME: temporarily disable this on macOS because of #3605.
+              if system == "x86_64-linux"
+              then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
+              else null;
             # Disabled because the latest stable version doesn't handle
             # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
             # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
@@ -492,8 +503,8 @@
       });
 
       packages = forAllSystems (system: {
-        inherit (nixpkgsFor.${system}) nix nix-benchmarks;
-      } // nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
+        inherit (nixpkgsFor.${system}) nix;
+      } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = let
           nixpkgs = nixpkgsFor.${system}.pkgsStatic;
         in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation {
@@ -531,8 +542,49 @@
           stripAllList = ["bin"];
 
           strictDeps = true;
+
+          hardeningDisable = [ "pie" ];
         };
-      });
+      } // builtins.listToAttrs (map (crossSystem: {
+        name = "nix-${crossSystem}";
+        value = let
+          nixpkgsCross = import nixpkgs {
+            inherit system crossSystem;
+            overlays = [ self.overlay ];
+          };
+        in with commonDeps nixpkgsCross; nixpkgsCross.stdenv.mkDerivation {
+          name = "nix-${version}";
+
+          src = self;
+
+          VERSION_SUFFIX = versionSuffix;
+
+          outputs = [ "out" "dev" "doc" ];
+
+          nativeBuildInputs = nativeBuildDeps;
+          buildInputs = buildDeps ++ propagatedDeps;
+
+          configureFlags = [ "--sysconfdir=/etc" "--disable-doc-gen" ];
+
+          enableParallelBuilding = true;
+
+          makeFlags = "profiledir=$(out)/etc/profile.d";
+
+          doCheck = true;
+
+          installFlags = "sysconfdir=$(out)/etc";
+
+          postInstall = ''
+            mkdir -p $doc/nix-support
+            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+            mkdir -p $out/nix-support
+            echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products
+          '';
+
+          doInstallCheck = true;
+          installCheckFlags = "sysconfdir=$(out)/etc";
+        };
+      }) crossSystems)));
 
       defaultPackage = forAllSystems (system: self.packages.${system}.nix);
 

maintainers/upload-release.pl

@@ -83,12 +83,12 @@ sub downloadFile {
 
     if (!-e $tmpFile) {
         print STDERR "downloading $srcFile to $tmpFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
+        system("NIX_REMOTE=https://cache.nixos.org/ nix store cat '$srcFile' > '$tmpFile'") == 0
             or die "unable to fetch $srcFile\n";
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
+    my $sha256_actual = `nix hash file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
         print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
@@ -110,6 +110,9 @@ downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
 downloadFile("binaryTarball.x86_64-darwin", "1");
+downloadFile("binaryTarball.aarch64-darwin", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
+downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
 downloadFile("installerScript", "1");
 
 for my $fn (glob "$tmpDir/*") {
@@ -153,6 +156,7 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
            "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
            "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
            "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+           "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
            "}\n");
 
 system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;

misc/fish/completion.fish

@@ -0,0 +1,37 @@
+function _nix_complete
+  # Get the current command up to a cursor.
+  # - Behaves correctly even with pipes and nested in commands like env.
+  # - TODO: Returns the command verbatim (does not interpolate variables).
+  #   That might not be optimal for arguments like -f.
+  set -l nix_args (commandline --current-process --tokenize --cut-at-cursor)
+  # --cut-at-cursor with --tokenize removes the current token so we need to add it separately.
+  # https://github.com/fish-shell/fish-shell/issues/7375
+  # Can be an empty string.
+  set -l current_token (commandline --current-token --cut-at-cursor)
+
+  # Nix wants the index of the argv item to complete but the $nix_args variable
+  # also contains the program name (argv[0]) so we would need to subtract 1.
+  # But the variable also misses the current token so it cancels out.
+  set -l nix_arg_to_complete (count $nix_args)
+
+  env NIX_GET_COMPLETIONS=$nix_arg_to_complete $nix_args $current_token
+end
+
+function _nix_accepts_files
+  set -l response (_nix_complete)
+  # First line is either filenames or no-filenames.
+  test $response[1] = 'filenames'
+end
+
+function _nix
+  set -l response (_nix_complete)
+  # Skip the first line since it handled by _nix_accepts_files.
+  # Tail lines each contain a command followed by a tab character and, optionally, a description.
+  # This is also the format fish expects.
+  string collect -- $response[2..-1]
+end
+
+# Disable file path completion if paths do not belong in the current context.
+complete --command nix --condition 'not _nix_accepts_files' --no-files
+
+complete --command nix --arguments '(_nix)'

misc/fish/local.mk

@@ -0,0 +1 @@
+$(eval $(call install-file-as, $(d)/completion.fish, $(datarootdir)/fish/vendor_completions.d/nix.fish, 0644))

misc/launchd/local.mk

@@ -1,4 +1,4 @@
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 
   $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons))
 

misc/systemd/local.mk

@@ -1,4 +1,4 @@
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
 

misc/upstart/local.mk

@@ -1,4 +1,4 @@
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644)))
 

misc/zsh/completion.zsh

@@ -1,3 +1,5 @@
+#compdef nix
+
 function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
@@ -18,4 +20,4 @@ function _nix() {
   _describe 'nix' suggestions
 }
 
-compdef _nix nix
+_nix "$@"

mk/lib.mk

@@ -10,8 +10,25 @@ bin-scripts :=
 noinst-scripts :=
 man-pages :=
 install-tests :=
-OS = $(shell uname -s)
 
+ifdef HOST_OS
+  HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
+  ifeq ($(HOST_KERNEL), cygwin)
+    HOST_CYGWIN = 1
+  endif
+  ifeq ($(patsubst darwin%,,$(HOST_KERNEL)),)
+    HOST_DARWIN = 1
+  endif
+  ifeq ($(patsubst freebsd%,,$(HOST_KERNEL)),)
+    HOST_FREEBSD = 1
+  endif
+  ifeq ($(HOST_KERNEL), linux)
+    HOST_LINUX = 1
+  endif
+  ifeq ($(patsubst solaris%,,$(HOST_KERNEL)),)
+    HOST_SOLARIS = 1
+  endif
+endif
 
 # Hack to define a literal space.
 space :=
@@ -50,16 +67,16 @@ endif
 BUILD_SHARED_LIBS ?= 1
 
 ifeq ($(BUILD_SHARED_LIBS), 1)
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
     GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE
   else
     GLOBAL_CFLAGS += -fPIC
     GLOBAL_CXXFLAGS += -fPIC
   endif
-  ifneq ($(OS), Darwin)
-   ifneq ($(OS), SunOS)
-    ifneq ($(OS), FreeBSD)
+  ifndef HOST_DARWIN
+   ifndef HOST_SOLARIS
+    ifndef HOST_FREEBSD
      GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries
     endif
    endif

mk/libraries.mk

@@ -1,9 +1,9 @@
 libs-list :=
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
   SO_EXT = dylib
 else
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     SO_EXT = dll
   else
     SO_EXT = so
@@ -59,7 +59,7 @@ define build-library
   $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs))))
   _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH))
 
-  ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+  ifdef HOST_CYGWIN
     $(1)_INSTALL_DIR ?= $$(bindir)
   else
     $(1)_INSTALL_DIR ?= $$(libdir)
@@ -73,18 +73,18 @@ define build-library
   ifeq ($(BUILD_SHARED_LIBS), 1)
 
     ifdef $(1)_ALLOW_UNDEFINED
-      ifeq ($(OS), Darwin)
+      ifdef HOST_DARWIN
         $(1)_LDFLAGS += -undefined suppress -flat_namespace
       endif
     else
-      ifneq ($(OS), Darwin)
-        ifneq (CYGWIN,$(findstring CYGWIN,$(OS)))
+      ifndef HOST_DARWIN
+        ifndef HOST_CYGWIN
           $(1)_LDFLAGS += -Wl,-z,defs
         endif
       endif
     endif
 
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       $(1)_LDFLAGS += -Wl,-soname=$$($(1)_NAME).$(SO_EXT)
     endif
 
@@ -93,7 +93,7 @@ define build-library
     $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
 	$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
     endif
     $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
@@ -108,7 +108,7 @@ define build-library
 	$$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED))
 
     $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
-    ifneq ($(OS), Darwin)
+    ifndef HOST_DARWIN
       ifeq ($(SET_RPATH_TO_LIBS), 1)
         $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR)
       else
@@ -125,8 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
-	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
+	$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/tests.mk

@@ -13,3 +13,7 @@ define run-install-test
 endef
 
 .PHONY: check installcheck
+
+print-top-help += \
+  echo "  check: Run unit tests"; \
+  echo "  installcheck: Run functional tests";

nix-rust/local.mk

@@ -11,12 +11,12 @@ libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
 libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust
 libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust
 
-ifeq ($(OS), Linux)
+ifdef HOST_LINUX
 libnixrust_LDFLAGS_USE += -ldl
 libnixrust_LDFLAGS_USE_INSTALLED += -ldl
 endif
 
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
 else
 libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
@@ -31,7 +31,7 @@ $(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
 
 $(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
 	$(target-gen) cp $^ $@
-ifeq ($(OS), Darwin)
+ifdef HOST_DARWIN
 	install_name_tool -id $@ $@
 endif
 
@@ -40,7 +40,7 @@ clean: clean-rust
 clean-rust:
 	$(suppress) rm -rfv nix-rust/target
 
-ifneq ($(OS), Darwin)
+ifndef HOST_DARWIN
 check: rust-tests
 
 rust-tests:

perl/Makefile

@@ -1,6 +1,6 @@
 makefiles = local.mk
 
-GLOBAL_CXXFLAGS += -g -Wall -std=c++17
+GLOBAL_CXXFLAGS += -g -Wall -std=c++17 -I ../src
 
 -include Makefile.config
 

perl/Makefile.config.in

@@ -1,3 +1,4 @@
+HOST_OS = @host_os@
 CC = @CC@
 CFLAGS = @CFLAGS@
 CXX = @CXX@

perl/configure.ac

@@ -7,6 +7,8 @@ CXXFLAGS=
 AC_PROG_CC
 AC_PROG_CXX
 
+AC_CANONICAL_HOST
+
 # Use 64-bit file system calls so that we can support files > 2 GiB.
 AC_SYS_LARGEFILE
 

perl/lib/Nix/Store.pm

@@ -22,6 +22,7 @@ our @EXPORT = qw(
     derivationFromPath
     addTempRoot
     getBinDir getStoreDir
+    queryRawRealisation
 );
 
 our $VERSION = '0.15';

perl/lib/Nix/Store.xs

@@ -15,6 +15,7 @@
 #include "crypto.hh"
 
 #include <sodium.h>
+#include <nlohmann/json.hpp>
 
 
 using namespace nix;
@@ -120,6 +121,18 @@ SV * queryPathInfo(char * path, int base32)
             croak("%s", e.what());
         }
 
+SV * queryRawRealisation(char * outputId)
+    PPCODE:
+      try {
+        auto realisation = store()->queryRealisation(DrvOutput::parse(outputId));
+        if (realisation)
+            XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0)));
+        else
+            XPUSHs(sv_2mortal(newSVpv("", 0)));
+      } catch (Error & e) {
+        croak("%s", e.what());
+      }
+
 
 SV * queryPathFromHashPart(char * hashPart)
     PPCODE:

perl/local.mk

@@ -28,7 +28,7 @@ Store_CXXFLAGS = \
 
 Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS)
 
-ifeq (CYGWIN,$(findstring CYGWIN,$(OS)))
+ifdef HOST_CYGWIN
   archlib = $(shell perl -E 'use Config; print $$Config{archlib};')
   libperl = $(shell perl -E 'use Config; print $$Config{libperl};')
   Store_LDFLAGS += $(shell find ${archlib} -name ${libperl})

scripts/create-darwin-volume.sh

@@ -715,7 +715,8 @@ create_volume() {
     # 6) getting special w/ awk may be fragile, but doing it to:
     #    - save time over running slow diskutil commands
     #    - skirt risk we grab wrong volume if multiple match
-    /usr/sbin/diskutil apfs addVolume "$NIX_VOLUME_USE_DISK" "$NIX_VOLUME_FS" "$NIX_VOLUME_LABEL" -nomount | /usr/bin/awk '/Created new APFS Volume/ {print $5}'
+    _sudo "to create a new APFS volume '$NIX_VOLUME_LABEL' on $NIX_VOLUME_USE_DISK" \
+        /usr/sbin/diskutil apfs addVolume "$NIX_VOLUME_USE_DISK" "$NIX_VOLUME_FS" "$NIX_VOLUME_LABEL" -nomount | /usr/bin/awk '/Created new APFS Volume/ {print $5}'
 }
 
 volume_uuid_from_special() {
@@ -738,7 +739,6 @@ await_volume() {
 setup_volume() {
     local use_special use_uuid profile_packages
     task "Creating a Nix volume" >&2
-    # DOING: I'm tempted to wrap this call in a grep to get the new disk special without doing anything too complex, but this sudo wrapper *is* a little complex, so it'll be a PITA unless maybe we can skip sudo on this. Let's just try it without.
 
     use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}"
 
@@ -759,6 +759,11 @@ setup_volume() {
 
     await_volume
 
+    if [ "$(/usr/sbin/diskutil info -plist "$NIX_ROOT" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        _sudo "to set enableOwnership (enabling users to own files)" \
+            /usr/sbin/diskutil enableOwnership "$NIX_ROOT"
+    fi
+
     # TODO: below is a vague kludge for now; I just don't know
     # what if any safe action there is to take here. Also, the
     # reminder isn't very helpful.

scripts/install-darwin-multi-user.sh

@@ -13,11 +13,22 @@ NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 read_only_root() {
     # this touch command ~should~ always produce an error
     # as of this change I confirmed /usr/bin/touch emits:
+    # "touch: /: Operation not permitted" Monterey
     # "touch: /: Read-only file system" Catalina+ and Big Sur
     # "touch: /: Permission denied" Mojave
     # (not matching prefix for compat w/ coreutils touch in case using
     # an explicit path causes problems; its prefix differs)
-    [[ "$(/usr/bin/touch / 2>&1)" = *"Read-only file system" ]]
+    case "$(/usr/bin/touch / 2>&1)" in
+        *"Read-only file system") # Catalina, Big Sur
+            return 0
+            ;;
+        *"Operation not permitted") # Monterey
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
 
     # Avoiding the slow semantic way to get this information (~330ms vs ~8ms)
     # unless using touch causes problems. Just in case, that approach is:
@@ -67,7 +78,7 @@ poly_service_installed_check() {
 poly_service_uninstall_directions() {
     echo "$1. Remove macOS-specific components:"
     if should_create_volume && test_nix_volume_mountd_installed; then
-        darwin_volume_uninstall_directions
+        nix_volume_mountd_uninstall_directions
     fi
     if test_nix_daemon_installed; then
         nix_daemon_uninstall_directions
@@ -206,4 +217,8 @@ poly_prepare_to_install() {
 EOF
         setup_darwin_volume
     fi
+
+    if [ "$(diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        failure "This script needs a /nix volume with global permissions! This may require running sudo diskutil enableOwnership /nix."
+    fi
 }

scripts/install-multi-user.sh

@@ -33,7 +33,7 @@ NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 readonly NIX_ROOT="/nix"
 readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv" "/etc/bash.bashrc" "/etc/zsh/zshenv")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/etc/bash.bashrc" "/etc/zsh/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -701,7 +701,10 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
+              cp -RLp ./store/* "$NIX_ROOT/store/"
+
+        _sudo "to make the new store non-writable at $NIX_ROOT/store" \
+              chmod -R ugo-w "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"

scripts/install-nix-from-closure.sh

@@ -106,12 +106,11 @@ while [ $# -gt 0 ]; do
                 echo ""
                 echo " --no-channel-add:    Don't add any channels. nixpkgs-unstable is installed by default."
                 echo ""
-                echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable"
-                echo "                      is installed by default."
+                echo " --no-modify-profile: Don't modify the user profile to automatically load nix."
                 echo ""
                 echo " --daemon-user-count: Number of build users to create. Defaults to 32."
                 echo ""
-                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf"
+                echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix/nix.conf"
                 echo ""
                 if [ -n "${INVOKED_FROM_INSTALL_IN:-}" ]; then
                     echo " --tarball-url-prefix URL: Base URL to download the Nix tarball from."

scripts/install.in

@@ -40,13 +40,23 @@ case "$(uname -s).$(uname -m)" in
         path=@tarballPath_aarch64-linux@
         system=aarch64-linux
         ;;
+    Linux.armv6l_linux)
+        hash=@tarballHash_armv6l-linux@
+        path=@tarballPath_armv6l-linux@
+        system=armv6l-linux
+        ;;
+    Linux.armv7l_linux)
+        hash=@tarballHash_armv7l-linux@
+        path=@tarballPath_armv7l-linux@
+        system=armv7l-linux
+        ;;
     Darwin.x86_64)
         hash=@tarballHash_x86_64-darwin@
         path=@tarballPath_x86_64-darwin@
         system=x86_64-darwin
         ;;
     Darwin.arm64|Darwin.aarch64)
-        hash=@binaryTarball_aarch64-darwin@
+        hash=@tarballHash_aarch64-darwin@
         path=@tarballPath_aarch64-darwin@
         system=aarch64-darwin
         ;;
@@ -66,14 +76,21 @@ fi
 
 tarball=$tmpDir/nix-@nixVersion@-$system.tar.xz
 
-require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
 if [ "$(uname -s)" != "Darwin" ]; then
     require_util xz "unpack the binary tarball"
 fi
 
+if command -v wget > /dev/null 2>&1; then
+    fetch() { wget "$1" -O "$2"; }
+elif command -v curl > /dev/null 2>&1; then
+    fetch() { curl -L "$1" -o "$2"; }
+else
+    oops "you don't have wget or curl installed, which I need to download the binary tarball"
+fi
+
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
-curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
+fetch "$url" "$tarball" || oops "failed to download '$url'"
 
 if command -v sha256sum > /dev/null 2>&1; then
     hash2="$(sha256sum -b "$tarball" | cut -c1-64)"

src/build-remote/build-remote.cc

@@ -130,11 +130,14 @@ static int main_build_remote(int argc, char * * argv)
                 for (auto & m : machines) {
                     debug("considering building on remote machine '%s'", m.storeUri);
 
-                    if (m.enabled && std::find(m.systemTypes.begin(),
-                            m.systemTypes.end(),
-                            neededSystem) != m.systemTypes.end() &&
+                    if (m.enabled
+                        && (neededSystem == "builtin"
+                            || std::find(m.systemTypes.begin(),
+                                m.systemTypes.end(),
+                                neededSystem) != m.systemTypes.end()) &&
                         m.allSupported(requiredFeatures) &&
-                        m.mandatoryMet(requiredFeatures)) {
+                        m.mandatoryMet(requiredFeatures))
+                    {
                         rightType = true;
                         AutoCloseFD free;
                         uint64_t load = 0;
@@ -270,14 +273,23 @@ connected:
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
+            copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
 
         auto drv = store->readDerivation(*drvPath);
         auto outputHashes = staticOutputHashes(*store, drv);
-        drv.inputSrcs = store->parseStorePathSet(inputs);
+
+        // Hijack the inputs paths of the derivation to include all the paths
+        // that come from the `inputDrvs` set.
+        // We don’t do that for the derivations whose `inputDrvs` is empty
+        // because
+        // 1. It’s not needed
+        // 2. Changing the `inputSrcs` set changes the associated output ids,
+        //  which break CA derivations
+        if (!drv.inputDrvs.empty())
+            drv.inputSrcs = store->parseStorePathSet(inputs);
 
         auto result = sshStore->buildDerivation(*drvPath, drv);
 
@@ -312,7 +324,7 @@ connected:
             if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
                 for (auto & path : missingPaths)
                     localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
+            copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
         }
         // XXX: Should be done as part of `copyPaths`
         for (auto & realisation : missingRealisations) {

src/libcmd/command.cc

@@ -54,6 +54,30 @@ void StoreCommand::run()
     run(getStore());
 }
 
+EvalCommand::EvalCommand()
+{
+}
+
+EvalCommand::~EvalCommand()
+{
+    if (evalState)
+        evalState->printStats();
+}
+
+ref<Store> EvalCommand::getEvalStore()
+{
+    if (!evalStore)
+        evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
+    return ref<Store>(evalStore);
+}
+
+ref<EvalState> EvalCommand::getEvalState()
+{
+    if (!evalState)
+        evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
+    return ref<EvalState>(evalState);
+}
+
 BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     : recursive(recursive)
 {
@@ -91,12 +115,12 @@ void BuiltPathsCommand::run(ref<Store> store)
         for (auto & p : store->queryAllValidPaths())
             paths.push_back(BuiltPath::Opaque{p});
     } else {
-        paths = toBuiltPaths(store, realiseMode, operateOn, installables);
+        paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations
             StorePathSet pathsRoots, pathsClosure;
-            for (auto & root: paths) {
+            for (auto & root : paths) {
                 auto rootFromThis = root.outPaths();
                 pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
             }
@@ -114,17 +138,20 @@ StorePathsCommand::StorePathsCommand(bool recursive)
 {
 }
 
-void StorePathsCommand::run(ref<Store> store, BuiltPaths paths)
+void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
 {
-    StorePaths storePaths;
-    for (auto& builtPath : paths)
-        for (auto& p : builtPath.outPaths())
-            storePaths.push_back(p);
+    StorePathSet storePaths;
+    for (auto & builtPath : paths)
+        for (auto & p : builtPath.outPaths())
+            storePaths.insert(p);
 
-    run(store, std::move(storePaths));
+    auto sorted = store->topoSortPaths(storePaths);
+    std::reverse(sorted.begin(), sorted.end());
+
+    run(store, std::move(sorted));
 }
 
-void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
 {
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
@@ -176,10 +203,10 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
-            [&](BuiltPath::Opaque bo) {
+            [&](const BuiltPath::Opaque & bo) {
                 result.push_back(bo.path);
             },
-            [&](BuiltPath::Built bfd) {
+            [&](const BuiltPath::Built & bfd) {
                 for (auto & output : bfd.outputs) {
                     result.push_back(output.second);
                 }
@@ -188,7 +215,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
     }
 
     if (result.size() != 1)
-        throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
+        throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
     updateProfile(result[0]);
 }

src/libcmd/command.hh

@@ -45,11 +45,18 @@ private:
 
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
-    ref<EvalState> getEvalState();
-
-    std::shared_ptr<EvalState> evalState;
+    EvalCommand();
 
     ~EvalCommand();
+
+    ref<Store> getEvalStore();
+
+    ref<EvalState> getEvalState();
+
+private:
+    std::shared_ptr<Store> evalStore;
+
+    std::shared_ptr<EvalState> evalState;
 };
 
 struct MixFlakeOptions : virtual Args, EvalCommand
@@ -101,6 +108,8 @@ enum class Realise {
        exists. */
     Derivation,
     /* Evaluate in dry-run mode. Postcondition: nothing. */
+    // FIXME: currently unused, but could be revived if we can
+    // evaluate derivations in-memory.
     Nothing
 };
 
@@ -160,7 +169,7 @@ public:
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, BuiltPaths paths) = 0;
+    virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
 
     void run(ref<Store> store) override;
 
@@ -173,9 +182,9 @@ struct StorePathsCommand : public BuiltPathsCommand
 
     using BuiltPathsCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+    virtual void run(ref<Store> store, std::vector<StorePath> && storePaths) = 0;
 
-    void run(ref<Store> store, BuiltPaths paths) override;
+    void run(ref<Store> store, BuiltPaths && paths) override;
 };
 
 /* A command that operates on exactly one store path. */
@@ -185,7 +194,7 @@ struct StorePathCommand : public StorePathsCommand
 
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store, std::vector<StorePath> storePaths) override;
+    void run(ref<Store> store, std::vector<StorePath> && storePaths) override;
 };
 
 /* A helper class for registering commands globally. */
@@ -216,26 +225,37 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-BuiltPaths build(ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store, Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode = bmNormal);
 
-std::set<StorePath> toStorePaths(ref<Store> store,
-    Realise mode, OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
-
-StorePath toStorePath(ref<Store> store,
-    Realise mode, OperateOn operateOn,
-    std::shared_ptr<Installable> installable);
-
-std::set<StorePath> toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables,
-    bool useDeriver = false);
-
-BuiltPaths toBuiltPaths(
+std::set<StorePath> toStorePaths(
+    ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
+    const std::vector<std::shared_ptr<Installable>> & installables);
+
+StorePath toStorePath(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
+    std::shared_ptr<Installable> installable);
+
+std::set<StorePath> toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    bool useDeriver = false);
+
+BuiltPaths toBuiltPaths(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
+    const std::vector<std::shared_ptr<Installable>> & installables);
 
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */

src/libcmd/installables.cc

@@ -58,9 +58,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "no-registries",
-        .description = "Don't allow lookups in the flake registries.",
+        .description =
+          "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
         .category = category,
-        .handler = {&lockFlags.useRegistries, false}
+        .handler = {[&]() {
+            lockFlags.useRegistries = false;
+            warn("'--no-registries' is deprecated; use '--no-use-registries'");
+        }}
     });
 
     addFlag({
@@ -171,14 +175,50 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 
 void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
-    if (file) return; // FIXME
+    if (file) {
+        evalSettings.pureEval = false;
+        auto state = getEvalState();
+        Expr *e = state->parseExprFromFile(
+            resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
+        );
 
-    completeFlakeRefWithFragment(
-        getEvalState(),
-        lockFlags,
-        getDefaultFlakeAttrPathPrefixes(),
-        getDefaultFlakeAttrPaths(),
-        prefix);
+        Value root;
+        state->eval(e, root);
+
+        auto autoArgs = getAutoArgs(*state);
+
+        std::string prefix_ = std::string(prefix);
+        auto sep = prefix_.rfind('.');
+        std::string searchWord;
+        if (sep != std::string::npos) {
+            searchWord = prefix_.substr(sep, std::string::npos);
+            prefix_ = prefix_.substr(0, sep);
+        } else {
+            searchWord = prefix_;
+            prefix_ = "";
+        }
+
+        Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first);
+        state->forceValue(v1);
+        Value v2;
+        state->autoCallFunction(*autoArgs, v1, v2);
+
+        if (v2.type() == nAttrs) {
+            for (auto & i : *v2.attrs) {
+                std::string name = i.name;
+                if (name.find(searchWord) == 0) {
+                    completions->add(i.name);
+                }
+            }
+        }
+    } else {
+        completeFlakeRefWithFragment(
+            getEvalState(),
+            lockFlags,
+            getDefaultFlakeAttrPathPrefixes(),
+            getDefaultFlakeAttrPaths(),
+            prefix);
+    }
 }
 
 void completeFlakeRefWithFragment(
@@ -249,19 +289,6 @@ void completeFlakeRefWithFragment(
     completeFlakeRef(evalState->store, prefix);
 }
 
-ref<EvalState> EvalCommand::getEvalState()
-{
-    if (!evalState)
-        evalState = std::make_shared<EvalState>(searchPath, getStore());
-    return ref<EvalState>(evalState);
-}
-
-EvalCommand::~EvalCommand()
-{
-    if (evalState)
-        evalState->printStats();
-}
-
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (prefix == "")
@@ -302,14 +329,6 @@ Installable::getCursors(EvalState & state)
     return {{evalCache->getRoot(), ""}};
 }
 
-std::pair<Value*, Pos> Installable::toValue(EvalState & state)
-{
-    auto values = toValues(state);
-    if (values.empty())
-        throw Error("cannot find flake attribute '%s'", what());
-    return {std::get<0>(values[0]), std::get<1>(values[0])};
-}
-
 std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
 Installable::getCursor(EvalState & state)
 {
@@ -359,6 +378,7 @@ DerivedPaths InstallableValue::toDerivedPaths()
     DerivedPaths res;
 
     std::map<StorePath, std::set<std::string>> drvsToOutputs;
+    RealisedPath::Set drvsToCopy;
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
@@ -366,6 +386,7 @@ DerivedPaths InstallableValue::toDerivedPaths()
         if (outputName == "")
             throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
         drvsToOutputs[drv.drvPath].insert(outputName);
+        drvsToCopy.insert(drv.drvPath);
     }
 
     for (auto & i : drvsToOutputs)
@@ -386,11 +407,11 @@ struct InstallableAttrPath : InstallableValue
 
     std::string what() override { return attrPath; }
 
-    std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override
+    std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
         state.forceValue(*vRes);
-        return {{vRes, pos, attrPath}};
+        return {vRes, pos};
     }
 
     virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
@@ -436,10 +457,12 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
     callFlake(state, lockedFlake, *vFlake);
 
-    auto vRes = state.allocValue();
-    auto gotField = state.lazyGetAttrField(*vFlake, {state.symbols.create("outputs")}, noPos, *vRes);
-    assert(gotField != EvalState::LazyValueType::Missing);
-    return vRes;
+    auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+    assert(aOutputs);
+
+    state.forceValue(*aOutputs->value);
+
+    return aOutputs->value;
 }
 
 ref<eval_cache::EvalCache> openEvalCache(
@@ -506,34 +529,25 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
-        auto emptyArgs = state->allocBindings(0);
-        try {
-            auto [drvValue, pos] = findAlongAttrPath(
-                *state,
-                attrPath,
-                *emptyArgs,
-                *getFlakeOutputs(*state, *lockedFlake)
-            );
-            Value * v = state->allocValue();
-            if (!state->getAttrField(*drvValue, {state->sDrvPath}, pos, *v))
-                break;
-            auto drvPath = state->forceString(*v);
-            if (!state->getAttrField(*drvValue, {state->sOutPath}, pos, *v))
-                break;
-            auto outPath = state->forceString(*v);
-            if (!state->getAttrField(*drvValue, {state->sOutputName}, pos, *v))
-                break;
-            auto outputName = state->forceString(*v);
+        auto attr = root->findAlongAttrPath(
+            parseAttrPath(*state, attrPath),
+            true
+        );
 
-            auto drvInfo = DerivationInfo{
-                state->store->parseStorePath(drvPath),
-                state->store->maybeParseStorePath(outPath),
-                outputName
-            };
+        if (!attr) continue;
 
-            return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
-        } catch (AttrPathNotFound & e) {
-        }
+        if (!attr->isDerivation())
+            throw Error("flake output attribute '%s' is not a derivation", attrPath);
+
+        auto drvPath = attr->forceDerivation();
+
+        auto drvInfo = DerivationInfo{
+            std::move(drvPath),
+            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
+            attr->getAttr(state->sOutputName)->getString()
+        };
+
+        return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
     }
 
     throw Error("flake '%s' does not provide attribute %s",
@@ -547,22 +561,25 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
     return res;
 }
 
-std::vector<std::tuple<Value *, Pos, std::string>>
-InstallableFlake::toValues(EvalState & state)
+std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 {
-    auto vOutputs = getFlakeOutputs(state, *getLockedFlake());
-    auto emptyArgs = state.allocBindings(0);
+    auto lockedFlake = getLockedFlake();
 
-    std::vector<std::tuple<Value *, Pos, std::string>> res;
+    auto vOutputs = getFlakeOutputs(state, *lockedFlake);
+
+    auto emptyArgs = state.allocBindings(0);
 
     for (auto & attrPath : getActualAttrPaths()) {
         try {
             auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
-            res.push_back({v, pos, attrPath});
-        } catch (Error &) {}
+            state.forceValue(*v);
+            return {v, pos};
+        } catch (AttrPathNotFound & e) {
+        }
     }
 
-    return res;
+    throw Error("flake '%s' does not provide attribute %s",
+        flakeRef, showAttrPaths(getActualAttrPaths()));
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
@@ -585,10 +602,10 @@ InstallableFlake::getCursors(EvalState & state)
 
 std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
 {
+    flake::LockFlags lockFlagsApplyConfig = lockFlags;
+    lockFlagsApplyConfig.applyNixConfig = true;
     if (!_lockedFlake) {
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
-        _lockedFlake->flake.config.apply();
-        // FIXME: send new config to the daemon.
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }
@@ -637,6 +654,17 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         for (auto & s : ss) {
             std::exception_ptr ex;
 
+            if (s.find('/') != std::string::npos) {
+                try {
+                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
+                    continue;
+                } catch (BadStorePath &) {
+                } catch (...) {
+                    if (!ex)
+                        ex = std::current_exception();
+                }
+            }
+
             try {
                 auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
@@ -651,25 +679,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
                 ex = std::current_exception();
             }
 
-            if (s.find('/') != std::string::npos) {
-                try {
-                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
-                    continue;
-                } catch (BadStorePath &) {
-                } catch (...) {
-                    if (!ex)
-                        ex = std::current_exception();
-                }
-            }
-
             std::rethrow_exception(ex);
-
-            /*
-            throw Error(
-                pathExists(s)
-                ? "path '%s' is not a flake or a store path"
-                : "don't know how to handle argument '%s'", s);
-            */
         }
     }
 
@@ -684,25 +694,24 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths)
+BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
 {
     BuiltPaths res;
-    for (auto& b : hopefullyBuiltPaths)
+    for (const auto & b : hopefullyBuiltPaths)
         std::visit(
             overloaded{
-                [&](DerivedPath::Opaque bo) {
+                [&](const DerivedPath::Opaque & bo) {
                     res.push_back(BuiltPath::Opaque{bo.path});
                 },
-                [&](DerivedPath::Built bfd) {
+                [&](const DerivedPath::Built & bfd) {
                     OutputPathMap outputs;
-                    auto drv = store->readDerivation(bfd.drvPath);
-                    auto outputHashes = staticOutputHashes(*store, drv);
+                    auto drv = evalStore->readDerivation(bfd.drvPath);
+                    auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
                     auto drvOutputs = drv.outputsAndOptPaths(*store);
-                    for (auto& output : bfd.outputs) {
+                    for (auto & output : bfd.outputs) {
                         if (!outputHashes.count(output))
                             throw Error(
-                                "the derivation '%s' doesn't have an output "
-                                "named '%s'",
+                                "the derivation '%s' doesn't have an output named '%s'",
                                 store->printStorePath(bfd.drvPath), output);
                         if (settings.isExperimentalFeatureEnabled(
                                 "ca-derivations")) {
@@ -713,7 +722,7 @@ BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths)
                             if (!realisation)
                                 throw Error(
                                     "cannot operate on an output of unbuilt "
-                                    "content-addresed derivation '%s'",
+                                    "content-addressed derivation '%s'",
                                     outputId.to_string());
                             outputs.insert_or_assign(
                                 output, realisation->outPath);
@@ -734,8 +743,12 @@ BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths)
     return res;
 }
 
-BuiltPaths build(ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode)
 {
     if (mode == Realise::Nothing)
         settings.readOnlyMode = true;
@@ -747,23 +760,24 @@ BuiltPaths build(ref<Store> store, Realise mode,
         pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
     }
 
-    if (mode == Realise::Nothing)
+    if (mode == Realise::Nothing || mode == Realise::Derivation)
         printMissing(store, pathsToBuild, lvlError);
     else if (mode == Realise::Outputs)
-        store->buildPaths(pathsToBuild, bMode);
+        store->buildPaths(pathsToBuild, bMode, evalStore);
 
-    return getBuiltPaths(store, pathsToBuild);
+    return getBuiltPaths(evalStore, store, pathsToBuild);
 }
 
 BuiltPaths toBuiltPaths(
+    ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
-    if (operateOn == OperateOn::Output) {
-        return build(store, mode, installables);
-    } else {
+    if (operateOn == OperateOn::Output)
+        return build(evalStore, store, mode, installables);
+    else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
@@ -774,23 +788,27 @@ BuiltPaths toBuiltPaths(
     }
 }
 
-StorePathSet toStorePaths(ref<Store> store,
+StorePathSet toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
     Realise mode, OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
     StorePathSet outPaths;
-    for (auto & path : toBuiltPaths(store, mode, operateOn, installables)) {
+    for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
         auto thisOutPaths = path.outPaths();
         outPaths.insert(thisOutPaths.begin(), thisOutPaths.end());
     }
     return outPaths;
 }
 
-StorePath toStorePath(ref<Store> store,
+StorePath toStorePath(
+    ref<Store> evalStore,
+    ref<Store> store,
     Realise mode, OperateOn operateOn,
     std::shared_ptr<Installable> installable)
 {
-    auto paths = toStorePaths(store, mode, operateOn, {installable});
+    auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
 
     if (paths.size() != 1)
         throw Error("argument '%s' should evaluate to one store path", installable->what());
@@ -798,15 +816,17 @@ StorePath toStorePath(ref<Store> store,
     return *paths.begin();
 }
 
-StorePathSet toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables, bool useDeriver)
+StorePathSet toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    bool useDeriver)
 {
     StorePathSet drvPaths;
 
-    for (auto & i : installables)
-        for (auto & b : i->toDerivedPaths())
+    for (const auto & i : installables)
+        for (const auto & b : i->toDerivedPaths())
             std::visit(overloaded {
-                [&](DerivedPath::Opaque bo) {
+                [&](const DerivedPath::Opaque & bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
                     auto derivers = store->queryValidDerivers(bo.path);
@@ -815,7 +835,7 @@ StorePathSet toDerivations(ref<Store> store,
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
                 },
-                [&](DerivedPath::Built bfd) {
+                [&](const DerivedPath::Built & bfd) {
                     drvPaths.insert(bfd.drvPath);
                 },
             }, b.raw());

src/libcmd/installables.hh

@@ -26,7 +26,7 @@ struct App
 struct UnresolvedApp
 {
     App unresolved;
-    App resolve(ref<Store>);
+    App resolve(ref<Store> evalStore, ref<Store> store);
 };
 
 struct Installable
@@ -41,11 +41,10 @@ struct Installable
 
     UnresolvedApp toApp(EvalState & state);
 
-    virtual std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state)
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {
         throw Error("argument '%s' cannot be evaluated", what());
     }
-    std::pair<Value *, Pos> toValue(EvalState & state);
 
     /* Return a value only if this installable is a store path or a
        symlink to it. */
@@ -110,7 +109,7 @@ struct InstallableFlake : InstallableValue
 
     std::vector<DerivationInfo> toDerivations() override;
 
-    std::vector<std::tuple<Value *, Pos, std::string>> toValues(EvalState & state) override;
+    std::pair<Value *, Pos> toValue(EvalState & state) override;
 
     std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
     getCursors(EvalState & state) override;

src/libcmd/local.mk

@@ -8,8 +8,8 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS = -llowdown
+libcmd_LDFLAGS += -llowdown -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/markdown.cc

@@ -12,7 +12,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     struct lowdown_opts opts {
         .type = LOWDOWN_TERM,
         .maxdepth = 20,
-        .cols = std::min(getWindowSize().second, (unsigned short) 80),
+        .cols = std::max(getWindowSize().second, (unsigned short) 80),
         .hmargin = 0,
         .vmargin = 0,
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
@@ -25,7 +25,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     Finally freeDoc([&]() { lowdown_doc_free(doc); });
 
     size_t maxn = 0;
-    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size(), nullptr);
     if (!node)
         throw Error("cannot parse Markdown document");
     Finally freeNode([&]() { lowdown_node_free(node); });
@@ -40,11 +40,11 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot allocate Markdown output buffer");
     Finally freeBuffer([&]() { lowdown_buf_free(buf); });
 
-    int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
+    int rndr_res = lowdown_term_rndr(buf, renderer, node);
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return std::string(buf->data, buf->size);
+    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
 }
 
 }

src/libexpr/attr-path.cc

@@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s)
             ++i;
             while (1) {
                 if (i == s.end())
-                    throw Error("missing closing quote in selection path '%1%'", s);
+                    throw ParseError("missing closing quote in selection path '%1%'", s);
                 if (*i == '"') break;
                 cur.push_back(*i++);
             }
@@ -58,23 +58,26 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
         Value * vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
+        state.forceValue(*v);
 
         /* It should evaluate to either a set or an expression,
            according to what is specified in the attrPath. */
 
         if (!attrIndex) {
 
+            if (v->type() != nAttrs)
+                throw TypeError(
+                    "the expression selected by the selection path '%1%' should be a set but is %2%",
+                    attrPath,
+                    showType(*v));
             if (attr.empty())
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
-            auto v2 = state.allocValue();
-            auto gotField = state.lazyGetAttrField(*v, {state.symbols.create(attr)}, pos, *v2) != EvalState::LazyValueType::Missing;
-            if (!gotField)
+            Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
+            if (a == v->attrs->end())
                 throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
-            v = v2;
-            /* Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); */
-            /* v = &*a->value; */
-            /* pos = *a->pos; */
+            v = &*a->value;
+            pos = *a->pos;
         }
 
         else {
@@ -97,7 +100,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 }
 
 
-Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
+Pos findPackageFilename(EvalState & state, Value & v, std::string what)
 {
     Value * v2;
     try {
@@ -113,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     auto colon = pos.rfind(':');
     if (colon == std::string::npos)
-        throw Error("cannot parse meta.position attribute '%s'", pos);
+        throw ParseError("cannot parse meta.position attribute '%s'", pos);
 
     std::string filename(pos, 0, colon);
     unsigned int lineno;
     try {
         lineno = std::stoi(std::string(pos, colon + 1));
     } catch (std::invalid_argument & e) {
-        throw Error("cannot parse line number '%s'", pos);
+        throw ParseError("cannot parse line number '%s'", pos);
     }
 
     Symbol file = state.symbols.create(filename);

src/libexpr/attr-path.hh

@@ -14,7 +14,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
     Bindings & autoArgs, Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
-Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
+Pos findPackageFilename(EvalState & state, Value & v, std::string what);
 
 std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
 

src/libexpr/attr-set.hh

@@ -2,7 +2,6 @@
 
 #include "nixexpr.hh"
 #include "symbol-table.hh"
-#include "value-cache.hh"
 
 #include <algorithm>
 #include <optional>
@@ -18,8 +17,8 @@ struct Attr
 {
     Symbol name;
     Value * value;
-    Pos * pos;
-    Attr(Symbol name, Value * value, Pos * pos = &noPos)
+    ptr<Pos> pos;
+    Attr(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos))
         : name(name), value(value), pos(pos) { };
     Attr() : pos(&noPos) { };
     bool operator < (const Attr & a) const
@@ -36,14 +35,13 @@ class Bindings
 {
 public:
     typedef uint32_t size_t;
-    Pos *pos;
-    ValueCache eval_cache;
+    ptr<Pos> pos;
 
 private:
     size_t size_, capacity_;
     Attr attrs[0];
 
-    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
+    Bindings(size_t capacity) : pos(&noPos), size_(0), capacity_(capacity) { }
     Bindings(const Bindings & bindings) = delete;
 
 public:

src/libexpr/common-eval-args.cc

@@ -61,6 +61,14 @@ MixEvalArgs::MixEvalArgs()
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
         }}
     });
+
+    addFlag({
+        .longName = "eval-store",
+        .description = "The Nix store to use for evaluations.",
+        .category = category,
+        .labels = {"store-url"},
+        .handler = {&evalStoreUrl},
+    });
 }
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)

src/libexpr/common-eval-args.hh

@@ -16,8 +16,9 @@ struct MixEvalArgs : virtual Args
 
     Strings searchPath;
 
-private:
+    std::optional<std::string> evalStoreUrl;
 
+private:
     std::map<std::string, std::string> autoArgs;
 };
 

src/libexpr/context.cc

@@ -1,21 +0,0 @@
-#include "context.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
-    } else
-        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
-}
-
-std::string encodeContext(std::string_view name, std::string_view path)
-{
-    return "!" + std::string(name) + "!" + std::string(path);
-}
-
-}

src/libexpr/context.hh

@@ -1,11 +0,0 @@
-#include "util.hh"
-
-namespace nix {
-
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-std::pair<string, string> decodeContext(std::string_view s);
-
-std::string encodeContext(std::string_view name, std::string_view path);
-
-}

src/libexpr/eval-inline.hh

@@ -46,12 +46,6 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     }
     else if (v.isApp())
         callFunction(*v.app.left, *v.app.right, v, noPos);
-    else if (v.isCachedThunk()) {
-        auto evalCache = v.getEvalCache();
-        v.mkThunk(v.cachedThunk.thunk->env, v.cachedThunk.thunk->expr);
-        forceValue(v, pos);
-        v.setEvalCache(evalCache);
-    }
     else if (v.isBlackhole())
         throwEvalError(pos, "infinite recursion encountered");
 }

src/libexpr/eval.cc

@@ -1,5 +1,4 @@
 #include "eval.hh"
-#include "value-cache.hh"
 #include "hash.hh"
 #include "util.hh"
 #include "store-api.hh"
@@ -9,7 +8,6 @@
 #include "filetransfer.hh"
 #include "json.hh"
 #include "function-trace.hh"
-#include "value-cache.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -66,7 +64,11 @@ static char * dupStringWithLen(const char * s, size_t size)
 
 RootValue allocRootValue(Value * v)
 {
+#if HAVE_BOEHMGC
     return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
+#else
+    return std::make_shared<Value *>(v);
+#endif
 }
 
 
@@ -124,7 +126,6 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         str << "]";
         break;
     case tThunk:
-    case tCachedThunk:
     case tApp:
         str << "<CODE>";
         break;
@@ -196,7 +197,7 @@ string showType(const Value & v)
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tThunk: case tCachedThunk: return "a thunk";
+        case tThunk: return "a thunk";
         case tApp: return "a function application";
         case tBlackhole: return "a black hole";
     default:
@@ -219,7 +220,7 @@ bool Value::isTrivial() const
     return
         internalType != tApp
         && internalType != tPrimOpApp
-        && ((internalType != tThunk && internalType != tCachedThunk)
+        && (internalType != tThunk
             || (dynamic_cast<ExprAttrs *>(thunk.expr)
                 && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
             || dynamic_cast<ExprLambda *>(thunk.expr)
@@ -236,22 +237,34 @@ static void * oomHandler(size_t requested)
 }
 
 class BoehmGCStackAllocator : public StackAllocator {
-  boost::coroutines2::protected_fixedsize_stack stack {
-    // We allocate 8 MB, the default max stack size on NixOS.
-    // A smaller stack might be quicker to allocate but reduces the stack
-    // depth available for source filter expressions etc.
-    std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
+    boost::coroutines2::protected_fixedsize_stack stack {
+        // We allocate 8 MB, the default max stack size on NixOS.
+        // A smaller stack might be quicker to allocate but reduces the stack
+        // depth available for source filter expressions etc.
+        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
     };
 
+    // This is specific to boost::coroutines2::protected_fixedsize_stack.
+    // The stack protection page is included in sctx.size, so we have to
+    // subtract one page size from the stack size.
+    std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) {
+        return sctx.size - boost::context::stack_traits::page_size();
+    }
+
   public:
     boost::context::stack_context allocate() override {
         auto sctx = stack.allocate();
-        GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+
+        // Stacks generally start at a high address and grow to lower addresses.
+        // Architectures that do the opposite are rare; in fact so rare that
+        // boost_routine does not implement it.
+        // So we subtract the stack size.
+        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
         return sctx;
     }
 
     void deallocate(boost::context::stack_context sctx) override {
-        GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
         stack.deallocate(sctx);
     }
 
@@ -365,7 +378,10 @@ static Strings parseNixPath(const string & s)
 }
 
 
-EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
+EvalState::EvalState(
+    const Strings & _searchPath,
+    ref<Store> store,
+    std::shared_ptr<Store> buildStore)
     : sWith(symbols.create("<with>"))
     , sOutPath(symbols.create("outPath"))
     , sDrvPath(symbols.create("drvPath"))
@@ -398,6 +414,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sEpsilon(symbols.create(""))
     , repair(NoRepair)
     , store(store)
+    , buildStore(buildStore ? buildStore : store)
     , regexCache(makeRegexCache())
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)
@@ -428,12 +445,12 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
                     StorePathSet closure;
                     store->computeFSClosure(store->toStorePath(r.second).first, closure);
                     for (auto & path : closure)
-                        allowedPaths->insert(store->printStorePath(path));
+                        allowPath(path);
                 } catch (InvalidPath &) {
-                    allowedPaths->insert(r.second);
+                    allowPath(r.second);
                 }
             } else
-                allowedPaths->insert(r.second);
+                allowPath(r.second);
         }
     }
 
@@ -445,11 +462,37 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
 
 EvalState::~EvalState()
 {
-    for (auto [_, cache] : evalCache) {
-        cache->commit();
+}
+
+
+void EvalState::requireExperimentalFeatureOnEvaluation(
+    const std::string & feature,
+    const std::string_view fName,
+    const Pos & pos)
+{
+    if (!settings.isExperimentalFeatureEnabled(feature)) {
+        throw EvalError({
+            .msg = hintfmt(
+                "Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.",
+                feature,
+                fName
+            ),
+            .errPos = pos
+        });
     }
 }
 
+void EvalState::allowPath(const Path & path)
+{
+    if (allowedPaths)
+        allowedPaths->insert(path);
+}
+
+void EvalState::allowPath(const StorePath & storePath)
+{
+    if (allowedPaths)
+        allowedPaths->insert(store->toRealPath(storePath));
+}
 
 Path EvalState::checkSourcePath(const Path & path_)
 {
@@ -756,7 +799,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
         }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
-            if (countCalls && j->pos) attrSelects[*j->pos]++;
+            if (countCalls) attrSelects[*j->pos]++;
             return j->value;
         }
         if (!env->prevWith)
@@ -766,18 +809,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 }
 
 
-std::atomic<uint64_t> nrValuesFreed{0};
-
-void finalizeValue(void * obj, void * data)
-{
-    nrValuesFreed++;
-}
-
 Value * EvalState::allocValue()
 {
     nrValues++;
     auto v = (Value *) allocBytes(sizeof(Value));
-    //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr);
     return v;
 }
 
@@ -819,9 +854,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 }
 
 
-void EvalState::mkPos(Value & v, Pos * pos)
+void EvalState::mkPos(Value & v, ptr<Pos> pos)
 {
-    if (pos && pos->file.set()) {
+    if (pos->file.set()) {
         mkAttrs(v, 3);
         mkString(*allocAttr(v, sFile), pos->file);
         mkInt(*allocAttr(v, sLine), pos->line);
@@ -844,39 +879,37 @@ Value * Expr::maybeThunk(EvalState & state, Env & env)
 }
 
 
-unsigned long nrAvoided = 0;
-
 Value * ExprVar::maybeThunk(EvalState & state, Env & env)
 {
     Value * v = state.lookupVar(&env, *this, true);
     /* The value might not be initialised in the environment yet.
        In that case, ignore it. */
-    if (v) { nrAvoided++; return v; }
+    if (v) { state.nrAvoided++; return v; }
     return Expr::maybeThunk(state, env);
 }
 
 
 Value * ExprString::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
 Value * ExprInt::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
 Value * ExprFloat::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
 Value * ExprPath::maybeThunk(EvalState & state, Env & env)
 {
-    nrAvoided++;
+    state.nrAvoided++;
     return &v;
 }
 
@@ -891,38 +924,23 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
         return;
     }
 
-    Path path2 = resolveExprPath(path);
-    if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) {
+    Path resolvedPath = resolveExprPath(path);
+    if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) {
         v = i->second;
         return;
     }
 
-    printTalkative("evaluating file '%1%'", path2);
+    printTalkative("evaluating file '%1%'", resolvedPath);
     Expr * e = nullptr;
 
-    auto j = fileParseCache.find(path2);
+    auto j = fileParseCache.find(resolvedPath);
     if (j != fileParseCache.end())
         e = j->second;
 
     if (!e)
-        e = parseExprFromFile(checkSourcePath(path2));
+        e = parseExprFromFile(checkSourcePath(resolvedPath));
 
-    fileParseCache[path2] = e;
-
-    try {
-        // Enforce that 'flake.nix' is a direct attrset, not a
-        // computation.
-        if (mustBeTrivial &&
-            !(dynamic_cast<ExprAttrs *>(e)))
-            throw Error("file '%s' must be an attribute set", path);
-        eval(e, v);
-    } catch (Error & e) {
-        addErrorTrace(e, "while evaluating the file '%1%':", path2);
-        throw;
-    }
-
-    fileEvalCache[path2] = v;
-    if (path != path2) fileEvalCache[path] = v;
+    cacheFile(path, resolvedPath, e, v, mustBeTrivial);
 }
 
 
@@ -933,6 +951,32 @@ void EvalState::resetFileCache()
 }
 
 
+void EvalState::cacheFile(
+    const Path & path,
+    const Path & resolvedPath,
+    Expr * e,
+    Value & v,
+    bool mustBeTrivial)
+{
+    fileParseCache[resolvedPath] = e;
+
+    try {
+        // Enforce that 'flake.nix' is a direct attrset, not a
+        // computation.
+        if (mustBeTrivial &&
+            !(dynamic_cast<ExprAttrs *>(e)))
+            throw EvalError("file '%s' must be an attribute set", path);
+        eval(e, v);
+    } catch (Error & e) {
+        addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath);
+        throw;
+    }
+
+    fileEvalCache[resolvedPath] = v;
+    if (path != resolvedPath) fileEvalCache[path] = v;
+}
+
+
 void EvalState::eval(Expr * e, Value & v)
 {
     e->eval(*this, baseEnv, v);
@@ -1023,7 +1067,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
             } else
                 vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
             env2.values[displ++] = vAttr;
-            v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos));
+            v.attrs->push_back(Attr(i.first, vAttr, ptr(&i.second.pos)));
         }
 
         /* If the rec contains an attribute called `__overrides', then
@@ -1055,7 +1099,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 
     else
         for (auto & i : attrs)
-            v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos));
+            v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), ptr(&i.second.pos)));
 
     /* Dynamic attrs apply *after* rec and __overrides. */
     for (auto & i : dynamicAttrs) {
@@ -1072,11 +1116,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
-        v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos));
+        v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), ptr(&i.pos)));
         v.attrs->sort(); // FIXME: inefficient
     }
 
-    v.attrs->pos = &pos;
+    v.attrs->pos = ptr(&pos);
 }
 
 
@@ -1113,235 +1157,66 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v)
     v = *v2;
 }
 
-unsigned long nrLookups = 0;
 
-std::pair<ValueCache::CacheResult, ValueCache> ValueCache::getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest)
+static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
 {
-    if (!rawCache)
-        return { {NoCacheKey}, ValueCache(nullptr) };
-    auto resultingCursor = rawCache->findAlongAttrPath(selector);
-    if (!resultingCursor)
-        return { {CacheMiss}, ValueCache(nullptr) };
-
-    auto cachedValue = resultingCursor->getCachedValue();
-    auto cacheResult = std::visit(
-        overloaded{
-            [&](tree_cache::attributeSet_t) { return ValueCache::CacheResult{ Forward }; },
-            [&](tree_cache::unknown_t) { return ValueCache::CacheResult{ UnCacheable }; },
-            [&](tree_cache::thunk_t) { return ValueCache::CacheResult{ CacheMiss }; },
-            [&](tree_cache::failed_t x) -> ValueCache::CacheResult {throw EvalError(x.error); },
-            [&](tree_cache::missing_t x) {
-                return ValueCache::CacheResult{
-                    .returnCode = CacheHit,
-                    .lastQueriedSymbolIfMissing = x.attrName
-                };
-            },
-            [&](tree_cache::string_t s) {
-                PathSet context;
-                for (auto& [pathName, outputName] : s.second) {
-                    // If the cached value depends on some non-existent
-                    // path, we need to discard it and force the evaluation
-                    // to bring back the context in the store
-                    if (!state.store->isValidPath(
-                            state.store->parseStorePath(pathName)))
-                        return ValueCache::CacheResult{UnCacheable};
-                    context.insert("!" + outputName + "!" + pathName);
-                }
-                mkString(dest, s.first, context);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<bool> b) {
-                dest.mkBool(b.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<int64_t> i) {
-                dest.mkInt(i.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-            [&](tree_cache::wrapped_basetype<double> d) {
-                dest.mkFloat(d.value);
-                return ValueCache::CacheResult{CacheHit};
-            },
-        },
-        cachedValue);
-
-    return { cacheResult, ValueCache(resultingCursor) };
-}
-
-void EvalState::updateCacheStats(ValueCache::CacheResult cacheResult)
-{
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            nrCacheHits++;
-            break;
-        case ValueCache::CacheMiss:
-            nrCacheMisses++;
-            break;
-        case ValueCache::UnCacheable:
-            nrUncacheable++;
-            break;
-        case ValueCache::NoCacheKey:
-            nrUncached++;
-            break;
-        case ValueCache::Forward:
-            nrCacheHits++;
-            break;
-    };
-}
-
-EvalState::LazyValueType EvalState::lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    auto eval_cache = attrs.getEvalCache();
-    if (eval_cache.isEmpty()) {
-        forceValue(attrs, pos);
-        eval_cache = attrs.getEvalCache();
-    }
-    auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
-    updateCacheStats(cacheResult);
-
-    auto delayValue = [&](ValueCache & cursor) {
-            auto recordAsVar = new ExprCastedVar(&attrs);
-            auto accessExpr = new ExprSelect(pos, recordAsVar, selector);
-            auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
-            thunk->expr = accessExpr;
-            thunk->env = &baseEnv;
-
-            dest.mkCachedThunk(
-                thunk,
-                new ValueCache(cursor)
-            );
-    };
-
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            if (cacheResult.lastQueriedSymbolIfMissing)
-                return LazyValueType::Missing;
-            return LazyValueType::PlainValue;
-        case ValueCache::UnCacheable:
-            delayValue(resultingCursor);
-            return LazyValueType::DelayedUncacheable;
-        case ValueCache::Forward:
-            delayValue(resultingCursor);
-            return LazyValueType::DelayedAttr;
-        default:
-            if(getAttrField(attrs, selector, pos, dest))
-                return LazyValueType::PlainValue;
-            else
-                return LazyValueType::Missing;
-    }
-
-}
-
-bool EvalState::getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    Pos * pos2 = 0;
-
-    auto eval_cache = attrs.getEvalCache();
-    if (eval_cache.isEmpty()) {
-        forceValue(attrs, pos);
-        eval_cache = attrs.getEvalCache();
-    }
-    auto [ cacheResult, resultingCursor ] = eval_cache.getValue(*this, selector, dest);
-    updateCacheStats(cacheResult);
-    switch (cacheResult.returnCode) {
-        case ValueCache::CacheHit:
-            if (cacheResult.lastQueriedSymbolIfMissing)
-                return false;
-            return true;
-        case ValueCache::CacheMiss:
-            resultingCursor = eval_cache;
-            break;
-        case ValueCache::Forward: // FIXME: Handle properly
-        case ValueCache::NoCacheKey:
-        case ValueCache::UnCacheable:
-            ;
-    }
-
-    forceValue(attrs, pos);
-
-    Value * vAttrs = &attrs;
-    try {
-        for (auto & name : selector) {
-            nrLookups++;
-            Bindings::iterator j;
-            if (vAttrs->type() != nAttrs ||
-                (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
-                return false;
-            }
-            vAttrs = j->value;
-            pos2 = j->pos;
-            try {
-                forceValue(*vAttrs, pos2 != NULL ? *pos2 : pos );
-            } catch (EvalError & e) {
-                resultingCursor.addFailedChild(name, e);
-                throw;
-            }
-            if (cacheResult.returnCode == ValueCache::CacheMiss) {
-                resultingCursor = resultingCursor.addChild(name, *vAttrs);
-                vAttrs->setEvalCache(resultingCursor);
-            }
-            if (countCalls && pos2) attrSelects[*pos2]++;
+    std::ostringstream out;
+    bool first = true;
+    for (auto & i : attrPath) {
+        if (!first) out << '.'; else first = false;
+        try {
+            out << getName(i, state, env);
+        } catch (Error & e) {
+            assert(!i.symbol.set());
+            out << "\"${" << *i.expr << "}\"";
         }
-
-    } catch (Error & e) {
-        if (pos2 && pos2->file != sDerivationNix) {
-            vector<string> strSelector;
-            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", concatStringsSep(".", selector));
-        }
-        throw;
     }
-
-    if (cacheResult.returnCode == ValueCache::Forward)
-        vAttrs->setEvalCache(resultingCursor);
-
-    dest = *vAttrs;
-    return true;
+    return out.str();
 }
 
-void EvalState::getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest)
-{
-    if (!getAttrField(attrs, selector, pos, dest))
-        throw Error("Missing attribute path '%s'", "ImTooLazyToImplementThisRightNow");
-}
-
-std::vector<Symbol> EvalState::getFields(Value & attrs, const Pos & pos)
-{
-    auto eval_cache = attrs.getEvalCache();
-    if (eval_cache.isEmpty()) {
-        forceValue(attrs, pos);
-        eval_cache = attrs.getEvalCache();
-    }
-    if (auto attrNames = eval_cache.listChildren(symbols)) {
-        return *attrNames;
-    }
-
-    forceAttrs(attrs);
-    std::vector<Symbol> res;
-    for (auto & attr : *attrs.attrs)
-        res.push_back(attr.name);
-    return res;
-}
 
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
     Value vTmp;
+    ptr<Pos> pos2(&noPos);
+    Value * vAttrs = &vTmp;
 
     e->eval(state, env, vTmp);
 
-    std::vector<Symbol> selector;
-    for (auto & i : attrPath)
-        selector.push_back(getName(i, state, env));
+    try {
 
-    bool gotField = state.getAttrField(vTmp, selector, pos, v);
-    if (!gotField) {
-        if (def) {
-            def->eval(state, env, v);
-            return;
-        } else {
-            throwEvalError(pos, "Missing field");
+        for (auto & i : attrPath) {
+            state.nrLookups++;
+            Bindings::iterator j;
+            Symbol name = getName(i, state, env);
+            if (def) {
+                state.forceValue(*vAttrs, pos);
+                if (vAttrs->type() != nAttrs ||
+                    (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                {
+                    def->eval(state, env, v);
+                    return;
+                }
+            } else {
+                state.forceAttrs(*vAttrs, pos);
+                if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
+                    throwEvalError(pos, "attribute '%1%' missing", name);
+            }
+            vAttrs = j->value;
+            pos2 = j->pos;
+            if (state.countCalls) state.attrSelects[*pos2]++;
         }
+
+        state.forceValue(*vAttrs, (*pos2 != noPos ? *pos2 : this->pos ) );
+
+    } catch (Error & e) {
+        if (*pos2 != noPos && pos2->file != state.sDerivationNix)
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+                showAttrPath(state, env, attrPath));
+        throw;
     }
 
+    v = *vAttrs;
 }
 
 
@@ -1453,13 +1328,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     auto size =
         (lambda.arg.empty() ? 0 : 1) +
-        (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
+        (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
     Env & env2(allocEnv(size));
     env2.up = fun.lambda.env;
 
     size_t displ = 0;
 
-    if (!lambda.matchAttrs)
+    if (!lambda.hasFormals())
         env2.values[displ++] = &arg;
 
     else {
@@ -1524,38 +1399,9 @@ void EvalState::incrFunctionCall(ExprLambda * fun)
     functionCalls[fun]++;
 }
 
-std::optional<tree_cache::AttrValue> ValueCache::getRawValue()
-{
-    if (!rawCache)
-        return std::nullopt;
-    return rawCache->getCachedValue();
-}
-
-std::optional<std::vector<Symbol>> ValueCache::listChildren(SymbolTable& symbols)
-{
-    auto ret = std::vector<Symbol>();
-    if (rawCache) {
-        auto cachedValue = rawCache->getCachedValue();
-            if (std::get_if<tree_cache::attributeSet_t>(&cachedValue)) {
-                for (auto & fieldStr : rawCache->getChildren())
-                    ret.push_back(symbols.create(fieldStr));
-            }
-            return ret;
-    }
-    return std::nullopt;
-}
 
 void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 {
-    if (auto evalCache = fun.getEvalCache(); !evalCache.isEmpty()) {
-        if (auto cacheValue = evalCache.getRawValue()) {
-            if (std::holds_alternative<tree_cache::attributeSet_t>(*cacheValue)) {
-                res = fun;
-                return;
-            }
-        }
-    }
-
     forceValue(fun);
 
     if (fun.type() == nAttrs) {
@@ -1568,7 +1414,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         }
     }
 
-    if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
+    if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) {
         res = fun;
         return;
     }
@@ -1770,7 +1616,6 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
            and none of the strings are allowed to have contexts. */
         if (first) {
             firstType = vTmp.type();
-            first = false;
         }
 
         if (firstType == nInt) {
@@ -1791,7 +1636,12 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
             } else
                 throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
-            s << state.coerceToString(pos, vTmp, context, false, firstType == nString);
+            /* skip canonization of first path, which would only be not
+            canonized in the first place if it's coming from a ./${foo} type
+            path */
+            s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first);
+
+        first = false;
     }
 
     if (firstType == nInt)
@@ -1810,7 +1660,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 
 void ExprPos::eval(EvalState & state, Env & env, Value & v)
 {
-    state.mkPos(v, &pos);
+    state.mkPos(v, ptr(&pos));
 }
 
 
@@ -1901,6 +1751,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
 }
 
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+    if (s.at(0) == '!') {
+        size_t index = s.find("!", 1);
+        return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+    } else
+        return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -1919,102 +1781,6 @@ std::vector<std::pair<Path, std::string>> Value::getContext()
     return res;
 }
 
-ValueCache & Value::getEvalCache()
-{
-    if (internalType == tAttrs) {
-        return attrs->eval_cache;
-    } else if (internalType == tCachedThunk) {
-        return *cachedThunk.cache;
-    } else {
-        return ValueCache::empty;
-    }
-}
-
-ValueCache ValueCache::empty = ValueCache(nullptr);
-
-tree_cache::AttrValue cachedValueFor(Value& v)
-{
-    tree_cache::AttrValue valueToCache;
-    switch (v.type()) {
-        case nThunk:
-            valueToCache = tree_cache::thunk_t{};
-            break;
-        case nNull:
-        case nList:
-        case nFunction:
-        case nExternal:
-            valueToCache = tree_cache::unknown_t{};
-            break;
-        case nBool:
-            valueToCache = tree_cache::wrapped_basetype<bool>{v.boolean};
-            break;
-        case nString:
-            valueToCache = tree_cache::string_t{
-                v.string.s,
-                v.getContext()
-            };
-            break;
-        case nPath:
-            valueToCache = tree_cache::string_t{
-                v.path,
-                {}
-            };
-            break;
-        case nAttrs:
-            valueToCache = tree_cache::attributeSet_t{};
-            break;
-        case nInt:
-            valueToCache = tree_cache::wrapped_basetype<int64_t>{v.integer};
-            break;
-        case nFloat:
-            valueToCache = tree_cache::wrapped_basetype<double>{v.fpoint};
-            break;
-    };
-    return valueToCache;
-}
-
-ValueCache ValueCache::addChild(const Symbol& name, Value& value)
-{
-    if (!rawCache)
-        return ValueCache::empty;
-
-    auto cachedValue = cachedValueFor(value);
-    auto ret = ValueCache(rawCache->addChild(name, cachedValue));
-    if (value.type() == nAttrs)
-        ret.addAttrSetChilds(*value.attrs);
-    return ret;
-}
-
-
-ValueCache ValueCache::addFailedChild(const Symbol& name, const Error & error)
-{
-    if (!rawCache) return ValueCache::empty;
-    return ValueCache(rawCache->addChild(name, tree_cache::failed_t{ .error = error.msg() }));
-}
-
-void ValueCache::addAttrSetChilds(Bindings & children)
-{
-    if (!rawCache) return;
-    for (auto & attr : children) {
-        // We could in theory directly store the value, but that would cause
-        // an infinite recursion in case of a cyclic attrset.
-        // So in a first time, just store a thunk
-        if (attr.value->type() == nAttrs)
-            rawCache->addChild(attr.name, tree_cache::thunk_t{});
-        else
-            addChild(attr.name, *attr.value);
-    }
-
-}
-
-void Value::setEvalCache(ValueCache & newCache)
-{
-    if (internalType == tAttrs) {
-        attrs->eval_cache = newCache;
-    } else if (internalType == tCachedThunk) {
-        cachedThunk.cache = &newCache;
-    }
-}
 
 string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
 {
@@ -2064,7 +1830,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
 }
 
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
-    bool coerceMore, bool copyToStore)
+    bool coerceMore, bool copyToStore, bool canonicalizePath)
 {
     forceValue(v, pos);
 
@@ -2076,7 +1842,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     }
 
     if (v.type() == nPath) {
-        Path path(canonPath(v.path));
+        Path path(canonicalizePath ? canonPath(v.path) : v.path);
         return copyToStore ? copyPathToStore(context, path) : path;
     }
 
@@ -2135,6 +1901,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
             : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
+        allowPath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
     }
@@ -2297,13 +2064,6 @@ void EvalState::printStats()
         topObj.attr("nrLookups", nrLookups);
         topObj.attr("nrPrimOpCalls", nrPrimOpCalls);
         topObj.attr("nrFunctionCalls", nrFunctionCalls);
-        {
-            auto cache = topObj.object("evalCache");
-            cache.attr("nrCacheMisses", nrCacheMisses);
-            cache.attr("nrCacheHits", nrCacheHits);
-            cache.attr("nrUncached", nrUncached);
-            cache.attr("nrUncacheable", nrUncacheable);
-        }
 #if HAVE_BOEHMGC
         {
             auto gc = topObj.object("gc");
@@ -2404,21 +2164,6 @@ Strings EvalSettings::getDefaultNixPath()
     return res;
 }
 
-std::shared_ptr<tree_cache::Cache> EvalState::openTreeCache(Hash cacheKey)
-{
-    if (auto iter = evalCache.find(cacheKey); iter != evalCache.end())
-        return iter->second;
-
-    if (!(evalSettings.useEvalCache && evalSettings.pureEval))
-        return nullptr;
-    auto thisCache = tree_cache::Cache::tryCreate(
-            cacheKey,
-            symbols
-    );
-    evalCache.insert({cacheKey, thisCache});
-    return thisCache;
-}
-
 EvalSettings evalSettings;
 
 static GlobalConfig::Register rEvalSettings(&evalSettings);

src/libexpr/eval.hh

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "attr-set.hh"
-#include "context.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
@@ -95,8 +94,14 @@ public:
 
     Value vEmptySet;
 
+    /* Store used to materialise .drv files. */
     const ref<Store> store;
 
+    /* Store used to build stuff. */
+    const ref<Store> buildStore;
+
+    RootValue vCallFlake = nullptr;
+    RootValue vImportedDrvToDerivation = nullptr;
 
 private:
     SrcToStore srcToStore;
@@ -116,7 +121,6 @@ private:
     typedef std::map<Path, Value> FileEvalCache;
 #endif
     FileEvalCache fileEvalCache;
-    std::map<Hash, std::shared_ptr<tree_cache::Cache>> evalCache;
 
     SearchPath searchPath;
 
@@ -130,15 +134,31 @@ private:
 
 public:
 
-    EvalState(const Strings & _searchPath, ref<Store> store);
+    EvalState(
+        const Strings & _searchPath,
+        ref<Store> store,
+        std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    std::shared_ptr<tree_cache::Cache> openTreeCache(Hash);
+    void requireExperimentalFeatureOnEvaluation(
+        const std::string & feature,
+        const std::string_view fName,
+        const Pos & pos
+    );
 
     void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
 
+    /* Allow access to a path. */
+    void allowPath(const Path & path);
+
+    /* Allow access to a store path. Note that this gets remapped to
+       the real store path if `store` is a chroot store. */
+    void allowPath(const StorePath & storePath);
+
+    /* Check whether access to a path is allowed and throw an error if
+       not. Otherwise return the canonicalised path. */
     Path checkSourcePath(const Path & path);
 
     void checkURI(const std::string & uri);
@@ -167,6 +187,14 @@ public:
        trivial (i.e. doesn't require arbitrary computation). */
     void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
 
+    /* Like `cacheFile`, but with an already parsed expression. */
+    void cacheFile(
+        const Path & path,
+        const Path & resolvedPath,
+        Expr * e,
+        Value & v,
+        bool mustBeTrivial = false);
+
     void resetFileCache();
 
     /* Look up a file in the search path. */
@@ -221,7 +249,8 @@ public:
        booleans and lists to a string.  If `copyToStore' is set,
        referenced paths are copied to the Nix store as a side effect. */
     string coerceToString(const Pos & pos, Value & v, PathSet & context,
-        bool coerceMore = false, bool copyToStore = true);
+        bool coerceMore = false, bool copyToStore = true,
+        bool canonicalizePath = true);
 
     string copyPathToStore(PathSet & context, const Path & path);
 
@@ -305,7 +334,7 @@ public:
     void mkList(Value & v, size_t length);
     void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
-    void mkPos(Value & v, Pos * pos);
+    void mkPos(Value & v, ptr<Pos> pos);
 
     void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
@@ -314,25 +343,21 @@ public:
 
     void realiseContext(const PathSet & context);
 
-    void updateCacheStats(ValueCache::CacheResult);
-
 private:
 
     unsigned long nrEnvs = 0;
     unsigned long nrValuesInEnvs = 0;
     unsigned long nrValues = 0;
     unsigned long nrListElems = 0;
+    unsigned long nrLookups = 0;
     unsigned long nrAttrsets = 0;
     unsigned long nrAttrsInAttrsets = 0;
+    unsigned long nrAvoided = 0;
     unsigned long nrOpUpdates = 0;
     unsigned long nrOpUpdateValuesCopied = 0;
     unsigned long nrListConcats = 0;
     unsigned long nrPrimOpCalls = 0;
     unsigned long nrFunctionCalls = 0;
-    unsigned long nrCacheHits = 0;
-    unsigned long nrCacheMisses = 0;
-    unsigned long nrUncacheable = 0;
-    unsigned long nrUncached = 0;
 
     bool countCalls;
 
@@ -349,30 +374,14 @@ private:
 
     friend struct ExprOpUpdate;
     friend struct ExprOpConcatLists;
+    friend struct ExprVar;
+    friend struct ExprString;
+    friend struct ExprInt;
+    friend struct ExprFloat;
+    friend struct ExprPath;
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
     friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
-
-public:
-
-    bool getAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-
-    enum struct LazyValueType {
-        PlainValue,
-        DelayedUncacheable,
-        DelayedAttr,
-        Missing,
-    };
-
-    // Similar to `getAttrField`, but if the cache says that the result is an
-    // attribute set, just return a thunk to it rather than forcing it.
-    LazyValueType lazyGetAttrField(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-
-    // Similar to `getAttrField`, but throws an `Error` if the field can’t be
-    // found
-    void getAttrFieldThrow(Value & attrs, const std::vector<Symbol> & selector, const Pos & pos, Value & dest);
-
-    std::vector<Symbol> getFields(Value & attrs, const Pos & pos);
 };
 
 
@@ -380,6 +389,10 @@ public:
 string showType(ValueType type);
 string showType(const Value & v);
 
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+   name>. */
+std::pair<string, string> decodeContext(std::string_view s);
+
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
 

src/libexpr/flake/config.cc

@@ -29,7 +29,7 @@ static void writeTrustedList(const TrustedList & trustedList)
 
 void ConfigFile::apply()
 {
-    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
+    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"};
 
     for (auto & [name, value] : settings) {
 

src/libexpr/flake/flake.cc

@@ -1,4 +1,5 @@
 #include "flake.hh"
+#include "eval.hh"
 #include "lockfile.hh"
 #include "primops.hh"
 #include "eval-inline.hh"
@@ -63,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     debug("got tree '%s' from '%s'",
         state.store->printStorePath(tree.storePath), lockedRef);
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+    state.allowPath(tree.storePath);
 
     assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
 
@@ -88,10 +88,12 @@ static void expectType(EvalState & state, ValueType type,
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos);
+    EvalState & state, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir);
 
 static FlakeInput parseFlakeInput(EvalState & state,
-    const std::string & inputName, Value * value, const Pos & pos)
+    const std::string & inputName, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir)
 {
     expectType(state, nAttrs, *value, pos);
 
@@ -115,7 +117,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nBool, *attr.value, *attr.pos);
                 input.isFlake = attr.value->boolean;
             } else if (attr.name == sInputs) {
-                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
+                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir);
             } else if (attr.name == sFollows) {
                 expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
@@ -153,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         if (!attrs.empty())
             throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
         if (url)
-            input.ref = parseFlakeRef(*url, {}, true);
+            input.ref = parseFlakeRef(*url, baseDir, true);
     }
 
     if (!input.follows && !input.ref)
@@ -163,7 +165,8 @@ static FlakeInput parseFlakeInput(EvalState & state,
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos)
+    EvalState & state, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir)
 {
     std::map<FlakeId, FlakeInput> inputs;
 
@@ -174,7 +177,8 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
             parseFlakeInput(state,
                 inputAttr.name,
                 inputAttr.value,
-                *inputAttr.pos));
+                *inputAttr.pos,
+                baseDir));
     }
 
     return inputs;
@@ -190,7 +194,8 @@ static Flake getFlake(
         state, originalRef, allowLookup, flakeCache);
 
     // Guard against symlink attacks.
-    auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
+    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir);
+    auto flakeFile = canonPath(flakeDir + "/flake.nix");
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
             lockedRef, state.store->printStorePath(sourceInfo.storePath));
@@ -218,14 +223,14 @@ static Flake getFlake(
     auto sInputs = state.symbols.create("inputs");
 
     if (auto inputs = vInfo.attrs->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
+        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir);
 
     auto sOutputs = state.symbols.create("outputs");
 
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
         expectType(state, nFunction, *outputs->value, *outputs->pos);
 
-        if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
+        if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) {
             for (auto & formal : outputs->value->lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {
@@ -296,7 +301,14 @@ LockedFlake lockFlake(
 
     FlakeCache flakeCache;
 
-    auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
+    auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
+
+    auto flake = getFlake(state, topRef, useRegistries, flakeCache);
+
+    if (lockFlags.applyNixConfig) {
+        flake.config.apply();
+        // FIXME: send new config to the daemon.
+    }
 
     try {
 
@@ -317,25 +329,38 @@ LockedFlake lockFlake(
 
         std::vector<FlakeRef> parents;
 
+        struct LockParent {
+            /* The path to this parent. */
+            InputPath path;
+
+            /* Whether we are currently inside a top-level lockfile
+               (inputs absolute) or subordinate lockfile (inputs
+               relative). */
+            bool absolute;
+        };
+
         std::function<void(
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
-            std::shared_ptr<const Node> oldNode)>
+            std::shared_ptr<const Node> oldNode,
+            const LockParent & parent,
+            const Path & parentPath)>
             computeLocks;
 
         computeLocks = [&](
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
-            std::shared_ptr<const Node> oldNode)
+            std::shared_ptr<const Node> oldNode,
+            const LockParent & parent,
+            const Path & parentPath)
         {
             debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
             /* Get the overrides (i.e. attributes of the form
                'inputs.nixops.inputs.nixpkgs.url = ...'). */
-            // FIXME: check this
-            for (auto & [id, input] : flake.inputs) {
+            for (auto & [id, input] : flakeInputs) {
                 for (auto & [idOverride, inputOverride] : input.overrides) {
                     auto inputPath(inputPathPrefix);
                     inputPath.push_back(id);
@@ -359,22 +384,31 @@ LockedFlake lockFlake(
                        ancestors? */
                     auto i = overrides.find(inputPath);
                     bool hasOverride = i != overrides.end();
-                    if (hasOverride) overridesUsed.insert(inputPath);
+                    if (hasOverride) {
+                        overridesUsed.insert(inputPath);
+                        // Respect the “flakeness” of the input even if we
+                        // override it
+                        i->second.isFlake = input2.isFlake;
+                    }
                     auto & input = hasOverride ? i->second : input2;
 
                     /* Resolve 'follows' later (since it may refer to an input
                        path we haven't processed yet. */
                     if (input.follows) {
                         InputPath target;
-                        if (hasOverride || input.absolute)
-                            /* 'follows' from an override is relative to the
-                               root of the graph. */
+
+                        if (parent.absolute && !hasOverride) {
                             target = *input.follows;
-                        else {
-                            /* Otherwise, it's relative to the current flake. */
-                            target = inputPathPrefix;
+                        } else {
+                            if (hasOverride) {
+                                target = inputPathPrefix;
+                                target.pop_back();
+                            } else
+                                target = parent.path;
+
                             for (auto & i : *input.follows) target.push_back(i);
                         }
+
                         debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
                         node->inputs.insert_or_assign(id, target);
                         continue;
@@ -420,7 +454,7 @@ LockedFlake lockFlake(
                         if (hasChildUpdate) {
                             auto inputFlake = getFlake(
                                 state, oldLock->lockedRef, false, flakeCache);
-                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
+                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath);
                         } else {
                             /* No need to fetch this flake, we can be
                                lazy. However there may be new overrides on the
@@ -437,12 +471,11 @@ LockedFlake lockFlake(
                                 } else if (auto follows = std::get_if<1>(&i.second)) {
                                     fakeInputs.emplace(i.first, FlakeInput {
                                         .follows = *follows,
-                                        .absolute = true
                                     });
                                 }
                             }
 
-                            computeLocks(fakeInputs, childNode, inputPath, oldLock);
+                            computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath);
                         }
 
                     } else {
@@ -454,7 +487,15 @@ LockedFlake lockFlake(
                             throw Error("cannot update flake input '%s' in pure mode", inputPathS);
 
                         if (input.isFlake) {
-                            auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
+                            Path localPath = parentPath;
+                            FlakeRef localRef = *input.ref;
+
+                            // If this input is a path, recurse it down.
+                            // This allows us to resolve path inputs relative to the current flake.
+                            if (localRef.input.getType() == "path")
+                                localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
+
+                            auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache);
 
                             /* Note: in case of an --override-input, we use
                                the *original* ref (input2.ref) for the
@@ -475,6 +516,13 @@ LockedFlake lockFlake(
                             parents.push_back(*input.ref);
                             Finally cleanup([&]() { parents.pop_back(); });
 
+                            // Follows paths from existing inputs in the top-level lockfile are absolute,
+                            // whereas paths in subordinate lockfiles are relative to those lockfiles.
+                            LockParent newParent {
+                                .path = inputPath,
+                                .absolute = oldLock ? true : false
+                            };
+
                             /* Recursively process the inputs of this
                                flake. Also, unless we already have this flake
                                in the top-level lock file, use this flake's
@@ -484,12 +532,13 @@ LockedFlake lockFlake(
                                 oldLock
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
                                 : LockFile::read(
-                                    inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+                                    inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root,
+                                newParent, localPath);
                         }
 
                         else {
                             auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
-                                state, *input.ref, lockFlags.useRegistries, flakeCache);
+                                state, *input.ref, useRegistries, flakeCache);
                             node->inputs.insert_or_assign(id,
                                 std::make_shared<LockedNode>(lockedRef, *input.ref, false));
                         }
@@ -502,9 +551,17 @@ LockedFlake lockFlake(
             }
         };
 
+        LockParent parent {
+            .path = {},
+            .absolute = true
+        };
+
+        // Bring in the current ref for relative path resolution if we have it
+        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir);
+
         computeLocks(
             flake.inputs, newLockFile.root, {},
-            lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath);
 
         for (auto & i : lockFlags.inputOverrides)
             if (!overridesUsed.count(i.first))
@@ -554,8 +611,8 @@ LockedFlake lockFlake(
                         topRef.input.markChangedFile(
                             (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
                             lockFlags.commitLockFile
-                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
-                                    relPath, lockFileExists ? "Update" : "Add", diff))
+                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s",
+                                    relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true)))
                             : std::nullopt);
 
                         /* Rewriting the lockfile changed the top-level
@@ -563,7 +620,7 @@ LockedFlake lockFlake(
                            also just clear the 'rev' field... */
                         auto prevLockedRef = flake.lockedRef;
                         FlakeCache dummyCache;
-                        flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
+                        flake = getFlake(state, topRef, useRegistries, dummyCache);
 
                         if (lockFlags.commitLockFile &&
                             flake.lockedRef.input.getRev() &&
@@ -580,8 +637,10 @@ LockedFlake lockFlake(
                     }
                 } else
                     throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
-            } else
+            } else {
                 warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
+                flake.forceDirty = true;
+            }
         }
 
         return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
@@ -596,48 +655,40 @@ void callFlake(EvalState & state,
     const LockedFlake & lockedFlake,
     Value & vRes)
 {
-    auto lockExpr = new ExprString(
-        state.symbols.create(lockedFlake.lockFile.to_string())
-    );
-    auto subdirExpr = new ExprString(
-        state.symbols.create(lockedFlake.flake.lockedRef.subdir)
-    );
-
+    auto vLocks = state.allocValue();
     auto vRootSrc = state.allocValue();
-    emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
+    auto vRootSubdir = state.allocValue();
+    auto vTmp1 = state.allocValue();
+    auto vTmp2 = state.allocValue();
 
-    static Expr * callFlakeExpr = nullptr;
-    if (!callFlakeExpr) {
-        callFlakeExpr = state.parseExprFromString(
+    mkString(*vLocks, lockedFlake.lockFile.to_string());
+
+    emitTreeAttrs(
+        state,
+        *lockedFlake.flake.sourceInfo,
+        lockedFlake.flake.lockedRef.input,
+        *vRootSrc,
+        false,
+        lockedFlake.flake.forceDirty);
+
+    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+
+    if (!state.vCallFlake) {
+        state.vCallFlake = allocRootValue(state.allocValue());
+        state.eval(state.parseExprFromString(
             #include "call-flake.nix.gen.hh"
-            , "/");
+            , "/"), **state.vCallFlake);
     }
 
-    auto resExpr = new ExprApp(
-        new ExprApp(
-            new ExprApp(
-                callFlakeExpr,
-                lockExpr
-            ),
-            new ExprCastedVar(vRootSrc)
-        ),
-        subdirExpr
-    );
-    auto thunk = (Thunk*)allocBytes(sizeof(Thunk));
-    thunk->expr = resExpr;
-    thunk->env = &state.baseEnv;
-    auto fingerprint = lockedFlake.getFingerprint();
-    auto treeCache = state.openTreeCache(fingerprint);
-    auto cacheRoot = treeCache ? treeCache->getRoot() : nullptr;
-    auto evalCache = new ValueCache(cacheRoot);
-    vRes.mkCachedThunk(
-        thunk,
-        evalCache
-    );
+    state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
+    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.requireExperimentalFeatureOnEvaluation("flakes", "builtins.getFlake", pos);
+
     auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
     if (evalSettings.pureEval && !flakeRef.input.isImmutable())
@@ -647,13 +698,13 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
         lockFlake(state, flakeRef,
             LockFlags {
                 .updateLockFile = false,
-                .useRegistries = !evalSettings.pureEval,
+                .useRegistries = !evalSettings.pureEval && settings.useRegistries,
                 .allowMutable  = !evalSettings.pureEval,
             }),
         v);
 }
 
-static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
+static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
 
 }
 
@@ -663,8 +714,9 @@ Fingerprint LockedFlake::getFingerprint() const
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
     return hashString(htSHA256,
-        fmt("%s;%d;%d;%s",
+        fmt("%s;%s;%d;%d;%s",
             flake.sourceInfo->storePath.to_string(),
+            flake.lockedRef.subdir,
             flake.lockedRef.input.getRevCount().value_or(0),
             flake.lockedRef.input.getLastModified().value_or(0),
             lockFile));

src/libexpr/flake/flake.hh

@@ -43,7 +43,6 @@ struct FlakeInput
     std::optional<FlakeRef> ref;
     bool isFlake = true;  // true = process flake to get outputs, false = (fetched) static source path
     std::optional<InputPath> follows;
-    bool absolute = false; // whether 'follows' is relative to the flake root
     FlakeInputs overrides;
 };
 
@@ -59,9 +58,10 @@ struct ConfigFile
 /* The contents of a flake.nix file. */
 struct Flake
 {
-    FlakeRef originalRef;   // the original flake specification (by the user)
-    FlakeRef resolvedRef;   // registry references and caching resolved to the specific underlying flake
-    FlakeRef lockedRef;     // the specific local store result of invoking the fetcher
+    FlakeRef originalRef; // the original flake specification (by the user)
+    FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
+    FlakeRef lockedRef; // the specific local store result of invoking the fetcher
+    bool forceDirty = false; // pretend that 'lockedRef' is dirty
     std::optional<std::string> description;
     std::shared_ptr<const fetchers::Tree> sourceInfo;
     FlakeInputs inputs;
@@ -102,7 +102,11 @@ struct LockFlags
 
     /* Whether to use the registries to lookup indirect flake
        references like 'nixpkgs'. */
-    bool useRegistries = true;
+    std::optional<bool> useRegistries = std::nullopt;
+
+    /* Whether to apply flake's nixConfig attribute to the configuration */
+
+    bool applyNixConfig = false;
 
     /* Whether mutable flake references (i.e. those without a Git
        revision or similar) without a corresponding lock are
@@ -137,6 +141,8 @@ void emitTreeAttrs(
     EvalState & state,
     const fetchers::Tree & tree,
     const fetchers::Input & input,
-    Value & v, bool emptyRevFallback = false);
+    Value & v,
+    bool emptyRevFallback = false,
+    bool forceDirty = false);
 
 }

src/libexpr/flake/flakeref.cc

@@ -172,8 +172,12 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         auto parsedURL = parseURL(url);
         std::string fragment;
         std::swap(fragment, parsedURL.fragment);
+
+        auto input = Input::fromURL(parsedURL);
+        input.parent = baseDir;
+
         return std::make_pair(
-            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+            FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")),
             fragment);
     }
 }

src/libexpr/flake/lockfile.cc

@@ -2,6 +2,8 @@
 #include "store-api.hh"
 #include "url-parts.hh"
 
+#include <iomanip>
+
 #include <nlohmann/json.hpp>
 
 namespace nix::flake {
@@ -268,10 +270,20 @@ std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
     return res;
 }
 
+static std::string describe(const FlakeRef & flakeRef)
+{
+    auto s = fmt("'%s'", flakeRef.to_string());
+
+    if (auto lastModified = flakeRef.input.getLastModified())
+        s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d"));
+
+    return s;
+}
+
 std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
 {
     if (auto node = std::get_if<0>(&edge))
-        stream << "'" << (*node)->lockedRef << "'";
+        stream << describe((*node)->lockedRef);
     else if (auto follows = std::get_if<1>(&edge))
         stream << fmt("follows '%s'", printInputPath(*follows));
     return stream;
@@ -299,14 +311,15 @@ std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
 
     while (i != oldFlat.end() || j != newFlat.end()) {
         if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
-            res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
+            res += fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n    %s\n",
+                printInputPath(j->first), j->second);
             ++j;
         } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
-            res += fmt("* Removed '%s'\n", printInputPath(i->first));
+            res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputPath(i->first));
             ++i;
         } else {
             if (!equals(i->second, j->second)) {
-                res += fmt("* Updated '%s': %s -> %s\n",
+                res += fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n    %s\n  → %s\n",
                     printInputPath(i->first),
                     i->second,
                     j->second);

src/libexpr/lexer.l

@@ -9,6 +9,9 @@
 %s DEFAULT
 %x STRING
 %x IND_STRING
+%x INPATH
+%x INPATH_SLASH
+%x PATH_START
 
 
 %{
@@ -25,6 +28,8 @@ using namespace nix;
 
 namespace nix {
 
+// backup to recover from yyless(0)
+YYLTYPE prev_yylloc;
 
 static void initLoc(YYLTYPE * loc)
 {
@@ -35,14 +40,18 @@ static void initLoc(YYLTYPE * loc)
 
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
+    prev_yylloc = *loc;
+
     loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
 
-    while (len--) {
+    for (size_t i = 0; i < len; i++) {
        switch (*s++) {
        case '\r':
-           if (*s == '\n') /* cr/lf */
+           if (*s == '\n') { /* cr/lf */
+               i++;
                s++;
+           }
            /* fall through */
        case '\n':
            ++loc->last_line;
@@ -95,9 +104,12 @@ ANY         .|\n
 ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
 INT         [0-9]+
 FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH        [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
-HPATH       \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
-SPATH       \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\>
+PATH_CHAR   [a-zA-Z0-9\.\_\-\+]
+PATH        {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
+PATH_SEG    {PATH_CHAR}*\/
+HPATH       \~(\/{PATH_CHAR}+)+\/?
+HPATH_START \~\/
+SPATH       \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
 URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
 
 
@@ -198,17 +210,75 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 
+{PATH_SEG}\$\{ |
+{HPATH_START}\$\{ {
+  PUSH_STATE(PATH_START);
+  yyless(0);
+  *yylloc = prev_yylloc;
+}
+
+<PATH_START>{PATH_SEG} {
+  POP_STATE();
+  PUSH_STATE(INPATH_SLASH);
+  yylval->path = strdup(yytext);
+  return PATH;
+}
+
+<PATH_START>{HPATH_START} {
+  POP_STATE();
+  PUSH_STATE(INPATH_SLASH);
+  yylval->path = strdup(yytext);
+  return HPATH;
+}
+
+{PATH} {
+  if (yytext[yyleng-1] == '/')
+    PUSH_STATE(INPATH_SLASH);
+  else
+    PUSH_STATE(INPATH);
+  yylval->path = strdup(yytext);
+  return PATH;
+}
+{HPATH} {
+  if (yytext[yyleng-1] == '/')
+    PUSH_STATE(INPATH_SLASH);
+  else
+    PUSH_STATE(INPATH);
+  yylval->path = strdup(yytext);
+  return HPATH;
+}
+
+<INPATH,INPATH_SLASH>\$\{ {
+  POP_STATE();
+  PUSH_STATE(INPATH);
+  PUSH_STATE(DEFAULT);
+  return DOLLAR_CURLY;
+}
+<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
+  POP_STATE();
+  if (yytext[yyleng-1] == '/')
+      PUSH_STATE(INPATH_SLASH);
+  else
+      PUSH_STATE(INPATH);
+  yylval->e = new ExprString(data->symbols.create(string(yytext)));
+  return STR;
+}
+<INPATH>{ANY} |
+<INPATH><<EOF>> {
+  /* if we encounter a non-path character we inform the parser that the path has
+     ended with a PATH_END token and re-parse this character in the default
+     context (it may be ')', ';', or something of that sort) */
+  POP_STATE();
+  yyless(0);
+  *yylloc = prev_yylloc;
+  return PATH_END;
+}
+
+<INPATH_SLASH>{ANY} |
+<INPATH_SLASH><<EOF>> {
+  throw ParseError("path has a trailing slash");
+}
 
-{PATH}      { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path '%s' has a trailing slash", yytext);
-              yylval->path = strdup(yytext);
-              return PATH;
-            }
-{HPATH}     { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path '%s' has a trailing slash", yytext);
-              yylval->path = strdup(yytext);
-              return HPATH;
-            }
 {SPATH}     { yylval->path = strdup(yytext); return SPATH; }
 {URI}       { yylval->uri = strdup(yytext); return URI; }
 

src/libexpr/local.mk

@@ -15,8 +15,8 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS = -lboost_context
-ifeq ($(OS), Linux)
+libexpr_LDFLAGS += -lboost_context -pthread
+ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
 
@@ -35,7 +35,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))

src/libexpr/nixexpr.cc

@@ -124,7 +124,7 @@ void ExprList::show(std::ostream & str) const
 void ExprLambda::show(std::ostream & str) const
 {
     str << "(";
-    if (matchAttrs) {
+    if (hasFormals()) {
         str << "{ ";
         bool first = true;
         for (auto & i : formals->formals) {
@@ -348,7 +348,7 @@ void ExprLambda::bindVars(const StaticEnv & env)
 
     if (!arg.empty()) newEnv.vars[arg] = displ++;
 
-    if (matchAttrs) {
+    if (hasFormals()) {
         for (auto & i : formals->formals)
             newEnv.vars[i.name] = displ++;
 
@@ -440,18 +440,6 @@ string ExprLambda::showNamePos() const
     return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
 }
 
-void ExprCastedVar::show(std::ostream & str) const {
-    std::set<const Value*> active;
-    printValue(str, active, *v);
-}
-
-void ExprCastedVar::bindVars(const StaticEnv & env) {}
-void ExprCastedVar::eval(EvalState & state, Env & env, Value & v) {
-    v = std::move(*this->v);
-}
-Value * ExprCastedVar::maybeThunk(EvalState & state, Env & env) {
-    return v;
-}
 
 
 /* Symbol table. */

src/libexpr/nixexpr.hh

@@ -165,12 +165,7 @@ struct ExprSelect : Expr
     Expr * e, * def;
     AttrPath attrPath;
     ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { };
-    ExprSelect(const Pos & pos, Expr * e, const Symbol & name)
-        : ExprSelect(pos, e, std::vector{name}) {};
-    ExprSelect(const Pos & pos, Expr * e, const std::vector<Symbol> & symbolicAttrPath) : pos(pos), e(e), def(0) {
-        for (auto & name : symbolicAttrPath)
-            attrPath.push_back(AttrName(name));
-    };
+    ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); };
     COMMON_METHODS
 };
 
@@ -238,11 +233,10 @@ struct ExprLambda : Expr
     Pos pos;
     Symbol name;
     Symbol arg;
-    bool matchAttrs;
     Formals * formals;
     Expr * body;
-    ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
-        : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
+    ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body)
+        : pos(pos), arg(arg), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
             throw ParseError({
@@ -252,6 +246,7 @@ struct ExprLambda : Expr
     };
     void setName(Symbol & name);
     string showNamePos() const;
+    inline bool hasFormals() const { return formals != nullptr; }
     COMMON_METHODS
 };
 
@@ -339,14 +334,6 @@ struct ExprPos : Expr
     COMMON_METHODS
 };
 
-struct ExprCastedVar : Expr
-{
-    Value * v;
-    ExprCastedVar(Value * v) : v(v) {};
-    Value * maybeThunk(EvalState & state, Env & env);
-    COMMON_METHODS
-};
-
 
 /* Static environments are used to map variable names onto (level,
    displacement) pairs used to obtain the value of the variable at

src/libexpr/parser.y

@@ -290,13 +290,13 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %type <formal> formal
 %type <attrNames> attrs attrpath
 %type <string_parts> string_parts_interpolated ind_string_parts
-%type <e> string_parts string_attr
+%type <e> path_start string_parts string_attr
 %type <id> attr
 %token <id> ID ATTRPATH
 %token <e> STR IND_STR
 %token <n> INT
 %token <nf> FLOAT
-%token <path> PATH HPATH SPATH
+%token <path> PATH HPATH SPATH PATH_END
 %token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
 %token DOLLAR_CURLY /* == ${ */
@@ -324,13 +324,13 @@ expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); }
   | '{' formals '}' '@' ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); }
   | ID '@' '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
@@ -405,8 +405,11 @@ expr_simple
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = stripIndentation(CUR_POS, data->symbols, *$2);
   }
-  | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
-  | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); }
+  | path_start PATH_END { $$ = $1; }
+  | path_start string_parts_interpolated PATH_END {
+      $2->insert($2->begin(), $1);
+      $$ = new ExprConcatStrings(CUR_POS, false, $2);
+  }
   | SPATH {
       string path($1 + 1, strlen($1) - 2);
       $$ = new ExprApp(CUR_POS,
@@ -452,6 +455,20 @@ string_parts_interpolated
     }
   ;
 
+path_start
+  : PATH {
+    Path path(absPath($1, data->basePath));
+    /* add back in the trailing '/' to the first segment */
+    if ($1[strlen($1)-1] == '/' && strlen($1) > 1)
+      path += "/";
+    $$ = new ExprPath(path);
+  }
+  | HPATH {
+    Path path(getHome() + string($1 + 1));
+    $$ = new ExprPath(path);
+  }
+  ;
+
 ind_string_parts
   : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
   | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
@@ -735,7 +752,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, path };
         else {
             logWarning({
-                .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second)
             });
             res = { false, "" };
         }

src/libexpr/primops.cc

@@ -52,16 +52,13 @@ void EvalState::realiseContext(const PathSet & context)
     if (drvs.empty()) return;
 
     if (!evalSettings.enableImportFromDerivation)
-        throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
+        throw Error(
+            "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             store->printStorePath(drvs.begin()->drvPath));
 
-    /* For performance, prefetch all substitute info. */
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
+    /* Build/substitute the context. */
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
     store->buildPaths(buildReqs);
 
     /* Add the output of this derivations to the allowed
@@ -124,7 +121,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         });
     } catch (Error & e) {
         e.addTrace(pos, "while importing '%s'", path);
-        throw e;
+        throw;
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -160,16 +157,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         }
         w.attrs->sort();
 
-        static RootValue fun;
-        if (!fun) {
-            fun = allocRootValue(state.allocValue());
+        if (!state.vImportedDrvToDerivation) {
+            state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
             state.eval(state.parseExprFromString(
                 #include "imported-drv-to-derivation.nix.gen.hh"
-                , "/"), **fun);
+                , "/"), **state.vImportedDrvToDerivation);
         }
 
-        state.forceFunction(**fun, pos);
-        mkApp(v, **fun, w);
+        state.forceFunction(**state.vImportedDrvToDerivation, pos);
+        mkApp(v, **state.vImportedDrvToDerivation, w);
         state.forceAttrs(v, pos);
     }
 
@@ -414,7 +410,7 @@ static RegisterPrimOp primop_isNull({
       Return `true` if *e* evaluates to `null`, and `false` otherwise.
 
       > **Warning**
-      > 
+      >
       > This function is *deprecated*; just write `e == null` instead.
     )",
     .fun = prim_isNull,
@@ -1174,7 +1170,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         // hash per output.
         auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
         std::visit(overloaded {
-            [&](Hash h) {
+            [&](Hash & h) {
                 for (auto & i : outputs) {
                     auto outPath = state.store->makeOutputPath(i, h, drvName);
                     drv.env[i] = state.store->printStorePath(outPath);
@@ -1186,11 +1182,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         });
                 }
             },
-            [&](CaOutputHashes) {
+            [&](CaOutputHashes &) {
                 // Shouldn't happen as the toplevel derivation is not CA.
                 assert(false);
             },
-            [&](DeferredHash _) {
+            [&](DeferredHash &) {
                 for (auto & i : outputs) {
                     drv.outputs.insert_or_assign(i,
                         DerivationOutput {
@@ -1493,15 +1489,20 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     string type = state.forceStringNoCtx(*args[0], pos);
     std::optional<HashType> ht = parseHashType(type);
     if (!ht)
-      throw Error({
-          .msg = hintfmt("unknown hash type '%1%'", type),
-          .errPos = pos
-      });
+        throw Error({
+            .msg = hintfmt("unknown hash type '%1%'", type),
+            .errPos = pos
+        });
 
-    PathSet context; // discarded
-    Path p = state.coerceToPath(pos, *args[1], context);
+    PathSet context;
+    Path path = state.coerceToPath(pos, *args[1], context);
+    try {
+        state.realiseContext(context);
+    } catch (InvalidPathError & e) {
+        throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos);
+    }
 
-    mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+    mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -1712,7 +1713,7 @@ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Va
         parseJSON(state, s, v);
     } catch (JSONParseError &e) {
         e.addTrace(pos, "while decoding a JSON string");
-        throw e;
+        throw;
     }
 }
 
@@ -1842,50 +1843,81 @@ static RegisterPrimOp primop_toFile({
     .fun = prim_toFile,
 });
 
-static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
-    Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
+static void addPath(
+    EvalState & state,
+    const Pos & pos,
+    const string & name,
+    Path path,
+    Value * filterFun,
+    FileIngestionMethod method,
+    const std::optional<Hash> expectedHash,
+    Value & v,
+    const PathSet & context)
 {
-    const auto path = evalSettings.pureEval && expectedHash ?
-        path_ :
-        state.checkSourcePath(path_);
-    PathFilter filter = filterFun ? ([&](const Path & path) {
-        auto st = lstat(path);
+    try {
+        // FIXME: handle CA derivation outputs (where path needs to
+        // be rewritten to the actual output).
+        state.realiseContext(context);
 
-        /* Call the filter function.  The first argument is the path,
-           the second is a string indicating the type of the file. */
-        Value arg1;
-        mkString(arg1, path);
+        if (state.store->isInStore(path)) {
+            auto [storePath, subPath] = state.store->toStorePath(path);
+            auto info = state.store->queryPathInfo(storePath);
+            if (!info->references.empty())
+                throw EvalError("store path '%s' is not allowed to have references",
+                    state.store->printStorePath(storePath));
+            path = state.store->toRealPath(storePath) + subPath;
+        }
 
-        Value fun2;
-        state.callFunction(*filterFun, arg1, fun2, noPos);
+        path = evalSettings.pureEval && expectedHash
+            ? path
+            : state.checkSourcePath(path);
 
-        Value arg2;
-        mkString(arg2,
-            S_ISREG(st.st_mode) ? "regular" :
-            S_ISDIR(st.st_mode) ? "directory" :
-            S_ISLNK(st.st_mode) ? "symlink" :
-            "unknown" /* not supported, will fail! */);
+        PathFilter filter = filterFun ? ([&](const Path & path) {
+            auto st = lstat(path);
 
-        Value res;
-        state.callFunction(fun2, arg2, res, noPos);
+            /* Call the filter function.  The first argument is the path,
+               the second is a string indicating the type of the file. */
+            Value arg1;
+            mkString(arg1, path);
 
-        return state.forceBool(res, pos);
-    }) : defaultPathFilter;
+            Value fun2;
+            state.callFunction(*filterFun, arg1, fun2, noPos);
 
-    std::optional<StorePath> expectedStorePath;
-    if (expectedHash)
-        expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
-    Path dstPath;
-    if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-        dstPath = state.store->printStorePath(settings.readOnlyMode
-            ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
-            : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
-        if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
-            throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
-    } else
-        dstPath = state.store->printStorePath(*expectedStorePath);
+            Value arg2;
+            mkString(arg2,
+                S_ISREG(st.st_mode) ? "regular" :
+                S_ISDIR(st.st_mode) ? "directory" :
+                S_ISLNK(st.st_mode) ? "symlink" :
+                "unknown" /* not supported, will fail! */);
 
-    mkString(v, dstPath, {dstPath});
+            Value res;
+            state.callFunction(fun2, arg2, res, noPos);
+
+            return state.forceBool(res, pos);
+        }) : defaultPathFilter;
+
+        std::optional<StorePath> expectedStorePath;
+        if (expectedHash)
+            expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
+
+        Path dstPath;
+        if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+            dstPath = state.store->printStorePath(settings.readOnlyMode
+                ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
+            if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+                throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
+        } else
+            dstPath = state.store->printStorePath(*expectedStorePath);
+
+        mkString(v, dstPath, {dstPath});
+
+        state.allowPath(dstPath);
+
+    } catch (Error & e) {
+        e.addTrace(pos, "while adding path '%s'", path);
+        throw;
+    }
 }
 
 
@@ -1893,11 +1925,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
-    if (!context.empty())
-        throw EvalError({
-            .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-            .errPos = pos
-        });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type() != nFunction)
@@ -1908,13 +1935,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
             .errPos = pos
         });
 
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
     .name = "__filterSource",
     .args = {"e1", "e2"},
     .doc = R"(
+      > **Warning**
+      >
+      > `filterSource` should not be used to filter store paths. Since
+      > `filterSource` uses the name of the input directory while naming
+      > the output directory, doing so will produce a directory name in
+      > the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is
+      > the name of the input directory. Since `<hash>` depends on the
+      > unfiltered directory, the name of the output directory will
+      > indirectly depend on files that are filtered out by the
+      > function. This will trigger a rebuild even when a filtered out
+      > file is changed. Use `builtins.path` instead, which allows
+      > specifying the name of the output directory.
+
       This function allows you to copy sources into the Nix store while
       filtering certain files. For instance, suppose that you want to use
       the directory `source-dir` as an input to a Nix expression, e.g.
@@ -1961,18 +2001,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Value * filterFun = nullptr;
     auto method = FileIngestionMethod::Recursive;
     std::optional<Hash> expectedHash;
+    PathSet context;
 
     for (auto & attr : *args[0]->attrs) {
         const string & n(attr.name);
-        if (n == "path") {
-            PathSet context;
+        if (n == "path")
             path = state.coerceToPath(*attr.pos, *attr.value, context);
-            if (!context.empty())
-                throw EvalError({
-                    .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-                    .errPos = *attr.pos
-                });
-        } else if (attr.name == state.sName)
+        else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
             state.forceValue(*attr.value, pos);
@@ -1995,7 +2030,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     if (name.empty())
         name = baseNameOf(path);
 
-    addPath(state, pos, name, path, filterFun, method, expectedHash, v);
+    addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
 }
 
 static RegisterPrimOp primop_path({
@@ -2109,7 +2144,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
         pos
     );
     // !!! add to stack trace?
-    if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
+    if (state.countCalls && *i->pos != noPos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value, pos);
     v = *i->value;
 }
@@ -2360,7 +2395,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
             .errPos = pos
         });
 
-    if (!args[0]->lambda.fun->matchAttrs) {
+    if (!args[0]->lambda.fun->hasFormals()) {
         state.mkAttrs(v, 0);
         return;
     }
@@ -2369,7 +2404,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     for (auto & i : args[0]->lambda.fun->formals->formals) {
         // !!! should optimise booleans (allocate only once)
         Value * value = state.allocValue();
-        v.attrs->push_back(Attr(i.name, value, &i.pos));
+        v.attrs->push_back(Attr(i.name, value, ptr(&i.pos)));
         mkBool(*value, i.def);
     }
     v.attrs->sort();
@@ -2515,7 +2550,7 @@ static RegisterPrimOp primop_tail({
       the argument isn’t a list or is an empty list.
 
       > **Warning**
-      > 
+      >
       > This function should generally be avoided since it's inefficient:
       > unlike Haskell's `tail`, it takes O(n) time, so recursing over a
       > list by repeatedly calling `tail` takes O(n^2) time.
@@ -2897,7 +2932,7 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
             state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)));
         } catch (TypeError &e) {
             e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap"));
-            throw e;
+            throw;
         }
         len += lists[n].listSize();
     }
@@ -3109,7 +3144,7 @@ static RegisterPrimOp primop_toString({
 
         - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`.
 
-        - A set containing `{ __toString = self: ...; }`.
+        - A set containing `{ __toString = self: ...; }` or `{ outPath = ...; }`.
 
         - An integer.
 
@@ -3194,7 +3229,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
 
-    mkString(v, hashString(*ht, s).to_string(Base16, false), context);
+    mkString(v, hashString(*ht, s).to_string(Base16, false));
 }
 
 static RegisterPrimOp primop_hashString({
@@ -3601,15 +3636,13 @@ static RegisterPrimOp primop_splitVersion({
 RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
 
 
-RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
-    std::optional<std::string> requiredFeature)
+RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
 {
     if (!primOps) primOps = new PrimOps;
     primOps->push_back({
         .name = name,
         .args = {},
         .arity = arity,
-        .requiredFeature = std::move(requiredFeature),
         .fun = fun
     });
 }
@@ -3645,9 +3678,7 @@ void EvalState::createBaseEnv()
     if (!evalSettings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
-    }
 
-    if (!evalSettings.pureEval) {
         mkString(v, settings.thisSystem.get());
         addConstant("__currentSystem", v);
     }
@@ -3685,14 +3716,13 @@ void EvalState::createBaseEnv()
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
-            if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
-                addPrimOp({
-                    .fun = primOp.fun,
-                    .arity = std::max(primOp.args.size(), primOp.arity),
-                    .name = symbols.create(primOp.name),
-                    .args = std::move(primOp.args),
-                    .doc = primOp.doc,
-                });
+            addPrimOp({
+                .fun = primOp.fun,
+                .arity = std::max(primOp.args.size(), primOp.arity),
+                .name = symbols.create(primOp.name),
+                .args = std::move(primOp.args),
+                .doc = primOp.doc,
+            });
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */

src/libexpr/primops.hh

@@ -15,7 +15,6 @@ struct RegisterPrimOp
         std::vector<std::string> args;
         size_t arity = 0;
         const char * doc;
-        std::optional<std::string> requiredFeature;
         PrimOpFun fun;
     };
 
@@ -28,8 +27,7 @@ struct RegisterPrimOp
     RegisterPrimOp(
         std::string name,
         size_t arity,
-        PrimOpFun fun,
-        std::optional<std::string> requiredFeature = {});
+        PrimOpFun fun);
 
     RegisterPrimOp(Info && info);
 };

src/libexpr/primops/fetchMercurial.cc

@@ -62,6 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     fetchers::Attrs attrs;
     attrs.insert_or_assign("type", "hg");
     attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
+    attrs.insert_or_assign("name", name);
     if (ref) attrs.insert_or_assign("ref", *ref);
     if (rev) attrs.insert_or_assign("rev", rev->gitRev());
     auto input = fetchers::Input::fromAttrs(std::move(attrs));
@@ -83,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
         mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
     v.attrs->sort();
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+    state.allowPath(tree.storePath);
 }
 
 static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -7,6 +7,7 @@
 
 #include <ctime>
 #include <iomanip>
+#include <regex>
 
 namespace nix {
 
@@ -15,7 +16,8 @@ void emitTreeAttrs(
     const fetchers::Tree & tree,
     const fetchers::Input & input,
     Value & v,
-    bool emptyRevFallback)
+    bool emptyRevFallback,
+    bool forceDirty)
 {
     assert(input.isImmutable());
 
@@ -32,24 +34,28 @@ void emitTreeAttrs(
     mkString(*state.allocAttr(v, state.symbols.create("narHash")),
         narHash->to_string(SRI, true));
 
-    if (auto rev = input.getRev()) {
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
-    } else if (emptyRevFallback) {
-        // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
-        auto emptyHash = Hash(htSHA1);
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
-    }
-
     if (input.getType() == "git")
         mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
             fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
 
-    if (auto revCount = input.getRevCount())
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
-    else if (emptyRevFallback)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
+    if (!forceDirty) {
+
+        if (auto rev = input.getRev()) {
+            mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
+            mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
+        } else if (emptyRevFallback) {
+            // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
+            auto emptyHash = Hash(htSHA1);
+            mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
+            mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
+        }
+
+        if (auto revCount = input.getRevCount())
+            mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
+        else if (emptyRevFallback)
+            mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
+
+    }
 
     if (auto lastModified = input.getLastModified()) {
         mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
@@ -60,25 +66,33 @@ void emitTreeAttrs(
     v.attrs->sort();
 }
 
-std::string fixURI(std::string uri, EvalState &state)
+std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
 {
     state.checkURI(uri);
-    return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+    return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
 }
 
-void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
+std::string fixURIForGit(std::string uri, EvalState & state)
 {
-    string n(name);
-    attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
+    static std::regex scp_uri("([^/].*)@(.*):(.*)");
+    if (uri[0] != '/' && std::regex_match(uri, scp_uri))
+        return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
+    else
+        return fixURI(uri, state);
 }
 
+struct FetchTreeParams {
+    bool emptyRevFallback = false;
+    bool allowNameArgument = false;
+};
+
 static void fetchTree(
-    EvalState &state,
-    const Pos &pos,
-    Value **args,
-    Value &v,
-    const std::optional<std::string> type,
-    bool emptyRevFallback = false
+    EvalState & state,
+    const Pos & pos,
+    Value * * args,
+    Value & v,
+    std::optional<std::string> type,
+    const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
     PathSet context;
@@ -90,17 +104,33 @@ static void fetchTree(
 
         fetchers::Attrs attrs;
 
+        if (auto aType = args[0]->attrs->get(state.sType)) {
+            if (type)
+                throw Error({
+                    .msg = hintfmt("unexpected attribute 'type'"),
+                    .errPos = pos
+                });
+            type = state.forceStringNoCtx(*aType->value, *aType->pos);
+        } else if (!type)
+            throw Error({
+                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .errPos = pos
+            });
+
+        attrs.emplace("type", type.value());
+
         for (auto & attr : *args[0]->attrs) {
+            if (attr.name == state.sType) continue;
             state.forceValue(*attr.value);
-            if (attr.value->type() == nPath || attr.value->type() == nString)
-                addURI(
-                    state,
-                    attrs,
-                    attr.name,
-                    state.coerceToString(*attr.pos, *attr.value, context, false, false)
-                );
-            else if (attr.value->type() == nString)
-                addURI(state, attrs, attr.name, attr.value->string.s);
+            if (attr.value->type() == nPath || attr.value->type() == nString) {
+                auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false);
+                attrs.emplace(attr.name,
+                    attr.name == "url"
+                    ? type == "git"
+                      ? fixURIForGit(s, state)
+                      : fixURI(s, state)
+                    : s);
+            }
             else if (attr.value->type() == nBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
             else if (attr.value->type() == nInt)
@@ -110,26 +140,24 @@ static void fetchTree(
                     attr.name, showType(*attr.value));
         }
 
-        if (type)
-            attrs.emplace("type", type.value());
-
-        if (!attrs.count("type"))
-            throw Error({
-                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                .errPos = pos
-            });
+        if (!params.allowNameArgument)
+            if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
+                throw Error({
+                    .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+                    .errPos = pos
+                });
 
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
-        auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
+        auto url = state.coerceToString(pos, *args[0], context, false, false);
 
         if (type == "git") {
             fetchers::Attrs attrs;
             attrs.emplace("type", "git");
-            attrs.emplace("url", url);
+            attrs.emplace("url", fixURIForGit(url, state));
             input = fetchers::Input::fromAttrs(std::move(attrs));
         } else {
-            input = fetchers::Input::fromURL(url);
+            input = fetchers::Input::fromURL(fixURI(url, state));
         }
     }
 
@@ -141,16 +169,15 @@ static void fetchTree(
 
     auto [tree, input2] = input.fetch(state.store);
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+    state.allowPath(tree.storePath);
 
-    emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
+    emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
 }
 
 static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     settings.requireExperimentalFeature("flakes");
-    fetchTree(state, pos, args, v, std::nullopt);
+    fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
 }
 
 // FIXME: document
@@ -206,20 +233,18 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
         ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
-    auto path = state.store->toRealPath(storePath);
-
     if (expectedHash) {
         auto hash = unpack
             ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, path);
+            : hashFile(htSHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash)
             throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(path);
+    state.allowPath(storePath);
 
+    auto path = state.store->printStorePath(storePath);
     mkString(v, path, PathSet({path}));
 }
 
@@ -292,7 +317,7 @@ static RegisterPrimOp primop_fetchTarball({
 
 static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
 {
-    fetchTree(state, pos, args, v, "git", true);
+    fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
 }
 
 static RegisterPrimOp primop_fetchGit({

src/libexpr/tree-cache.cc

@@ -1,392 +0,0 @@
-#include "tree-cache.hh"
-#include "sqlite.hh"
-#include "store-api.hh"
-#include "context.hh"
-
-namespace nix::tree_cache {
-
-static const char * schema = R"sql(
-create table if not exists Attributes (
-    id          integer primary key autoincrement not null,
-    parent      integer not null,
-    name        text,
-    type        integer not null,
-    value       text,
-    context     text,
-    unique      (parent, name)
-);
-
-create index if not exists IndexByParent on Attributes(parent, name);
-)sql";
-
-struct AttrDb
-{
-    std::atomic_bool failed{false};
-
-    struct State
-    {
-        SQLite db;
-        SQLiteStmt insertAttribute;
-        SQLiteStmt updateAttribute;
-        SQLiteStmt insertAttributeWithContext;
-        SQLiteStmt queryAttribute;
-        SQLiteStmt queryAttributes;
-        std::unique_ptr<SQLiteTxn> txn;
-    };
-
-    std::unique_ptr<Sync<State>> _state;
-
-    AttrDb(const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
-    {
-        auto state(_state->lock());
-
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
-        createDirs(cacheDir);
-
-        Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
-
-        state->db = SQLite(dbPath);
-        state->db.isCache();
-        state->db.exec(schema);
-
-        state->insertAttribute.create(state->db,
-            "insert into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
-
-        state->updateAttribute.create(state->db,
-            "update Attributes set type = ?, value = ?, context = ? where id = ?");
-
-        state->insertAttributeWithContext.create(state->db,
-            "insert into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
-
-        state->queryAttribute.create(state->db,
-            "select id, type, value, context from Attributes where parent = ? and name = ?");
-
-        state->queryAttributes.create(state->db,
-            "select name, type from Attributes where parent = ?");
-
-        state->txn = std::make_unique<SQLiteTxn>(state->db);
-    }
-
-    ~AttrDb()
-    {
-        try {
-            auto state(_state->lock());
-            if (!failed)
-                state->txn->commit();
-            state->txn.reset();
-        } catch (...) {
-            ignoreException();
-        }
-    }
-
-    template<typename F>
-    AttrId doSQLite(F && fun)
-    {
-        if (failed) return 0;
-        try {
-            return fun();
-        } catch (SQLiteError &) {
-            ignoreException();
-            failed = true;
-            return 0;
-        }
-    }
-
-    /**
-     * Store a leaf of the tree in the db
-     */
-    AttrId addEntry(
-        const AttrKey & key,
-        const AttrValue & value)
-    {
-        return doSQLite([&]()
-        {
-            auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->insertAttributeWithContext.use()
-                (key.first)
-                (key.second)
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                .exec();
-            AttrId rowId = state->db.getLastInsertedRowId();
-            assert(rowId);
-            return rowId;
-        });
-    }
-
-    std::optional<AttrId> getId(const AttrKey& key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return std::nullopt;
-
-        return (AttrType) queryAttribute.getInt(0);
-    }
-
-    AttrId setOrUpdate(const AttrKey& key, const AttrValue& value)
-    {
-        debug("cache: miss for the attribute %s", key.second);
-        if (auto existingId = getId(key)) {
-            setValue(*existingId, value);
-            return *existingId;
-        }
-        return addEntry(key, value);
-    }
-
-    void setValue(const AttrId & id, const AttrValue & value)
-    {
-        auto state(_state->lock());
-            auto rawValue = RawValue::fromVariant(value);
-
-            state->updateAttribute.use()
-                (rawValue.type)
-                (rawValue.value.value_or(""), rawValue.value.has_value())
-                (rawValue.serializeContext())
-                (id)
-                .exec();
-    }
-
-    std::optional<std::pair<AttrId, AttrValue>> getValue(AttrKey key)
-    {
-        auto state(_state->lock());
-
-        auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
-        if (!queryAttribute.next()) return {};
-
-        auto rowId = (AttrType) queryAttribute.getInt(0);
-        auto type = (AttrType) queryAttribute.getInt(1);
-
-        switch (type) {
-            case AttrType::Attrs: {
-                return {{rowId, attributeSet_t()}};
-            }
-            case AttrType::String: {
-                std::vector<std::pair<Path, std::string>> context;
-                if (!queryAttribute.isNull(3))
-                    for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(s));
-                return {{rowId, string_t{queryAttribute.getStr(2), context}}};
-            }
-            case AttrType::Bool:
-                return {{rowId, wrapped_basetype<bool>{queryAttribute.getInt(2) != 0}}};
-            case AttrType::Int:
-                return {{rowId, wrapped_basetype<int64_t>{queryAttribute.getInt(2)}}};
-            case AttrType::Double:
-                return {{rowId, wrapped_basetype<double>{(double)queryAttribute.getInt(2)}}};
-            case AttrType::Unknown:
-                return {{rowId, unknown_t{}}};
-            case AttrType::Thunk:
-                return {{rowId, thunk_t{}}};
-            case AttrType::Missing:
-                return {{rowId, missing_t{key.second}}};
-            case AttrType::Failed:
-                return {{rowId, failed_t{queryAttribute.getStr(2)}}};
-            default:
-                throw Error("unexpected type in evaluation cache");
-        }
-    }
-
-    std::vector<std::string> getChildren(AttrId parentId)
-    {
-        std::vector<std::string> res;
-        auto state(_state->lock());
-
-        auto queryAttributes(state->queryAttributes.use()(parentId));
-
-        while (queryAttributes.next()) {
-            if (queryAttributes.getInt(1) != AttrType::Missing)
-            res.push_back(queryAttributes.getStr(0));
-        }
-
-        return res;
-    }
-};
-
-Cache::Cache(const Hash & useCache,
-        SymbolTable & symbols)
-    : db(std::make_shared<AttrDb>(useCache))
-    , symbols(symbols)
-    , rootSymbol(symbols.create("%root%"))
-{
-}
-
-std::shared_ptr<Cache> Cache::tryCreate(const Hash & useCache, SymbolTable & symbols)
-{
-    try {
-        return std::make_shared<Cache>(useCache, symbols);
-    } catch (SQLiteError &) {
-        ignoreException();
-        return nullptr;
-    }
-}
-
-void Cache::commit()
-{
-    if (db) {
-        debug("Saving the cache");
-        auto state(db->_state->lock());
-        if (state->txn->active) {
-            state->txn->commit();
-            state->txn.reset();
-            state->txn = std::make_unique<SQLiteTxn>(state->db);
-        }
-    }
-}
-
-Cursor::Ref Cache::getRoot()
-{
-    return new Cursor(ref(shared_from_this()), std::nullopt, thunk_t{});
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrValue& value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({root->db->setOrUpdate(getKey(), value), value})
-{
-}
-
-Cursor::Cursor(
-    ref<Cache> root,
-    const Parent & parent,
-    const AttrId & id,
-    const AttrValue & value
-    )
-    : root(root)
-    , parentId(parent ? std::optional{parent->first.cachedValue.first} : std::nullopt)
-    , label(parent ? parent->second : root->rootSymbol)
-    , cachedValue({id, value})
-{
-}
-
-
-AttrKey Cursor::getKey()
-{
-    if (!parentId)
-        return {0, root->rootSymbol};
-    return {*parentId, label};
-}
-
-AttrValue Cursor::getCachedValue()
-{
-    return cachedValue.second;
-}
-
-void Cursor::setValue(const AttrValue & v)
-{
-    root->db->setValue(cachedValue.first, v);
-    cachedValue.second = v;
-}
-
-Cursor::Ref Cursor::addChild(const Symbol & attrPath, const AttrValue & v)
-{
-    Parent parent = {{*this, attrPath}};
-    auto childCursor = new Cursor(
-        root,
-        parent,
-        v
-    );
-    return childCursor;
-}
-
-std::vector<std::string> Cursor::getChildren()
-{
-    return root->db->getChildren(cachedValue.first);
-}
-
-std::optional<std::vector<std::string>> Cursor::getChildrenAtPath(const std::vector<Symbol> & attrPath)
-{
-    auto cursorAtPath = findAlongAttrPath(attrPath);
-    if (cursorAtPath)
-        return cursorAtPath->getChildren();
-    return std::nullopt;
-}
-
-Cursor::Ref Cursor::maybeGetAttr(const Symbol & name)
-{
-    auto rawAttr = root->db->getValue({cachedValue.first, name});
-    if (rawAttr) {
-        Parent parent = {{*this, name}};
-        debug("cache: hit for the attribute %s", cachedValue.first);
-        return new Cursor (
-            root, parent, rawAttr->first,
-            rawAttr->second);
-    }
-    if (std::holds_alternative<attributeSet_t>(cachedValue.second)) {
-        // If the parent is an attribute set but we're not present in the db,
-        // then we're not a member of this attribute set. So mark as missing
-        return addChild(name, missing_t{name});
-    }
-    return nullptr;
-}
-
-Cursor::Ref Cursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
-{
-    auto currentCursor = this;
-    for (auto & currentAccessor : attrPath) {
-        currentCursor = currentCursor->maybeGetAttr(currentAccessor);
-        if (!currentCursor)
-            break;
-        if (std::holds_alternative<missing_t>(currentCursor->cachedValue.second))
-            break;
-        if (std::holds_alternative<failed_t>(currentCursor->cachedValue.second))
-            break;
-    }
-    return currentCursor;
-}
-
-const RawValue RawValue::fromVariant(const AttrValue & value)
-{
-    RawValue res;
-    std::visit(overloaded{
-      [&](attributeSet_t x) { res.type = AttrType::Attrs; },
-      [&](string_t x) {
-        res.type = AttrType::String;
-        res.value = x.first;
-        res.context = x.second;
-      },
-      [&](wrapped_basetype<bool> x) {
-        res.type = AttrType::Bool;
-        res.value = x.value ? "1" : "0";
-      },
-      [&](wrapped_basetype<int64_t> x) {
-        res.type = AttrType::Int;
-        res.value = std::to_string(x.value);
-      },
-      [&](wrapped_basetype<double> x) {
-        res.type = AttrType::Double;
-        res.value = std::to_string(x.value);
-      },
-      [&](unknown_t x) { res.type = AttrType::Unknown; },
-      [&](missing_t x) { res.type = AttrType::Missing; },
-      [&](thunk_t x) { res.type = AttrType::Thunk; },
-      [&](failed_t x) {
-          res.type = AttrType::Failed;
-          res.value = x.error;
-      }
-    }, value);
-    return res;
-}
-
-std::string RawValue::serializeContext() const
-{
-    std::string res;
-    for (auto & elt : context) {
-        res.append(encodeContext(elt.second, elt.first));
-        res.push_back(' ');
-    }
-    if (!res.empty())
-        res.pop_back(); // Remove the trailing space
-    return res;
-}
-
-}

src/libexpr/tree-cache.hh

@@ -1,156 +0,0 @@
-/**
- * caching for a tree-like data structure (like Nix values)
- *
- * The cache is an sqlite db whose rows are the nodes of the tree, with a
- * pointer to their parent (except for the root of course)
- */
-
-#pragma once
-
-#include "sync.hh"
-#include "hash.hh"
-#include "symbol-table.hh"
-
-#include <functional>
-#include <variant>
-
-namespace nix::tree_cache {
-
-struct AttrDb;
-class Cursor;
-
-class Cache : public std::enable_shared_from_this<Cache>
-{
-private:
-    friend class Cursor;
-
-    /**
-     * The database holding the cache
-     */
-    std::shared_ptr<AttrDb> db;
-
-    SymbolTable & symbols;
-
-    /**
-     * Distinguished symbol indicating the root of the tree
-     */
-    const Symbol rootSymbol;
-
-public:
-
-    Cache(
-        const Hash & useCache,
-        SymbolTable & symbols
-    );
-
-    static std::shared_ptr<Cache> tryCreate(const Hash & useCache, SymbolTable & symbols);
-
-    Cursor * getRoot();
-
-    /**
-     * Flush the cache to disk
-     */
-    void commit();
-};
-
-enum AttrType {
-    Unknown = 0,
-    Attrs = 1,
-    String = 2,
-    Bool = 3,
-    Int = 4,
-    Double = 5,
-    Thunk = 6,
-    Missing = 7, // Missing fields of attribute sets
-    Failed = 8,
-};
-
-struct attributeSet_t {};
-struct unknown_t {};
-struct thunk_t {};
-struct failed_t { string error; };
-struct missing_t { Symbol attrName; };
-
-// Putting several different primitive types in an `std::variant` partially
-// breaks the `std::visit(overloaded{...` hackery because of the implicit cast
-// from one to another which breaks the exhaustiveness check.
-// So we wrap them in a trivial class just to force the disambiguation
-template<typename T>
-struct wrapped_basetype{ T value; };
-
-typedef uint64_t AttrId;
-
-typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
-
-typedef std::variant<
-    attributeSet_t,
-    string_t,
-    unknown_t,
-    thunk_t,
-    missing_t,
-    failed_t,
-    wrapped_basetype<bool>,
-    wrapped_basetype<int64_t>,
-    wrapped_basetype<double>
-> AttrValue;
-
-struct RawValue {
-    AttrType type;
-    std::optional<std::string> value;
-    std::vector<std::pair<Path, std::string>> context;
-
-    std::string serializeContext() const;
-
-    static const RawValue fromVariant(const AttrValue&);
-    AttrValue toVariant() const;
-};
-
-/**
- * View inside the cache.
- *
- * A `Cursor` represents a node in the cached tree (be it a leaf or not)
- */
-class Cursor : public std::enable_shared_from_this<Cursor>
-{
-    /**
-     * The overall cache of which this cursor is a view
-     */
-    ref<Cache> root;
-
-    typedef std::optional<std::pair<Cursor&, Symbol>> Parent;
-
-    std::optional<AttrId> parentId;
-    Symbol label;
-
-    std::pair<AttrId, AttrValue> cachedValue;
-
-    /**
-     * Get the identifier for this node in the database
-     */
-    AttrKey getKey();
-
-public:
-
-    using Ref = Cursor*;
-
-    // Create a new cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrValue&);
-    // Build a cursor from an existing cache entry
-    Cursor(ref<Cache> root, const Parent & parent, const AttrId& id, const AttrValue&);
-
-    AttrValue getCachedValue();
-
-    void setValue(const AttrValue & v);
-
-    Ref addChild(const Symbol & attrPath, const AttrValue & v);
-
-    Ref findAlongAttrPath(const std::vector<Symbol> & attrPath);
-    Ref maybeGetAttr(const Symbol & attrPath);
-
-
-    std::vector<std::string> getChildren();
-    std::optional<std::vector<std::string>> getChildrenAtPath(const std::vector<Symbol> & attrPath);
-};
-
-}

src/libexpr/value-cache.hh

@@ -1,51 +0,0 @@
-#pragma once
-#include "tree-cache.hh"
-
-namespace nix {
-struct Value;
-class EvalState;
-class Bindings;
-
-class ValueCache {
-    tree_cache::Cursor::Ref rawCache;
-
-public:
-
-    ValueCache(tree_cache::Cursor::Ref rawCache) : rawCache(rawCache) {}
-
-    static ValueCache empty;
-
-    bool isEmpty () { return rawCache == nullptr; }
-
-    enum ReturnCode {
-        // The cache result was an attribute set, so we forward it later in the
-        // chain
-        Forward,
-        CacheMiss,
-        CacheHit,
-        UnCacheable,
-        NoCacheKey,
-    };
-
-    struct CacheResult {
-        ReturnCode returnCode;
-
-        // In case the query returns a `missing_t`, the symbol that's missing
-        std::optional<Symbol> lastQueriedSymbolIfMissing;
-    };
-    std::pair<CacheResult, ValueCache> getValue(EvalState & state, const std::vector<Symbol> & selector, Value & dest);
-
-    ValueCache addChild(const Symbol & attrName, Value & value);
-    ValueCache addFailedChild(const Symbol & attrName, const Error & error);
-    ValueCache addNumChild(SymbolTable & symbols, int idx, const Value & value);
-    void addAttrSetChilds(Bindings & children);
-    void addListChilds(SymbolTable & symbols, Value** elems, int listSize);
-
-    std::optional<std::vector<Symbol>> listChildren(SymbolTable&);
-    std::optional<std::vector<Symbol>> listChildrenAtPath(SymbolTable&, const std::vector<Symbol> & attrPath);
-
-    std::optional<tree_cache::AttrValue> getRawValue();
-
-    ValueCache() : rawCache(nullptr) {}
-};
-}

src/libexpr/value-to-xml.cc

@@ -42,7 +42,7 @@ static void showAttrs(EvalState & state, bool strict, bool location,
 
         XMLAttrs xmlAttrs;
         xmlAttrs["name"] = i;
-        if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
+        if (location && a.pos != ptr(&noPos)) posToXML(xmlAttrs, *a.pos);
 
         XMLOpenElement _(doc, "attr", xmlAttrs);
         printValueAsXML(state, strict, location,
@@ -135,7 +135,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
             if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
             XMLOpenElement _(doc, "function", xmlAttrs);
 
-            if (v.lambda.fun->matchAttrs) {
+            if (v.lambda.fun->hasFormals()) {
                 XMLAttrs attrs;
                 if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
                 if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";

src/libexpr/value.hh

@@ -23,7 +23,6 @@ typedef enum {
     tApp,
     tLambda,
     tBlackhole,
-    tCachedThunk,
     tPrimOp,
     tPrimOpApp,
     tExternal,
@@ -57,7 +56,6 @@ struct Pos;
 class EvalState;
 class XMLWriter;
 class JSONPlaceholder;
-class ValueCache;
 
 
 typedef int64_t NixInt;
@@ -105,10 +103,6 @@ class ExternalValueBase
 
 std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
 
-struct Thunk {
-    Env * env;
-    Expr * expr;
-};
 
 struct Value
 {
@@ -128,7 +122,6 @@ public:
     inline bool isThunk() const { return internalType == tThunk; };
     inline bool isApp() const { return internalType == tApp; };
     inline bool isBlackhole() const { return internalType == tBlackhole; };
-    inline bool isCachedThunk() const { return internalType == tCachedThunk; };
 
     // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
@@ -172,13 +165,10 @@ public:
             Value * * elems;
         } bigList;
         Value * smallList[2];
-        Thunk thunk;
         struct {
-            Thunk * thunk;
-            // This is a pointer only to prevent a recursive import as
-            // `EvalCache` is already a pointer so would fit very nicely here.
-            ValueCache * cache;
-        } cachedThunk;
+            Env * env;
+            Expr * expr;
+        } thunk;
         struct {
             Value * left, * right;
         } app;
@@ -209,7 +199,7 @@ public:
             case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
             case tExternal: return nExternal;
             case tFloat: return nFloat;
-            case tThunk: case tApp: case tBlackhole: case tCachedThunk: return nThunk;
+            case tThunk: case tApp: case tBlackhole: return nThunk;
         }
         abort();
     }
@@ -282,13 +272,6 @@ public:
         thunk.expr = ex;
     }
 
-    inline void mkCachedThunk(Thunk * t, ValueCache * cache)
-    {
-        internalType = tCachedThunk;
-        cachedThunk.thunk = t;
-        cachedThunk.cache = cache;
-    }
-
     inline void mkApp(Value * l, Value * r)
     {
         internalType = tApp;
@@ -366,9 +349,6 @@ public:
     bool isTrivial() const;
 
     std::vector<std::pair<Path, std::string>> getContext();
-
-    ValueCache & getEvalCache();
-    void setEvalCache(ValueCache &);
 };
 
 

src/libfetchers/fetchers.cc

@@ -200,12 +200,17 @@ void Input::markChangedFile(
     return scheme->markChangedFile(*this, file, commitMsg);
 }
 
+std::string Input::getName() const
+{
+    return maybeGetStrAttr(attrs, "name").value_or("source");
+}
+
 StorePath Input::computeStorePath(Store & store) const
 {
     auto narHash = getNarHash();
     if (!narHash)
         throw Error("cannot compute store path for mutable input '%s'", to_string());
-    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+    return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName());
 }
 
 std::string Input::getType() const

src/libfetchers/fetchers.hh

@@ -38,6 +38,9 @@ struct Input
     bool immutable = false;
     bool direct = true;
 
+    /* path of the parent of this input, used for relative path resolution */
+    std::optional<Path> parent;
+
 public:
     static Input fromURL(const std::string & url);
 
@@ -81,6 +84,8 @@ public:
         std::string_view file,
         std::optional<std::string> commitMsg) const;
 
+    std::string getName() const;
+
     StorePath computeStorePath(Store & store) const;
 
     // Convenience functions for common attributes.

src/libfetchers/git.cc

@@ -4,6 +4,7 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "pathlocks.hh"
 
 #include <sys/time.h>
 #include <sys/wait.h>
@@ -12,6 +13,12 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
+// Explicit initial branch of our bare repo to suppress warnings from new version of git.
+// The value itself does not matter, since we always fetch a specific revision or branch.
+// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
+// old version of git, which will ignore unrecognized `-c` options.
+const std::string gitInitialBranch = "__nix_dummy_branch";
+
 static std::string readHead(const Path & path)
 {
     return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
@@ -60,7 +67,7 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -167,10 +174,10 @@ struct GitInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        std::string name = input.getName();
+
         bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
         bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
         bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
@@ -270,7 +277,7 @@ struct GitInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 // FIXME: maybe we should use the timestamp of the last
                 // modified dirty file?
@@ -317,11 +324,17 @@ struct GitInputScheme : InputScheme
             Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
             repoDir = cacheDir;
 
+            Path cacheDirLock = cacheDir + ".lock";
+            createDirs(dirOf(cacheDir));
+            AutoCloseFD lock = openLockFile(cacheDirLock, true);
+            lockFile(lock.get(), ltWrite, true);
+
             if (!pathExists(cacheDir)) {
-                createDirs(dirOf(cacheDir));
-                runProgram("git", true, { "init", "--bare", repoDir });
+                runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
             }
 
+            deleteLockFile(cacheDirLock, lock.get());
+
             Path localRefFile =
                 input.getRef()->compare(0, 5, "refs/") == 0
                 ? cacheDir + "/" + *input.getRef()
@@ -406,17 +419,14 @@ struct GitInputScheme : InputScheme
         AutoDelete delTmpDir(tmpDir, true);
         PathFilter filter = defaultPathFilter;
 
-        RunOptions checkCommitOpts(
-            "git",
-            { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
-        );
-        checkCommitOpts.searchPath = true;
-        checkCommitOpts.mergeStderrToStdout = true;
-
-        auto result = runProgram(checkCommitOpts);
+        auto result = runProgram(RunOptions {
+            .program = "git",
+            .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
+            .mergeStderrToStdout = true
+        });
         if (WEXITSTATUS(result.first) == 128
-            && result.second.find("bad file") != std::string::npos
-        ) {
+            && result.second.find("bad file") != std::string::npos)
+        {
             throw Error(
                 "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
                     "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
@@ -432,7 +442,7 @@ struct GitInputScheme : InputScheme
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);
 
-            runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+            runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
             // TODO: repoDir might lack the ref (it only checks if rev
             // exists, see FIXME above) so use a big hammer and fetch
             // everything to ensure we get the rev.
@@ -448,9 +458,11 @@ struct GitInputScheme : InputScheme
             // FIXME: should pipe this, or find some better way to extract a
             // revision.
             auto source = sinkToSource([&](Sink & sink) {
-                RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
-                gitOptions.standardOut = &sink;
-                runProgram2(gitOptions);
+                runProgram2({
+                    .program = "git",
+                    .args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
+                    .standardOut = &sink
+                });
             });
 
             unpackTarfile(*source, tmpDir);

src/libfetchers/github.cc

@@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+        auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
 
         input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
 
@@ -273,9 +273,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
     void clone(const Input & input, const Path & destDir) override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+        Input::fromURL(fmt("git+https://%s/%s/%s.git",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
-            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
     }
 };
@@ -341,9 +341,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
-        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+        Input::fromURL(fmt("git+https://%s/%s/%s.git",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
-            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
     }
 };

src/libfetchers/local.mk

@@ -8,4 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
+libfetchers_LDFLAGS += -pthread
+
 libfetchers_LIBS = libutil libstore

src/libfetchers/mercurial.cc

@@ -11,34 +11,32 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
-namespace {
+static RunOptions hgOptions(const Strings & args)
+{
+    auto env = getEnv();
+    // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
+    env["HGPLAIN"] = "";
 
-RunOptions hgOptions(const Strings & args) {
-	RunOptions opts("hg", args);
-	opts.searchPath = true;
-
-	auto env = getEnv();
-	// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
-	env["HGPLAIN"] = "";
-	opts.environment = env;
-
-	return opts;
+    return {
+        .program = "hg",
+        .searchPath = true,
+        .args = args,
+        .environment = env
+    };
 }
 
 // runProgram wrapper that uses hgOptions instead of stock RunOptions.
-string runHg(const Strings & args, const std::optional<std::string> & input = {})
+static string runHg(const Strings & args, const std::optional<std::string> & input = {})
 {
-	RunOptions opts = hgOptions(args);
-	opts.input = input;
+    RunOptions opts = hgOptions(args);
+    opts.input = input;
 
-	auto res = runProgram(opts);
+    auto res = runProgram(std::move(opts));
 
-	if (!statusOk(res.first))
-		throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
-
-	return res.second;
-}
+    if (!statusOk(res.first))
+        throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
 
+    return res.second;
 }
 
 struct MercurialInputScheme : InputScheme
@@ -74,7 +72,7 @@ struct MercurialInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "hg") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name")
                 throw Error("unsupported Mercurial input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -147,10 +145,10 @@ struct MercurialInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto name = "source";
-
         Input input(_input);
 
+        auto name = input.getName();
+
         auto [isLocal, actualUrl_] = getActualUrl(input);
         auto actualUrl = actualUrl_; // work around clang bug
 
@@ -193,7 +191,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
+                auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
 
                 return {
                     Tree(store->toRealPath(storePath), std::move(storePath)),
@@ -253,9 +251,7 @@ struct MercurialInputScheme : InputScheme
            have to pull again. */
         if (!(input.getRev()
                 && pathExists(cacheDir)
-                && runProgram(
-                    hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
-                    .killStderr(true)).second == "1"))
+                && runProgram(hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })).second == "1"))
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
 

src/libfetchers/path.cc

@@ -82,18 +82,38 @@ struct PathInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
+        std::string absPath;
         auto path = getStrAttr(input.attrs, "path");
 
-        // FIXME: check whether access to 'path' is allowed.
+        if (path[0] != '/') {
+            if (!input.parent)
+                throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
 
-        auto storePath = store->maybeParseStorePath(path);
+            auto parent = canonPath(*input.parent);
+
+            // the path isn't relative, prefix it
+            absPath = nix::absPath(path, parent);
+
+            // for security, ensure that if the parent is a store path, it's inside it
+            if (store->isInStore(parent)) {
+                auto storePath = store->printStorePath(store->toStorePath(parent).first);
+                if (!isInDir(absPath, storePath))
+                    throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
+            }
+        } else
+            absPath = path;
+
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath));
+
+        // FIXME: check whether access to 'path' is allowed.
+        auto storePath = store->maybeParseStorePath(absPath);
 
         if (storePath)
             store->addTempRoot(*storePath);
 
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
             // FIXME: try to substitute storePath.
-            storePath = store->addToStore("source", path);
+            storePath = store->addToStore("source", absPath);
 
         return {
             Tree(store->toRealPath(*storePath), std::move(*storePath)),

src/libfetchers/registry.cc

@@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry()
     return userRegistry;
 }
 
+std::shared_ptr<Registry> getCustomRegistry(const Path & p)
+{
+    static auto customRegistry =
+        Registry::read(p, Registry::Custom);
+    return customRegistry;
+}
+
 static std::shared_ptr<Registry> flagRegistry =
     std::make_shared<Registry>(Registry::Flag);
 

src/libfetchers/registry.hh

@@ -14,6 +14,7 @@ struct Registry
         User = 1,
         System = 2,
         Global = 3,
+        Custom = 4,
     };
 
     RegistryType type;
@@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries;
 
 std::shared_ptr<Registry> getUserRegistry();
 
+std::shared_ptr<Registry> getCustomRegistry(const Path & p);
+
 Path getUserRegistryPath();
 
 Registries getRegistries(ref<Store> store);

src/libfetchers/tarball.cc

@@ -196,7 +196,7 @@ struct TarballInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
+            if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name")
                 throw Error("unsupported tarball input attribute '%s'", name);
 
         Input input;
@@ -226,7 +226,7 @@ struct TarballInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
         return {std::move(tree), input};
     }
 };

src/libmain/local.mk

@@ -8,10 +8,10 @@ libmain_SOURCES := $(wildcard $(d)/*.cc)
 
 libmain_CXXFLAGS += -I src/libutil -I src/libstore
 
-libmain_LDFLAGS = $(OPENSSL_LIBS)
+libmain_LDFLAGS += $(OPENSSL_LIBS)
 
 libmain_LIBS = libstore libutil
 
 libmain_ALLOW_UNDEFINED = 1
 
-$(eval $(call install-file-in, $(d)/nix-main.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))

src/libmain/progress-bar.cc

@@ -484,7 +484,7 @@ Logger * makeProgressBar(bool printBuildLogs)
 {
     return new ProgressBar(
         printBuildLogs,
-        isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"
+        shouldANSI()
     );
 }
 

src/libmain/shared.cc

@@ -15,6 +15,9 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <signal.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
 
 #include <openssl/crypto.h>
 
@@ -110,6 +113,31 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
 }
 #endif
 
+static std::once_flag dns_resolve_flag;
+
+static void preloadNSS() {
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    std::call_once(dns_resolve_flag, []() {
+        struct addrinfo *res = NULL;
+
+        /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN.
+           We need the resolution to be done locally, as nscd socket will not be accessible in the
+           sandbox. */
+        char * previous_env = getenv("LOCALDOMAIN");
+        setenv("LOCALDOMAIN", "invalid", 1);
+        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) {
+            if (res) freeaddrinfo(res);
+        }
+        if (previous_env) {
+             setenv("LOCALDOMAIN", previous_env, 1);
+        } else {
+             unsetenv("LOCALDOMAIN");
+        }
+    });
+}
 
 static void sigHandler(int signo) { }
 
@@ -176,6 +204,8 @@ void initNix()
     if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
+
+    preloadNSS();
 }
 
 
@@ -238,7 +268,7 @@ LegacyArgs::LegacyArgs(const std::string & programName,
     addFlag({
         .longName = "no-gc-warning",
         .description = "Disable warnings about not using `--add-root`.",
-        .handler = {&gcWarning, true},
+        .handler = {&gcWarning, false},
     });
 
     addFlag({

src/libstore/binary-cache-store.cc

@@ -52,9 +52,9 @@ void BinaryCacheStore::init()
                     throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
                         getUri(), value, storeDir);
             } else if (name == "WantMassQuery") {
-                wantMassQuery.setDefault(value == "1" ? "true" : "false");
+                wantMassQuery.setDefault(value == "1");
             } else if (name == "Priority") {
-                priority.setDefault(fmt("%d", std::stoi(value)));
+                priority.setDefault(std::stoi(value));
             }
         }
     }
@@ -130,17 +130,6 @@ AutoCloseFD openFile(const Path & path)
     return fd;
 }
 
-struct FileSource : FdSource
-{
-    AutoCloseFD fd2;
-
-    FileSource(const Path & path)
-        : fd2(openFile(path))
-    {
-        fd = fd2.get();
-    }
-};
-
 ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
     std::function<ValidPathInfo(HashResult)> mkInfo)

src/libstore/build/derivation-goal.cc

@@ -143,7 +143,6 @@ void DerivationGoal::work()
     (this->*state)();
 }
 
-
 void DerivationGoal::addWantedOutputs(const StringSet & outputs)
 {
     /* If we already want all outputs, there is nothing to do. */
@@ -166,7 +165,7 @@ void DerivationGoal::getDerivation()
     /* The first thing to do is to make sure that the derivation
        exists.  If it doesn't, it may be created through a
        substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
+    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
         loadDerivation();
         return;
     }
@@ -189,12 +188,12 @@ void DerivationGoal::loadDerivation()
     /* `drvPath' should already be a root, but let's be on the safe
        side: if the user forgot to make it a root, we wouldn't want
        things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
+    worker.evalStore.addTempRoot(drvPath);
 
-    assert(worker.store.isValidPath(drvPath));
+    assert(worker.evalStore.isValidPath(drvPath));
 
     /* Get the derivation. */
-    drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
+    drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
 
     haveDerivation();
 }
@@ -213,8 +212,8 @@ void DerivationGoal::haveDerivation()
         if (i.second.second)
             worker.store.addTempRoot(*i.second.second);
 
-    auto outputHashes = staticOutputHashes(worker.store, *drv);
-    for (auto &[outputName, outputHash] : outputHashes)
+    auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
+    for (auto & [outputName, outputHash] : outputHashes)
       initialOutputs.insert({
             outputName,
             InitialOutput{
@@ -338,6 +337,15 @@ void DerivationGoal::gaveUpOnSubstitution()
         for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
             addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
 
+    /* Copy the input sources from the eval store to the build
+       store. */
+    if (&worker.evalStore != &worker.store) {
+        RealisedPath::Set inputSrcs;
+        for (auto & i : drv->inputSrcs)
+            inputSrcs.insert(i);
+        copyClosure(worker.evalStore, worker.store, inputSrcs);
+    }
+
     for (auto & i : drv->inputSrcs) {
         if (worker.store.isValidPath(i)) continue;
         if (!settings.useSubstitutes)
@@ -479,8 +487,8 @@ void DerivationGoal::inputsRealised()
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
+            assert(worker.evalStore.isValidPath(drvPath));
+            auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
             for (auto & j : wantedDepOutputs) {
                 if (outputs.count(j) > 0) {
                     auto optRealizedInput = outputs.at(j);
@@ -545,7 +553,7 @@ void DerivationGoal::tryToBuild()
     PathSet lockFiles;
     /* FIXME: Should lock something like the drv itself so we don't build same
        CA drv concurrently */
-    if (dynamic_cast<LocalStore *>(&worker.store))
+    if (dynamic_cast<LocalStore *>(&worker.store)) {
         /* If we aren't a local store, we might need to use the local store as
            a build remote, but that would cause a deadlock. */
         /* FIXME: Make it so we can use ourselves as a build remote even if we
@@ -553,9 +561,15 @@ void DerivationGoal::tryToBuild()
         /* FIXME: find some way to lock for scheduling for the other stores so
            a forking daemon with --store still won't farm out redundant builds.
            */
-        for (auto & i : drv->outputsAndOptPaths(worker.store))
+        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
             if (i.second.second)
                 lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
+            else
+                lockFiles.insert(
+                    worker.store.Store::toRealPath(drvPath) + "." + i.first
+                );
+        }
+    }
 
     if (!outputLocks.lockPaths(lockFiles, "", false)) {
         if (!actLock)
@@ -602,7 +616,9 @@ void DerivationGoal::tryToBuild()
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
+    bool buildLocally =
+        (buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
+        && settings.maxBuildJobs.get() != 0;
 
     if (!buildLocally) {
         switch (tryBuildHook()) {
@@ -739,6 +755,64 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 {
 }
 
+void runPostBuildHook(
+    Store & store,
+    Logger & logger,
+    const StorePath & drvPath,
+    StorePathSet outputPaths
+)
+{
+    auto hook = settings.postBuildHook;
+    if (hook == "")
+        return;
+
+    Activity act(logger, lvlInfo, actPostBuildHook,
+            fmt("running post-build-hook '%s'", settings.postBuildHook),
+            Logger::Fields{store.printStorePath(drvPath)});
+    PushActivity pact(act.id);
+    std::map<std::string, std::string> hookEnvironment = getEnv();
+
+    hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath));
+    hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
+    hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue());
+
+    struct LogSink : Sink {
+        Activity & act;
+        std::string currentLine;
+
+        LogSink(Activity & act) : act(act) { }
+
+        void operator() (std::string_view data) override {
+            for (auto c : data) {
+                if (c == '\n') {
+                    flushLine();
+                } else {
+                    currentLine += c;
+                }
+            }
+        }
+
+        void flushLine() {
+            act.result(resPostBuildLogLine, currentLine);
+            currentLine.clear();
+        }
+
+        ~LogSink() {
+            if (currentLine != "") {
+                currentLine += '\n';
+                flushLine();
+            }
+        }
+    };
+    LogSink sink(act);
+
+    runProgram2({
+        .program = settings.postBuildHook,
+        .environment = hookEnvironment,
+        .standardOut = &sink,
+        .mergeStderrToStdout = true,
+    });
+}
 
 void DerivationGoal::buildDone()
 {
@@ -804,57 +878,15 @@ void DerivationGoal::buildDone()
            being valid. */
         registerOutputs();
 
-        if (settings.postBuildHook != "") {
-            Activity act(*logger, lvlInfo, actPostBuildHook,
-                fmt("running post-build-hook '%s'", settings.postBuildHook),
-                Logger::Fields{worker.store.printStorePath(drvPath)});
-            PushActivity pact(act.id);
-            StorePathSet outputPaths;
-            for (auto i : drv->outputs) {
-                outputPaths.insert(finalOutputs.at(i.first));
-            }
-            std::map<std::string, std::string> hookEnvironment = getEnv();
-
-            hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
-            hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths))));
-
-            RunOptions opts(settings.postBuildHook, {});
-            opts.environment = hookEnvironment;
-
-            struct LogSink : Sink {
-                Activity & act;
-                std::string currentLine;
-
-                LogSink(Activity & act) : act(act) { }
-
-                void operator() (std::string_view data) override {
-                    for (auto c : data) {
-                        if (c == '\n') {
-                            flushLine();
-                        } else {
-                            currentLine += c;
-                        }
-                    }
-                }
-
-                void flushLine() {
-                    act.result(resPostBuildLogLine, currentLine);
-                    currentLine.clear();
-                }
-
-                ~LogSink() {
-                    if (currentLine != "") {
-                        currentLine += '\n';
-                        flushLine();
-                    }
-                }
-            };
-            LogSink sink(act);
-
-            opts.standardOut = &sink;
-            opts.mergeStderrToStdout = true;
-            runProgram2(opts);
-        }
+        StorePathSet outputPaths;
+        for (auto & [_, path] : finalOutputs)
+            outputPaths.insert(path);
+        runPostBuildHook(
+            worker.store,
+            *logger,
+            drvPath,
+            outputPaths
+        );
 
         if (buildMode == bmCheck) {
             cleanupPostOutputsRegisteredModeCheck();
@@ -910,6 +942,8 @@ void DerivationGoal::resolvedFinished() {
 
     auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
 
+    StorePathSet outputPaths;
+
     // `wantedOutputs` might be empty, which means “all the outputs”
     auto realWantedOutputs = wantedOutputs;
     if (realWantedOutputs.empty())
@@ -927,8 +961,10 @@ void DerivationGoal::resolvedFinished() {
             auto newRealisation = *realisation;
             newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
             newRealisation.signatures.clear();
+            newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
             signRealisation(newRealisation);
             worker.store.registerDrvOutput(newRealisation);
+            outputPaths.insert(realisation->outPath);
         } else {
             // If we don't have a realisation, then it must mean that something
             // failed when building the resolved drv
@@ -936,6 +972,13 @@ void DerivationGoal::resolvedFinished() {
         }
     }
 
+    runPostBuildHook(
+        worker.store,
+        *logger,
+        drvPath,
+        outputPaths
+    );
+
     // This is potentially a bit fishy in terms of error reporting. Not sure
     // how to do it in a cleaner way
     amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
@@ -968,7 +1011,7 @@ HookReply DerivationGoal::tryBuildHook()
                     return readLine(worker.hook->fromHook.readSide.get());
                 } catch (Error & e) {
                     e.addTrace({}, "while reading the response from the build hook");
-                    throw e;
+                    throw;
                 }
             }();
             if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
@@ -1014,7 +1057,7 @@ HookReply DerivationGoal::tryBuildHook()
         machineName = readLine(hook->fromHook.readSide.get());
     } catch (Error & e) {
         e.addTrace({}, "while reading the machine name from the build hook");
-        throw e;
+        throw;
     }
 
     /* Tell the hook all the inputs that have to be copied to the
@@ -1048,42 +1091,6 @@ HookReply DerivationGoal::tryBuildHook()
 }
 
 
-StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
-{
-    StorePathSet paths;
-
-    for (auto & storePath : storePaths) {
-        if (!inputPaths.count(storePath))
-            throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath));
-
-        worker.store.computeFSClosure({storePath}, paths);
-    }
-
-    /* If there are derivations in the graph, then include their
-       outputs as well.  This is useful if you want to do things
-       like passing all build-time dependencies of some path to a
-       derivation that builds a NixOS DVD image. */
-    auto paths2 = paths;
-
-    for (auto & j : paths2) {
-        if (j.isDerivation()) {
-            Derivation drv = worker.store.derivationFromPath(j);
-            for (auto & k : drv.outputsAndOptPaths(worker.store)) {
-                if (!k.second.second)
-                    /* FIXME: I am confused why we are calling
-                       `computeFSClosure` on the output path, rather than
-                       derivation itself. That doesn't seem right to me, so I
-                       won't try to implemented this for CA derivations. */
-                    throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
-                worker.store.computeFSClosure(*k.second.second, paths);
-            }
-        }
-    }
-
-    return paths;
-}
-
-
 void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output

src/libstore/build/drv-output-substitution-goal.cc

@@ -17,6 +17,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker
 void DrvOutputSubstitutionGoal::init()
 {
     trace("init");
+
+    /* If the derivation already exists, we’re done */
+    if (worker.store.queryRealisation(id)) {
+        amDone(ecSuccess);
+        return;
+    }
+
     subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
     tryNext();
 }
@@ -53,6 +60,26 @@ void DrvOutputSubstitutionGoal::tryNext()
         return;
     }
 
+    for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
+        if (depId != id) {
+            if (auto localOutputInfo = worker.store.queryRealisation(depId);
+                localOutputInfo && localOutputInfo->outPath != depPath) {
+                warn(
+                    "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
+                    "Local:  %s\n"
+                    "Remote: %s",
+                    sub->getUri(),
+                    depId.to_string(),
+                    worker.store.printStorePath(localOutputInfo->outPath),
+                    worker.store.printStorePath(depPath)
+                );
+                tryNext();
+                return;
+            }
+            addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
+        }
+    }
+
     addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
 
     if (waitees.empty()) outPathValid();

src/libstore/build/entry-points.cc

@@ -6,17 +6,17 @@
 
 namespace nix {
 
-void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
+void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore)
 {
-    Worker worker(*this);
+    Worker worker(*this, evalStore ? *evalStore : *this);
 
     Goals goals;
-    for (auto & br : reqs) {
+    for (const auto & br : reqs) {
         std::visit(overloaded {
-            [&](DerivedPath::Built bfd) {
+            [&](const DerivedPath::Built & bfd) {
                 goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
             },
-            [&](DerivedPath::Opaque bo) {
+            [&](const DerivedPath::Opaque & bo) {
                 goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
             },
         }, br.raw());
@@ -51,7 +51,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
     BuildMode buildMode)
 {
-    Worker worker(*this);
+    Worker worker(*this, *this);
     auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode);
 
     BuildResult result;
@@ -93,7 +93,7 @@ void Store::ensurePath(const StorePath & path)
     /* If the path is already valid, we're done. */
     if (isValidPath(path)) return;
 
-    Worker worker(*this);
+    Worker worker(*this, *this);
     GoalPtr goal = worker.makePathSubstitutionGoal(path);
     Goals goals = {goal};
 
@@ -111,7 +111,7 @@ void Store::ensurePath(const StorePath & path)
 
 void LocalStore::repairPath(const StorePath & path)
 {
-    Worker worker(*this);
+    Worker worker(*this, *this);
     GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
     Goals goals = {goal};
 

src/libstore/build/goal.cc

@@ -13,11 +13,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p)
 {
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
+    if (goals.find(p) != goals.end())
+        return;
+    goals.insert(p);
 }
 
 
@@ -46,10 +44,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
         /* If we failed and keepGoing is not set, we remove all
            remaining waitees. */
         for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
+            goal->waiters.extract(shared_from_this());
         }
         waitees.clear();
 

src/libstore/build/goal.hh

@@ -19,7 +19,7 @@ struct CompareGoalPtrs {
 
 /* Set of goals. */
 typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
+typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
 
 /* A map of paths to goals (and the other way around). */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;

src/libstore/build/local-derivation-goal.cc

@@ -17,16 +17,14 @@
 #include <regex>
 #include <queue>
 
-#include <sys/types.h>
-#include <sys/socket.h>
 #include <sys/un.h>
-#include <netdb.h>
 #include <fcntl.h>
 #include <termios.h>
 #include <unistd.h>
 #include <sys/mman.h>
 #include <sys/utsname.h>
 #include <sys/resource.h>
+#include <sys/socket.h>
 
 #if HAVE_STATVFS
 #include <sys/statvfs.h>
@@ -34,7 +32,6 @@
 
 /* Includes required for chroot support. */
 #if __linux__
-#include <sys/socket.h>
 #include <sys/ioctl.h>
 #include <net/if.h>
 #include <netinet/ip.h>
@@ -70,12 +67,14 @@ void handleDiffHook(
     auto diffHook = settings.diffHook;
     if (diffHook != "" && settings.runDiffHook) {
         try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-            auto diffRes = runProgram(diffHookOptions);
+            auto diffRes = runProgram(RunOptions {
+                .program = diffHook,
+                .searchPath = true,
+                .args = {tryA, tryB, drvPath, tmpDir},
+                .uid = uid,
+                .gid = gid,
+                .chdir = "/"
+            });
             if (!statusOk(diffRes.first))
                 throw ExecError(diffRes.first,
                     "diff-hook program '%1%' %2%",
@@ -292,7 +291,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         auto & localStore = getLocalStore();
         uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
         struct statvfs st;
-        if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
+        if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 &&
             (uint64_t) st.f_bavail * st.f_bsize < required)
             diskFull = true;
         if (statvfs(tmpDir.c_str(), &st) == 0 &&
@@ -344,23 +343,6 @@ int childEntry(void * arg)
 }
 
 
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
 static void linkOrCopy(const Path & from, const Path & to)
 {
     if (link(from.c_str(), to.c_str()) == -1) {
@@ -389,9 +371,6 @@ void LocalDerivationGoal::startBuilder()
             settings.thisSystem,
             concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
 
-    if (drv->isBuiltin())
-        preloadNSS();
-
 #if __APPLE__
     additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
 #endif
@@ -417,7 +396,7 @@ void LocalDerivationGoal::startBuilder()
     }
 
     auto & localStore = getLocalStore();
-    if (localStore.storeDir != localStore.realStoreDir) {
+    if (localStore.storeDir != localStore.realStoreDir.get()) {
         #if __linux__
             useChroot = true;
         #else
@@ -518,7 +497,7 @@ void LocalDerivationGoal::startBuilder()
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,
                 worker.store.makeValidityRegistration(
-                    exportReferences({storePath}), false, false));
+                    worker.store.exportReferences({storePath}, inputPaths), false, false));
         }
     }
 
@@ -732,6 +711,7 @@ void LocalDerivationGoal::startBuilder()
     if (!builderOut.readSide)
         throw SysError("opening pseudoterminal master");
 
+    // FIXME: not thread-safe, use ptsname_r
     std::string slaveName(ptsname(builderOut.readSide.get()));
 
     if (buildUser) {
@@ -775,7 +755,6 @@ void LocalDerivationGoal::startBuilder()
     result.startTime = time(0);
 
     /* Fork a child to build the package. */
-    ProcessOptions options;
 
 #if __linux__
     if (useChroot) {
@@ -818,8 +797,6 @@ void LocalDerivationGoal::startBuilder()
 
         userNamespaceSync.create();
 
-        options.allowVfork = false;
-
         Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
         static bool userNamespacesEnabled =
             pathExists(maxUserNamespaces)
@@ -877,7 +854,7 @@ void LocalDerivationGoal::startBuilder()
             writeFull(builderOut.writeSide.get(),
                 fmt("%d %d\n", usingUserNamespace, child));
             _exit(0);
-        }, options);
+        });
 
         int res = helper.wait();
         if (res != 0 && settings.sandboxFallback) {
@@ -941,10 +918,9 @@ void LocalDerivationGoal::startBuilder()
 #endif
     {
     fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
         pid = startProcess([&]() {
             runChild();
-        }, options);
+        });
     }
 
     /* parent */
@@ -959,9 +935,12 @@ void LocalDerivationGoal::startBuilder()
             try {
                 return readLine(builderOut.readSide.get());
             } catch (Error & e) {
-                e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
+                auto status = pid.wait();
+                e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)",
+                    worker.store.printStorePath(drvPath),
+                    statusToString(status),
                     concatStringsSep("|", msgs));
-                throw e;
+                throw;
             }
         }();
         if (string(msg, 0, 1) == "\2") break;
@@ -1084,123 +1063,38 @@ void LocalDerivationGoal::initEnv()
 }
 
 
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
 void LocalDerivationGoal::writeStructuredAttrs()
 {
-    auto structuredAttrs = parsedDrv->getStructuredAttrs();
-    if (!structuredAttrs) return;
+    if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) {
+        auto json = structAttrsJson.value();
+        nlohmann::json rewritten;
+        for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
+            /* The placeholder must have a rewrite, so we use it to cover both the
+               cases where we know or don't know the output path ahead of time. */
+            rewritten[i] = rewriteStrings(v, inputRewrites);
+        }
 
-    auto json = *structuredAttrs;
+        json["outputs"] = rewritten;
 
-    /* Add an "outputs" object containing the output paths. */
-    nlohmann::json outputs;
-    for (auto & i : drv->outputs) {
-        /* The placeholder must have a rewrite, so we use it to cover both the
-           cases where we know or don't know the output path ahead of time. */
-        outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
+        auto jsonSh = writeStructuredAttrsShell(json);
+
+        writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
+        chownToBuilder(tmpDir + "/.attrs.sh");
+        env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh";
+        writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
+        chownToBuilder(tmpDir + "/.attrs.json");
+        env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json";
     }
-    json["outputs"] = outputs;
-
-    /* Handle exportReferencesGraph. */
-    auto e = json.find("exportReferencesGraph");
-    if (e != json.end() && e->is_object()) {
-        for (auto i = e->begin(); i != e->end(); ++i) {
-            std::ostringstream str;
-            {
-                JSONPlaceholder jsonRoot(str, true);
-                StorePathSet storePaths;
-                for (auto & p : *i)
-                    storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
-                worker.store.pathInfoToJSON(jsonRoot,
-                    exportReferences(storePaths), false, true);
-            }
-            json[i.key()] = nlohmann::json::parse(str.str()); // urgh
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.json");
-
-    /* As a convenience to bash scripts, write a shell file that
-       maps all attributes that are representable in bash -
-       namely, strings, integers, nulls, Booleans, and arrays and
-       objects consisting entirely of those values. (So nested
-       arrays or objects are not supported.) */
-
-    auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
-        if (value.is_string())
-            return shellEscape(value);
-
-        if (value.is_number()) {
-            auto f = value.get<float>();
-            if (std::ceil(f) == f)
-                return std::to_string(value.get<int>());
-        }
-
-        if (value.is_null())
-            return std::string("''");
-
-        if (value.is_boolean())
-            return value.get<bool>() ? std::string("1") : std::string("");
-
-        return {};
-    };
-
-    std::string jsonSh;
-
-    for (auto i = json.begin(); i != json.end(); ++i) {
-
-        if (!std::regex_match(i.key(), shVarName)) continue;
-
-        auto & value = i.value();
-
-        auto s = handleSimpleType(value);
-        if (s)
-            jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
-        else if (value.is_array()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += *s3; s2 += ' ';
-            }
-
-            if (good)
-                jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
-        }
-
-        else if (value.is_object()) {
-            std::string s2;
-            bool good = true;
-
-            for (auto i = value.begin(); i != value.end(); ++i) {
-                auto s3 = handleSimpleType(i.value());
-                if (!s3) { good = false; break; }
-                s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
-            }
-
-            if (good)
-                jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
-        }
-    }
-
-    writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-    chownToBuilder(tmpDir + "/.attrs.sh");
 }
 
 
 static StorePath pathPartOfReq(const DerivedPath & req)
 {
     return std::visit(overloaded {
-        [&](DerivedPath::Opaque bo) {
+        [&](const DerivedPath::Opaque & bo) {
             return bo.path;
         },
-        [&](DerivedPath::Built bfd)  {
+        [&](const DerivedPath::Built & bfd)  {
             return bfd.drvPath;
         },
     }, req.raw());
@@ -1333,13 +1227,20 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
     // XXX: This should probably be allowed if the realisation corresponds to
     // an allowed derivation
-    { throw Error("queryRealisation"); }
-
-    void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
     {
+        if (!goal.isAllowed(id))
+            throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
+        return next->queryRealisation(id);
+    }
+
+    void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
+    {
+        assert(!evalStore);
+
         if (buildMode != bmNormal) throw Error("unsupported build mode");
 
         StorePathSet newPaths;
+        std::set<Realisation> newRealisations;
 
         for (auto & req : paths) {
             if (!goal.isAllowed(req))
@@ -1352,16 +1253,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             auto p =  std::get_if<DerivedPath::Built>(&path);
             if (!p) continue;
             auto & bfd = *p;
+            auto drv = readDerivation(bfd.drvPath);
+            auto drvHashes = staticOutputHashes(*this, drv);
             auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
             for (auto & [outputName, outputPath] : outputs)
-                if (wantOutput(outputName, bfd.outputs))
+                if (wantOutput(outputName, bfd.outputs)) {
                     newPaths.insert(outputPath);
+                    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                        auto thisRealisation = next->queryRealisation(
+                            DrvOutput{drvHashes.at(outputName), outputName}
+                        );
+                        assert(thisRealisation);
+                        newRealisations.insert(*thisRealisation);
+                    }
+                }
         }
 
         StorePathSet closure;
         next->computeFSClosure(newPaths, closure);
         for (auto & path : closure)
             goal.addDependency(path);
+        for (auto & real : Realisation::closure(*next, newRealisations))
+            goal.addedDrvOutputs.insert(real.id);
     }
 
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
@@ -1737,7 +1650,7 @@ void LocalDerivationGoal::runChild()
                 /* N.B. it is realistic that these paths might not exist. It
                    happens when testing Nix building fixed-output derivations
                    within a pure derivation. */
-                for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" })
+                for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" })
                     if (pathExists(path))
                         ss.push_back(path);
             }
@@ -1919,7 +1832,7 @@ void LocalDerivationGoal::runChild()
         /* Fill in the arguments. */
         Strings args;
 
-        const char *builder = "invalid";
+        std::string builder = "invalid";
 
         if (drv->isBuiltin()) {
             ;
@@ -2045,13 +1958,13 @@ void LocalDerivationGoal::runChild()
                 }
                 args.push_back(drv->builder);
             } else {
-                builder = drv->builder.c_str();
+                builder = drv->builder;
                 args.push_back(std::string(baseNameOf(drv->builder)));
             }
         }
 #else
         else {
-            builder = drv->builder.c_str();
+            builder = drv->builder;
             args.push_back(std::string(baseNameOf(drv->builder)));
         }
 #endif
@@ -2107,9 +2020,9 @@ void LocalDerivationGoal::runChild()
             posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
         }
 
-        posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+        posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 #else
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+        execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 #endif
 
         throw SysError("executing '%1%'", drv->builder);
@@ -2224,8 +2137,7 @@ void LocalDerivationGoal::registerOutputs()
 
         /* Pass blank Sink as we are not ready to hash data at this stage. */
         NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
+        auto references = scanForReferences(blank, actualPath, referenceablePaths);
 
         outputReferencesIfUnregistered.insert_or_assign(
             outputName,
@@ -2239,8 +2151,8 @@ void LocalDerivationGoal::registerOutputs()
                 /* Since we'll use the already installed versions of these, we
                    can treat them as leaves and ignore any references they
                    have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
+                [&](const AlreadyRegistered &) { return StringSet {}; },
+                [&](const PerhapsNeedToRegister & refs) {
                     StringSet referencedOutputs;
                     /* FIXME build inverted map up front so no quadratic waste here */
                     for (auto & r : refs.refs)
@@ -2276,11 +2188,11 @@ void LocalDerivationGoal::registerOutputs()
         };
 
         std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
+            [&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> {
                 finish(skippedFinalPath.path);
                 return std::nullopt;
             },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
+            [&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> {
                 return r.refs;
             },
         }, outputReferencesIfUnregistered.at(outputName));
@@ -2292,7 +2204,7 @@ void LocalDerivationGoal::registerOutputs()
         auto rewriteOutput = [&]() {
             /* Apply hash rewriting if necessary. */
             if (!outputRewrites.empty()) {
-                warn("rewriting hashes in '%1%'; cross fingers", actualPath);
+                debug("rewriting hashes in '%1%'; cross fingers", actualPath);
 
                 /* FIXME: this is in-memory. */
                 StringSink sink;
@@ -2396,7 +2308,7 @@ void LocalDerivationGoal::registerOutputs()
         };
 
         ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
+            [&](const DerivationOutputInputAddressed & output) {
                 /* input-addressed case */
                 auto requiredFinalPath = output.path;
                 /* Preemptively add rewrite rule for final hash, as that is
@@ -2415,14 +2327,14 @@ void LocalDerivationGoal::registerOutputs()
                     newInfo0.references.insert(newInfo0.path);
                 return newInfo0;
             },
-            [&](DerivationOutputCAFixed dof) {
+            [&](const DerivationOutputCAFixed & dof) {
                 auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
                     .method = dof.hash.method,
                     .hashType = dof.hash.hash.type,
                 });
 
                 /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
+                const Hash & wanted = dof.hash.hash;
                 assert(newInfo0.ca);
                 auto got = getContentAddressHash(*newInfo0.ca);
                 if (wanted != got) {
@@ -2464,6 +2376,7 @@ void LocalDerivationGoal::registerOutputs()
            floating CA derivations and hash-mismatching fixed-output
            derivations. */
         PathLocks dynamicOutputLock;
+        dynamicOutputLock.setDeletion(true);
         auto optFixedPath = output.path(worker.store, drv->name, outputName);
         if (!optFixedPath ||
             worker.store.printStorePath(*optFixedPath) != finalDestPath)
@@ -2487,6 +2400,7 @@ void LocalDerivationGoal::registerOutputs()
                 assert(newInfo.ca);
             } else {
                 auto destPath = worker.store.toRealPath(finalDestPath);
+                deletePath(destPath);
                 movePath(actualPath, destPath);
                 actualPath = destPath;
             }
@@ -2556,7 +2470,13 @@ void LocalDerivationGoal::registerOutputs()
         infos.emplace(outputName, std::move(newInfo));
     }
 
-    if (buildMode == bmCheck) return;
+    if (buildMode == bmCheck) {
+        // In case of FOD mismatches on `--check` an error must be thrown as this is also
+        // a source for non-determinism.
+        if (delayedException)
+            std::rethrow_exception(delayedException);
+        return;
+    }
 
     /* Apply output checks. */
     checkOutputs(infos);

src/libstore/build/local-derivation-goal.hh

@@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Paths that were added via recursive Nix calls. */
     StorePathSet addedPaths;
 
+    /* Realisations that were added via recursive Nix calls. */
+    std::set<DrvOutput> addedDrvOutputs;
+
     /* Recursive Nix calls are only allowed to build or realize paths
        in the original input closure or added via a recursive Nix call
        (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
@@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal
     {
         return inputPaths.count(path) || addedPaths.count(path);
     }
+    bool isAllowed(const DrvOutput & id)
+    {
+        return addedDrvOutputs.count(id);
+    }
+
     bool isAllowed(const DerivedPath & req);
 
     friend struct RestrictedStore;