Parse '(f x) y' the same as 'f x y'

Previously an expression like 'x: y: ...' would create two
environments with one value. Now it creates one environment with two
values. This reduces the number of allocations and the distance in the
environment chain that variable lookups need to traverse.

On

  $ nix-instantiate --dry-run '<nixpkgs/nixos/release-combined.nix>' -A nixos.tests.simple.x86_64-linux

this gives a ~30% reduction in the number of Env allocations.

.github/workflows/test.yml

@@ -1,26 +1,32 @@
 name: "Test"
+
 on:
   pull_request:
   push:
+
 jobs:
+
   tests:
+    needs: [check_cachix]
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
-
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.3.5
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v14.1
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v10
+      if: needs.check_cachix.outputs.secret == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
+
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest
@@ -32,6 +38,7 @@ jobs:
         env:
           _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
         run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}"
+
   installer:
     needs: [tests, check_cachix]
     if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
@@ -39,11 +46,11 @@ jobs:
     outputs:
       installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.3.5
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v14.1
     - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -51,6 +58,7 @@ jobs:
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - id: prepare-installer
       run: scripts/prepare-installer-for-github-actions
+
   installer_test:
     needs: [installer, check_cachix]
     if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
@@ -59,9 +67,9 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.3.5
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v14.1
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"

.gitignore

@@ -40,6 +40,7 @@ perl/Makefile.config
 
 # /src/libstore/
 *.gen.*
+/src/libstore/tests/libstore-tests
 
 # /src/libutil/
 /src/libutil/tests/libutil-tests
@@ -57,9 +58,6 @@ perl/Makefile.config
 
 /src/nix-prefetch-url/nix-prefetch-url
 
-# /src/nix-daemon/
-/src/nix-daemon/nix-daemon
-
 /src/nix-collect-garbage/nix-collect-garbage
 
 # /src/nix-channel/

.version

@@ -1 +1 @@
-2.4
+2.5

Makefile

@@ -4,6 +4,7 @@ makefiles = \
   src/libutil/local.mk \
   src/libutil/tests/local.mk \
   src/libstore/local.mk \
+  src/libstore/tests/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \

boehmgc-coroutine-sp-fallback.diff

@@ -1,8 +1,8 @@
 diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index 1cee6a0b..46c3acd9 100644
+index 4b2c429..1fb4c52 100644
 --- a/pthread_stop_world.c
 +++ b/pthread_stop_world.c
-@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void)
      struct GC_traced_stack_sect_s *traced_stack_sect;
      pthread_t self = pthread_self();
      word total_size = 0;
@@ -11,7 +11,7 @@ index 1cee6a0b..46c3acd9 100644
  
      if (!EXPECT(GC_thr_initialized, TRUE))
        GC_thr_init();
-@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void)
            hi = p->altstack + p->altstack_size;
            /* FIXME: Need to scan the normal stack too, but how ? */
            /* FIXME: Assume stack grows down */
@@ -22,6 +22,9 @@ index 1cee6a0b..46c3acd9 100644
 +          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
 +            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
 +          }
++          if (pthread_attr_destroy(&pattr)) {
++            ABORT("GC_push_all_stacks: pthread_attr_destroy failed!");
++          }
 +          // When a thread goes into a coroutine, we lose its original sp until
 +          // control flow returns to the thread.
 +          // While in the coroutine, the sp points outside the thread stack,

configure.ac

@@ -32,14 +32,6 @@ AC_ARG_WITH(system, AS_HELP_STRING([--with-system=SYSTEM],[Platform identifier (
         system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";;
    esac])
 
-sys_name=$(uname -s | tr 'A-Z ' 'a-z_')
-
-case $sys_name in
-    cygwin*)
-        sys_name=cygwin
-        ;;
-esac
-
 AC_MSG_RESULT($system)
 AC_SUBST(system)
 AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')])
@@ -63,10 +55,12 @@ AC_SYS_LARGEFILE
 
 # Solaris-specific stuff.
 AC_STRUCT_DIRENT_D_TYPE
-if test "$sys_name" = sunos; then
+case "$host_os" in
+  solaris*)
     # Solaris requires -lsocket -lnsl for network functions
     LDFLAGS="-lsocket -lnsl $LDFLAGS"
-fi
+    ;;
+esac
 
 
 # Check for pubsetbuf.
@@ -210,21 +204,24 @@ AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
 
 
 # Look for libseccomp, required for Linux sandboxing.
-if test "$sys_name" = linux; then
-  AC_ARG_ENABLE([seccomp-sandboxing],
-                AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)
-                              ]))
-  if test "x$enable_seccomp_sandboxing" != "xno"; then
-    PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
-                      [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
-    have_seccomp=1
-    AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
-  else
+case "$host_os" in
+  linux*)
+    AC_ARG_ENABLE([seccomp-sandboxing],
+                  AS_HELP_STRING([--disable-seccomp-sandboxing],[Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)
+                                ]))
+    if test "x$enable_seccomp_sandboxing" != "xno"; then
+      PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp],
+                        [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
+      have_seccomp=1
+      AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
+    else
+      have_seccomp=
+    fi
+    ;;
+  *)
     have_seccomp=
-  fi
-else
-  have_seccomp=
-fi
+    ;;
+esac
 AC_SUBST(HAVE_SECCOMP, [$have_seccomp])
 
 
@@ -263,6 +260,8 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
+# Look for lowdown library.
+PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.8.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])
@@ -274,9 +273,11 @@ AC_CHECK_FUNCS([strsignal posix_fallocate sysconf])
 
 # This is needed if bzip2 is a static library, and the Nix libraries
 # are dynamic.
-if test "$(uname)" = "Darwin"; then
+case "${host_os}" in
+  darwin*)
     LDFLAGS="-all_load $LDFLAGS"
-fi
+    ;;
+esac
 
 
 AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]),

doc/manual/generate-manpage.nix

@@ -1,4 +1,4 @@
-command:
+{ command, renderLinks ? false }:
 
 with builtins;
 with import ./utils.nix;
@@ -20,7 +20,11 @@ let
            categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
            listCommands = cmds:
              concatStrings (map (name:
-               "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n")
+               "* "
+               + (if renderLinks
+                  then "[`${command} ${name}`](./${appendName filename name}.md)"
+                  else "`${command} ${name}`")
+               + " - ${cmds.${name}.description}\n")
                (attrNames cmds));
          in
          "where *subcommand* is one of the following:\n\n"
@@ -89,7 +93,7 @@ let
 in
 
 let
-  manpages = processCommand { filename = "nix"; command = "nix"; def = command; };
+  manpages = processCommand { filename = "nix"; command = "nix"; def = builtins.fromJSON command; };
   summary = concatStrings (map (manpage: "    - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages);
 in
 (listToAttrs manpages) // { "SUMMARY.md" = summary; }

doc/manual/local.mk

@@ -44,7 +44,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@
-	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))'
+	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }'
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
@@ -78,24 +78,23 @@ man: doc/manual/generated/man1/nix3-manpages
 all: doc/manual/generated/man1/nix3-manpages
 
 $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages
-	@mkdir -p $$(dirname $@)
-	$(trace-install) install -m 0644 $$(dirname $<)/* $$(dirname $@)
+	@mkdir -p $(DESTDIR)$$(dirname $@)
+	$(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@)
 
 doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
-	@mkdir -p $$(dirname $@)
+	@mkdir -p $(DESTDIR)$$(dirname $@)
 	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
 	  name=$$(basename $$i .md); \
 	  tmpFile=$$(mktemp); \
 	  if [[ $$name = SUMMARY ]]; then continue; fi; \
 	  printf "Title: %s\n\n" "$$name" > $$tmpFile; \
 	  cat $$i >> $$tmpFile; \
-	  lowdown -sT man -M section=1 $$tmpFile -o $$(dirname $@)/$$name.1; \
+	  lowdown -sT man -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \
 	  rm $$tmpFile; \
 	done
-	touch $@
+	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
-	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual
-	@cp doc/manual/highlight.pack.js $(docdir)/manual/highlight.js
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md)
+	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual
 
 endif

doc/manual/src/SUMMARY.md.in

@@ -70,6 +70,8 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
+  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)
   - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md)

doc/manual/src/command-ref/conf-file-prefix.md

@@ -17,7 +17,7 @@ By default Nix reads settings from the following places:
 
     Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
     and `XDG_CONFIG_HOME`. If these are unset, it will look in
-    `$HOME/.config/nix.conf`.
+    `$HOME/.config/nix/nix.conf`.
 
   - If `NIX_CONFIG` is set, its contents is treated as the contents of
     a configuration file.

doc/manual/src/command-ref/nix-env.md

@@ -401,7 +401,7 @@ of a derivation `x` by looking at their respective `name` attributes.
 The names (e.g., `gcc-3.3.1` are split into two parts: the package name
 (`gcc`), and the version (`3.3.1`). The version part starts after the
 first dash not followed by a letter. `x` is considered an upgrade of `y`
-if their package names match, and the version of `y` is higher that that
+if their package names match, and the version of `y` is higher than that
 of `x`.
 
 The versions are compared by splitting them into contiguous components

doc/manual/src/command-ref/nix-shell.md

@@ -11,8 +11,8 @@
   [`--command` *cmd*]
   [`--run` *cmd*]
   [`--exclude` *regexp*]
-  [--pure]
-  [--keep *name*]
+  [`--pure`]
+  [`--keep` *name*]
   {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]}
 
 # Description
@@ -110,13 +110,19 @@ shell in which to build it:
 
 ```console
 $ nix-shell '<nixpkgs>' -A pan
-[nix-shell]$ unpackPhase
+[nix-shell]$ eval ${unpackPhase:-unpackPhase}
 [nix-shell]$ cd pan-*
-[nix-shell]$ configurePhase
-[nix-shell]$ buildPhase
+[nix-shell]$ eval ${configurePhase:-configurePhase}
+[nix-shell]$ eval ${buildPhase:-buildPhase}
 [nix-shell]$ ./pan/gui/pan
 ```
 
+The reason we use form `eval ${configurePhase:-configurePhase}` here is because
+those packages that override these phases do so by exporting the overridden
+values in the environment variable of the same name.
+Here bash is being told to either evaluate the contents of 'configurePhase',
+if it exists as a variable, otherwise evaluate the configurePhase function.
+
 To clear the environment first, and do some additional automatic
 initialisation of the interactive shell:
 

doc/manual/src/command-ref/nix-store.md

@@ -125,7 +125,7 @@ Special exit codes:
 
   - `104`\
     Not deterministic, the build succeeded in check mode but the
-    resulting output is not binary reproducable.
+    resulting output is not binary reproducible.
 
 With the `--keep-going` flag it's possible for multiple failures to
 occur, in this case the 1xx status codes are or combined using binary

doc/manual/src/contributing/cli-guideline.md

@@ -3,7 +3,7 @@
 ## Goals
 
 Purpose of this document is to provide a clear direction to **help design
-delightful command line** experience. This document contain guidelines to
+delightful command line** experience. This document contains guidelines to
 follow to ensure a consistent and approachable user experience.
 
 ## Overview
@@ -115,7 +115,7 @@ The rules are:
 
 - Help is shown by using `--help` or `help` command (eg `nix` `--``help` or
   `nix help`).
-- For non-COMMANDs (eg. `nix` `--``help`  and `nix store` `--``help`) we **show
+- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show
   a summary** of most common use cases. Summary is presented on the STDOUT
   without any use of PAGER.
 - For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the
@@ -230,8 +230,8 @@ Now **Learn** part of the output is where you educate users. You should only
 show it when you know that a build will take some time and not annoy users of
 the builds that take only few seconds.
 
-Every feature like this should go though a intensive review and testing to
-collect as much a feedback as possible and to fine tune every little detail. If
+Every feature like this should go through an intensive review and testing to
+collect as much feedback as possible and to fine tune every little detail. If
 done right this can be an awesome features beginners and advance users will
 love, but if not done perfectly it will annoy users and leave bad impression.
 
@@ -272,11 +272,11 @@ not know which `ARGUMENTS` and `OPTIONS` are required or which values are
 possible for those options.
 
 In cases, the user might not provide the input or they provide wrong input,
-rather then show the error, prompt a user with an option to find and select
+rather than show the error, prompt a user with an option to find and select
 correct input (see examples).
 
 Prompting is of course not required when TTY is not attached to STDIN. This
-would mean that scripts wont need to handle prompt, but rather handle errors.
+would mean that scripts won't need to handle prompt, but rather handle errors.
 
 A place to use prompt and provide user with interactive select
 
@@ -300,7 +300,7 @@ going to happen.
 ```shell
 $ nix build --option substitutors https://cache.example.org
 ------------------------------------------------------------------------
-  Warning! A security related question need to be answered.
+  Warning! A security related question needs to be answered.
 ------------------------------------------------------------------------
   The following substitutors will be used to in `my-project`: 
     - https://cache.example.org
@@ -311,14 +311,14 @@ $ nix build --option substitutors https://cache.example.org
 
 # Output
 
-Terminal output can be quite limiting in many ways. Which should forces us to
+Terminal output can be quite limiting in many ways. Which should force us to
 think about the experience even more. As with every design the output is a
 compromise between being terse and being verbose, between showing help to
 beginners and annoying advance users. For this it is important that we know
 what are the priorities.
 
 Nix command line should be first and foremost written with beginners in mind.
-But users wont stay beginners for long and what was once useful might quickly
+But users won't stay beginners for long and what was once useful might quickly
 become annoying. There is no golden rule that we can give in this guideline
 that would make it easier how to draw a line and find best compromise.
 
@@ -508,7 +508,7 @@ can, with a few key strokes, be changed into and advance introspection tool.
 
 ### Progress
 
-For longer running commands we should provide and overview of the progress.
+For longer running commands we should provide and overview the progress.
 This is shown best in `nix build` example:
 
 ```shell
@@ -553,7 +553,7 @@ going to happen.
 ```shell
 $ nix build --option substitutors https://cache.example.org
 ------------------------------------------------------------------------
-  Warning! A security related question need to be answered.
+  Warning! A security related question needs to be answered.
 ------------------------------------------------------------------------
   The following substitutors will be used to in `my-project`: 
     - https://cache.example.org

doc/manual/src/expressions/advanced-attributes.md

@@ -237,7 +237,7 @@ Derivations can declare some infrequently used optional attributes.
   - `preferLocalBuild`\
     If this attribute is set to `true` and [distributed building is
     enabled](../advanced-topics/distributed-builds.md), then, if
-    possible, the derivaton will be built locally instead of forwarded
+    possible, the derivation will be built locally instead of forwarded
     to a remote machine. This is appropriate for trivial builders
     where the cost of doing a download or remote build would exceed
     the cost of building locally.

doc/manual/src/expressions/expression-syntax.md

@@ -26,7 +26,7 @@ elements (referenced from the figure by number):
     called with three arguments: `stdenv`, `fetchurl`, and `perl`. They
     are needed to build Hello, but we don't know how to build them here;
     that's why they are function arguments. `stdenv` is a package that
-    is used by almost all Nix Packages packages; it provides a
+    is used by almost all Nix Packages; it provides a
     “standard” environment consisting of the things you would expect
     in a basic Unix environment: a C/C++ compiler (GCC, to be precise),
     the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`,

doc/manual/src/expressions/language-values.md

@@ -64,7 +64,7 @@ Nix has the following basic data types:
     the start of each line. To be precise, it strips from each line a
     number of spaces equal to the minimal indentation of the string as a
     whole (disregarding the indentation of empty lines). For instance,
-    the first and second line are indented two space, while the third
+    the first and second line are indented two spaces, while the third
     line is indented four spaces. Thus, two spaces are stripped from
     each line, so the resulting string is
     
@@ -139,6 +139,13 @@ Nix has the following basic data types:
     environment variable `NIX_PATH` will be searched for the given file
     or directory name.
 
+    Antiquotation is supported in any paths except those in angle brackets.
+    `./${foo}-${bar}.nix` is a more convenient way of writing 
+    `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At
+    least one slash must appear *before* any antiquotations for this to be
+    recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division
+    operation. `./a.${foo}/b.${bar}` is a path.
+
   - *Booleans* with values `true` and `false`.
 
   - The null value, denoted as `null`.

doc/manual/src/installation/building-source.md

@@ -1,9 +1,9 @@
 # Building Nix from Source
 
-After unpacking or checking out the Nix sources, issue the following
-commands:
+After cloning Nix's Git repository, issue the following commands:
 
 ```console
+$ ./bootstrap.sh
 $ ./configure options...
 $ make
 $ make install
@@ -11,13 +11,6 @@ $ make install
 
 Nix requires GNU Make so you may need to invoke `gmake` instead.
 
-When building from the Git repository, these should be preceded by the
-command:
-
-```console
-$ ./bootstrap.sh
-```
-
 The installation path can be specified by passing the `--prefix=prefix`
 to `configure`. The default installation directory is `/usr/local`. You
 can change this to any location you like. You must have write permission

doc/manual/src/installation/env-variables.md

@@ -40,7 +40,7 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
 > **Note**
 > 
 > You must not add the export and then do the install, as the Nix
-> installer will detect the presense of Nix configuration, and abort.
+> installer will detect the presence of Nix configuration, and abort.
 
 ## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon
 

doc/manual/src/installation/installing-source.md

@@ -1,4 +1,4 @@
 # Installing Nix from Source
 
-If no binary package is available, you can download and compile a source
-distribution.
+If no binary package is available or if you want to hack on Nix, you
+can build Nix from its Git repository.

doc/manual/src/installation/obtaining-source.md

@@ -1,14 +1,9 @@
-# Obtaining a Source Distribution
+# Obtaining the Source
 
-The source tarball of the most recent stable release can be downloaded
-from the [Nix homepage](http://nixos.org/nix/download.html). You can
-also grab the [most recent development
-release](http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents).
-
-Alternatively, the most recent sources of Nix can be obtained from its
-[Git repository](https://github.com/NixOS/nix). For example, the
-following command will check out the latest revision into a directory
-called `nix`:
+The most recent sources of Nix can be obtained from its [Git
+repository](https://github.com/NixOS/nix). For example, the following
+command will check out the latest revision into a directory called
+`nix`:
 
 ```console
 $ git clone https://github.com/NixOS/nix

doc/manual/src/installation/prerequisites-source.md

@@ -2,9 +2,8 @@
 
   - GNU Autoconf (<https://www.gnu.org/software/autoconf/>) and the
     autoconf-archive macro collection
-    (<https://www.gnu.org/software/autoconf-archive/>). These are only
-    needed to run the bootstrap script, and are not necessary if your
-    source distribution came with a pre-built `./configure` script.
+    (<https://www.gnu.org/software/autoconf-archive/>). These are
+    needed to run the bootstrap script.
 
   - GNU Make.
 
@@ -26,15 +25,6 @@
     available for download from the official repository
     <https://github.com/google/brotli>.
 
-  - The bzip2 compressor program and the `libbz2` library. Thus you must
-    have bzip2 installed, including development headers and libraries.
-    If your distribution does not provide these, you can obtain bzip2
-    from
-    <https://sourceware.org/bzip2/>.
-
-  - `liblzma`, which is provided by XZ Utils. If your distribution does
-    not provide this, you can get it from <https://tukaani.org/xz/>.
-
   - cURL and its library. If your distribution does not provide it, you
     can get it from <https://curl.haxx.se/>.
 
@@ -61,8 +51,7 @@
     you need version 2.5.35, which is available on
     [SourceForge](http://lex.sourceforge.net/). Slightly older versions
     may also work, but ancient versions like the ubiquitous 2.5.4a
-    won't. Note that these are only required if you modify the parser or
-    when you are building from the Git repository.
+    won't.
 
   - The `libseccomp` is used to provide syscall filtering on Linux. This
     is an optional dependency and can be disabled passing a

doc/manual/src/introduction.md

@@ -127,7 +127,7 @@ $ nix-env --install firefox
 
 _could_ cause quite a bit of build activity, as not only Firefox but
 also all its dependencies (all the way up to the C library and the
-compiler) would have to built, at least if they are not already in the
+compiler) would have to be built, at least if they are not already in the
 Nix store.  This is a _source deployment model_.  For most users,
 building from source is not very pleasant as it takes far too long.
 However, Nix can automatically skip building from source and instead

doc/manual/src/release-notes/rl-2.4.md

@@ -1,8 +1,539 @@
-# Release 2.4 (202X-XX-XX)
+# Release 2.4 (2021-11-01)
 
-  - It is now an error to modify the `plugin-files` setting via a
-    command-line flag that appears after the first non-flag argument
-    to any command, including a subcommand to `nix`. For example,
-    `nix-instantiate default.nix --plugin-files ""` must now become
-    `nix-instantiate --plugin-files "" default.nix`.
-  - Plugins that add new `nix` subcommands are now actually respected.
+This is the first release in more than two years and is the result of
+more than 2800 commits from 195 contributors since release 2.3.
+
+## Highlights
+
+* Nix's **error messages** have been improved a lot. For instance,
+  evaluation errors now point out the location of the error:
+
+  ```
+  $ nix build
+  error: undefined variable 'bzip3'
+
+         at /nix/store/449lv242z0zsgwv95a8124xi11sp419f-source/flake.nix:88:13:
+
+             87|           [ curl
+             88|             bzip3 xz brotli editline
+               |             ^
+             89|             openssl sqlite
+  ```
+
+* The **`nix` command** has seen a lot of work and is now almost at
+  feature parity with the old command-line interface (the `nix-*`
+  commands). It aims to be [more modern, consistent and pleasant to
+  use](../contributing/cli-guideline.md) than the old CLI. It is still
+  marked as experimental but its interface should not change much
+  anymore in future releases.
+
+* **Flakes** are a new format to package Nix-based projects in a more
+  discoverable, composable, consistent and reproducible way. A flake
+  is just a repository or tarball containing a file named `flake.nix`
+  that specifies dependencies on other flakes and returns any Nix
+  assets such as packages, Nixpkgs overlays, NixOS modules or CI
+  tests. The new `nix` CLI is primarily based around flakes; for
+  example, a command like `nix run nixpkgs#hello` runs the `hello`
+  application from the `nixpkgs` flake.
+
+  Flakes are currently marked as experimental. For an introduction,
+  see [this blog
+  post](https://www.tweag.io/blog/2020-05-25-flakes/). For detailed
+  information about flake syntax and semantics, see the [`nix flake`
+  manual page](../command-ref/new-cli/nix3-flake.md).
+
+* Nix's store can now be **content-addressed**, meaning that the hash
+  component of a store path is the hash of the path's
+  contents. Previously Nix could only build **input-addressed** store
+  paths, where the hash is computed from the derivation dependency
+  graph. Content-addressing allows deduplication, early cutoff in
+  build systems, and unprivileged closure copying. This is still [an
+  experimental
+  feature](https://discourse.nixos.org/t/content-addressed-nix-call-for-testers/12881).
+
+* The Nix manual has been converted into Markdown, making it easier to
+  contribute. In addition, every `nix` subcommand now has a manual
+  page, documenting every option.
+
+* A new setting that allows **experimental features** to be enabled
+  selectively. This allows us to merge unstable features into Nix more
+  quickly and do more frequent releases.
+
+## Other features
+
+* There are many new `nix` subcommands:
+
+  - `nix develop` is intended to replace `nix-shell`. It has a number
+    of new features:
+
+    * It automatically sets the output environment variables (such as
+      `$out`) to writable locations (such as `./outputs/out`).
+
+    * It can store the environment in a profile. This is useful for
+      offline work.
+
+    * It can run specific phases directly. For instance, `nix develop
+      --build` runs `buildPhase`.
+
+    - It allows dependencies in the Nix store to be "redirected" to
+      arbitrary directories using the `--redirect` flag. This is
+      useful if you want to hack on a package *and* some of its
+      dependencies at the same time.
+
+  - `nix print-dev-env` prints the environment variables and bash
+    functions defined by a derivation. This is useful for users of
+    other shells than bash (especially with `--json`).
+
+  - `nix shell` was previously named `nix run` and is intended to
+    replace `nix-shell -p`, but without the `stdenv` overhead. It
+    simply starts a shell where some packages have been added to
+    `$PATH`.
+
+  - `nix run` (not to be confused with the old subcommand that has
+    been renamed to `nix shell`) runs an "app", a flake output that
+    specifies a command to run, or an eponymous program from a
+    package. For example, `nix run nixpkgs#hello` runs the `hello`
+    program from the `hello` package in `nixpkgs`.
+
+  - `nix flake` is the container for flake-related operations, such as
+    creating a new flake, querying the contents of a flake or updating
+    flake lock files.
+
+  - `nix registry` allows you to query and update the flake registry,
+    which maps identifiers such as `nixpkgs` to concrete flake URLs.
+
+  - `nix profile` is intended to replace `nix-env`. Its main advantage
+    is that it keeps track of the provenance of installed packages
+    (e.g. exactly which flake version a package came from). It also
+    has some helpful subcommands:
+
+    * `nix profile history` shows what packages were added, upgraded
+      or removed between each version of a profile.
+
+    * `nix profile diff-closures` shows the changes between the
+      closures of each version of a profile. This allows you to
+      discover the addition or removal of dependencies or size
+      changes.
+
+    **Warning**: after a profile has been updated using `nix profile`,
+    it is no longer usable with `nix-env`.
+
+  - `nix store diff-closures` shows the differences between the
+    closures of two store paths in terms of the versions and sizes of
+    dependencies in the closures.
+
+  - `nix store make-content-addressable` rewrites an arbitrary closure
+    to make it content-addressed. Such paths can be copied into other
+    stores without requiring signatures.
+
+  - `nix bundle` uses the [`nix-bundle`
+    program](https://github.com/matthewbauer/nix-bundle) to convert a
+    closure into a self-extracting executable.
+
+  - Various other replacements for the old CLI, e.g. `nix store gc`,
+    `nix store delete`, `nix store repair`, `nix nar dump-path`, `nix
+    store prefetch-file`, `nix store prefetch-tarball`, `nix key` and
+    `nix daemon`.
+
+* Nix now has an **evaluation cache** for flake outputs. For example,
+  a second invocation of the command `nix run nixpkgs#firefox` will
+  not need to evaluate the `firefox` attribute because it's already in
+  the evaluation cache. This is made possible by the hermetic
+  evaluation model of flakes.
+
+* The new `--offline` flag disables substituters and causes all
+  locally cached tarballs and repositories to be considered
+  up-to-date.
+
+* The new `--refresh` flag causes all locally cached tarballs and
+  repositories to be considered out-of-date.
+
+* Many `nix` subcommands now have a `--json` option to produce
+  machine-readable output.
+
+* `nix repl` has a new `:doc` command to show documentation about
+  builtin functions (e.g. `:doc builtins.map`).
+
+* Binary cache stores now have an option `index-debug-info` to create
+  an index of DWARF debuginfo files for use by
+  [`dwarffs`](https://github.com/edolstra/dwarffs).
+
+* To support flakes, Nix now has an extensible mechanism for fetching
+  source trees. Currently it has the following backends:
+
+  * Git repositories
+
+  * Mercurial repositories
+
+  * GitHub and GitLab repositories (an optimisation for faster
+    fetching than Git)
+
+  * Tarballs
+
+  * Arbitrary directories
+
+  The fetcher infrastructure is exposed via flake input specifications
+  and via the `fetchTree` built-in.
+
+* **Languages changes**: the only new language feature is that you can
+  now have antiquotations in paths, e.g. `./${foo}` instead of `./. +
+  foo`.
+
+* **New built-in functions**:
+
+  - `builtins.fetchTree` allows fetching a source tree using any
+    backends supported by the fetcher infrastructure. It subsumes the
+    functionality of existing built-ins like `fetchGit`,
+    `fetchMercurial` and `fetchTarball`.
+
+  - `builtins.getFlake` fetches a flake and returns its output
+    attributes. This function should not be used inside flakes! Use
+    flake inputs instead.
+
+  - `builtins.floor` and `builtins.ceil` round a floating-point number
+    down and up, respectively.
+
+* Experimental support for recursive Nix. This means that Nix
+  derivations can now call Nix to build other derivations. This is not
+  in a stable state yet and not well
+  [documented](https://github.com/NixOS/nix/commit/c4d7c76b641d82b2696fef73ce0ac160043c18da).
+
+* The new experimental feature `no-url-literals` disables URL
+  literals. This helps to implement [RFC
+  45](https://github.com/NixOS/rfcs/pull/45).
+
+* Nix now uses `libarchive` to decompress and unpack tarballs and zip
+  files, so `tar` is no longer required.
+
+* The priority of substituters can now be overridden using the
+  `priority` substituter setting (e.g. `--substituters
+  'http://cache.nixos.org?priority=100 daemon?priority=10'`).
+
+* `nix edit` now supports non-derivation attributes, e.g. `nix edit
+  .#nixosConfigurations.bla`.
+
+* The `nix` command now provides command line completion for `bash`,
+  `zsh` and `fish`. Since the support for getting completions is built
+  into `nix`, it's easy to add support for other shells.
+
+* The new `--log-format` flag selects what Nix's output looks like. It
+  defaults to a terse progress indicator. There is a new
+  `internal-json` output format for use by other programs.
+
+* `nix eval` has a new `--apply` flag that applies a function to the
+  evaluation result.
+
+* `nix eval` has a new `--write-to` flag that allows it to write a
+  nested attribute set of string leaves to a corresponding directory
+  tree.
+
+* Memory improvements: many operations that add paths to the store or
+  copy paths between stores now run in constant memory.
+
+* Many `nix` commands now support the flag `--derivation` to operate
+  on a `.drv` file itself instead of its outputs.
+
+* There is a new store called `dummy://` that does not support
+  building or adding paths. This is useful if you want to use the Nix
+  evaluator but don't have a Nix store.
+
+* The `ssh-ng://` store now allows substituting paths on the remote,
+  as `ssh://` already did.
+
+* When auto-calling a function with an ellipsis, all arguments are now
+  passed.
+
+* New `nix-shell` features:
+
+  - It preserves the `PS1` environment variable if
+    `NIX_SHELL_PRESERVE_PROMPT` is set.
+
+  - With `-p`, it passes any `--arg`s as Nixpkgs arguments.
+
+  - Support for structured attributes.
+
+* `nix-prefetch-url` has a new `--executable` flag.
+
+* On `x86_64` systems, [`x86_64` microarchitecture
+  levels](https://lwn.net/Articles/844831/) are mapped to additional
+  system types (e.g. `x86_64-v1-linux`).
+
+* The new `--eval-store` flag allows you to use a different store for
+  evaluation than for building or storing the build result. This is
+  primarily useful when you want to query whether something exists in
+  a read-only store, such as a binary cache:
+
+  ```
+  # nix path-info --json --store https://cache.nixos.org \
+    --eval-store auto nixpkgs#hello
+  ```
+
+  (Here `auto` indicates the local store.)
+
+* The Nix daemon has a new low-latency mechanism for copying
+  closures. This is useful when building on remote stores such as
+  `ssh-ng://`.
+
+* Plugins can now register `nix` subcommands.
+
+## Incompatible changes
+
+* The `nix` command is now marked as an experimental feature. This
+  means that you need to add
+
+  ```
+  experimental-features = nix-command
+  ```
+
+  to your `nix.conf` if you want to use it, or pass
+  `--extra-experimental-features nix-command` on the command line.
+
+* The `nix` command no longer has a syntax for referring to packages
+  in a channel. This means that the following no longer works:
+
+  ```console
+  nix build nixpkgs.hello # Nix 2.3
+  ```
+
+  Instead, you can either use the `#` syntax to select a package from
+  a flake, e.g.
+
+  ```console
+  nix build nixpkgs#hello
+  ```
+
+  Or, if you want to use the `nixpkgs` channel in the `NIX_PATH`
+  environment variable:
+
+  ```console
+  nix build -f '<nixpkgs>' hello
+  ```
+
+* The old `nix run` has been renamed to `nix shell`, while there is a
+  new `nix run` that runs a default command. So instead of
+
+  ```console
+  nix run nixpkgs.hello -c hello # Nix 2.3
+  ```
+
+  you should use
+
+  ```console
+  nix shell nixpkgs#hello -c hello
+  ```
+
+  or just
+
+  ```console
+  nix run nixpkgs#hello
+  ```
+
+  if the command you want to run has the same name as the package.
+
+* It is now an error to modify the `plugin-files` setting via a
+  command-line flag that appears after the first non-flag argument to
+  any command, including a subcommand to `nix`. For example,
+  `nix-instantiate default.nix --plugin-files ""` must now become
+  `nix-instantiate --plugin-files "" default.nix`.
+
+* We no longer release source tarballs. If you want to build from
+  source, please build from the tags in the Git repository.
+
+## Contributors
+
+This release has contributions from
+Adam Höse,
+Albert Safin,
+Alex Kovar,
+Alex Zero,
+Alexander Bantyev,
+Alexandre Esteves,
+Alyssa Ross,
+Anatole Lucet,
+Anders Kaseorg,
+Andreas Rammhold,
+Antoine Eiche,
+Antoine Martin,
+Arnout Engelen,
+Arthur Gautier,
+aszlig,
+Ben Burdette,
+Benjamin Hipple,
+Bernardo Meurer,
+Björn Gohla,
+Bjørn Forsman,
+Bob van der Linden,
+Brian Leung,
+Brian McKenna,
+Brian Wignall,
+Bruce Toll,
+Bryan Richter,
+Calle Rosenquist,
+Calvin Loncaric,
+Carlo Nucera,
+Carlos D'Agostino,
+Chaz Schlarp,
+Christian Höppner,
+Christian Kampka,
+Chua Hou,
+Chuck,
+Cole Helbling,
+Daiderd Jordan,
+Dan Callahan,
+Dani,
+Daniel Fitzpatrick,
+Danila Fedorin,
+Daniël de Kok,
+Danny Bautista,
+DavHau,
+David McFarland,
+Dima,
+Domen Kožar,
+Dominik Schrempf,
+Dominique Martinet,
+dramforever,
+Dustin DeWeese,
+edef,
+Eelco Dolstra,
+Emilio Karakey,
+Emily,
+Eric Culp,
+Ersin Akinci,
+Fabian Möller,
+Farid Zakaria,
+Federico Pellegrin,
+Finn Behrens,
+Florian Franzen,
+Félix Baylac-Jacqué,
+Gabriel Gonzalez,
+Geoff Reedy,
+Georges Dubus,
+Graham Christensen,
+Greg Hale,
+Greg Price,
+Gregor Kleen,
+Gregory Hale,
+Griffin Smith,
+Guillaume Bouchard,
+Harald van Dijk,
+illustris,
+Ivan Zvonimir Horvat,
+Jade,
+Jake Waksbaum,
+jakobrs,
+James Ottaway,
+Jan Tojnar,
+Janne Heß,
+Jaroslavas Pocepko,
+Jarrett Keifer,
+Jeremy Schlatter,
+Joachim Breitner,
+Joe Hermaszewski,
+Joe Pea,
+John Ericson,
+Jonathan Ringer,
+Josef Kemetmüller,
+Joseph Lucas,
+Jude Taylor,
+Julian Stecklina,
+Julien Tanguy,
+Jörg Thalheim,
+Kai Wohlfahrt,
+keke,
+Keshav Kini,
+Kevin Quick,
+Kevin Stock,
+Kjetil Orbekk,
+Krzysztof Gogolewski,
+kvtb,
+Lars Mühmel,
+Leonhard Markert,
+Lily Ballard,
+Linus Heckemann,
+Lorenzo Manacorda,
+Lucas Desgouilles,
+Lucas Franceschino,
+Lucas Hoffmann,
+Luke Granger-Brown,
+Madeline Haraj,
+Marwan Aljubeh,
+Mat Marini,
+Mateusz Piotrowski,
+Matthew Bauer,
+Matthew Kenigsberg,
+Mauricio Scheffer,
+Maximilian Bosch,
+Michael Adler,
+Michael Bishop,
+Michael Fellinger,
+Michael Forney,
+Michael Reilly,
+mlatus,
+Mykola Orliuk,
+Nathan van Doorn,
+Naïm Favier,
+ng0,
+Nick Van den Broeck,
+Nicolas Stig124 Formichella,
+Niels Egberts,
+Niklas Hambüchen,
+Nikola Knezevic,
+oxalica,
+p01arst0rm,
+Pamplemousse,
+Patrick Hilhorst,
+Paul Opiyo,
+Pavol Rusnak,
+Peter Kolloch,
+Philipp Bartsch,
+Philipp Middendorf,
+Piotr Szubiakowski,
+Profpatsch,
+Puck Meerburg,
+Ricardo M. Correia,
+Rickard Nilsson,
+Robert Hensing,
+Robin Gloster,
+Rodrigo,
+Rok Garbas,
+Ronnie Ebrin,
+Rovanion Luckey,
+Ryan Burns,
+Ryan Mulligan,
+Ryne Everett,
+Sam Doshi,
+Sam Lidder,
+Samir Talwar,
+Samuel Dionne-Riel,
+Sebastian Ullrich,
+Sergei Trofimovich,
+Sevan Janiyan,
+Shao Cheng,
+Shea Levy,
+Silvan Mosberger,
+Stefan Frijters,
+Stefan Jaax,
+sternenseemann,
+Steven Shaw,
+Stéphan Kochen,
+SuperSandro2000,
+Suraj Barkale,
+Taeer Bar-Yam,
+Thomas Churchman,
+Théophane Hufschmitt,
+Timothy DeHerrera,
+Timothy Klim,
+Tobias Möst,
+Tobias Pflug,
+Tom Bereknyei,
+Travis A. Everett,
+Ujjwal Jain,
+Vladimír Čunát,
+Wil Taylor,
+Will Dietz,
+Yaroslav Bolyukin,
+Yestin L. Harrison,
+YI,
+Yorick van Pelt,
+Yuriy Taraday and
+zimbatm.

doc/manual/src/release-notes/rl-next.md

@@ -0,0 +1,5 @@
+# Release 2.5 (2021-XX-XX)
+
+* Binary cache stores now have a setting `compression-level`.
+
+* `nix develop` now has a flag `--unpack` to run `unpackPhase`.

flake.lock

@@ -3,27 +3,26 @@
     "lowdown-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1617481909,
-        "narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=",
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
         "owner": "kristapsdz",
         "repo": "lowdown",
-        "rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
         "type": "github"
       },
       "original": {
         "owner": "kristapsdz",
-        "ref": "VERSION_0_8_4",
         "repo": "lowdown",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1624862269,
-        "narHash": "sha256-JFcsh2+7QtfKdJFoPibLFPLgIW6Ycnv8Bts9a7RYme0=",
+        "lastModified": 1632864508,
+        "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f77036342e2b690c61c97202bf48f2ce13acc022",
+        "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
         "type": "github"
       },
       "original": {

flake.nix

@@ -2,7 +2,7 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small";
-  inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; };
+  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
   outputs = { self, nixpkgs, lowdown-src }:
 
@@ -61,6 +61,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
+            "--with-boost=${boost}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
             "LDFLAGS=-fuse-ld=gold"
           ];
@@ -70,7 +71,7 @@
           [
             buildPackages.bison
             buildPackages.flex
-            (lib.getBin buildPackages.lowdown)
+            (lib.getBin buildPackages.lowdown-nix)
             buildPackages.mdbook
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
@@ -78,7 +79,7 @@
 
             # Tests
             buildPackages.git
-            buildPackages.mercurial
+            buildPackages.mercurial # FIXME: remove? only needed for tests
             buildPackages.jq
           ]
           ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
@@ -89,7 +90,7 @@
             openssl sqlite
             libarchive
             boost
-            lowdown
+            lowdown-nix
             gmock
           ]
           ++ lib.optionals stdenv.isLinux [libseccomp]
@@ -126,8 +127,7 @@
           ''
             mkdir -p $out/nix-support
 
-            # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
-            # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
+            # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
             tarballPath() {
               # Remove the store prefix
               local path=''${1#${builtins.storeDir}/}
@@ -153,13 +153,15 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
-      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
+      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
-        # Must keep this name short as OSX has a rather strict limit on the
-        # socket path length, and this name appears in the path of the
-        # nix-daemon socket used in the tests
-        name = "nix-tests";
+        name =
+          "nix-tests"
+          + optionalString
+            (versionAtLeast daemon.version "2.4pre20211005" &&
+             versionAtLeast client.version "2.4pre20211005")
+            "-${client.version}-against-${daemon.version}";
         inherit version;
 
         src = self;
@@ -178,8 +180,8 @@
         installPhase = ''
           mkdir -p $out
         '';
-        installCheckPhase = "make installcheck";
 
+        installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES";
       };
 
       binaryTarball = buildPackages: nix: pkgs: let
@@ -259,11 +261,11 @@
       # 'nix.perl-bindings' packages.
       overlay = final: prev: {
 
-        # An older version of Nix to test against when using the daemon.
-        # Currently using `nixUnstable` as the stable one doesn't respect
-        # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests.
         nixStable = prev.nix;
 
+        # Forward from the previous stage as we don’t want it to pick the lowdown override
+        nixUnstable = prev.nixUnstable;
+
         nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
@@ -349,15 +351,8 @@
 
         };
 
-        lowdown = with final; stdenv.mkDerivation rec {
-          name = "lowdown-0.8.4";
-
-          /*
-          src = fetchurl {
-            url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz";
-            hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw==";
-          };
-          */
+        lowdown-nix = with final; stdenv.mkDerivation rec {
+          name = "lowdown-0.9.0";
 
           src = lowdown-src;
 
@@ -450,6 +445,12 @@
           inherit (self) overlay;
         };
 
+        tests.nssPreload = (import ./tests/nss-preload.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
         tests.githubFlakes = (import ./tests/github-flakes.nix rec {
           system = "x86_64-linux";
           inherit nixpkgs;
@@ -488,24 +489,27 @@
             '';
         */
 
+        installTests = forAllSystems (system:
+          let pkgs = nixpkgsFor.${system}; in
+          pkgs.runCommand "install-tests" {
+            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
+            againstCurrentUnstable =
+              # FIXME: temporarily disable this on macOS because of #3605.
+              if system == "x86_64-linux"
+              then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
+              else null;
+            # Disabled because the latest stable version doesn't handle
+            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
+            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
+          } "touch $out");
+
       };
 
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
-        installTests =
-          let pkgs = nixpkgsFor.${system}; in
-          pkgs.runCommand "install-tests" {
-            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
-            againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
-            # Disabled because the latest stable version doesn't handle
-            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
-            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
-          } "touch $out";
-      } // (if system == "x86_64-linux" then (builtins.listToAttrs (map (crossSystem: {
-        name = "binaryTarball-${crossSystem}";
-        value = self.hydraJobs.binaryTarballCross.${system}.${crossSystem};
-      }) crossSystems)) else {}));
+        installTests = self.hydraJobs.installTests.${system};
+      });
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;

maintainers/upload-release.pl

@@ -19,6 +19,8 @@ my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
 my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
 
+my $isLatest = ($ENV{'IS_LATEST'} // "") eq "1";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -35,16 +37,18 @@ sub fetch {
 my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
 my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
 #print Dumper($evalInfo);
+my $flakeUrl = $evalInfo->{flake} or die;
+my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die);
+my $nixRev = $flakeInfo->{revision} or die;
 
-my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die;
+my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
+#print Dumper($buildInfo);
 
-my $tarballInfo = decode_json(fetch("$evalUrl/job/tarball", 'application/json'));
-
-my $releaseName = $tarballInfo->{releasename};
+my $releaseName = $buildInfo->{nixname};
 $releaseName =~ /nix-(.*)$/ or die;
 my $version = $1;
 
-print STDERR "Nix revision is $nixRev, version is $version\n";
+print STDERR "Flake URL is $flakeUrl, Nix revision is $nixRev, version is $version\n";
 
 my $releaseDir = "nix/$releaseName";
 
@@ -83,12 +87,12 @@ sub downloadFile {
 
     if (!-e $tmpFile) {
         print STDERR "downloading $srcFile to $tmpFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0
+        system("NIX_REMOTE=https://cache.nixos.org/ nix store cat '$srcFile' > '$tmpFile'") == 0
             or die "unable to fetch $srcFile\n";
     }
 
     my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
-    my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`;
+    my $sha256_actual = `nix hash file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
     if ($sha256_expected ne $sha256_actual) {
         print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
@@ -104,8 +108,6 @@ sub downloadFile {
     return $sha256_expected;
 }
 
-downloadFile("tarball", "2"); # .tar.bz2
-my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
@@ -134,42 +136,38 @@ for my $fn (glob "$tmpDir/*") {
     }
 }
 
-exit if $version =~ /pre/;
-
 # Update nix-fallback-paths.nix.
-system("cd $nixpkgsDir && git pull") == 0 or die;
+if ($isLatest) {
+    system("cd $nixpkgsDir && git pull") == 0 or die;
 
-sub getStorePath {
-    my ($jobName) = @_;
-    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    for my $product (values %{$buildInfo->{buildproducts}}) {
-        next unless $product->{type} eq "nix-build";
-        next if $product->{path} =~ /[a-z]+$/;
-        return $product->{path};
+    sub getStorePath {
+        my ($jobName) = @_;
+        my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+        return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
     }
-    die;
+
+    write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
+               "{\n" .
+               "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
+               "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
+               "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+               "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+               "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
+               "}\n");
+
+    system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
 }
 
-write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
-           "{\n" .
-           "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
-           "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
-           "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
-           "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
-           "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
-           "}\n");
-
-system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
-
 # Update the "latest" symlink.
 $channelsBucket->add_key(
     "nix-latest/install", "",
     { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
-    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr
+    if $isLatest;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
 system("git push --tags") == 0 or die;
-system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;

mk/libraries.mk

@@ -125,8 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
-	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
+	$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/tests.mk

@@ -13,3 +13,7 @@ define run-install-test
 endef
 
 .PHONY: check installcheck
+
+print-top-help += \
+  echo "  check: Run unit tests"; \
+  echo "  installcheck: Run functional tests";

perl/lib/Nix/Store.pm

@@ -22,6 +22,7 @@ our @EXPORT = qw(
     derivationFromPath
     addTempRoot
     getBinDir getStoreDir
+    queryRawRealisation
 );
 
 our $VERSION = '0.15';

perl/lib/Nix/Store.xs

@@ -15,6 +15,7 @@
 #include "crypto.hh"
 
 #include <sodium.h>
+#include <nlohmann/json.hpp>
 
 
 using namespace nix;
@@ -120,6 +121,18 @@ SV * queryPathInfo(char * path, int base32)
             croak("%s", e.what());
         }
 
+SV * queryRawRealisation(char * outputId)
+    PPCODE:
+      try {
+        auto realisation = store()->queryRealisation(DrvOutput::parse(outputId));
+        if (realisation)
+            XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0)));
+        else
+            XPUSHs(sv_2mortal(newSVpv("", 0)));
+      } catch (Error & e) {
+        croak("%s", e.what());
+      }
+
 
 SV * queryPathFromHashPart(char * hashPart)
     PPCODE:

scripts/create-darwin-volume.sh

@@ -715,7 +715,8 @@ create_volume() {
     # 6) getting special w/ awk may be fragile, but doing it to:
     #    - save time over running slow diskutil commands
     #    - skirt risk we grab wrong volume if multiple match
-    /usr/sbin/diskutil apfs addVolume "$NIX_VOLUME_USE_DISK" "$NIX_VOLUME_FS" "$NIX_VOLUME_LABEL" -nomount | /usr/bin/awk '/Created new APFS Volume/ {print $5}'
+    _sudo "to create a new APFS volume '$NIX_VOLUME_LABEL' on $NIX_VOLUME_USE_DISK" \
+        /usr/sbin/diskutil apfs addVolume "$NIX_VOLUME_USE_DISK" "$NIX_VOLUME_FS" "$NIX_VOLUME_LABEL" -nomount | /usr/bin/awk '/Created new APFS Volume/ {print $5}'
 }
 
 volume_uuid_from_special() {
@@ -738,7 +739,6 @@ await_volume() {
 setup_volume() {
     local use_special use_uuid profile_packages
     task "Creating a Nix volume" >&2
-    # DOING: I'm tempted to wrap this call in a grep to get the new disk special without doing anything too complex, but this sudo wrapper *is* a little complex, so it'll be a PITA unless maybe we can skip sudo on this. Let's just try it without.
 
     use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}"
 
@@ -759,6 +759,11 @@ setup_volume() {
 
     await_volume
 
+    if [ "$(/usr/sbin/diskutil info -plist "$NIX_ROOT" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        _sudo "to set enableOwnership (enabling users to own files)" \
+            /usr/sbin/diskutil enableOwnership "$NIX_ROOT"
+    fi
+
     # TODO: below is a vague kludge for now; I just don't know
     # what if any safe action there is to take here. Also, the
     # reminder isn't very helpful.

scripts/install-darwin-multi-user.sh

@@ -13,11 +13,22 @@ NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 read_only_root() {
     # this touch command ~should~ always produce an error
     # as of this change I confirmed /usr/bin/touch emits:
+    # "touch: /: Operation not permitted" Monterey
     # "touch: /: Read-only file system" Catalina+ and Big Sur
     # "touch: /: Permission denied" Mojave
     # (not matching prefix for compat w/ coreutils touch in case using
     # an explicit path causes problems; its prefix differs)
-    [[ "$(/usr/bin/touch / 2>&1)" = *"Read-only file system" ]]
+    case "$(/usr/bin/touch / 2>&1)" in
+        *"Read-only file system") # Catalina, Big Sur
+            return 0
+            ;;
+        *"Operation not permitted") # Monterey
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
 
     # Avoiding the slow semantic way to get this information (~330ms vs ~8ms)
     # unless using touch causes problems. Just in case, that approach is:
@@ -67,7 +78,7 @@ poly_service_installed_check() {
 poly_service_uninstall_directions() {
     echo "$1. Remove macOS-specific components:"
     if should_create_volume && test_nix_volume_mountd_installed; then
-        darwin_volume_uninstall_directions
+        nix_volume_mountd_uninstall_directions
     fi
     if test_nix_daemon_installed; then
         nix_daemon_uninstall_directions
@@ -206,4 +217,8 @@ poly_prepare_to_install() {
 EOF
         setup_darwin_volume
     fi
+
+    if [ "$(diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        failure "This script needs a /nix volume with global permissions! This may require running sudo diskutil enableOwnership /nix."
+    fi
 }

scripts/install-multi-user.sh

@@ -33,7 +33,7 @@ NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 readonly NIX_ROOT="/nix"
 readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv" "/etc/bash.bashrc" "/etc/zsh/zshenv")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/etc/bash.bashrc" "/etc/zsh/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -599,7 +599,7 @@ manager. This will happen in a few stages:
 1. Make sure your computer doesn't already have Nix. If it does, I
    will show you instructions on how to clean up your old install.
 
-2. Show you what we are going to install and where. Then we will ask
+2. Show you what I am going to install and where. Then I will ask
    if you are ready to continue.
 
 3. Create the system users and groups that the Nix daemon uses to run
@@ -614,14 +614,14 @@ manager. This will happen in a few stages:
 
 EOF
 
-    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
+    if ui_confirm "Would you like to see a more detailed list of what I will do?"; then
         cat <<EOF
 
-We will:
+I will:
 
  - make sure your computer doesn't already have Nix files
    (if it does, I will tell you how to clean them up.)
- - create local users (see the list above for the users we'll make)
+ - create local users (see the list above for the users I'll make)
  - create a local group ($NIX_BUILD_GROUP_NAME)
  - install Nix in to $NIX_ROOT
  - create a configuration file in /etc/nix
@@ -656,7 +656,7 @@ run in a headless fashion, like this:
 
   $ curl -L https://nixos.org/nix/install | sh
 
-or maybe in a CI pipeline. Because of that, we're going to skip the
+or maybe in a CI pipeline. Because of that, I'm going to skip the
 verbose output in the interest of brevity.
 
 If you would like to
@@ -670,7 +670,7 @@ EOF
     fi
 
     cat <<EOF
-This script is going to call sudo a lot. Every time we do, it'll
+This script is going to call sudo a lot. Every time I do, it'll
 output exactly what it'll do, and why.
 
 Just like this:
@@ -682,15 +682,15 @@ EOF
     cat <<EOF
 
 This might look scary, but everything can be undone by running just a
-few commands. We used to ask you to confirm each time sudo ran, but it
+few commands. I used to ask you to confirm each time sudo ran, but it
 was too many times. Instead, I'll just ask you this one time:
 
 EOF
-    if ui_confirm "Can we use sudo?"; then
+    if ui_confirm "Can I use sudo?"; then
         ok "Yay! Thanks! Let's get going!"
     else
         failure <<EOF
-That is okay, but we can't install.
+That is okay, but I can't install.
 EOF
     fi
 }
@@ -701,7 +701,10 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/"
+              cp -RLp ./store/* "$NIX_ROOT/store/"
+
+        _sudo "to make the new store non-writable at $NIX_ROOT/store" \
+              chmod -R ugo-w "$NIX_ROOT/store/"
 
         if [ -d "$NIX_INSTALLED_NIX" ]; then
             echo "      Alright! We have our first nix at $NIX_INSTALLED_NIX"
@@ -808,8 +811,8 @@ main() {
     # pre-Catalina macOS if run as root user.
     if [ $EUID -eq 0 ]; then
         failure <<EOF
-Please do not run this script with root privileges. We will call sudo
-when we need to.
+Please do not run this script with root privileges. I will call sudo
+when I need to.
 EOF
     fi
 

scripts/install-nix-from-closure.sh

@@ -215,7 +215,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break
@@ -226,7 +226,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break

scripts/install.in

@@ -76,14 +76,21 @@ fi
 
 tarball=$tmpDir/nix-@nixVersion@-$system.tar.xz
 
-require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
 if [ "$(uname -s)" != "Darwin" ]; then
     require_util xz "unpack the binary tarball"
 fi
 
+if command -v wget > /dev/null 2>&1; then
+    fetch() { wget "$1" -O "$2"; }
+elif command -v curl > /dev/null 2>&1; then
+    fetch() { curl -L "$1" -o "$2"; }
+else
+    oops "you don't have wget or curl installed, which I need to download the binary tarball"
+fi
+
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
-curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
+fetch "$url" "$tarball" || oops "failed to download '$url'"
 
 if command -v sha256sum > /dev/null 2>&1; then
     hash2="$(sha256sum -b "$tarball" | cut -c1-64)"

src/build-remote/build-remote.cc

@@ -18,6 +18,7 @@
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"
+#include "experimental-features.hh"
 
 using namespace nix;
 using std::cin;
@@ -130,11 +131,14 @@ static int main_build_remote(int argc, char * * argv)
                 for (auto & m : machines) {
                     debug("considering building on remote machine '%s'", m.storeUri);
 
-                    if (m.enabled && std::find(m.systemTypes.begin(),
-                            m.systemTypes.end(),
-                            neededSystem) != m.systemTypes.end() &&
+                    if (m.enabled
+                        && (neededSystem == "builtin"
+                            || std::find(m.systemTypes.begin(),
+                                m.systemTypes.end(),
+                                neededSystem) != m.systemTypes.end()) &&
                         m.allSupported(requiredFeatures) &&
-                        m.mandatoryMet(requiredFeatures)) {
+                        m.mandatoryMet(requiredFeatures))
+                    {
                         rightType = true;
                         AutoCloseFD free;
                         uint64_t load = 0;
@@ -295,7 +299,7 @@ connected:
 
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
             for (auto & outputName : wantedOutputs) {
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
@@ -327,7 +331,7 @@ connected:
         for (auto & realisation : missingRealisations) {
             // Should hold, because if the feature isn't enabled the set
             // of missing realisations should be empty
-            settings.requireExperimentalFeature("ca-derivations");
+            settings.requireExperimentalFeature(Xp::CaDerivations);
             store->registerDrvOutput(realisation);
         }
 

src/libcmd/command.cc

@@ -120,7 +120,7 @@ void BuiltPathsCommand::run(ref<Store> store)
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations
             StorePathSet pathsRoots, pathsClosure;
-            for (auto & root: paths) {
+            for (auto & root : paths) {
                 auto rootFromThis = root.outPaths();
                 pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
             }
@@ -138,17 +138,20 @@ StorePathsCommand::StorePathsCommand(bool recursive)
 {
 }
 
-void StorePathsCommand::run(ref<Store> store, BuiltPaths paths)
+void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
 {
-    StorePaths storePaths;
-    for (auto& builtPath : paths)
-        for (auto& p : builtPath.outPaths())
-            storePaths.push_back(p);
+    StorePathSet storePaths;
+    for (auto & builtPath : paths)
+        for (auto & p : builtPath.outPaths())
+            storePaths.insert(p);
 
-    run(store, std::move(storePaths));
+    auto sorted = store->topoSortPaths(storePaths);
+    std::reverse(sorted.begin(), sorted.end());
+
+    run(store, std::move(sorted));
 }
 
-void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
 {
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
@@ -200,10 +203,10 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
-            [&](BuiltPath::Opaque bo) {
+            [&](const BuiltPath::Opaque & bo) {
                 result.push_back(bo.path);
             },
-            [&](BuiltPath::Built bfd) {
+            [&](const BuiltPath::Built & bfd) {
                 for (auto & output : bfd.outputs) {
                     result.push_back(output.second);
                 }

src/libcmd/command.hh

@@ -108,6 +108,8 @@ enum class Realise {
        exists. */
     Derivation,
     /* Evaluate in dry-run mode. Postcondition: nothing. */
+    // FIXME: currently unused, but could be revived if we can
+    // evaluate derivations in-memory.
     Nothing
 };
 
@@ -167,7 +169,7 @@ public:
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, BuiltPaths paths) = 0;
+    virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
 
     void run(ref<Store> store) override;
 
@@ -180,9 +182,9 @@ struct StorePathsCommand : public BuiltPathsCommand
 
     using BuiltPathsCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+    virtual void run(ref<Store> store, std::vector<StorePath> && storePaths) = 0;
 
-    void run(ref<Store> store, BuiltPaths paths) override;
+    void run(ref<Store> store, BuiltPaths && paths) override;
 };
 
 /* A command that operates on exactly one store path. */
@@ -192,7 +194,7 @@ struct StorePathCommand : public StorePathsCommand
 
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store, std::vector<StorePath> storePaths) override;
+    void run(ref<Store> store, std::vector<StorePath> && storePaths) override;
 };
 
 /* A helper class for registering commands globally. */
@@ -223,15 +225,18 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store, Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode = bmNormal);
 
 std::set<StorePath> toStorePaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
+    const std::vector<std::shared_ptr<Installable>> & installables);
 
 StorePath toStorePath(
     ref<Store> evalStore,
@@ -240,8 +245,9 @@ StorePath toStorePath(
     OperateOn operateOn,
     std::shared_ptr<Installable> installable);
 
-std::set<StorePath> toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables,
+std::set<StorePath> toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
     bool useDeriver = false);
 
 BuiltPaths toBuiltPaths(
@@ -249,7 +255,7 @@ BuiltPaths toBuiltPaths(
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
+    const std::vector<std::shared_ptr<Installable>> & installables);
 
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */

src/libcmd/installables.cc

@@ -654,6 +654,17 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         for (auto & s : ss) {
             std::exception_ptr ex;
 
+            if (s.find('/') != std::string::npos) {
+                try {
+                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
+                    continue;
+                } catch (BadStorePath &) {
+                } catch (...) {
+                    if (!ex)
+                        ex = std::current_exception();
+                }
+            }
+
             try {
                 auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
@@ -668,25 +679,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
                 ex = std::current_exception();
             }
 
-            if (s.find('/') != std::string::npos) {
-                try {
-                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
-                    continue;
-                } catch (BadStorePath &) {
-                } catch (...) {
-                    if (!ex)
-                        ex = std::current_exception();
-                }
-            }
-
             std::rethrow_exception(ex);
-
-            /*
-            throw Error(
-                pathExists(s)
-                ? "path '%s' is not a flake or a store path"
-                : "don't know how to handle argument '%s'", s);
-            */
         }
     }
 
@@ -704,13 +697,13 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
 BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
 {
     BuiltPaths res;
-    for (auto & b : hopefullyBuiltPaths)
+    for (const auto & b : hopefullyBuiltPaths)
         std::visit(
             overloaded{
-                [&](DerivedPath::Opaque bo) {
+                [&](const DerivedPath::Opaque & bo) {
                     res.push_back(BuiltPath::Opaque{bo.path});
                 },
-                [&](DerivedPath::Built bfd) {
+                [&](const DerivedPath::Built & bfd) {
                     OutputPathMap outputs;
                     auto drv = evalStore->readDerivation(bfd.drvPath);
                     auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
@@ -721,7 +714,7 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
                                 "the derivation '%s' doesn't have an output named '%s'",
                                 store->printStorePath(bfd.drvPath), output);
                         if (settings.isExperimentalFeatureEnabled(
-                                "ca-derivations")) {
+                                Xp::CaDerivations)) {
                             auto outputId =
                                 DrvOutput{outputHashes.at(output), output};
                             auto realisation =
@@ -729,7 +722,7 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
                             if (!realisation)
                                 throw Error(
                                     "cannot operate on an output of unbuilt "
-                                    "content-addresed derivation '%s'",
+                                    "content-addressed derivation '%s'",
                                     outputId.to_string());
                             outputs.insert_or_assign(
                                 output, realisation->outPath);
@@ -750,8 +743,12 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
     return res;
 }
 
-BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode)
 {
     if (mode == Realise::Nothing)
         settings.readOnlyMode = true;
@@ -763,7 +760,7 @@ BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
         pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
     }
 
-    if (mode == Realise::Nothing)
+    if (mode == Realise::Nothing || mode == Realise::Derivation)
         printMissing(store, pathsToBuild, lvlError);
     else if (mode == Realise::Outputs)
         store->buildPaths(pathsToBuild, bMode, evalStore);
@@ -776,7 +773,7 @@ BuiltPaths toBuiltPaths(
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
     if (operateOn == OperateOn::Output)
         return build(evalStore, store, mode, installables);
@@ -795,7 +792,7 @@ StorePathSet toStorePaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
     StorePathSet outPaths;
     for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
@@ -819,15 +816,17 @@ StorePath toStorePath(
     return *paths.begin();
 }
 
-StorePathSet toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables, bool useDeriver)
+StorePathSet toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    bool useDeriver)
 {
     StorePathSet drvPaths;
 
-    for (auto & i : installables)
-        for (auto & b : i->toDerivedPaths())
+    for (const auto & i : installables)
+        for (const auto & b : i->toDerivedPaths())
             std::visit(overloaded {
-                [&](DerivedPath::Opaque bo) {
+                [&](const DerivedPath::Opaque & bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
                     auto derivers = store->queryValidDerivers(bo.path);
@@ -836,7 +835,7 @@ StorePathSet toDerivations(ref<Store> store,
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
                 },
-                [&](DerivedPath::Built bfd) {
+                [&](const DerivedPath::Built & bfd) {
                     drvPaths.insert(bfd.drvPath);
                 },
             }, b.raw());

src/libcmd/local.mk

@@ -8,8 +8,8 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS = -llowdown
+libcmd_LDFLAGS += -llowdown -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))

src/libcmd/markdown.cc

@@ -12,7 +12,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     struct lowdown_opts opts {
         .type = LOWDOWN_TERM,
         .maxdepth = 20,
-        .cols = std::min(getWindowSize().second, (unsigned short) 80),
+        .cols = std::max(getWindowSize().second, (unsigned short) 80),
         .hmargin = 0,
         .vmargin = 0,
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
@@ -25,7 +25,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     Finally freeDoc([&]() { lowdown_doc_free(doc); });
 
     size_t maxn = 0;
-    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size(), nullptr);
     if (!node)
         throw Error("cannot parse Markdown document");
     Finally freeNode([&]() { lowdown_node_free(node); });
@@ -40,11 +40,11 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot allocate Markdown output buffer");
     Finally freeBuffer([&]() { lowdown_buf_free(buf); });
 
-    int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
+    int rndr_res = lowdown_term_rndr(buf, renderer, node);
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return std::string(buf->data, buf->size);
+    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
 }
 
 }

src/libexpr/attr-set.hh

@@ -17,8 +17,8 @@ struct Attr
 {
     Symbol name;
     Value * value;
-    Pos * pos;
-    Attr(Symbol name, Value * value, Pos * pos = &noPos)
+    ptr<Pos> pos;
+    Attr(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos))
         : name(name), value(value), pos(pos) { };
     Attr() : pos(&noPos) { };
     bool operator < (const Attr & a) const
@@ -35,13 +35,13 @@ class Bindings
 {
 public:
     typedef uint32_t size_t;
-    Pos *pos;
+    ptr<Pos> pos;
 
 private:
     size_t size_, capacity_;
     Attr attrs[0];
 
-    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
+    Bindings(size_t capacity) : pos(&noPos), size_(0), capacity_(capacity) { }
     Bindings(const Bindings & bindings) = delete;
 
 public:

src/libexpr/eval.cc

@@ -127,6 +127,7 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
         break;
     case tThunk:
     case tApp:
+    case tPartialApp:
         str << "<CODE>";
         break;
     case tLambda:
@@ -445,12 +446,12 @@ EvalState::EvalState(
                     StorePathSet closure;
                     store->computeFSClosure(store->toStorePath(r.second).first, closure);
                     for (auto & path : closure)
-                        allowedPaths->insert(store->printStorePath(path));
+                        allowPath(path);
                 } catch (InvalidPath &) {
-                    allowedPaths->insert(r.second);
+                    allowPath(r.second);
                 }
             } else
-                allowedPaths->insert(r.second);
+                allowPath(r.second);
         }
     }
 
@@ -465,6 +466,35 @@ EvalState::~EvalState()
 }
 
 
+void EvalState::requireExperimentalFeatureOnEvaluation(
+    const ExperimentalFeature & feature,
+    const std::string_view fName,
+    const Pos & pos)
+{
+    if (!settings.isExperimentalFeatureEnabled(feature)) {
+        throw EvalError({
+            .msg = hintfmt(
+                "Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.",
+                feature,
+                fName
+            ),
+            .errPos = pos
+        });
+    }
+}
+
+void EvalState::allowPath(const Path & path)
+{
+    if (allowedPaths)
+        allowedPaths->insert(path);
+}
+
+void EvalState::allowPath(const StorePath & storePath)
+{
+    if (allowedPaths)
+        allowedPaths->insert(store->toRealPath(storePath));
+}
+
 Path EvalState::checkSourcePath(const Path & path_)
 {
     if (!allowedPaths) return path_;
@@ -554,14 +584,20 @@ Value * EvalState::addConstant(const string & name, Value & v)
 {
     Value * v2 = allocValue();
     *v2 = v;
-    staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
-    baseEnv.values[baseEnvDispl++] = v2;
-    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2));
+    addConstant(name, v2);
     return v2;
 }
 
 
+void EvalState::addConstant(const string & name, Value * v)
+{
+    staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl);
+    baseEnv.values[baseEnvDispl++] = v;
+    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
+}
+
+
 Value * EvalState::addPrimOp(const string & name,
     size_t arity, PrimOpFun primOp)
 {
@@ -580,7 +616,7 @@ Value * EvalState::addPrimOp(const string & name,
 
     Value * v = allocValue();
     v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym });
-    staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
+    staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl);
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
     return v;
@@ -606,7 +642,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
 
     Value * v = allocValue();
     v->mkPrimOp(new PrimOp(std::move(primOp)));
-    staticBaseEnv.vars[envName] = baseEnvDispl;
+    staticBaseEnv.vars.emplace_back(envName, baseEnvDispl);
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
     return v;
@@ -756,7 +792,7 @@ void mkPath(Value & v, const char * s)
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
-    for (size_t l = var.level; l; --l, env = env->up) ;
+    for (auto l = var.level; l; --l, env = env->up) ;
 
     if (!var.fromWith) return env->values[var.displ];
 
@@ -770,7 +806,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
         }
         Bindings::iterator j = env->values[0]->attrs->find(var.name);
         if (j != env->values[0]->attrs->end()) {
-            if (countCalls && j->pos) attrSelects[*j->pos]++;
+            if (countCalls) attrSelects[*j->pos]++;
             return j->value;
         }
         if (!env->prevWith)
@@ -825,9 +861,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 }
 
 
-void EvalState::mkPos(Value & v, Pos * pos)
+void EvalState::mkPos(Value & v, ptr<Pos> pos)
 {
-    if (pos && pos->file.set()) {
+    if (pos->file.set()) {
         mkAttrs(v, 3);
         mkString(*allocAttr(v, sFile), pos->file);
         mkInt(*allocAttr(v, sLine), pos->line);
@@ -895,23 +931,41 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
         return;
     }
 
-    Path path2 = resolveExprPath(path);
-    if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) {
+    Path resolvedPath = resolveExprPath(path);
+    if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) {
         v = i->second;
         return;
     }
 
-    printTalkative("evaluating file '%1%'", path2);
+    printTalkative("evaluating file '%1%'", resolvedPath);
     Expr * e = nullptr;
 
-    auto j = fileParseCache.find(path2);
+    auto j = fileParseCache.find(resolvedPath);
     if (j != fileParseCache.end())
         e = j->second;
 
     if (!e)
-        e = parseExprFromFile(checkSourcePath(path2));
+        e = parseExprFromFile(checkSourcePath(resolvedPath));
 
-    fileParseCache[path2] = e;
+    cacheFile(path, resolvedPath, e, v, mustBeTrivial);
+}
+
+
+void EvalState::resetFileCache()
+{
+    fileEvalCache.clear();
+    fileParseCache.clear();
+}
+
+
+void EvalState::cacheFile(
+    const Path & path,
+    const Path & resolvedPath,
+    Expr * e,
+    Value & v,
+    bool mustBeTrivial)
+{
+    fileParseCache[resolvedPath] = e;
 
     try {
         // Enforce that 'flake.nix' is a direct attrset, not a
@@ -921,19 +975,12 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
             throw EvalError("file '%s' must be an attribute set", path);
         eval(e, v);
     } catch (Error & e) {
-        addErrorTrace(e, "while evaluating the file '%1%':", path2);
+        addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath);
         throw;
     }
 
-    fileEvalCache[path2] = v;
-    if (path != path2) fileEvalCache[path] = v;
-}
-
-
-void EvalState::resetFileCache()
-{
-    fileEvalCache.clear();
-    fileParseCache.clear();
+    fileEvalCache[resolvedPath] = v;
+    if (path != resolvedPath) fileEvalCache[path] = v;
 }
 
 
@@ -1018,7 +1065,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
-        size_t displ = 0;
+        Displacement displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && !i.second.inherited) {
@@ -1027,7 +1074,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
             } else
                 vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
             env2.values[displ++] = vAttr;
-            v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos));
+            v.attrs->push_back(Attr(i.first, vAttr, ptr(&i.second.pos)));
         }
 
         /* If the rec contains an attribute called `__overrides', then
@@ -1059,7 +1106,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 
     else
         for (auto & i : attrs)
-            v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos));
+            v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), ptr(&i.second.pos)));
 
     /* Dynamic attrs apply *after* rec and __overrides. */
     for (auto & i : dynamicAttrs) {
@@ -1076,11 +1123,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 
         i.valueExpr->setName(nameSym);
         /* Keep sorted order so find can catch duplicates */
-        v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos));
+        v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), ptr(&i.pos)));
         v.attrs->sort(); // FIXME: inefficient
     }
 
-    v.attrs->pos = &pos;
+    v.attrs->pos = ptr(&pos);
 }
 
 
@@ -1094,7 +1141,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
-    size_t displ = 0;
+    Displacement displ = 0;
     for (auto & i : attrs->attrs)
         env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
 
@@ -1138,7 +1185,7 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa
 void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 {
     Value vTmp;
-    Pos * pos2 = 0;
+    ptr<Pos> pos2(&noPos);
     Value * vAttrs = &vTmp;
 
     e->eval(state, env, vTmp);
@@ -1164,13 +1211,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
             }
             vAttrs = j->value;
             pos2 = j->pos;
-            if (state.countCalls && pos2) state.attrSelects[*pos2]++;
+            if (state.countCalls) state.attrSelects[*pos2]++;
         }
 
-        state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) );
+        state.forceValue(*vAttrs, (*pos2 != noPos ? *pos2 : this->pos ) );
 
     } catch (Error & e) {
-        if (pos2 && pos2->file != state.sDerivationNix)
+        if (*pos2 != noPos && pos2->file != state.sDerivationNix)
             addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
                 showAttrPath(state, env, attrPath));
         throw;
@@ -1211,144 +1258,236 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-void ExprApp::eval(EvalState & state, Env & env, Value & v)
-{
-    /* FIXME: vFun prevents GCC from doing tail call optimisation. */
-    Value vFun;
-    e1->eval(state, env, vFun);
-    state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos);
-}
-
-
-void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
-{
-    /* Figure out the number of arguments still needed. */
-    size_t argsDone = 0;
-    Value * primOp = &fun;
-    while (primOp->isPrimOpApp()) {
-        argsDone++;
-        primOp = primOp->primOpApp.left;
-    }
-    assert(primOp->isPrimOp());
-    auto arity = primOp->primOp->arity;
-    auto argsLeft = arity - argsDone;
-
-    if (argsLeft == 1) {
-        /* We have all the arguments, so call the primop. */
-
-        /* Put all the arguments in an array. */
-        Value * vArgs[arity];
-        auto n = arity - 1;
-        vArgs[n--] = &arg;
-        for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left)
-            vArgs[n--] = arg->primOpApp.right;
-
-        /* And call the primop. */
-        nrPrimOpCalls++;
-        if (countCalls) primOpCalls[primOp->primOp->name]++;
-        primOp->primOp->fun(*this, pos, vArgs, v);
-    } else {
-        Value * fun2 = allocValue();
-        *fun2 = fun;
-        v.mkPrimOpApp(fun2, &arg);
-    }
-}
-
-void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
+void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos)
 {
     auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr;
 
     forceValue(fun, pos);
 
-    if (fun.isPrimOp() || fun.isPrimOpApp()) {
-        callPrimOp(fun, arg, v, pos);
-        return;
-    }
+    Value vCur(fun);
 
-    if (fun.type() == nAttrs) {
-      auto found = fun.attrs->find(sFunctor);
-      if (found != fun.attrs->end()) {
-        /* fun may be allocated on the stack of the calling function,
-         * but for functors we may keep a reference, so heap-allocate
-         * a copy and use that instead.
-         */
-        auto & fun2 = *allocValue();
-        fun2 = fun;
-        /* !!! Should we use the attr pos here? */
-        Value v2;
-        callFunction(*found->value, fun2, v2, pos);
-        return callFunction(v2, arg, v, pos);
-      }
-    }
+    auto makeAppChain = [&]()
+    {
+        vRes = vCur;
+        for (size_t i = 0; i < nrArgs; ++i) {
+            auto fun2 = allocValue();
+            *fun2 = vRes;
+            vRes.mkPrimOpApp(fun2, args[i]);
+        }
+    };
 
-    if (!fun.isLambda())
-        throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
+    auto callLambda = [&](Env * env, ExprLambda & lambda, Value * * args)
+    {
+        Env & env2(allocEnv(lambda.envSize));
+        env2.up = env;
 
-    ExprLambda & lambda(*fun.lambda.fun);
+        Displacement displ = 0;
 
-    auto size =
-        (lambda.arg.empty() ? 0 : 1) +
-        (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
-    Env & env2(allocEnv(size));
-    env2.up = fun.lambda.env;
+        for (auto & arg : lambda.args) {
+            auto vArg = *args++;
 
-    size_t displ = 0;
+            if (arg.arg != sEpsilon)
+                env2.values[displ++] = vArg;
 
-    if (!lambda.matchAttrs)
-        env2.values[displ++] = &arg;
+            if (arg.formals) {
+                forceAttrs(*vArg, pos);
 
-    else {
-        forceAttrs(arg, pos);
+                /* For each formal argument, get the actual argument.  If
+                   there is no matching actual argument but the formal
+                   argument has a default, use the default. */
+                size_t attrsUsed = 0;
+                for (auto & i : arg.formals->formals) {
+                    auto j = vArg->attrs->get(i.name);
+                    if (!j) {
+                        if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
+                            lambda, i.name);
+                        env2.values[displ++] = i.def->maybeThunk(*this, env2);
+                    } else {
+                        attrsUsed++;
+                        env2.values[displ++] = j->value;
+                    }
+                }
 
-        if (!lambda.arg.empty())
-            env2.values[displ++] = &arg;
-
-        /* For each formal argument, get the actual argument.  If
-           there is no matching actual argument but the formal
-           argument has a default, use the default. */
-        size_t attrsUsed = 0;
-        for (auto & i : lambda.formals->formals) {
-            Bindings::iterator j = arg.attrs->find(i.name);
-            if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
-                    lambda, i.name);
-                env2.values[displ++] = i.def->maybeThunk(*this, env2);
-            } else {
-                attrsUsed++;
-                env2.values[displ++] = j->value;
+                /* Check that each actual argument is listed as a formal
+                   argument (unless the attribute match specifies a `...'). */
+                if (!arg.formals->ellipsis && attrsUsed != vArg->attrs->size()) {
+                    /* Nope, so show the first unexpected argument to the
+                       user. */
+                    for (auto & i : *vArg->attrs)
+                        if (arg.formals->argNames.find(i.name) == arg.formals->argNames.end())
+                            throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
+                    abort(); // can't happen
+                }
             }
         }
 
-        /* Check that each actual argument is listed as a formal
-           argument (unless the attribute match specifies a `...'). */
-        if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) {
-            /* Nope, so show the first unexpected argument to the
-               user. */
-            for (auto & i : *arg.attrs)
-                if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
-            abort(); // can't happen
-        }
-    }
+        assert(displ == lambda.envSize);
 
-    nrFunctionCalls++;
-    if (countCalls) incrFunctionCall(&lambda);
+        nrFunctionCalls++;
+        if (countCalls) incrFunctionCall(&lambda);
 
-    /* Evaluate the body.  This is conditional on showTrace, because
-       catching exceptions makes this function not tail-recursive. */
-    if (loggerSettings.showTrace.get())
+        /* Evaluate the body. */
         try {
-            lambda.body->eval(*this, env2, v);
+            lambda.body->eval(*this, env2, vCur);
         } catch (Error & e) {
-            addErrorTrace(e, lambda.pos, "while evaluating %s",
-              (lambda.name.set()
-                  ? "'" + (string) lambda.name + "'"
-                  : "anonymous lambda"));
-            addErrorTrace(e, pos, "from call site%s", "");
+            if (loggerSettings.showTrace) {
+                addErrorTrace(e, lambda.pos, "while evaluating %s",
+                    (lambda.name.set()
+                        ? "'" + (string) lambda.name + "'"
+                        : "anonymous lambda"));
+                addErrorTrace(e, pos, "from call site%s", "");
+            }
             throw;
         }
-    else
-        fun.lambda.fun->body->eval(*this, env2, v);
+    };
+
+    while (nrArgs > 0) {
+
+        if (vCur.isLambda()) {
+
+            ExprLambda & lambda(*vCur.lambda.fun);
+
+            if (nrArgs < lambda.args.size()) {
+                vRes = vCur;
+                for (size_t i = 0; i < nrArgs; ++i) {
+                    auto fun2 = allocValue();
+                    *fun2 = vRes;
+                    vRes.mkPartialApp(fun2, args[i]);
+                }
+                return;
+            } else {
+                callLambda(vCur.lambda.env, lambda, args);
+                nrArgs -= lambda.args.size();
+                args += lambda.args.size();
+            }
+        }
+
+        else if (vCur.isPartialApp()) {
+            /* Figure out the number of arguments still needed. */
+            size_t argsDone = 0;
+            Value * lambda = &vCur;
+            while (lambda->isPartialApp()) {
+                argsDone++;
+                lambda = lambda->app.left;
+            }
+            assert(lambda->isLambda());
+            auto arity = lambda->lambda.fun->args.size();
+            auto argsLeft = arity - argsDone;
+
+            if (nrArgs < argsLeft) {
+                /* We still don't have enough arguments, so extend the tPartialApp chain. */
+                vRes = vCur;
+                for (size_t i = 0; i < nrArgs; ++i) {
+                    auto fun2 = allocValue();
+                    *fun2 = vRes;
+                    vRes.mkPartialApp(fun2, args[i]);
+                }
+                return;
+            } else {
+                /* We have all the arguments, so call the function
+                   with the previous and new arguments. */
+
+                Value * vArgs[arity];
+                auto n = argsDone;
+                for (Value * arg = &vCur; arg->isPartialApp(); arg = arg->app.left)
+                    vArgs[--n] = arg->app.right;
+
+                for (size_t i = 0; i < argsLeft; ++i)
+                    vArgs[argsDone + i] = args[i];
+
+                nrArgs -= argsLeft;
+                args += argsLeft;
+
+                callLambda(lambda->lambda.env, *lambda->lambda.fun, vArgs);
+            }
+        }
+
+        else if (vCur.isPrimOp()) {
+
+            size_t argsLeft = vCur.primOp->arity;
+
+            if (nrArgs < argsLeft) {
+                /* We don't have enough arguments, so create a tPrimOpApp chain. */
+                makeAppChain();
+                return;
+            } else {
+                /* We have all the arguments, so call the primop. */
+                nrPrimOpCalls++;
+                if (countCalls) primOpCalls[vCur.primOp->name]++;
+                vCur.primOp->fun(*this, pos, args, vCur);
+
+                nrArgs -= argsLeft;
+                args += argsLeft;
+            }
+        }
+
+        else if (vCur.isPrimOpApp()) {
+            /* Figure out the number of arguments still needed. */
+            size_t argsDone = 0;
+            Value * primOp = &vCur;
+            while (primOp->isPrimOpApp()) {
+                argsDone++;
+                primOp = primOp->primOpApp.left;
+            }
+            assert(primOp->isPrimOp());
+            auto arity = primOp->primOp->arity;
+            auto argsLeft = arity - argsDone;
+
+            if (nrArgs < argsLeft) {
+                /* We still don't have enough arguments, so extend the tPrimOpApp chain. */
+                makeAppChain();
+                return;
+            } else {
+                /* We have all the arguments, so call the primop with
+                   the previous and new arguments. */
+
+                Value * vArgs[arity];
+                auto n = argsDone;
+                for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
+                    vArgs[--n] = arg->primOpApp.right;
+
+                for (size_t i = 0; i < argsLeft; ++i)
+                    vArgs[argsDone + i] = args[i];
+
+                nrPrimOpCalls++;
+                if (countCalls) primOpCalls[primOp->primOp->name]++;
+                primOp->primOp->fun(*this, pos, vArgs, vCur);
+
+                nrArgs -= argsLeft;
+                args += argsLeft;
+            }
+        }
+
+        else if (vCur.type() == nAttrs) {
+            if (auto functor = vCur.attrs->get(sFunctor)) {
+                /* 'vCur" may be allocated on the stack of the calling
+                   function, but for functors we may keep a reference,
+                   so heap-allocate a copy and use that instead. */
+                Value * args2[] = {allocValue()};
+                *args2[0] = vCur;
+                /* !!! Should we use the attr pos here? */
+                callFunction(*functor->value, 1, args2, vCur, pos);
+            }
+        }
+
+        else
+            throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur);
+    }
+
+    vRes = vCur;
+}
+
+
+void ExprCall::eval(EvalState & state, Env & env, Value & v)
+{
+    Value vFun;
+    fun->eval(state, env, vFun);
+
+    Value * vArgs[args.size()];
+    for (size_t i = 0; i < args.size(); ++i)
+        vArgs[i] = args[i]->maybeThunk(state, env);
+
+    state.callFunction(vFun, args.size(), vArgs, v, pos);
 }
 
 
@@ -1374,42 +1513,48 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
         }
     }
 
-    if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
+    if (!fun.isLambda()) {
         res = fun;
         return;
     }
 
-    Value * actualArgs = allocValue();
-    mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size()));
+    Value * actualArgs[fun.lambda.fun->args.size()];
 
-    if (fun.lambda.fun->formals->ellipsis) {
-        // If the formals have an ellipsis (eg the function accepts extra args) pass
-        // all available automatic arguments (which includes arguments specified on
-        // the command line via --arg/--argstr)
-        for (auto& v : args) {
-            actualArgs->attrs->push_back(v);
+    for (const auto & [i, arg] : enumerate(fun.lambda.fun->args)) {
+        if (!arg.formals) {
+            res = fun;
+            return;
         }
-    } else {
-        // Otherwise, only pass the arguments that the function accepts
-        for (auto & i : fun.lambda.fun->formals->formals) {
-            Bindings::iterator j = args.find(i.name);
-            if (j != args.end()) {
-                actualArgs->attrs->push_back(*j);
-            } else if (!i.def) {
-                throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
+
+        actualArgs[i] = allocValue();
+        mkAttrs(*actualArgs[i], std::max(arg.formals->formals.size(), static_cast<size_t>(args.size())));
+
+        if (arg.formals->ellipsis) {
+            /* If the formals have an ellipsis (i.e. the function
+               accepts extra args), pass all available automatic
+               arguments. */
+            for (auto & v : args)
+                actualArgs[i]->attrs->push_back(v);
+        } else {
+            /* Otherwise, only pass the arguments that the function
+               accepts. */
+            for (auto & j : arg.formals->formals) {
+                if (auto attr = args.get(j.name))
+                    actualArgs[i]->attrs->push_back(*attr);
+                else if (!j.def)
+                    throwMissingArgumentError(j.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
 
 Nix attempted to evaluate a function as a top level expression; in
 this case it must have its arguments supplied either by default
 values, or passed explicitly with '--arg' or '--argstr'. See
-https://nixos.org/manual/nix/stable/#ss-functions.)", i.name);
-
+https://nixos.org/manual/nix/stable/#ss-functions.)", j.name);
             }
         }
+
+        actualArgs[i]->attrs->sort();
     }
 
-    actualArgs->attrs->sort();
-
-    callFunction(fun, *actualArgs, res, noPos);
+    callFunction(fun, fun.lambda.fun->args.size(), actualArgs, res, noPos);
 }
 
 
@@ -1576,7 +1721,6 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
            and none of the strings are allowed to have contexts. */
         if (first) {
             firstType = vTmp.type();
-            first = false;
         }
 
         if (firstType == nInt) {
@@ -1597,7 +1741,12 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
             } else
                 throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
         } else
-            s << state.coerceToString(pos, vTmp, context, false, firstType == nString);
+            /* skip canonization of first path, which would only be not
+            canonized in the first place if it's coming from a ./${foo} type
+            path */
+            s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first);
+
+        first = false;
     }
 
     if (firstType == nInt)
@@ -1616,7 +1765,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 
 void ExprPos::eval(EvalState & state, Env & env, Value & v)
 {
-    state.mkPos(v, &pos);
+    state.mkPos(v, ptr(&pos));
 }
 
 
@@ -1786,7 +1935,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v,
 }
 
 string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
-    bool coerceMore, bool copyToStore)
+    bool coerceMore, bool copyToStore, bool canonicalizePath)
 {
     forceValue(v, pos);
 
@@ -1798,7 +1947,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
     }
 
     if (v.type() == nPath) {
-        Path path(canonPath(v.path));
+        Path path(canonicalizePath ? canonPath(v.path) : v.path);
         return copyToStore ? copyPathToStore(context, path) : path;
     }
 
@@ -1857,6 +2006,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
             ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
             : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
         dstPath = store->printStorePath(p);
+        allowPath(p);
         srcToStore.insert_or_assign(path, std::move(p));
         printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
     }

src/libexpr/eval.hh

@@ -5,6 +5,7 @@
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "config.hh"
+#include "experimental-features.hh"
 
 #include <map>
 #include <optional>
@@ -100,6 +101,8 @@ public:
     /* Store used to build stuff. */
     const ref<Store> buildStore;
 
+    RootValue vCallFlake = nullptr;
+    RootValue vImportedDrvToDerivation = nullptr;
 
 private:
     SrcToStore srcToStore;
@@ -138,10 +141,25 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
+    void requireExperimentalFeatureOnEvaluation(
+        const ExperimentalFeature &,
+        const std::string_view fName,
+        const Pos & pos
+    );
+
     void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
 
+    /* Allow access to a path. */
+    void allowPath(const Path & path);
+
+    /* Allow access to a store path. Note that this gets remapped to
+       the real store path if `store` is a chroot store. */
+    void allowPath(const StorePath & storePath);
+
+    /* Check whether access to a path is allowed and throw an error if
+       not. Otherwise return the canonicalised path. */
     Path checkSourcePath(const Path & path);
 
     void checkURI(const std::string & uri);
@@ -170,6 +188,14 @@ public:
        trivial (i.e. doesn't require arbitrary computation). */
     void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
 
+    /* Like `cacheFile`, but with an already parsed expression. */
+    void cacheFile(
+        const Path & path,
+        const Path & resolvedPath,
+        Expr * e,
+        Value & v,
+        bool mustBeTrivial = false);
+
     void resetFileCache();
 
     /* Look up a file in the search path. */
@@ -224,7 +250,8 @@ public:
        booleans and lists to a string.  If `copyToStore' is set,
        referenced paths are copied to the Nix store as a side effect. */
     string coerceToString(const Pos & pos, Value & v, PathSet & context,
-        bool coerceMore = false, bool copyToStore = true);
+        bool coerceMore = false, bool copyToStore = true,
+        bool canonicalizePath = true);
 
     string copyPathToStore(PathSet & context, const Path & path);
 
@@ -250,6 +277,8 @@ private:
 
     Value * addConstant(const string & name, Value & v);
 
+    void addConstant(const string & name, Value * v);
+
     Value * addPrimOp(const string & name,
         size_t arity, PrimOpFun primOp);
 
@@ -289,8 +318,14 @@ public:
 
     bool isFunctor(Value & fun);
 
-    void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos);
-    void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos);
+    // FIXME: use std::span
+    void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos);
+
+    void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos)
+    {
+        Value * args[] = {&arg};
+        callFunction(fun, 1, args, vRes, pos);
+    }
 
     /* Automatically call a function for which each argument has a
        default value or has a binding in the `args' map. */
@@ -308,7 +343,7 @@ public:
     void mkList(Value & v, size_t length);
     void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
-    void mkPos(Value & v, Pos * pos);
+    void mkPos(Value & v, ptr<Pos> pos);
 
     void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 

src/libexpr/flake/config.cc

@@ -29,7 +29,7 @@ static void writeTrustedList(const TrustedList & trustedList)
 
 void ConfigFile::apply()
 {
-    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
+    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"};
 
     for (auto & [name, value] : settings) {
 

src/libexpr/flake/flake.cc

@@ -64,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     debug("got tree '%s' from '%s'",
         state.store->printStorePath(tree.storePath), lockedRef);
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+    state.allowPath(tree.storePath);
 
     assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
 
@@ -89,10 +88,12 @@ static void expectType(EvalState & state, ValueType type,
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos);
+    EvalState & state, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir);
 
 static FlakeInput parseFlakeInput(EvalState & state,
-    const std::string & inputName, Value * value, const Pos & pos)
+    const std::string & inputName, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir)
 {
     expectType(state, nAttrs, *value, pos);
 
@@ -116,7 +117,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nBool, *attr.value, *attr.pos);
                 input.isFlake = attr.value->boolean;
             } else if (attr.name == sInputs) {
-                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
+                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir);
             } else if (attr.name == sFollows) {
                 expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
@@ -154,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         if (!attrs.empty())
             throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
         if (url)
-            input.ref = parseFlakeRef(*url, {}, true);
+            input.ref = parseFlakeRef(*url, baseDir, true);
     }
 
     if (!input.follows && !input.ref)
@@ -164,7 +165,8 @@ static FlakeInput parseFlakeInput(EvalState & state,
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos)
+    EvalState & state, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir)
 {
     std::map<FlakeId, FlakeInput> inputs;
 
@@ -175,7 +177,8 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
             parseFlakeInput(state,
                 inputAttr.name,
                 inputAttr.value,
-                *inputAttr.pos));
+                *inputAttr.pos,
+                baseDir));
     }
 
     return inputs;
@@ -191,7 +194,8 @@ static Flake getFlake(
         state, originalRef, allowLookup, flakeCache);
 
     // Guard against symlink attacks.
-    auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
+    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir);
+    auto flakeFile = canonPath(flakeDir + "/flake.nix");
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
             lockedRef, state.store->printStorePath(sourceInfo.storePath));
@@ -219,15 +223,20 @@ static Flake getFlake(
     auto sInputs = state.symbols.create("inputs");
 
     if (auto inputs = vInfo.attrs->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
+        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir);
 
     auto sOutputs = state.symbols.create("outputs");
 
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
         expectType(state, nFunction, *outputs->value, *outputs->pos);
 
-        if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
-            for (auto & formal : outputs->value->lambda.fun->formals->formals) {
+        if (outputs->value->lambda.fun->args.size() != 1)
+            throw Error("the 'outputs' attribute of flake '%s' is not a unary function", lockedRef);
+
+        auto & arg = outputs->value->lambda.fun->args[0];
+
+        if (arg.formals) {
+            for (auto & formal : arg.formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {
                         .ref = parseFlakeRef(formal.name)
@@ -293,7 +302,7 @@ LockedFlake lockFlake(
     const FlakeRef & topRef,
     const LockFlags & lockFlags)
 {
-    settings.requireExperimentalFeature("flakes");
+    settings.requireExperimentalFeature(Xp::Flakes);
 
     FlakeCache flakeCache;
 
@@ -325,25 +334,38 @@ LockedFlake lockFlake(
 
         std::vector<FlakeRef> parents;
 
+        struct LockParent {
+            /* The path to this parent. */
+            InputPath path;
+
+            /* Whether we are currently inside a top-level lockfile
+               (inputs absolute) or subordinate lockfile (inputs
+               relative). */
+            bool absolute;
+        };
+
         std::function<void(
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
-            std::shared_ptr<const Node> oldNode)>
+            std::shared_ptr<const Node> oldNode,
+            const LockParent & parent,
+            const Path & parentPath)>
             computeLocks;
 
         computeLocks = [&](
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
-            std::shared_ptr<const Node> oldNode)
+            std::shared_ptr<const Node> oldNode,
+            const LockParent & parent,
+            const Path & parentPath)
         {
             debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
             /* Get the overrides (i.e. attributes of the form
                'inputs.nixops.inputs.nixpkgs.url = ...'). */
-            // FIXME: check this
-            for (auto & [id, input] : flake.inputs) {
+            for (auto & [id, input] : flakeInputs) {
                 for (auto & [idOverride, inputOverride] : input.overrides) {
                     auto inputPath(inputPathPrefix);
                     inputPath.push_back(id);
@@ -379,15 +401,19 @@ LockedFlake lockFlake(
                        path we haven't processed yet. */
                     if (input.follows) {
                         InputPath target;
-                        if (hasOverride || input.absolute)
-                            /* 'follows' from an override is relative to the
-                               root of the graph. */
+
+                        if (parent.absolute && !hasOverride) {
                             target = *input.follows;
-                        else {
-                            /* Otherwise, it's relative to the current flake. */
-                            target = inputPathPrefix;
+                        } else {
+                            if (hasOverride) {
+                                target = inputPathPrefix;
+                                target.pop_back();
+                            } else
+                                target = parent.path;
+
                             for (auto & i : *input.follows) target.push_back(i);
                         }
+
                         debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
                         node->inputs.insert_or_assign(id, target);
                         continue;
@@ -433,7 +459,7 @@ LockedFlake lockFlake(
                         if (hasChildUpdate) {
                             auto inputFlake = getFlake(
                                 state, oldLock->lockedRef, false, flakeCache);
-                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
+                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath);
                         } else {
                             /* No need to fetch this flake, we can be
                                lazy. However there may be new overrides on the
@@ -450,12 +476,11 @@ LockedFlake lockFlake(
                                 } else if (auto follows = std::get_if<1>(&i.second)) {
                                     fakeInputs.emplace(i.first, FlakeInput {
                                         .follows = *follows,
-                                        .absolute = true
                                     });
                                 }
                             }
 
-                            computeLocks(fakeInputs, childNode, inputPath, oldLock);
+                            computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath);
                         }
 
                     } else {
@@ -467,7 +492,15 @@ LockedFlake lockFlake(
                             throw Error("cannot update flake input '%s' in pure mode", inputPathS);
 
                         if (input.isFlake) {
-                            auto inputFlake = getFlake(state, *input.ref, useRegistries, flakeCache);
+                            Path localPath = parentPath;
+                            FlakeRef localRef = *input.ref;
+
+                            // If this input is a path, recurse it down.
+                            // This allows us to resolve path inputs relative to the current flake.
+                            if (localRef.input.getType() == "path")
+                                localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
+
+                            auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache);
 
                             /* Note: in case of an --override-input, we use
                                the *original* ref (input2.ref) for the
@@ -488,6 +521,13 @@ LockedFlake lockFlake(
                             parents.push_back(*input.ref);
                             Finally cleanup([&]() { parents.pop_back(); });
 
+                            // Follows paths from existing inputs in the top-level lockfile are absolute,
+                            // whereas paths in subordinate lockfiles are relative to those lockfiles.
+                            LockParent newParent {
+                                .path = inputPath,
+                                .absolute = oldLock ? true : false
+                            };
+
                             /* Recursively process the inputs of this
                                flake. Also, unless we already have this flake
                                in the top-level lock file, use this flake's
@@ -497,7 +537,8 @@ LockedFlake lockFlake(
                                 oldLock
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
                                 : LockFile::read(
-                                    inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+                                    inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root,
+                                newParent, localPath);
                         }
 
                         else {
@@ -515,9 +556,17 @@ LockedFlake lockFlake(
             }
         };
 
+        LockParent parent {
+            .path = {},
+            .absolute = true
+        };
+
+        // Bring in the current ref for relative path resolution if we have it
+        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir);
+
         computeLocks(
             flake.inputs, newLockFile.root, {},
-            lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath);
 
         for (auto & i : lockFlags.inputOverrides)
             if (!overridesUsed.count(i.first))
@@ -567,8 +616,8 @@ LockedFlake lockFlake(
                         topRef.input.markChangedFile(
                             (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
                             lockFlags.commitLockFile
-                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
-                                    relPath, lockFileExists ? "Update" : "Add", diff))
+                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s",
+                                    relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true)))
                             : std::nullopt);
 
                         /* Rewriting the lockfile changed the top-level
@@ -593,8 +642,10 @@ LockedFlake lockFlake(
                     }
                 } else
                     throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
-            } else
+            } else {
                 warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
+                flake.forceDirty = true;
+            }
         }
 
         return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
@@ -617,26 +668,32 @@ void callFlake(EvalState & state,
 
     mkString(*vLocks, lockedFlake.lockFile.to_string());
 
-    emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
+    emitTreeAttrs(
+        state,
+        *lockedFlake.flake.sourceInfo,
+        lockedFlake.flake.lockedRef.input,
+        *vRootSrc,
+        false,
+        lockedFlake.flake.forceDirty);
 
     mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
 
-    static RootValue vCallFlake = nullptr;
-
-    if (!vCallFlake) {
-        vCallFlake = allocRootValue(state.allocValue());
+    if (!state.vCallFlake) {
+        state.vCallFlake = allocRootValue(state.allocValue());
         state.eval(state.parseExprFromString(
             #include "call-flake.nix.gen.hh"
-            , "/"), **vCallFlake);
+            , "/"), **state.vCallFlake);
     }
 
-    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
     state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
     state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
+
     auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
     if (evalSettings.pureEval && !flakeRef.input.isImmutable())
@@ -646,13 +703,13 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
         lockFlake(state, flakeRef,
             LockFlags {
                 .updateLockFile = false,
-                .useRegistries = !evalSettings.pureEval && !settings.useRegistries,
+                .useRegistries = !evalSettings.pureEval && settings.useRegistries,
                 .allowMutable  = !evalSettings.pureEval,
             }),
         v);
 }
 
-static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
+static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
 
 }
 
@@ -662,8 +719,9 @@ Fingerprint LockedFlake::getFingerprint() const
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
     return hashString(htSHA256,
-        fmt("%s;%d;%d;%s",
+        fmt("%s;%s;%d;%d;%s",
             flake.sourceInfo->storePath.to_string(),
+            flake.lockedRef.subdir,
             flake.lockedRef.input.getRevCount().value_or(0),
             flake.lockedRef.input.getLastModified().value_or(0),
             lockFile));

src/libexpr/flake/flake.hh

@@ -43,7 +43,6 @@ struct FlakeInput
     std::optional<FlakeRef> ref;
     bool isFlake = true;  // true = process flake to get outputs, false = (fetched) static source path
     std::optional<InputPath> follows;
-    bool absolute = false; // whether 'follows' is relative to the flake root
     FlakeInputs overrides;
 };
 
@@ -59,9 +58,10 @@ struct ConfigFile
 /* The contents of a flake.nix file. */
 struct Flake
 {
-    FlakeRef originalRef;   // the original flake specification (by the user)
-    FlakeRef resolvedRef;   // registry references and caching resolved to the specific underlying flake
-    FlakeRef lockedRef;     // the specific local store result of invoking the fetcher
+    FlakeRef originalRef; // the original flake specification (by the user)
+    FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
+    FlakeRef lockedRef; // the specific local store result of invoking the fetcher
+    bool forceDirty = false; // pretend that 'lockedRef' is dirty
     std::optional<std::string> description;
     std::shared_ptr<const fetchers::Tree> sourceInfo;
     FlakeInputs inputs;
@@ -141,6 +141,8 @@ void emitTreeAttrs(
     EvalState & state,
     const fetchers::Tree & tree,
     const fetchers::Input & input,
-    Value & v, bool emptyRevFallback = false);
+    Value & v,
+    bool emptyRevFallback = false,
+    bool forceDirty = false);
 
 }

src/libexpr/flake/flakeref.cc

@@ -172,8 +172,12 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         auto parsedURL = parseURL(url);
         std::string fragment;
         std::swap(fragment, parsedURL.fragment);
+
+        auto input = Input::fromURL(parsedURL);
+        input.parent = baseDir;
+
         return std::make_pair(
-            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+            FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")),
             fragment);
     }
 }

src/libexpr/flake/lockfile.cc

@@ -2,6 +2,8 @@
 #include "store-api.hh"
 #include "url-parts.hh"
 
+#include <iomanip>
+
 #include <nlohmann/json.hpp>
 
 namespace nix::flake {
@@ -268,10 +270,20 @@ std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
     return res;
 }
 
+static std::string describe(const FlakeRef & flakeRef)
+{
+    auto s = fmt("'%s'", flakeRef.to_string());
+
+    if (auto lastModified = flakeRef.input.getLastModified())
+        s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d"));
+
+    return s;
+}
+
 std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
 {
     if (auto node = std::get_if<0>(&edge))
-        stream << "'" << (*node)->lockedRef << "'";
+        stream << describe((*node)->lockedRef);
     else if (auto follows = std::get_if<1>(&edge))
         stream << fmt("follows '%s'", printInputPath(*follows));
     return stream;
@@ -299,14 +311,15 @@ std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
 
     while (i != oldFlat.end() || j != newFlat.end()) {
         if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
-            res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
+            res += fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n    %s\n",
+                printInputPath(j->first), j->second);
             ++j;
         } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
-            res += fmt("* Removed '%s'\n", printInputPath(i->first));
+            res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputPath(i->first));
             ++i;
         } else {
             if (!equals(i->second, j->second)) {
-                res += fmt("* Updated '%s': %s -> %s\n",
+                res += fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n    %s\n  → %s\n",
                     printInputPath(i->first),
                     i->second,
                     j->second);

src/libexpr/lexer.l

@@ -9,6 +9,9 @@
 %s DEFAULT
 %x STRING
 %x IND_STRING
+%x INPATH
+%x INPATH_SLASH
+%x PATH_START
 
 
 %{
@@ -25,6 +28,8 @@ using namespace nix;
 
 namespace nix {
 
+// backup to recover from yyless(0)
+YYLTYPE prev_yylloc;
 
 static void initLoc(YYLTYPE * loc)
 {
@@ -35,6 +40,8 @@ static void initLoc(YYLTYPE * loc)
 
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
+    prev_yylloc = *loc;
+
     loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
 
@@ -57,6 +64,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
+// FIXME: optimize
 static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
 {
     string t;
@@ -97,9 +105,12 @@ ANY         .|\n
 ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
 INT         [0-9]+
 FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH        [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
-HPATH       \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
-SPATH       \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\>
+PATH_CHAR   [a-zA-Z0-9\.\_\-\+]
+PATH        {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
+PATH_SEG    {PATH_CHAR}*\/
+HPATH       \~(\/{PATH_CHAR}+)+\/?
+HPATH_START \~\/
+SPATH       \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
 URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
 
 
@@ -200,17 +211,75 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 
+{PATH_SEG}\$\{ |
+{HPATH_START}\$\{ {
+  PUSH_STATE(PATH_START);
+  yyless(0);
+  *yylloc = prev_yylloc;
+}
+
+<PATH_START>{PATH_SEG} {
+  POP_STATE();
+  PUSH_STATE(INPATH_SLASH);
+  yylval->path = strdup(yytext);
+  return PATH;
+}
+
+<PATH_START>{HPATH_START} {
+  POP_STATE();
+  PUSH_STATE(INPATH_SLASH);
+  yylval->path = strdup(yytext);
+  return HPATH;
+}
+
+{PATH} {
+  if (yytext[yyleng-1] == '/')
+    PUSH_STATE(INPATH_SLASH);
+  else
+    PUSH_STATE(INPATH);
+  yylval->path = strdup(yytext);
+  return PATH;
+}
+{HPATH} {
+  if (yytext[yyleng-1] == '/')
+    PUSH_STATE(INPATH_SLASH);
+  else
+    PUSH_STATE(INPATH);
+  yylval->path = strdup(yytext);
+  return HPATH;
+}
+
+<INPATH,INPATH_SLASH>\$\{ {
+  POP_STATE();
+  PUSH_STATE(INPATH);
+  PUSH_STATE(DEFAULT);
+  return DOLLAR_CURLY;
+}
+<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
+  POP_STATE();
+  if (yytext[yyleng-1] == '/')
+      PUSH_STATE(INPATH_SLASH);
+  else
+      PUSH_STATE(INPATH);
+  yylval->e = new ExprString(data->symbols.create(string(yytext)));
+  return STR;
+}
+<INPATH>{ANY} |
+<INPATH><<EOF>> {
+  /* if we encounter a non-path character we inform the parser that the path has
+     ended with a PATH_END token and re-parse this character in the default
+     context (it may be ')', ';', or something of that sort) */
+  POP_STATE();
+  yyless(0);
+  *yylloc = prev_yylloc;
+  return PATH_END;
+}
+
+<INPATH_SLASH>{ANY} |
+<INPATH_SLASH><<EOF>> {
+  throw ParseError("path has a trailing slash");
+}
 
-{PATH}      { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path '%s' has a trailing slash", yytext);
-              yylval->path = strdup(yytext);
-              return PATH;
-            }
-{HPATH}     { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path '%s' has a trailing slash", yytext);
-              yylval->path = strdup(yytext);
-              return HPATH;
-            }
 {SPATH}     { yylval->path = strdup(yytext); return SPATH; }
 {URI}       { yylval->uri = strdup(yytext); return URI; }
 

src/libexpr/local.mk

@@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS = -lboost_context
+libexpr_LDFLAGS += -lboost_context -pthread
 ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif
@@ -35,7 +35,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
 
 clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
 
-$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))

src/libexpr/nixexpr.cc

@@ -124,23 +124,36 @@ void ExprList::show(std::ostream & str) const
 void ExprLambda::show(std::ostream & str) const
 {
     str << "(";
-    if (matchAttrs) {
-        str << "{ ";
-        bool first = true;
-        for (auto & i : formals->formals) {
-            if (first) first = false; else str << ", ";
-            str << i.name;
-            if (i.def) str << " ? " << *i.def;
+    for (auto & arg : args) {
+        if (arg.formals) {
+            str << "{ ";
+            bool first = true;
+            for (auto & i : arg.formals->formals) {
+                if (first) first = false; else str << ", ";
+                str << i.name;
+                if (i.def) str << " ? " << *i.def;
+            }
+            if (arg.formals->ellipsis) {
+                if (!first) str << ", ";
+                str << "...";
+            }
+            str << " }";
+            if (!arg.arg.empty()) str << " @ ";
         }
-        if (formals->ellipsis) {
-            if (!first) str << ", ";
-            str << "...";
-        }
-        str << " }";
-        if (!arg.empty()) str << " @ ";
+        if (!arg.arg.empty()) str << arg.arg;
+        str << ": ";
     }
-    if (!arg.empty()) str << arg;
-    str << ": " << *body << ")";
+    str << *body << ")";
+}
+
+void ExprCall::show(std::ostream & str) const
+{
+    str << '(' << *fun;
+    for (auto e : args) {
+        str <<  ' ';
+        str << *e;
+    }
+    str << ')';
 }
 
 void ExprLet::show(std::ostream & str) const
@@ -263,14 +276,13 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
     const StaticEnv * curEnv;
-    unsigned int level;
+    Level level;
     int withLevel = -1;
     for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) {
         if (curEnv->isWith) {
             if (withLevel == -1) withLevel = level;
         } else {
-            StaticEnv::Vars::const_iterator i = curEnv->vars.find(name);
-            if (i != curEnv->vars.end()) {
+            if (auto i = curEnv->get(name)) {
                 fromWith = false;
                 this->level = level;
                 displ = i->second;
@@ -311,14 +323,16 @@ void ExprOpHasAttr::bindVars(const StaticEnv & env)
 void ExprAttrs::bindVars(const StaticEnv & env)
 {
     const StaticEnv * dynamicEnv = &env;
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0);
 
     if (recursive) {
         dynamicEnv = &newEnv;
 
-        unsigned int displ = 0;
+        Displacement displ = 0;
         for (auto & i : attrs)
-            newEnv.vars[i.first] = i.second.displ = displ++;
+            newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
+
+        // No need to sort newEnv since attrs is in sorted order.
 
         for (auto & i : attrs)
             i.second.e->bindVars(i.second.inherited ? env : newEnv);
@@ -342,30 +356,67 @@ void ExprList::bindVars(const StaticEnv & env)
 
 void ExprLambda::bindVars(const StaticEnv & env)
 {
-    StaticEnv newEnv(false, &env);
+    /* The parser adds arguments in reverse order. Let's fix that
+       now. */
+    std::reverse(args.begin(), args.end());
 
-    unsigned int displ = 0;
+    envSize = 0;
 
-    if (!arg.empty()) newEnv.vars[arg] = displ++;
-
-    if (matchAttrs) {
-        for (auto & i : formals->formals)
-            newEnv.vars[i.name] = displ++;
-
-        for (auto & i : formals->formals)
-            if (i.def) i.def->bindVars(newEnv);
+    for (auto & arg :args) {
+        if (!arg.arg.empty()) envSize++;
+        if (arg.formals) envSize += arg.formals->formals.size();
     }
 
+    StaticEnv newEnv(false, &env, envSize);
+
+    Displacement displ = 0;
+
+    for (auto & arg : args) {
+        if (!arg.arg.empty()) {
+            if (auto i = const_cast<StaticEnv::Vars::value_type *>(newEnv.get(arg.arg)))
+                i->second = displ++;
+            else
+                newEnv.vars.emplace_back(arg.arg, displ++);
+        }
+
+        if (arg.formals) {
+            for (auto & i : arg.formals->formals) {
+                if (auto j = const_cast<StaticEnv::Vars::value_type *>(newEnv.get(i.name)))
+                    j->second = displ++;
+                else
+                    newEnv.vars.emplace_back(i.name, displ++);
+            }
+
+            newEnv.sort();
+
+            for (auto & i : arg.formals->formals)
+                if (i.def) i.def->bindVars(newEnv);
+        }
+    }
+
+    assert(displ == envSize);
+
+    newEnv.sort();
+
     body->bindVars(newEnv);
 }
 
+void ExprCall::bindVars(const StaticEnv & env)
+{
+    fun->bindVars(env);
+    for (auto e : args)
+        e->bindVars(env);
+}
+
 void ExprLet::bindVars(const StaticEnv & env)
 {
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(false, &env, attrs->attrs.size());
 
-    unsigned int displ = 0;
+    Displacement displ = 0;
     for (auto & i : attrs->attrs)
-        newEnv.vars[i.first] = i.second.displ = displ++;
+        newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
+
+    // No need to sort newEnv since attrs->attrs is in sorted order.
 
     for (auto & i : attrs->attrs)
         i.second.e->bindVars(i.second.inherited ? env : newEnv);
@@ -379,7 +430,7 @@ void ExprWith::bindVars(const StaticEnv & env)
        level so that `lookupVar' can look up variables in the previous
        `with' if this one doesn't contain the desired attribute. */
     const StaticEnv * curEnv;
-    unsigned int level;
+    Level level;
     prevWith = 0;
     for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++)
         if (curEnv->isWith) {
@@ -452,5 +503,4 @@ size_t SymbolTable::totalSize() const
     return n;
 }
 
-
 }

src/libexpr/nixexpr.hh

@@ -4,8 +4,6 @@
 #include "symbol-table.hh"
 #include "error.hh"
 
-#include <map>
-
 
 namespace nix {
 
@@ -135,6 +133,9 @@ struct ExprPath : Expr
     Value * maybeThunk(EvalState & state, Env & env);
 };
 
+typedef uint32_t Level;
+typedef uint32_t Displacement;
+
 struct ExprVar : Expr
 {
     Pos pos;
@@ -150,8 +151,8 @@ struct ExprVar : Expr
        value is obtained by getting the attribute named `name' from
        the set stored in the environment that is `level' levels up
        from the current one.*/
-    unsigned int level;
-    unsigned int displ;
+    Level level;
+    Displacement displ;
 
     ExprVar(const Symbol & name) : name(name) { };
     ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
@@ -185,7 +186,7 @@ struct ExprAttrs : Expr
         bool inherited;
         Expr * e;
         Pos pos;
-        unsigned int displ; // displacement
+        Displacement displ; // displacement
         AttrDef(Expr * e, const Pos & pos, bool inherited=false)
             : inherited(inherited), e(e), pos(pos) { };
         AttrDef() { };
@@ -232,24 +233,38 @@ struct ExprLambda : Expr
 {
     Pos pos;
     Symbol name;
-    Symbol arg;
-    bool matchAttrs;
-    Formals * formals;
-    Expr * body;
-    ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
-        : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
+
+    struct Arg
     {
-        if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError({
-                .msg = hintfmt("duplicate formal function argument '%1%'", arg),
-                .errPos = pos
-            });
+        Symbol arg;
+        Formals * formals;
     };
+
+    std::vector<Arg> args;
+
+    Expr * body;
+
+    Displacement envSize = 0; // initialized by bindVars()
+
+    ExprLambda(const Pos & pos, Expr * body)
+        : pos(pos), body(body)
+    { };
     void setName(Symbol & name);
     string showNamePos() const;
     COMMON_METHODS
 };
 
+struct ExprCall : Expr
+{
+    Expr * fun;
+    std::vector<Expr *> args;
+    Pos pos;
+    ExprCall(const Pos & pos, Expr * fun, std::vector<Expr *> && args)
+        : fun(fun), args(args), pos(pos)
+    { }
+    COMMON_METHODS
+};
+
 struct ExprLet : Expr
 {
     ExprAttrs * attrs;
@@ -308,7 +323,6 @@ struct ExprOpNot : Expr
         void eval(EvalState & state, Env & env, Value & v); \
     };
 
-MakeBinOp(ExprApp, "")
 MakeBinOp(ExprOpEq, "==")
 MakeBinOp(ExprOpNEq, "!=")
 MakeBinOp(ExprOpAnd, "&&")
@@ -342,9 +356,28 @@ struct StaticEnv
 {
     bool isWith;
     const StaticEnv * up;
-    typedef std::map<Symbol, unsigned int> Vars;
+
+    // Note: these must be in sorted order.
+    typedef std::vector<std::pair<Symbol, Displacement>> Vars;
     Vars vars;
-    StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { };
+
+    StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
+        vars.reserve(expectedSize);
+    };
+
+    void sort()
+    {
+        std::sort(vars.begin(), vars.end(),
+            [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
+    }
+
+    const Vars::value_type * get(const Symbol & name) const
+    {
+        Vars::value_type key(name, 0);
+        auto i = std::lower_bound(vars.begin(), vars.end(), key);
+        if (i != vars.end() && i->first == name) return &*i;
+        return {};
+    }
 };
 
 

src/libexpr/parser.y

@@ -126,14 +126,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
                     auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
                         dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs[ad.first] = ad.second;
+                    jAttrs->attrs.emplace(ad.first, ad.second);
                 }
             } else {
                 dupAttr(attrPath, pos, j->second.pos);
             }
         } else {
             // This attr path is not defined. Let's create it.
-            attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos);
+            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
             e->setName(i->symbol);
         }
     } else {
@@ -154,6 +154,24 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 }
 
 
+static Expr * addArg(const Pos & pos, Expr * e, ExprLambda::Arg && arg)
+{
+    if (!arg.arg.empty() && arg.formals && arg.formals->argNames.count(arg.arg))
+        throw ParseError({
+            .msg = hintfmt("duplicate formal function argument '%1%'", arg.arg),
+            .errPos = pos
+        });
+
+    auto e2 = dynamic_cast<ExprLambda *>(e); // FIXME: slow?
+    if (!e2)
+        e2 = new ExprLambda(pos, e);
+    else
+        e2->pos = pos;
+    e2->args.emplace_back(std::move(arg));
+    return e2;
+}
+
+
 static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Expr *> & es)
 {
     if (es.empty()) return new ExprString(symbols.create(""));
@@ -283,20 +301,20 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 }
 
 %type <e> start expr expr_function expr_if expr_op
-%type <e> expr_app expr_select expr_simple
+%type <e> expr_select expr_simple expr_app
 %type <list> expr_list
 %type <attrs> binds
 %type <formals> formals
 %type <formal> formal
 %type <attrNames> attrs attrpath
 %type <string_parts> string_parts_interpolated ind_string_parts
-%type <e> string_parts string_attr
+%type <e> path_start string_parts string_attr
 %type <id> attr
 %token <id> ID ATTRPATH
 %token <e> STR IND_STR
 %token <n> INT
 %token <nf> FLOAT
-%token <path> PATH HPATH SPATH
+%token <path> PATH HPATH SPATH PATH_END
 %token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
 %token DOLLAR_CURLY /* == ${ */
@@ -324,13 +342,13 @@ expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
+    { $$ = addArg(CUR_POS, $3, {data->symbols.create($1), nullptr}); }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
+    { $$ = addArg(CUR_POS, $5, {data->state.sEpsilon, $2}); }
   | '{' formals '}' '@' ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
+    { $$ = addArg(CUR_POS, $7, {data->symbols.create($5), $2}); }
   | ID '@' '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
+    { $$ = addArg(CUR_POS, $7, {data->symbols.create($1), $4}); }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
@@ -353,13 +371,13 @@ expr_if
 
 expr_op
   : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); }
+  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); }
   | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
   | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
-  | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); }
-  | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); }
+  | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); }
+  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); }
+  | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); }
+  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); }
   | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); }
   | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); }
   | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); }
@@ -367,17 +385,22 @@ expr_op
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
     { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
-  | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); }
-  | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); }
-  | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); }
+  | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
+  | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
+  | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
   | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); }
   | expr_app
   ;
 
 expr_app
-  : expr_app expr_select
-    { $$ = new ExprApp(CUR_POS, $1, $2); }
-  | expr_select { $$ = $1; }
+  : expr_app expr_select {
+      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
+          e2->args.push_back($2);
+          $$ = $1;
+      } else
+          $$ = new ExprCall(CUR_POS, $1, {$2});
+  }
+  | expr_select
   ;
 
 expr_select
@@ -388,7 +411,7 @@ expr_select
   | /* Backwards compatibility: because Nixpkgs has a rarely used
        function named ‘or’, allow stuff like ‘map or [...]’. */
     expr_simple OR_KW
-    { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); }
+    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); }
   | expr_simple { $$ = $1; }
   ;
 
@@ -405,17 +428,20 @@ expr_simple
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = stripIndentation(CUR_POS, data->symbols, *$2);
   }
-  | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
-  | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); }
+  | path_start PATH_END { $$ = $1; }
+  | path_start string_parts_interpolated PATH_END {
+      $2->insert($2->begin(), $1);
+      $$ = new ExprConcatStrings(CUR_POS, false, $2);
+  }
   | SPATH {
       string path($1 + 1, strlen($1) - 2);
-      $$ = new ExprApp(CUR_POS,
-          new ExprApp(new ExprVar(data->symbols.create("__findFile")),
-              new ExprVar(data->symbols.create("__nixPath"))),
-          new ExprString(data->symbols.create(path)));
+      $$ = new ExprCall(CUR_POS,
+          new ExprVar(data->symbols.create("__findFile")),
+          {new ExprVar(data->symbols.create("__nixPath")),
+           new ExprString(data->symbols.create(path))});
   }
   | URI {
-      static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
+      static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals);
       if (noURLLiterals)
           throw ParseError({
               .msg = hintfmt("URL literals are disabled"),
@@ -438,7 +464,7 @@ expr_simple
 string_parts
   : STR
   | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
-  | { $$ = new ExprString(data->symbols.create("")); }
+  | { $$ = new ExprString(data->state.sEpsilon); }
   ;
 
 string_parts_interpolated
@@ -452,6 +478,20 @@ string_parts_interpolated
     }
   ;
 
+path_start
+  : PATH {
+    Path path(absPath($1, data->basePath));
+    /* add back in the trailing '/' to the first segment */
+    if ($1[strlen($1)-1] == '/' && strlen($1) > 1)
+      path += "/";
+    $$ = new ExprPath(path);
+  }
+  | HPATH {
+    Path path(getHome() + string($1 + 1));
+    $$ = new ExprPath(path);
+  }
+  ;
+
 ind_string_parts
   : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
   | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
@@ -466,7 +506,7 @@ binds
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos);
           Pos pos = makeCurPos(@3, data);
-          $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true);
+          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
       }
     }
   | binds INHERIT '(' expr ')' attrs ';'
@@ -475,7 +515,7 @@ binds
       for (auto & i : *$6) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos);
-          $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data));
+          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)));
       }
     }
   | { $$ = new ExprAttrs(makeCurPos(@0, data)); }
@@ -735,7 +775,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, path };
         else {
             logWarning({
-                .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second)
             });
             res = { false, "" };
         }

src/libexpr/primops.cc

@@ -52,16 +52,13 @@ void EvalState::realiseContext(const PathSet & context)
     if (drvs.empty()) return;
 
     if (!evalSettings.enableImportFromDerivation)
-        throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
+        throw Error(
+            "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             store->printStorePath(drvs.begin()->drvPath));
 
-    /* For performance, prefetch all substitute info. */
-    StorePathSet willBuild, willSubstitute, unknown;
-    uint64_t downloadSize, narSize;
+    /* Build/substitute the context. */
     std::vector<DerivedPath> buildReqs;
     for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
-    store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
     store->buildPaths(buildReqs);
 
     /* Add the output of this derivations to the allowed
@@ -124,7 +121,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         });
     } catch (Error & e) {
         e.addTrace(pos, "while importing '%s'", path);
-        throw e;
+        throw;
     }
 
     Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -160,16 +157,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
         }
         w.attrs->sort();
 
-        static RootValue fun;
-        if (!fun) {
-            fun = allocRootValue(state.allocValue());
+        if (!state.vImportedDrvToDerivation) {
+            state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
             state.eval(state.parseExprFromString(
                 #include "imported-drv-to-derivation.nix.gen.hh"
-                , "/"), **fun);
+                , "/"), **state.vImportedDrvToDerivation);
         }
 
-        state.forceFunction(**fun, pos);
-        mkApp(v, **fun, w);
+        state.forceFunction(**state.vImportedDrvToDerivation, pos);
+        mkApp(v, **state.vImportedDrvToDerivation, w);
         state.forceAttrs(v, pos);
     }
 
@@ -188,14 +184,17 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
             Env * env = &state.allocEnv(vScope->attrs->size());
             env->up = &state.baseEnv;
 
-            StaticEnv staticEnv(false, &state.staticBaseEnv);
+            StaticEnv staticEnv(false, &state.staticBaseEnv, vScope->attrs->size());
 
             unsigned int displ = 0;
             for (auto & attr : *vScope->attrs) {
-                staticEnv.vars[attr.name] = displ;
+                staticEnv.vars.emplace_back(attr.name, displ);
                 env->values[displ++] = attr.value;
             }
 
+            // No need to call staticEnv.sort(), because
+            // args[0]->attrs is already sorted.
+
             printTalkative("evaluating file '%1%'", realPath);
             Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv);
 
@@ -414,7 +413,7 @@ static RegisterPrimOp primop_isNull({
       Return `true` if *e* evaluates to `null`, and `false` otherwise.
 
       > **Warning**
-      > 
+      >
       > This function is *deprecated*; just write `e == null` instead.
     )",
     .fun = prim_isNull,
@@ -989,7 +988,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
             if (i->name == state.sContentAddressed) {
-                settings.requireExperimentalFeature("ca-derivations");
+                settings.requireExperimentalFeature(Xp::CaDerivations);
                 contentAddressed = state.forceBool(*i->value, pos);
             }
 
@@ -1174,7 +1173,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         // hash per output.
         auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
         std::visit(overloaded {
-            [&](Hash h) {
+            [&](Hash & h) {
                 for (auto & i : outputs) {
                     auto outPath = state.store->makeOutputPath(i, h, drvName);
                     drv.env[i] = state.store->printStorePath(outPath);
@@ -1186,11 +1185,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         });
                 }
             },
-            [&](CaOutputHashes) {
+            [&](CaOutputHashes &) {
                 // Shouldn't happen as the toplevel derivation is not CA.
                 assert(false);
             },
-            [&](DeferredHash _) {
+            [&](DeferredHash &) {
                 for (auto & i : outputs) {
                     drv.outputs.insert_or_assign(i,
                         DerivationOutput {
@@ -1493,15 +1492,20 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     string type = state.forceStringNoCtx(*args[0], pos);
     std::optional<HashType> ht = parseHashType(type);
     if (!ht)
-      throw Error({
-          .msg = hintfmt("unknown hash type '%1%'", type),
-          .errPos = pos
-      });
+        throw Error({
+            .msg = hintfmt("unknown hash type '%1%'", type),
+            .errPos = pos
+        });
 
-    PathSet context; // discarded
-    Path p = state.coerceToPath(pos, *args[1], context);
+    PathSet context;
+    Path path = state.coerceToPath(pos, *args[1], context);
+    try {
+        state.realiseContext(context);
+    } catch (InvalidPathError & e) {
+        throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos);
+    }
 
-    mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+    mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -1712,7 +1716,7 @@ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Va
         parseJSON(state, s, v);
     } catch (JSONParseError &e) {
         e.addTrace(pos, "while decoding a JSON string");
-        throw e;
+        throw;
     }
 }
 
@@ -1842,50 +1846,79 @@ static RegisterPrimOp primop_toFile({
     .fun = prim_toFile,
 });
 
-static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
-    Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
+static void addPath(
+    EvalState & state,
+    const Pos & pos,
+    const string & name,
+    Path path,
+    Value * filterFun,
+    FileIngestionMethod method,
+    const std::optional<Hash> expectedHash,
+    Value & v,
+    const PathSet & context)
 {
-    const auto path = evalSettings.pureEval && expectedHash ?
-        path_ :
-        state.checkSourcePath(path_);
-    PathFilter filter = filterFun ? ([&](const Path & path) {
-        auto st = lstat(path);
+    try {
+        // FIXME: handle CA derivation outputs (where path needs to
+        // be rewritten to the actual output).
+        state.realiseContext(context);
 
-        /* Call the filter function.  The first argument is the path,
-           the second is a string indicating the type of the file. */
-        Value arg1;
-        mkString(arg1, path);
+        if (state.store->isInStore(path)) {
+            auto [storePath, subPath] = state.store->toStorePath(path);
+            auto info = state.store->queryPathInfo(storePath);
+            if (!info->references.empty())
+                throw EvalError("store path '%s' is not allowed to have references",
+                    state.store->printStorePath(storePath));
+            path = state.store->toRealPath(storePath) + subPath;
+        }
 
-        Value fun2;
-        state.callFunction(*filterFun, arg1, fun2, noPos);
+        path = evalSettings.pureEval && expectedHash
+            ? path
+            : state.checkSourcePath(path);
 
-        Value arg2;
-        mkString(arg2,
-            S_ISREG(st.st_mode) ? "regular" :
-            S_ISDIR(st.st_mode) ? "directory" :
-            S_ISLNK(st.st_mode) ? "symlink" :
-            "unknown" /* not supported, will fail! */);
+        PathFilter filter = filterFun ? ([&](const Path & path) {
+            auto st = lstat(path);
 
-        Value res;
-        state.callFunction(fun2, arg2, res, noPos);
+            /* Call the filter function.  The first argument is the path,
+               the second is a string indicating the type of the file. */
+            Value arg1;
+            mkString(arg1, path);
 
-        return state.forceBool(res, pos);
-    }) : defaultPathFilter;
+            Value arg2;
+            mkString(arg2,
+                S_ISREG(st.st_mode) ? "regular" :
+                S_ISDIR(st.st_mode) ? "directory" :
+                S_ISLNK(st.st_mode) ? "symlink" :
+                "unknown" /* not supported, will fail! */);
 
-    std::optional<StorePath> expectedStorePath;
-    if (expectedHash)
-        expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
-    Path dstPath;
-    if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-        dstPath = state.store->printStorePath(settings.readOnlyMode
-            ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
-            : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
-        if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
-            throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
-    } else
-        dstPath = state.store->printStorePath(*expectedStorePath);
+            Value * args []{&arg1, &arg2};
+            Value res;
+            state.callFunction(*filterFun, 2, args, res, pos);
 
-    mkString(v, dstPath, {dstPath});
+            return state.forceBool(res, pos);
+        }) : defaultPathFilter;
+
+        std::optional<StorePath> expectedStorePath;
+        if (expectedHash)
+            expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
+
+        Path dstPath;
+        if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+            dstPath = state.store->printStorePath(settings.readOnlyMode
+                ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
+            if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+                throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
+        } else
+            dstPath = state.store->printStorePath(*expectedStorePath);
+
+        mkString(v, dstPath, {dstPath});
+
+        state.allowPath(dstPath);
+
+    } catch (Error & e) {
+        e.addTrace(pos, "while adding path '%s'", path);
+        throw;
+    }
 }
 
 
@@ -1893,11 +1926,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
 {
     PathSet context;
     Path path = state.coerceToPath(pos, *args[1], context);
-    if (!context.empty())
-        throw EvalError({
-            .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-            .errPos = pos
-        });
 
     state.forceValue(*args[0], pos);
     if (args[0]->type() != nFunction)
@@ -1908,13 +1936,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
             .errPos = pos
         });
 
-    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
+    addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
 }
 
 static RegisterPrimOp primop_filterSource({
     .name = "__filterSource",
     .args = {"e1", "e2"},
     .doc = R"(
+      > **Warning**
+      >
+      > `filterSource` should not be used to filter store paths. Since
+      > `filterSource` uses the name of the input directory while naming
+      > the output directory, doing so will produce a directory name in
+      > the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is
+      > the name of the input directory. Since `<hash>` depends on the
+      > unfiltered directory, the name of the output directory will
+      > indirectly depend on files that are filtered out by the
+      > function. This will trigger a rebuild even when a filtered out
+      > file is changed. Use `builtins.path` instead, which allows
+      > specifying the name of the output directory.
+
       This function allows you to copy sources into the Nix store while
       filtering certain files. For instance, suppose that you want to use
       the directory `source-dir` as an input to a Nix expression, e.g.
@@ -1961,18 +2002,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     Value * filterFun = nullptr;
     auto method = FileIngestionMethod::Recursive;
     std::optional<Hash> expectedHash;
+    PathSet context;
 
     for (auto & attr : *args[0]->attrs) {
         const string & n(attr.name);
-        if (n == "path") {
-            PathSet context;
+        if (n == "path")
             path = state.coerceToPath(*attr.pos, *attr.value, context);
-            if (!context.empty())
-                throw EvalError({
-                    .msg = hintfmt("string '%1%' cannot refer to other paths", path),
-                    .errPos = *attr.pos
-                });
-        } else if (attr.name == state.sName)
+        else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
         else if (n == "filter") {
             state.forceValue(*attr.value, pos);
@@ -1995,7 +2031,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
     if (name.empty())
         name = baseNameOf(path);
 
-    addPath(state, pos, name, path, filterFun, method, expectedHash, v);
+    addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
 }
 
 static RegisterPrimOp primop_path({
@@ -2109,7 +2145,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
         pos
     );
     // !!! add to stack trace?
-    if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
+    if (state.countCalls && *i->pos != noPos) state.attrSelects[*i->pos]++;
     state.forceValue(*i->value, pos);
     v = *i->value;
 }
@@ -2350,26 +2386,41 @@ static RegisterPrimOp primop_catAttrs({
 static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     state.forceValue(*args[0], pos);
+
     if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) {
         state.mkAttrs(v, 0);
         return;
     }
-    if (!args[0]->isLambda())
+
+    if (!args[0]->isLambda() && !args[0]->isPartialApp())
         throw TypeError({
             .msg = hintfmt("'functionArgs' requires a function"),
             .errPos = pos
         });
 
-    if (!args[0]->lambda.fun->matchAttrs) {
+    size_t argsDone = 0;
+    auto lambda = args[0];
+    while (lambda->isPartialApp()) {
+        argsDone++;
+        lambda = lambda->app.left;
+    }
+    assert(lambda->isLambda());
+
+    assert(argsDone < lambda->lambda.fun->args.size());
+
+    // FIXME: handle partially applied functions
+    auto formals = lambda->lambda.fun->args[argsDone].formals;
+
+    if (!formals) {
         state.mkAttrs(v, 0);
         return;
     }
 
-    state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size());
-    for (auto & i : args[0]->lambda.fun->formals->formals) {
+    state.mkAttrs(v, formals->formals.size());
+    for (auto & i : formals->formals) {
         // !!! should optimise booleans (allocate only once)
         Value * value = state.allocValue();
-        v.attrs->push_back(Attr(i.name, value, &i.pos));
+        v.attrs->push_back(Attr(i.name, value, ptr(&i.pos)));
         mkBool(*value, i.def);
     }
     v.attrs->sort();
@@ -2515,7 +2566,7 @@ static RegisterPrimOp primop_tail({
       the argument isn’t a list or is an empty list.
 
       > **Warning**
-      > 
+      >
       > This function should generally be avoided since it's inefficient:
       > unlike Haskell's `tail`, it takes O(n) time, so recursing over a
       > list by repeatedly calling `tail` takes O(n^2) time.
@@ -2657,10 +2708,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
         Value * vCur = args[1];
 
         for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
-            Value vTmp;
-            state.callFunction(*args[0], *vCur, vTmp, pos);
+            Value * vs []{vCur, args[2]->listElems()[n]};
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
-            state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
+            state.callFunction(*args[0], 2, vs, *vCur, pos);
         }
         state.forceValue(v, pos);
     } else {
@@ -2781,17 +2831,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
         v.listElems()[n] = args[1]->listElems()[n];
     }
 
-
     auto comparator = [&](Value * a, Value * b) {
         /* Optimization: if the comparator is lessThan, bypass
            callFunction. */
         if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan)
             return CompareValues()(a, b);
 
-        Value vTmp1, vTmp2;
-        state.callFunction(*args[0], *a, vTmp1, pos);
-        state.callFunction(vTmp1, *b, vTmp2, pos);
-        return state.forceBool(vTmp2, pos);
+        Value * vs[] = {a, b};
+        Value vBool;
+        state.callFunction(*args[0], 2, vs, vBool, pos);
+        return state.forceBool(vBool, pos);
     };
 
     /* FIXME: std::sort can segfault if the comparator is not a strict
@@ -2897,7 +2946,7 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V
             state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)));
         } catch (TypeError &e) {
             e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap"));
-            throw e;
+            throw;
         }
         len += lists[n].listSize();
     }
@@ -3194,7 +3243,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
 
-    mkString(v, hashString(*ht, s).to_string(Base16, false), context);
+    mkString(v, hashString(*ht, s).to_string(Base16, false));
 }
 
 static RegisterPrimOp primop_hashString({
@@ -3601,15 +3650,13 @@ static RegisterPrimOp primop_splitVersion({
 RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
 
 
-RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
-    std::optional<std::string> requiredFeature)
+RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
 {
     if (!primOps) primOps = new PrimOps;
     primOps->push_back({
         .name = name,
         .args = {},
         .arity = arity,
-        .requiredFeature = std::move(requiredFeature),
         .fun = fun
     });
 }
@@ -3645,9 +3692,7 @@ void EvalState::createBaseEnv()
     if (!evalSettings.pureEval) {
         mkInt(v, time(0));
         addConstant("__currentTime", v);
-    }
 
-    if (!evalSettings.pureEval) {
         mkString(v, settings.thisSystem.get());
         addConstant("__currentSystem", v);
     }
@@ -3685,26 +3730,31 @@ void EvalState::createBaseEnv()
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
-            if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
-                addPrimOp({
-                    .fun = primOp.fun,
-                    .arity = std::max(primOp.args.size(), primOp.arity),
-                    .name = symbols.create(primOp.name),
-                    .args = std::move(primOp.args),
-                    .doc = primOp.doc,
-                });
+            addPrimOp({
+                .fun = primOp.fun,
+                .arity = std::max(primOp.args.size(), primOp.arity),
+                .name = symbols.create(primOp.name),
+                .args = std::move(primOp.args),
+                .doc = primOp.doc,
+            });
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
     sDerivationNix = symbols.create("//builtin/derivation.nix");
-    eval(parse(
-        #include "primops/derivation.nix.gen.hh"
-        , foFile, sDerivationNix, "/", staticBaseEnv), v);
-    addConstant("derivation", v);
+    auto vDerivation = allocValue();
+    addConstant("derivation", vDerivation);
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
     baseEnv.values[0]->attrs->sort();
+
+    staticBaseEnv.sort();
+
+    /* Note: we have to initialize the 'derivation' constant *after*
+       building baseEnv/staticBaseEnv because it uses 'builtins'. */
+    eval(parse(
+        #include "primops/derivation.nix.gen.hh"
+        , foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/primops.hh

@@ -15,7 +15,6 @@ struct RegisterPrimOp
         std::vector<std::string> args;
         size_t arity = 0;
         const char * doc;
-        std::optional<std::string> requiredFeature;
         PrimOpFun fun;
     };
 
@@ -28,8 +27,7 @@ struct RegisterPrimOp
     RegisterPrimOp(
         std::string name,
         size_t arity,
-        PrimOpFun fun,
-        std::optional<std::string> requiredFeature = {});
+        PrimOpFun fun);
 
     RegisterPrimOp(Info && info);
 };

src/libexpr/primops/fetchMercurial.cc

@@ -15,7 +15,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     std::string name = "source";
     PathSet context;
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
 
     if (args[0]->type() == nAttrs) {
 
@@ -84,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
         mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
     v.attrs->sort();
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+    state.allowPath(tree.storePath);
 }
 
 static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);

src/libexpr/primops/fetchTree.cc

@@ -16,7 +16,8 @@ void emitTreeAttrs(
     const fetchers::Tree & tree,
     const fetchers::Input & input,
     Value & v,
-    bool emptyRevFallback)
+    bool emptyRevFallback,
+    bool forceDirty)
 {
     assert(input.isImmutable());
 
@@ -33,24 +34,28 @@ void emitTreeAttrs(
     mkString(*state.allocAttr(v, state.symbols.create("narHash")),
         narHash->to_string(SRI, true));
 
-    if (auto rev = input.getRev()) {
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
-    } else if (emptyRevFallback) {
-        // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
-        auto emptyHash = Hash(htSHA1);
-        mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
-        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
-    }
-
     if (input.getType() == "git")
         mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
             fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
 
-    if (auto revCount = input.getRevCount())
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
-    else if (emptyRevFallback)
-        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
+    if (!forceDirty) {
+
+        if (auto rev = input.getRev()) {
+            mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
+            mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
+        } else if (emptyRevFallback) {
+            // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
+            auto emptyHash = Hash(htSHA1);
+            mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
+            mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
+        }
+
+        if (auto revCount = input.getRevCount())
+            mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
+        else if (emptyRevFallback)
+            mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
+
+    }
 
     if (auto lastModified = input.getLastModified()) {
         mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
@@ -61,7 +66,7 @@ void emitTreeAttrs(
     v.attrs->sort();
 }
 
-std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
+std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
 {
     state.checkURI(uri);
     return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
@@ -76,46 +81,56 @@ std::string fixURIForGit(std::string uri, EvalState & state)
         return fixURI(uri, state);
 }
 
-void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
-{
-    string n(name);
-    attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
-}
-
 struct FetchTreeParams {
     bool emptyRevFallback = false;
     bool allowNameArgument = false;
 };
 
 static void fetchTree(
-    EvalState &state,
-    const Pos &pos,
-    Value **args,
-    Value &v,
-    const std::optional<std::string> type,
+    EvalState & state,
+    const Pos & pos,
+    Value * * args,
+    Value & v,
+    std::optional<std::string> type,
     const FetchTreeParams & params = FetchTreeParams{}
 ) {
     fetchers::Input input;
     PathSet context;
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
 
     if (args[0]->type() == nAttrs) {
         state.forceAttrs(*args[0], pos);
 
         fetchers::Attrs attrs;
 
+        if (auto aType = args[0]->attrs->get(state.sType)) {
+            if (type)
+                throw Error({
+                    .msg = hintfmt("unexpected attribute 'type'"),
+                    .errPos = pos
+                });
+            type = state.forceStringNoCtx(*aType->value, *aType->pos);
+        } else if (!type)
+            throw Error({
+                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+                .errPos = pos
+            });
+
+        attrs.emplace("type", type.value());
+
         for (auto & attr : *args[0]->attrs) {
-            state.forceValue(*attr.value);
-            if (attr.value->type() == nPath || attr.value->type() == nString)
-                addURI(
-                    state,
-                    attrs,
-                    attr.name,
-                    state.coerceToString(*attr.pos, *attr.value, context, false, false)
-                );
-            else if (attr.value->type() == nString)
-                addURI(state, attrs, attr.name, attr.value->string.s);
+            if (attr.name == state.sType) continue;
+            state.forceValue(*attr.value, *attr.pos);
+            if (attr.value->type() == nPath || attr.value->type() == nString) {
+                auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false);
+                attrs.emplace(attr.name,
+                    attr.name == "url"
+                    ? type == "git"
+                      ? fixURIForGit(s, state)
+                      : fixURI(s, state)
+                    : s);
+            }
             else if (attr.value->type() == nBool)
                 attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
             else if (attr.value->type() == nInt)
@@ -125,15 +140,6 @@ static void fetchTree(
                     attr.name, showType(*attr.value));
         }
 
-        if (type)
-            attrs.emplace("type", type.value());
-
-        if (!attrs.count("type"))
-            throw Error({
-                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                .errPos = pos
-            });
-
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 throw Error({
@@ -141,7 +147,6 @@ static void fetchTree(
                     .errPos = pos
                 });
 
-
         input = fetchers::Input::fromAttrs(std::move(attrs));
     } else {
         auto url = state.coerceToString(pos, *args[0], context, false, false);
@@ -164,15 +169,14 @@ static void fetchTree(
 
     auto [tree, input2] = input.fetch(state.store);
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+    state.allowPath(tree.storePath);
 
-    emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback);
+    emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
 }
 
 static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    settings.requireExperimentalFeature("flakes");
+    settings.requireExperimentalFeature(Xp::Flakes);
     fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
 }
 
@@ -185,7 +189,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     std::optional<std::string> url;
     std::optional<Hash> expectedHash;
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
 
     if (args[0]->type() == nAttrs) {
 
@@ -229,20 +233,18 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
         ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
-    auto path = state.store->toRealPath(storePath);
-
     if (expectedHash) {
         auto hash = unpack
             ? state.store->queryPathInfo(storePath)->narHash
-            : hashFile(htSHA256, path);
+            : hashFile(htSHA256, state.store->toRealPath(storePath));
         if (hash != *expectedHash)
             throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n  specified: %s\n  got:       %s",
                 *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
     }
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(path);
+    state.allowPath(storePath);
 
+    auto path = state.store->printStorePath(storePath);
     mkString(v, path, PathSet({path}));
 }
 
@@ -285,13 +287,13 @@ static RegisterPrimOp primop_fetchTarball({
       stdenv.mkDerivation { … }
       ```
 
-      The fetched tarball is cached for a certain amount of time (1 hour
-      by default) in `~/.cache/nix/tarballs/`. You can change the cache
-      timeout either on the command line with `--option tarball-ttl number
-      of seconds` or in the Nix configuration file with this option: ` 
-      number of seconds to cache `.
+      The fetched tarball is cached for a certain amount of time (1
+      hour by default) in `~/.cache/nix/tarballs/`. You can change the
+      cache timeout either on the command line with `--tarball-ttl`
+      *number-of-seconds* or in the Nix configuration file by adding
+      the line `tarball-ttl = ` *number-of-seconds*.
 
-      Note that when obtaining the hash with ` nix-prefetch-url ` the
+      Note that when obtaining the hash with `nix-prefetch-url` the
       option `--unpack` is required.
 
       This function can also verify the contents against a hash. In that
@@ -391,7 +393,7 @@ static RegisterPrimOp primop_fetchGit({
           ```
 
           > **Note**
-          > 
+          >
           > It is nice to always specify the branch which a revision
           > belongs to. Without the branch being specified, the fetcher
           > might fail if the default branch changes. Additionally, it can
@@ -428,12 +430,12 @@ static RegisterPrimOp primop_fetchGit({
           ```
 
           > **Note**
-          > 
+          >
           > Nix will refetch the branch in accordance with
           > the option `tarball-ttl`.
 
           > **Note**
-          > 
+          >
           > This behavior is disabled in *Pure evaluation mode*.
     )",
     .fun = prim_fetchGit,

src/libexpr/value-to-xml.cc

@@ -42,7 +42,7 @@ static void showAttrs(EvalState & state, bool strict, bool location,
 
         XMLAttrs xmlAttrs;
         xmlAttrs["name"] = i;
-        if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos);
+        if (location && a.pos != ptr(&noPos)) posToXML(xmlAttrs, *a.pos);
 
         XMLOpenElement _(doc, "attr", xmlAttrs);
         printValueAsXML(state, strict, location,
@@ -126,24 +126,28 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
         }
 
         case nFunction: {
+
             if (!v.isLambda()) {
-                // FIXME: Serialize primops and primopapps
+                // FIXME: Serialize primops and partial apps
                 doc.writeEmptyElement("unevaluated");
                 break;
             }
+
             XMLAttrs xmlAttrs;
             if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
             XMLOpenElement _(doc, "function", xmlAttrs);
 
-            if (v.lambda.fun->matchAttrs) {
+            auto & arg = v.lambda.fun->args[0];
+
+            if (arg.formals) {
                 XMLAttrs attrs;
-                if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
-                if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
+                if (arg.arg != state.sEpsilon) attrs["name"] = arg.arg;
+                if (arg.formals->ellipsis) attrs["ellipsis"] = "1";
                 XMLOpenElement _(doc, "attrspat", attrs);
-                for (auto & i : v.lambda.fun->formals->formals)
+                for (auto & i : arg.formals->formals)
                     doc.writeEmptyElement("attr", singletonAttrs("name", i.name));
             } else
-                doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg));
+                doc.writeEmptyElement("varpat", singletonAttrs("name", arg.arg));
 
             break;
         }

src/libexpr/value.hh

@@ -21,6 +21,7 @@ typedef enum {
     tListN,
     tThunk,
     tApp,
+    tPartialApp,
     tLambda,
     tBlackhole,
     tPrimOp,
@@ -125,6 +126,7 @@ public:
 
     // type() == nFunction
     inline bool isLambda() const { return internalType == tLambda; };
+    inline bool isPartialApp() const { return internalType == tPartialApp; };
     inline bool isPrimOp() const { return internalType == tPrimOp; };
     inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
 
@@ -196,7 +198,7 @@ public:
             case tNull: return nNull;
             case tAttrs: return nAttrs;
             case tList1: case tList2: case tListN: return nList;
-            case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
+            case tLambda: case tPartialApp: case tPrimOp: case tPrimOpApp: return nFunction;
             case tExternal: return nExternal;
             case tFloat: return nFloat;
             case tThunk: case tApp: case tBlackhole: return nThunk;
@@ -307,6 +309,13 @@ public:
         app.right = r;
     }
 
+    inline void mkPartialApp(Value * l, Value * r)
+    {
+        internalType = tPartialApp;
+        app.left = l;
+        app.right = r;
+    }
+
     inline void mkExternal(ExternalValueBase * e)
     {
         clearValue();

src/libfetchers/fetchers.hh

@@ -38,6 +38,9 @@ struct Input
     bool immutable = false;
     bool direct = true;
 
+    /* path of the parent of this input, used for relative path resolution */
+    std::optional<Path> parent;
+
 public:
     static Input fromURL(const std::string & url);
 

src/libfetchers/git.cc

@@ -4,6 +4,7 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "pathlocks.hh"
 
 #include <sys/time.h>
 #include <sys/wait.h>
@@ -12,6 +13,12 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
+// Explicit initial branch of our bare repo to suppress warnings from new version of git.
+// The value itself does not matter, since we always fetch a specific revision or branch.
+// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
+// old version of git, which will ignore unrecognized `-c` options.
+const std::string gitInitialBranch = "__nix_dummy_branch";
+
 static std::string readHead(const Path & path)
 {
     return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" }));
@@ -317,11 +324,17 @@ struct GitInputScheme : InputScheme
             Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
             repoDir = cacheDir;
 
+            Path cacheDirLock = cacheDir + ".lock";
+            createDirs(dirOf(cacheDir));
+            AutoCloseFD lock = openLockFile(cacheDirLock, true);
+            lockFile(lock.get(), ltWrite, true);
+
             if (!pathExists(cacheDir)) {
-                createDirs(dirOf(cacheDir));
-                runProgram("git", true, { "init", "--bare", repoDir });
+                runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
             }
 
+            deleteLockFile(cacheDirLock, lock.get());
+
             Path localRefFile =
                 input.getRef()->compare(0, 5, "refs/") == 0
                 ? cacheDir + "/" + *input.getRef()
@@ -406,17 +419,14 @@ struct GitInputScheme : InputScheme
         AutoDelete delTmpDir(tmpDir, true);
         PathFilter filter = defaultPathFilter;
 
-        RunOptions checkCommitOpts(
-            "git",
-            { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
-        );
-        checkCommitOpts.searchPath = true;
-        checkCommitOpts.mergeStderrToStdout = true;
-
-        auto result = runProgram(checkCommitOpts);
+        auto result = runProgram(RunOptions {
+            .program = "git",
+            .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() },
+            .mergeStderrToStdout = true
+        });
         if (WEXITSTATUS(result.first) == 128
-            && result.second.find("bad file") != std::string::npos
-        ) {
+            && result.second.find("bad file") != std::string::npos)
+        {
             throw Error(
                 "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
                     "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
@@ -432,7 +442,7 @@ struct GitInputScheme : InputScheme
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);
 
-            runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir });
+            runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
             // TODO: repoDir might lack the ref (it only checks if rev
             // exists, see FIXME above) so use a big hammer and fetch
             // everything to ensure we get the rev.
@@ -448,9 +458,11 @@ struct GitInputScheme : InputScheme
             // FIXME: should pipe this, or find some better way to extract a
             // revision.
             auto source = sinkToSource([&](Sink & sink) {
-                RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
-                gitOptions.standardOut = &sink;
-                runProgram2(gitOptions);
+                runProgram2({
+                    .program = "git",
+                    .args = { "-C", repoDir, "archive", input.getRev()->gitRev() },
+                    .standardOut = &sink
+                });
             });
 
             unpackTarfile(*source, tmpDir);

src/libfetchers/github.cc

@@ -273,9 +273,9 @@ struct GitHubInputScheme : GitArchiveInputScheme
     void clone(const Input & input, const Path & destDir) override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+        Input::fromURL(fmt("git+https://%s/%s/%s.git",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
-            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
     }
 };
@@ -341,9 +341,9 @@ struct GitLabInputScheme : GitArchiveInputScheme
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
-        Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+        Input::fromURL(fmt("git+https://%s/%s/%s.git",
                 host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
-            .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+            .applyOverrides(input.getRef(), input.getRev())
             .clone(destDir);
     }
 };

src/libfetchers/local.mk

@@ -8,4 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
+libfetchers_LDFLAGS += -pthread
+
 libfetchers_LIBS = libutil libstore

src/libfetchers/mercurial.cc

@@ -11,34 +11,32 @@ using namespace std::string_literals;
 
 namespace nix::fetchers {
 
-namespace {
+static RunOptions hgOptions(const Strings & args)
+{
+    auto env = getEnv();
+    // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
+    env["HGPLAIN"] = "";
 
-RunOptions hgOptions(const Strings & args) {
-	RunOptions opts("hg", args);
-	opts.searchPath = true;
-
-	auto env = getEnv();
-	// Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
-	env["HGPLAIN"] = "";
-	opts.environment = env;
-
-	return opts;
+    return {
+        .program = "hg",
+        .searchPath = true,
+        .args = args,
+        .environment = env
+    };
 }
 
 // runProgram wrapper that uses hgOptions instead of stock RunOptions.
-string runHg(const Strings & args, const std::optional<std::string> & input = {})
+static string runHg(const Strings & args, const std::optional<std::string> & input = {})
 {
-	RunOptions opts = hgOptions(args);
-	opts.input = input;
+    RunOptions opts = hgOptions(args);
+    opts.input = input;
 
-	auto res = runProgram(opts);
+    auto res = runProgram(std::move(opts));
 
-	if (!statusOk(res.first))
-		throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
-
-	return res.second;
-}
+    if (!statusOk(res.first))
+        throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
 
+    return res.second;
 }
 
 struct MercurialInputScheme : InputScheme
@@ -253,9 +251,7 @@ struct MercurialInputScheme : InputScheme
            have to pull again. */
         if (!(input.getRev()
                 && pathExists(cacheDir)
-                && runProgram(
-                    hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
-                    .killStderr(true)).second == "1"))
+                && runProgram(hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })).second == "1"))
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
 

src/libfetchers/path.cc

@@ -82,18 +82,38 @@ struct PathInputScheme : InputScheme
 
     std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
     {
+        std::string absPath;
         auto path = getStrAttr(input.attrs, "path");
 
-        // FIXME: check whether access to 'path' is allowed.
+        if (path[0] != '/') {
+            if (!input.parent)
+                throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
 
-        auto storePath = store->maybeParseStorePath(path);
+            auto parent = canonPath(*input.parent);
+
+            // the path isn't relative, prefix it
+            absPath = nix::absPath(path, parent);
+
+            // for security, ensure that if the parent is a store path, it's inside it
+            if (store->isInStore(parent)) {
+                auto storePath = store->printStorePath(store->toStorePath(parent).first);
+                if (!isInDir(absPath, storePath))
+                    throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath);
+            }
+        } else
+            absPath = path;
+
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s'", absPath));
+
+        // FIXME: check whether access to 'path' is allowed.
+        auto storePath = store->maybeParseStorePath(absPath);
 
         if (storePath)
             store->addTempRoot(*storePath);
 
         if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
             // FIXME: try to substitute storePath.
-            storePath = store->addToStore("source", path);
+            storePath = store->addToStore("source", absPath);
 
         return {
             Tree(store->toRealPath(*storePath), std::move(*storePath)),

src/libmain/local.mk

@@ -8,10 +8,10 @@ libmain_SOURCES := $(wildcard $(d)/*.cc)
 
 libmain_CXXFLAGS += -I src/libutil -I src/libstore
 
-libmain_LDFLAGS = $(OPENSSL_LIBS)
+libmain_LDFLAGS += $(OPENSSL_LIBS)
 
 libmain_LIBS = libstore libutil
 
 libmain_ALLOW_UNDEFINED = 1
 
-$(eval $(call install-file-in, $(d)/nix-main.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644))

src/libmain/shared.cc

@@ -15,6 +15,9 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <signal.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
 
 #include <openssl/crypto.h>
 
@@ -110,6 +113,31 @@ static void opensslLockCallback(int mode, int type, const char * file, int line)
 }
 #endif
 
+static std::once_flag dns_resolve_flag;
+
+static void preloadNSS() {
+    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
+       load its lookup libraries in the parent before any child gets a chance to. */
+    std::call_once(dns_resolve_flag, []() {
+        struct addrinfo *res = NULL;
+
+        /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN.
+           We need the resolution to be done locally, as nscd socket will not be accessible in the
+           sandbox. */
+        char * previous_env = getenv("LOCALDOMAIN");
+        setenv("LOCALDOMAIN", "invalid", 1);
+        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) {
+            if (res) freeaddrinfo(res);
+        }
+        if (previous_env) {
+             setenv("LOCALDOMAIN", previous_env, 1);
+        } else {
+             unsetenv("LOCALDOMAIN");
+        }
+    });
+}
 
 static void sigHandler(int signo) { }
 
@@ -176,6 +204,8 @@ void initNix()
     if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/"))
         unsetenv("TMPDIR");
 #endif
+
+    preloadNSS();
 }
 
 

src/libstore/binary-cache-store.cc

@@ -52,9 +52,9 @@ void BinaryCacheStore::init()
                     throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'",
                         getUri(), value, storeDir);
             } else if (name == "WantMassQuery") {
-                wantMassQuery.setDefault(value == "1" ? "true" : "false");
+                wantMassQuery.setDefault(value == "1");
             } else if (name == "Priority") {
-                priority.setDefault(fmt("%d", std::stoi(value)));
+                priority.setDefault(std::stoi(value));
             }
         }
     }
@@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    std::string hashPart(narInfo->path.hashPart());
-
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
+        state_->pathInfoCache.upsert(
+            std::string(narInfo->path.to_string()),
+            PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
     }
 
     if (diskCache)
-        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+        diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
 }
 
 AutoCloseFD openFile(const Path & path)
@@ -130,17 +130,6 @@ AutoCloseFD openFile(const Path & path)
     return fd;
 }
 
-struct FileSource : FdSource
-{
-    AutoCloseFD fd2;
-
-    FileSource(const Path & path)
-        : fd2(openFile(path))
-    {
-        fd = fd2.get();
-    }
-};
-
 ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
     std::function<ValidPathInfo(HashResult)> mkInfo)
@@ -160,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     {
     FdSink fileSink(fdTemp.get());
     TeeSink teeSinkCompressed { fileSink, fileHashSink };
-    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
+    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
     TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
     TeeSource teeSource { narSource, teeSinkUncompressed };
     narAccessor = makeNarAccessor(teeSource);

src/libstore/binary-cache-store.hh

@@ -15,13 +15,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
+    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
     const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
     const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
     const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
     const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
-        "enable multi-threading compression, available for xz only currently"};
+        "enable multi-threading compression for NARs, available for xz and zstd only currently"};
+    const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
+        "specify 'preset level' of compression to be used with NARs: "
+        "meaning and accepted range of values depends on compression method selected, "
+        "other than -1 which we reserve to indicate Nix defaults should be used"};
 };
 
 class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store

src/libstore/build/derivation-goal.cc

@@ -204,7 +204,7 @@ void DerivationGoal::haveDerivation()
     trace("have derivation");
 
     if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
+        settings.requireExperimentalFeature(Xp::CaDerivations);
 
     retrySubstitution = false;
 
@@ -453,7 +453,7 @@ void DerivationGoal::inputsRealised()
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
             ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
             || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
             /* We are be able to resolve this derivation based on the
@@ -566,7 +566,7 @@ void DerivationGoal::tryToBuild()
                 lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
             else
                 lockFiles.insert(
-                    worker.store.Store::toRealPath(drvPath) + "!" + i.first
+                    worker.store.Store::toRealPath(drvPath) + "." + i.first
                 );
         }
     }
@@ -616,7 +616,9 @@ void DerivationGoal::tryToBuild()
     /* Don't do a remote build if the derivation has the attribute
        `preferLocalBuild' set.  Also, check and repair modes are only
        supported for local builds. */
-    bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
+    bool buildLocally =
+        (buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
+        && settings.maxBuildJobs.get() != 0;
 
     if (!buildLocally) {
         switch (tryBuildHook()) {
@@ -774,9 +776,6 @@ void runPostBuildHook(
     hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths))));
     hookEnvironment.emplace("NIX_CONFIG", globalConfig.toKeyValue());
 
-    RunOptions opts(settings.postBuildHook, {});
-    opts.environment = hookEnvironment;
-
     struct LogSink : Sink {
         Activity & act;
         std::string currentLine;
@@ -807,9 +806,12 @@ void runPostBuildHook(
     };
     LogSink sink(act);
 
-    opts.standardOut = &sink;
-    opts.mergeStderrToStdout = true;
-    runProgram2(opts);
+    runProgram2({
+        .program = settings.postBuildHook,
+        .environment = hookEnvironment,
+        .standardOut = &sink,
+        .mergeStderrToStdout = true,
+    });
 }
 
 void DerivationGoal::buildDone()
@@ -1009,7 +1011,7 @@ HookReply DerivationGoal::tryBuildHook()
                     return readLine(worker.hook->fromHook.readSide.get());
                 } catch (Error & e) {
                     e.addTrace({}, "while reading the response from the build hook");
-                    throw e;
+                    throw;
                 }
             }();
             if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
@@ -1055,7 +1057,7 @@ HookReply DerivationGoal::tryBuildHook()
         machineName = readLine(hook->fromHook.readSide.get());
     } catch (Error & e) {
         e.addTrace({}, "while reading the machine name from the build hook");
-        throw e;
+        throw;
     }
 
     /* Tell the hook all the inputs that have to be copied to the
@@ -1273,7 +1275,7 @@ void DerivationGoal::checkPathValidity()
                     : PathStatus::Corrupt,
             };
         }
-        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
             auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
             if (auto real = worker.store.queryRealisation(drvOutput)) {
                 info.known = {

src/libstore/build/entry-points.cc

@@ -1,4 +1,3 @@
-#include "machines.hh"
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "derivation-goal.hh"
@@ -11,12 +10,12 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
     Worker worker(*this, evalStore ? *evalStore : *this);
 
     Goals goals;
-    for (auto & br : reqs) {
+    for (const auto & br : reqs) {
         std::visit(overloaded {
-            [&](DerivedPath::Built bfd) {
+            [&](const DerivedPath::Built & bfd) {
                 goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
             },
-            [&](DerivedPath::Opaque bo) {
+            [&](const DerivedPath::Opaque & bo) {
                 goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
             },
         }, br.raw());
@@ -74,7 +73,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
                     outputId,
                     Realisation{ outputId, *staticOutput.second}
                     );
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
             auto realisation = this->queryRealisation(outputId);
             if (realisation)
                 result.builtOutputs.insert_or_assign(

src/libstore/build/goal.cc

@@ -13,11 +13,9 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
 
 void addToWeakGoals(WeakGoals & goals, GoalPtr p)
 {
-    // FIXME: necessary?
-    // FIXME: O(n)
-    for (auto & i : goals)
-        if (i.lock() == p) return;
-    goals.push_back(p);
+    if (goals.find(p) != goals.end())
+        return;
+    goals.insert(p);
 }
 
 
@@ -46,10 +44,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
         /* If we failed and keepGoing is not set, we remove all
            remaining waitees. */
         for (auto & goal : waitees) {
-            WeakGoals waiters2;
-            for (auto & j : goal->waiters)
-                if (j.lock() != shared_from_this()) waiters2.push_back(j);
-            goal->waiters = waiters2;
+            goal->waiters.extract(shared_from_this());
         }
         waitees.clear();
 

src/libstore/build/goal.hh

@@ -19,7 +19,7 @@ struct CompareGoalPtrs {
 
 /* Set of goals. */
 typedef set<GoalPtr, CompareGoalPtrs> Goals;
-typedef list<WeakGoalPtr> WeakGoals;
+typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
 
 /* A map of paths to goals (and the other way around). */
 typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;

src/libstore/build/local-derivation-goal.cc

@@ -17,16 +17,14 @@
 #include <regex>
 #include <queue>
 
-#include <sys/types.h>
-#include <sys/socket.h>
 #include <sys/un.h>
-#include <netdb.h>
 #include <fcntl.h>
 #include <termios.h>
 #include <unistd.h>
 #include <sys/mman.h>
 #include <sys/utsname.h>
 #include <sys/resource.h>
+#include <sys/socket.h>
 
 #if HAVE_STATVFS
 #include <sys/statvfs.h>
@@ -34,7 +32,6 @@
 
 /* Includes required for chroot support. */
 #if __linux__
-#include <sys/socket.h>
 #include <sys/ioctl.h>
 #include <net/if.h>
 #include <netinet/ip.h>
@@ -70,12 +67,14 @@ void handleDiffHook(
     auto diffHook = settings.diffHook;
     if (diffHook != "" && settings.runDiffHook) {
         try {
-            RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
-            diffHookOptions.searchPath = true;
-            diffHookOptions.uid = uid;
-            diffHookOptions.gid = gid;
-            diffHookOptions.chdir = "/";
-            auto diffRes = runProgram(diffHookOptions);
+            auto diffRes = runProgram(RunOptions {
+                .program = diffHook,
+                .searchPath = true,
+                .args = {tryA, tryB, drvPath, tmpDir},
+                .uid = uid,
+                .gid = gid,
+                .chdir = "/"
+            });
             if (!statusOk(diffRes.first))
                 throw ExecError(diffRes.first,
                     "diff-hook program '%1%' %2%",
@@ -344,23 +343,6 @@ int childEntry(void * arg)
 }
 
 
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
-    /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
-       one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
-       been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
-       load its lookup libraries in the parent before any child gets a chance to. */
-    std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
-            if (res) freeaddrinfo(res);
-        }
-    });
-}
-
-
 static void linkOrCopy(const Path & from, const Path & to)
 {
     if (link(from.c_str(), to.c_str()) == -1) {
@@ -389,9 +371,6 @@ void LocalDerivationGoal::startBuilder()
             settings.thisSystem,
             concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
 
-    if (drv->isBuiltin())
-        preloadNSS();
-
 #if __APPLE__
     additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
 #endif
@@ -732,6 +711,7 @@ void LocalDerivationGoal::startBuilder()
     if (!builderOut.readSide)
         throw SysError("opening pseudoterminal master");
 
+    // FIXME: not thread-safe, use ptsname_r
     std::string slaveName(ptsname(builderOut.readSide.get()));
 
     if (buildUser) {
@@ -775,7 +755,6 @@ void LocalDerivationGoal::startBuilder()
     result.startTime = time(0);
 
     /* Fork a child to build the package. */
-    ProcessOptions options;
 
 #if __linux__
     if (useChroot) {
@@ -818,8 +797,6 @@ void LocalDerivationGoal::startBuilder()
 
         userNamespaceSync.create();
 
-        options.allowVfork = false;
-
         Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
         static bool userNamespacesEnabled =
             pathExists(maxUserNamespaces)
@@ -877,7 +854,7 @@ void LocalDerivationGoal::startBuilder()
             writeFull(builderOut.writeSide.get(),
                 fmt("%d %d\n", usingUserNamespace, child));
             _exit(0);
-        }, options);
+        });
 
         int res = helper.wait();
         if (res != 0 && settings.sandboxFallback) {
@@ -941,10 +918,9 @@ void LocalDerivationGoal::startBuilder()
 #endif
     {
     fallback:
-        options.allowVfork = !buildUser && !drv->isBuiltin();
         pid = startProcess([&]() {
             runChild();
-        }, options);
+        });
     }
 
     /* parent */
@@ -959,9 +935,12 @@ void LocalDerivationGoal::startBuilder()
             try {
                 return readLine(builderOut.readSide.get());
             } catch (Error & e) {
-                e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
+                auto status = pid.wait();
+                e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)",
+                    worker.store.printStorePath(drvPath),
+                    statusToString(status),
                     concatStringsSep("|", msgs));
-                throw e;
+                throw;
             }
         }();
         if (string(msg, 0, 1) == "\2") break;
@@ -1112,10 +1091,10 @@ void LocalDerivationGoal::writeStructuredAttrs()
 static StorePath pathPartOfReq(const DerivedPath & req)
 {
     return std::visit(overloaded {
-        [&](DerivedPath::Opaque bo) {
+        [&](const DerivedPath::Opaque & bo) {
             return bo.path;
         },
-        [&](DerivedPath::Built bfd)  {
+        [&](const DerivedPath::Built & bfd)  {
             return bfd.drvPath;
         },
     }, req.raw());
@@ -1280,7 +1259,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             for (auto & [outputName, outputPath] : outputs)
                 if (wantOutput(outputName, bfd.outputs)) {
                     newPaths.insert(outputPath);
-                    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
                         auto thisRealisation = next->queryRealisation(
                             DrvOutput{drvHashes.at(outputName), outputName}
                         );
@@ -1341,7 +1320,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
 
 void LocalDerivationGoal::startDaemon()
 {
-    settings.requireExperimentalFeature("recursive-nix");
+    settings.requireExperimentalFeature(Xp::RecursiveNix);
 
     Store::Params params;
     params["path-info-cache-size"] = "0";
@@ -1374,7 +1353,7 @@ void LocalDerivationGoal::startDaemon()
             AutoCloseFD remote = accept(daemonSocket.get(),
                 (struct sockaddr *) &remoteAddr, &remoteAddrLen);
             if (!remote) {
-                if (errno == EINTR) continue;
+                if (errno == EINTR || errno == EAGAIN) continue;
                 if (errno == EINVAL) break;
                 throw SysError("accepting connection");
             }
@@ -1853,7 +1832,7 @@ void LocalDerivationGoal::runChild()
         /* Fill in the arguments. */
         Strings args;
 
-        const char *builder = "invalid";
+        std::string builder = "invalid";
 
         if (drv->isBuiltin()) {
             ;
@@ -1979,13 +1958,13 @@ void LocalDerivationGoal::runChild()
                 }
                 args.push_back(drv->builder);
             } else {
-                builder = drv->builder.c_str();
+                builder = drv->builder;
                 args.push_back(std::string(baseNameOf(drv->builder)));
             }
         }
 #else
         else {
-            builder = drv->builder.c_str();
+            builder = drv->builder;
             args.push_back(std::string(baseNameOf(drv->builder)));
         }
 #endif
@@ -2041,9 +2020,9 @@ void LocalDerivationGoal::runChild()
             posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
         }
 
-        posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+        posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 #else
-        execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+        execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 #endif
 
         throw SysError("executing '%1%'", drv->builder);
@@ -2158,8 +2137,7 @@ void LocalDerivationGoal::registerOutputs()
 
         /* Pass blank Sink as we are not ready to hash data at this stage. */
         NullSink blank;
-        auto references = worker.store.parseStorePathSet(
-            scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
+        auto references = scanForReferences(blank, actualPath, referenceablePaths);
 
         outputReferencesIfUnregistered.insert_or_assign(
             outputName,
@@ -2173,8 +2151,8 @@ void LocalDerivationGoal::registerOutputs()
                 /* Since we'll use the already installed versions of these, we
                    can treat them as leaves and ignore any references they
                    have. */
-                [&](AlreadyRegistered _) { return StringSet {}; },
-                [&](PerhapsNeedToRegister refs) {
+                [&](const AlreadyRegistered &) { return StringSet {}; },
+                [&](const PerhapsNeedToRegister & refs) {
                     StringSet referencedOutputs;
                     /* FIXME build inverted map up front so no quadratic waste here */
                     for (auto & r : refs.refs)
@@ -2210,11 +2188,11 @@ void LocalDerivationGoal::registerOutputs()
         };
 
         std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
-            [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
+            [&](const AlreadyRegistered & skippedFinalPath) -> std::optional<StorePathSet> {
                 finish(skippedFinalPath.path);
                 return std::nullopt;
             },
-            [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
+            [&](const PerhapsNeedToRegister & r) -> std::optional<StorePathSet> {
                 return r.refs;
             },
         }, outputReferencesIfUnregistered.at(outputName));
@@ -2226,7 +2204,7 @@ void LocalDerivationGoal::registerOutputs()
         auto rewriteOutput = [&]() {
             /* Apply hash rewriting if necessary. */
             if (!outputRewrites.empty()) {
-                warn("rewriting hashes in '%1%'; cross fingers", actualPath);
+                debug("rewriting hashes in '%1%'; cross fingers", actualPath);
 
                 /* FIXME: this is in-memory. */
                 StringSink sink;
@@ -2330,7 +2308,7 @@ void LocalDerivationGoal::registerOutputs()
         };
 
         ValidPathInfo newInfo = std::visit(overloaded {
-            [&](DerivationOutputInputAddressed output) {
+            [&](const DerivationOutputInputAddressed & output) {
                 /* input-addressed case */
                 auto requiredFinalPath = output.path;
                 /* Preemptively add rewrite rule for final hash, as that is
@@ -2349,14 +2327,14 @@ void LocalDerivationGoal::registerOutputs()
                     newInfo0.references.insert(newInfo0.path);
                 return newInfo0;
             },
-            [&](DerivationOutputCAFixed dof) {
+            [&](const DerivationOutputCAFixed & dof) {
                 auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
                     .method = dof.hash.method,
                     .hashType = dof.hash.hash.type,
                 });
 
                 /* Check wanted hash */
-                Hash & wanted = dof.hash.hash;
+                const Hash & wanted = dof.hash.hash;
                 assert(newInfo0.ca);
                 auto got = getContentAddressHash(*newInfo0.ca);
                 if (wanted != got) {
@@ -2492,7 +2470,13 @@ void LocalDerivationGoal::registerOutputs()
         infos.emplace(outputName, std::move(newInfo));
     }
 
-    if (buildMode == bmCheck) return;
+    if (buildMode == bmCheck) {
+        // In case of FOD mismatches on `--check` an error must be thrown as this is also
+        // a source for non-determinism.
+        if (delayedException)
+            std::rethrow_exception(delayedException);
+        return;
+    }
 
     /* Apply output checks. */
     checkOutputs(infos);
@@ -2577,7 +2561,7 @@ void LocalDerivationGoal::registerOutputs()
        that for floating CA derivations, which otherwise couldn't be cached,
        but it's fine to do in all cases. */
 
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         for (auto& [outputName, newInfo] : infos) {
             auto thisRealisation = Realisation{
                 .id = DrvOutput{initialOutputs.at(outputName).outputHash,

src/libstore/build/worker.cc

@@ -239,7 +239,7 @@ void Worker::run(const Goals & _topGoals)
         }
     }
 
-    /* Call queryMissing() efficiently query substitutes. */
+    /* Call queryMissing() to efficiently query substitutes. */
     StorePathSet willBuild, willSubstitute, unknown;
     uint64_t downloadSize, narSize;
     store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);

src/libstore/content-address.cc

@@ -31,10 +31,10 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
 std::string renderContentAddress(ContentAddress ca)
 {
     return std::visit(overloaded {
-        [](TextHash th) {
+        [](TextHash & th) {
             return "text:" + th.hash.to_string(Base32, true);
         },
-        [](FixedOutputHash fsh) {
+        [](FixedOutputHash & fsh) {
             return makeFixedOutputCA(fsh.method, fsh.hash);
         }
     }, ca);
@@ -43,10 +43,10 @@ std::string renderContentAddress(ContentAddress ca)
 std::string renderContentAddressMethod(ContentAddressMethod cam)
 {
     return std::visit(overloaded {
-        [](TextHashMethod &th) {
+        [](TextHashMethod & th) {
             return std::string{"text:"} + printHashType(htSHA256);
         },
-        [](FixedOutputHashMethod &fshm) {
+        [](FixedOutputHashMethod & fshm) {
             return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType);
         }
     }, cam);
@@ -104,12 +104,12 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
 
     return std::visit(
         overloaded {
-            [&](TextHashMethod thm) {
+            [&](TextHashMethod & thm) {
                 return ContentAddress(TextHash {
                     .hash = Hash::parseNonSRIUnprefixed(rest, htSHA256)
                 });
             },
-            [&](FixedOutputHashMethod fohMethod) {
+            [&](FixedOutputHashMethod & fohMethod) {
                 return ContentAddress(FixedOutputHash {
                     .method = fohMethod.fileIngestionMethod,
                     .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)),
@@ -137,10 +137,10 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
 Hash getContentAddressHash(const ContentAddress & ca)
 {
     return std::visit(overloaded {
-        [](TextHash th) {
+        [](const TextHash & th) {
             return th.hash;
         },
-        [](FixedOutputHash fsh) {
+        [](const FixedOutputHash & fsh) {
             return fsh.hash;
         }
     }, ca);

src/libstore/daemon.cc

@@ -227,8 +227,15 @@ struct ClientSettings
             try {
                 if (name == "ssh-auth-sock") // obsolete
                     ;
+                else if (name == settings.experimentalFeatures.name) {
+                    // We don’t want to forward the experimental features to
+                    // the daemon, as that could cause some pretty weird stuff
+                    if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get())
+                        debug("Ignoring the client-specified experimental features");
+                }
                 else if (trusted
                     || name == settings.buildTimeout.name
+                    || name == settings.buildRepeat.name
                     || name == "connect-timeout"
                     || (name == "builders" && value == ""))
                     settings.set(name, value);
@@ -389,13 +396,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 FramedSource source(from);
                 // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
                 return std::visit(overloaded {
-                    [&](TextHashMethod &_) {
+                    [&](TextHashMethod &) {
                         // We could stream this by changing Store
                         std::string contents = source.drain();
                         auto path = store->addTextToStore(name, contents, refs, repair);
                         return store->queryPathInfo(path);
                     },
-                    [&](FixedOutputHashMethod &fohm) {
+                    [&](FixedOutputHashMethod & fohm) {
                         if (!refs.empty())
                             throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
                         auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
@@ -618,9 +625,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    // Obsolete.
     case wopSyncWithGC: {
         logger->startWork();
-        store->syncWithGC();
         logger->stopWork();
         to << 1;
         break;

src/libstore/derivations.cc

@@ -10,18 +10,18 @@ namespace nix {
 std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
 {
     return std::visit(overloaded {
-        [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
+        [](const DerivationOutputInputAddressed & doi) -> std::optional<StorePath> {
             return { doi.path };
         },
-        [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
+        [&](const DerivationOutputCAFixed & dof) -> std::optional<StorePath> {
             return {
                 dof.path(store, drvName, outputName)
             };
         },
-        [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
+        [](const DerivationOutputCAFloating & dof) -> std::optional<StorePath> {
             return std::nullopt;
         },
-        [](DerivationOutputDeferred) -> std::optional<StorePath> {
+        [](const DerivationOutputDeferred &) -> std::optional<StorePath> {
             return std::nullopt;
         },
     }, output);
@@ -187,7 +187,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
                 },
             };
         } else {
-            settings.requireExperimentalFeature("ca-derivations");
+            settings.requireExperimentalFeature(Xp::CaDerivations);
             assert(pathS == "");
             return DerivationOutput {
                 .output = DerivationOutputCAFloating {
@@ -332,22 +332,22 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
         std::visit(overloaded {
-            [&](DerivationOutputInputAddressed doi) {
+            [&](const DerivationOutputInputAddressed & doi) {
                 s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
             },
-            [&](DerivationOutputCAFixed dof) {
+            [&](const DerivationOutputCAFixed & dof) {
                 s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
                 s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
                 s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
             },
-            [&](DerivationOutputCAFloating dof) {
+            [&](const DerivationOutputCAFloating & dof) {
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                 s += ','; printUnquotedString(s, "");
             },
-            [&](DerivationOutputDeferred) {
+            [&](const DerivationOutputDeferred &) {
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
@@ -420,13 +420,13 @@ DerivationType BasicDerivation::type() const
     std::optional<HashType> floatingHashType;
     for (auto & i : outputs) {
         std::visit(overloaded {
-            [&](DerivationOutputInputAddressed _) {
+            [&](const DerivationOutputInputAddressed &) {
                inputAddressedOutputs.insert(i.first);
             },
-            [&](DerivationOutputCAFixed _) {
+            [&](const DerivationOutputCAFixed &) {
                 fixedCAOutputs.insert(i.first);
             },
-            [&](DerivationOutputCAFloating dof) {
+            [&](const DerivationOutputCAFloating & dof) {
                 floatingCAOutputs.insert(i.first);
                 if (!floatingHashType) {
                     floatingHashType = dof.hashType;
@@ -435,7 +435,7 @@ DerivationType BasicDerivation::type() const
                         throw Error("All floating outputs must use the same hash type");
                 }
             },
-            [&](DerivationOutputDeferred _) {
+            [&](const DerivationOutputDeferred &) {
                deferredIAOutputs.insert(i.first);
             },
         }, i.second.output);
@@ -538,15 +538,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
         const auto & res = pathDerivationModulo(store, i.first);
         std::visit(overloaded {
             // Regular non-CA derivation, replace derivation
-            [&](Hash drvHash) {
+            [&](const Hash & drvHash) {
                 inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
             },
-            [&](DeferredHash deferredHash) {
+            [&](const DeferredHash & deferredHash) {
                 isDeferred = true;
                 inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
             },
             // CA derivation's output hashes
-            [&](CaOutputHashes outputHashes) {
+            [&](const CaOutputHashes & outputHashes) {
                 std::set<std::string> justOut = { "out" };
                 for (auto & output : i.second) {
                     /* Put each one in with a single "out" output.. */
@@ -572,17 +572,17 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
 {
     std::map<std::string, Hash> res;
     std::visit(overloaded {
-        [&](Hash drvHash) {
+        [&](const Hash & drvHash) {
             for (auto & outputName : drv.outputNames()) {
                 res.insert({outputName, drvHash});
             }
         },
-        [&](DeferredHash deferredHash) {
+        [&](const DeferredHash & deferredHash) {
             for (auto & outputName : drv.outputNames()) {
                 res.insert({outputName, deferredHash.hash});
             }
         },
-        [&](CaOutputHashes outputHashes) {
+        [&](const CaOutputHashes & outputHashes) {
             res = outputHashes;
         },
     }, hashDerivationModulo(store, drv, true));
@@ -666,22 +666,22 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
     for (auto & i : drv.outputs) {
         out << i.first;
         std::visit(overloaded {
-            [&](DerivationOutputInputAddressed doi) {
+            [&](const DerivationOutputInputAddressed & doi) {
                 out << store.printStorePath(doi.path)
                     << ""
                     << "";
             },
-            [&](DerivationOutputCAFixed dof) {
+            [&](const DerivationOutputCAFixed & dof) {
                 out << store.printStorePath(dof.path(store, drv.name, i.first))
                     << dof.hash.printMethodAlgo()
                     << dof.hash.hash.to_string(Base16, false);
             },
-            [&](DerivationOutputCAFloating dof) {
+            [&](const DerivationOutputCAFloating & dof) {
                 out << ""
                     << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
                     << "";
             },
-            [&](DerivationOutputDeferred) {
+            [&](const DerivationOutputDeferred &) {
                 out << ""
                     << ""
                     << "";

src/libstore/derivations.hh

@@ -138,8 +138,8 @@ struct Derivation : BasicDerivation
 
     /* Return the underlying basic derivation but with these changes:
 
-	   1. Input drvs are emptied, but the outputs of them that were used are
-	      added directly to input sources.
+       1. Input drvs are emptied, but the outputs of them that were used are
+          added directly to input sources.
 
        2. Input placeholders are replaced with realized input store paths. */
     std::optional<BasicDerivation> tryResolve(Store & store);

src/libstore/derived-path.cc

@@ -24,8 +24,8 @@ StorePathSet BuiltPath::outPaths() const
 {
     return std::visit(
         overloaded{
-            [](BuiltPath::Opaque p) { return StorePathSet{p.path}; },
-            [](BuiltPath::Built b) {
+            [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; },
+            [](const BuiltPath::Built & b) {
                 StorePathSet res;
                 for (auto & [_, path] : b.outputs)
                     res.insert(path);
@@ -94,13 +94,13 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
     RealisedPath::Set res;
     std::visit(
         overloaded{
-            [&](BuiltPath::Opaque p) { res.insert(p.path); },
-            [&](BuiltPath::Built p) {
+            [&](const BuiltPath::Opaque & p) { res.insert(p.path); },
+            [&](const BuiltPath::Built & p) {
                 auto drvHashes =
                     staticOutputHashes(store, store.readDerivation(p.drvPath));
                 for (auto& [outputName, outputPath] : p.outputs) {
                     if (settings.isExperimentalFeatureEnabled(
-                            "ca-derivations")) {
+                                Xp::CaDerivations)) {
                         auto thisRealisation = store.queryRealisation(
                             DrvOutput{drvHashes.at(outputName), outputName});
                         assert(thisRealisation);  // We’ve built it, so we must h

src/libstore/filetransfer.cc

@@ -716,15 +716,24 @@ struct curlFileTransfer : public FileTransfer
     }
 };
 
+ref<curlFileTransfer> makeCurlFileTransfer()
+{
+    return make_ref<curlFileTransfer>();
+}
+
 ref<FileTransfer> getFileTransfer()
 {
-    static ref<FileTransfer> fileTransfer = makeFileTransfer();
+    static ref<curlFileTransfer> fileTransfer = makeCurlFileTransfer();
+
+    if (fileTransfer->state_.lock()->quit)
+        fileTransfer = makeCurlFileTransfer();
+
     return fileTransfer;
 }
 
 ref<FileTransfer> makeFileTransfer()
 {
-    return make_ref<curlFileTransfer>();
+    return makeCurlFileTransfer();
 }
 
 std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)

src/libstore/gc.cc

@@ -10,48 +10,22 @@
 #include <regex>
 #include <random>
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
+#include <climits>
 #include <errno.h>
 #include <fcntl.h>
+#include <poll.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/types.h>
+#include <sys/un.h>
 #include <unistd.h>
-#include <climits>
 
 namespace nix {
 
 
-static string gcLockName = "gc.lock";
-static string gcRootsDir = "gcroots";
-
-
-/* Acquire the global GC lock.  This is used to prevent new Nix
-   processes from starting after the temporary root files have been
-   read.  To be precise: when they try to create a new temporary root
-   file, they will block until the garbage collector has finished /
-   yielded the GC lock. */
-AutoCloseFD LocalStore::openGCLock(LockType lockType)
-{
-    Path fnGCLock = (format("%1%/%2%")
-        % stateDir % gcLockName).str();
-
-    debug(format("acquiring global GC lock '%1%'") % fnGCLock);
-
-    AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-    if (!fdGCLock)
-        throw SysError("opening global GC lock '%1%'", fnGCLock);
-
-    if (!lockFile(fdGCLock.get(), lockType, false)) {
-        printInfo("waiting for the big garbage collector lock...");
-        lockFile(fdGCLock.get(), lockType, true);
-    }
-
-    /* !!! Restrict read permission on the GC root.  Otherwise any
-       process that can open the file for reading can DoS the
-       collector. */
-
-    return fdGCLock;
-}
+static std::string gcSocketPath = "/gc-socket/socket";
+static std::string gcRootsDir = "gcroots";
 
 
 static void makeSymlink(const Path & link, const Path & target)
@@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target)
 }
 
 
-void LocalStore::syncWithGC()
-{
-    AutoCloseFD fdGCLock = openGCLock(ltRead);
-}
-
-
 void LocalStore::addIndirectRoot(const Path & path)
 {
     string hash = hashString(htSHA1, path).to_string(Base32, false);
@@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
                 "(are you running nix-build inside the store?)", gcRoot);
 
+    /* Register this root with the garbage collector, if it's
+       running. This should be superfluous since the caller should
+       have registered this root yet, but let's be on the safe
+       side. */
+    addTempRoot(storePath);
+
     /* Don't clobber the link if it already exists and doesn't
        point to the Nix store. */
     if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
@@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
     makeSymlink(gcRoot, printStorePath(storePath));
     addIndirectRoot(gcRoot);
 
-    /* Grab the global GC root, causing us to block while a GC is in
-       progress.  This prevents the set of permanent roots from
-       increasing while a GC is in progress. */
-    syncWithGC();
-
     return gcRoot;
 }
 
@@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path)
     if (!state->fdTempRoots) {
 
         while (1) {
-            AutoCloseFD fdGCLock = openGCLock(ltRead);
-
             if (pathExists(fnTempRoots))
                 /* It *must* be stale, since there can be no two
                    processes with the same pid. */
@@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path)
 
             state->fdTempRoots = openLockFile(fnTempRoots, true);
 
-            fdGCLock = -1;
-
-            debug(format("acquiring read lock on '%1%'") % fnTempRoots);
-            lockFile(state->fdTempRoots.get(), ltRead, true);
+            debug("acquiring write lock on '%s'", fnTempRoots);
+            lockFile(state->fdTempRoots.get(), ltWrite, true);
 
             /* Check whether the garbage collector didn't get in our
                way. */
@@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path)
 
     }
 
-    /* Upgrade the lock to a write lock.  This will cause us to block
-       if the garbage collector is holding our lock. */
-    debug(format("acquiring write lock on '%1%'") % fnTempRoots);
-    lockFile(state->fdTempRoots.get(), ltWrite, true);
+    if (!state->fdGCLock)
+        state->fdGCLock = openGCLock();
 
+ restart:
+    FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
+
+    if (!gcLock.acquired) {
+        /* We couldn't get a shared global GC lock, so the garbage
+           collector is running. So we have to connect to the garbage
+           collector and inform it about our root. */
+        if (!state->fdRootsSocket) {
+            auto socketPath = stateDir.get() + gcSocketPath;
+            debug("connecting to '%s'", socketPath);
+            state->fdRootsSocket = createUnixDomainSocket();
+            nix::connect(state->fdRootsSocket.get(), socketPath);
+        }
+
+        try {
+            debug("sending GC root '%s'", printStorePath(path));
+            writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
+            char c;
+            readFull(state->fdRootsSocket.get(), &c, 1);
+            assert(c == '1');
+            debug("got ack for GC root '%s'", printStorePath(path));
+        } catch (SysError & e) {
+            /* The garbage collector may have exited, so we need to
+               restart. */
+            if (e.errNo == EPIPE) {
+                debug("GC socket disconnected");
+                state->fdRootsSocket.close();
+                goto restart;
+            }
+        } catch (EndOfFile & e) {
+            debug("GC socket disconnected");
+            state->fdRootsSocket.close();
+            goto restart;
+        }
+    }
+
+    /* Append the store path to the temporary roots file. */
     string s = printStorePath(path) + '\0';
     writeFull(state->fdTempRoots.get(), s);
-
-    /* Downgrade to a read lock. */
-    debug(format("downgrading to read lock on '%1%'") % fnTempRoots);
-    lockFile(state->fdTempRoots.get(), ltRead, true);
 }
 
 
 static std::string censored = "{censored}";
 
 
-void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
+void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
 {
     /* Read the `temproots' directory for per-process temporary root
        files. */
@@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
         pid_t pid = std::stoi(i.name);
 
         debug(format("reading temporary root file '%1%'") % path);
-        FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
-        if (!*fd) {
+        AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666));
+        if (!fd) {
             /* It's okay if the file has disappeared. */
             if (errno == ENOENT) continue;
             throw SysError("opening temporary roots file '%1%'", path);
         }
 
-        /* This should work, but doesn't, for some reason. */
-        //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
-        //if (*fd == -1) continue;
-
         /* Try to acquire a write lock without blocking.  This can
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
-        if (lockFile(fd->get(), ltWrite, false)) {
+        if (lockFile(fd.get(), ltWrite, false)) {
             printInfo("removing stale temporary roots file '%1%'", path);
             unlink(path.c_str());
-            writeFull(fd->get(), "d");
+            writeFull(fd.get(), "d");
             continue;
         }
 
-        /* Acquire a read lock.  This will prevent the owning process
-           from upgrading to a write lock, therefore it will block in
-           addTempRoot(). */
-        debug(format("waiting for read lock on '%1%'") % path);
-        lockFile(fd->get(), ltRead, true);
-
         /* Read the entire file. */
-        string contents = readFile(fd->get());
+        string contents = readFile(fd.get());
 
         /* Extract the roots. */
         string::size_type pos = 0, end;
@@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
             tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
             pos = end + 1;
         }
-
-        fds.push_back(fd); /* keep open */
     }
 }
 
@@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor)
     Roots roots;
     findRootsNoTemp(roots, censor);
 
-    FDs fds;
-    findTempRoots(fds, roots, censor);
+    findTempRoots(roots, censor);
 
     return roots;
 }
@@ -455,265 +438,139 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 struct GCLimitReached { };
 
 
-struct LocalStore::GCState
-{
-    const GCOptions & options;
-    GCResults & results;
-    StorePathSet roots;
-    StorePathSet tempRoots;
-    StorePathSet dead;
-    StorePathSet alive;
-    bool gcKeepOutputs;
-    bool gcKeepDerivations;
-    uint64_t bytesInvalidated;
-    bool moveToTrash = true;
-    bool shouldDelete;
-    GCState(const GCOptions & options, GCResults & results)
-        : options(options), results(results), bytesInvalidated(0) { }
-};
-
-
-bool LocalStore::isActiveTempFile(const GCState & state,
-    const Path & path, const string & suffix)
-{
-    return hasSuffix(path, suffix)
-        && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size())));
-}
-
-
-void LocalStore::deleteGarbage(GCState & state, const Path & path)
-{
-    uint64_t bytesFreed;
-    deletePath(path, bytesFreed);
-    state.results.bytesFreed += bytesFreed;
-}
-
-
-void LocalStore::deletePathRecursive(GCState & state, const Path & path)
-{
-    checkInterrupt();
-
-    uint64_t size = 0;
-
-    auto storePath = maybeParseStorePath(path);
-    if (storePath && isValidPath(*storePath)) {
-        StorePathSet referrers;
-        queryReferrers(*storePath, referrers);
-        for (auto & i : referrers)
-            if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
-        size = queryPathInfo(*storePath)->narSize;
-        invalidatePathChecked(*storePath);
-    }
-
-    Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
-
-    struct stat st;
-    if (lstat(realPath.c_str(), &st)) {
-        if (errno == ENOENT) return;
-        throw SysError("getting status of %1%", realPath);
-    }
-
-    printInfo(format("deleting '%1%'") % path);
-
-    state.results.paths.insert(path);
-
-    /* If the path is not a regular file or symlink, move it to the
-       trash directory.  The move is to ensure that later (when we're
-       not holding the global GC lock) we can delete the path without
-       being afraid that the path has become alive again.  Otherwise
-       delete it right away. */
-    if (state.moveToTrash && S_ISDIR(st.st_mode)) {
-        // Estimate the amount freed using the narSize field.  FIXME:
-        // if the path was not valid, need to determine the actual
-        // size.
-        try {
-            if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
-                throw SysError("making '%1%' writable", realPath);
-            Path tmp = trashDir + "/" + std::string(baseNameOf(path));
-            if (rename(realPath.c_str(), tmp.c_str()))
-                throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
-            state.bytesInvalidated += size;
-        } catch (SysError & e) {
-            if (e.errNo == ENOSPC) {
-                printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
-                deleteGarbage(state, realPath);
-            }
-        }
-    } else
-        deleteGarbage(state, realPath);
-
-    if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
-        printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
-        throw GCLimitReached();
-    }
-}
-
-
-bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path)
-{
-    if (visited.count(path)) return false;
-
-    if (state.alive.count(path)) return true;
-
-    if (state.dead.count(path)) return false;
-
-    if (state.roots.count(path)) {
-        debug("cannot delete '%1%' because it's a root", printStorePath(path));
-        state.alive.insert(path);
-        return true;
-    }
-
-    visited.insert(path);
-
-    if (!isValidPath(path)) return false;
-
-    StorePathSet incoming;
-
-    /* Don't delete this path if any of its referrers are alive. */
-    queryReferrers(path, incoming);
-
-    /* If keep-derivations is set and this is a derivation, then
-       don't delete the derivation if any of the outputs are alive. */
-    if (state.gcKeepDerivations && path.isDerivation()) {
-        for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
-            if (maybeOutPath &&
-                isValidPath(*maybeOutPath) &&
-                queryPathInfo(*maybeOutPath)->deriver == path
-                )
-                incoming.insert(*maybeOutPath);
-    }
-
-    /* If keep-outputs is set, then don't delete this path if there
-       are derivers of this path that are not garbage. */
-    if (state.gcKeepOutputs) {
-        auto derivers = queryValidDerivers(path);
-        for (auto & i : derivers)
-            incoming.insert(i);
-    }
-
-    for (auto & i : incoming)
-        if (i != path)
-            if (canReachRoot(state, visited, i)) {
-                state.alive.insert(path);
-                return true;
-            }
-
-    return false;
-}
-
-
-void LocalStore::tryToDelete(GCState & state, const Path & path)
-{
-    checkInterrupt();
-
-    auto realPath = realStoreDir + "/" + std::string(baseNameOf(path));
-    if (realPath == linksDir || realPath == trashDir) return;
-
-    //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
-
-    auto storePath = maybeParseStorePath(path);
-
-    if (!storePath || !isValidPath(*storePath)) {
-        /* A lock file belonging to a path that we're building right
-           now isn't garbage. */
-        if (isActiveTempFile(state, path, ".lock")) return;
-
-        /* Don't delete .chroot directories for derivations that are
-           currently being built. */
-        if (isActiveTempFile(state, path, ".chroot")) return;
-
-        /* Don't delete .check directories for derivations that are
-           currently being built, because we may need to run
-           diff-hook. */
-        if (isActiveTempFile(state, path, ".check")) return;
-    }
-
-    StorePathSet visited;
-
-    if (storePath && canReachRoot(state, visited, *storePath)) {
-        debug("cannot delete '%s' because it's still reachable", path);
-    } else {
-        /* No path we visited was a root, so everything is garbage.
-           But we only delete ‘path’ and its referrers here so that
-           ‘nix-store --delete’ doesn't have the unexpected effect of
-           recursing into derivations and outputs. */
-        for (auto & i : visited)
-            state.dead.insert(i);
-        if (state.shouldDelete)
-            deletePathRecursive(state, path);
-    }
-}
-
-
-/* Unlink all files in /nix/store/.links that have a link count of 1,
-   which indicates that there are no other links and so they can be
-   safely deleted.  FIXME: race condition with optimisePath(): we
-   might see a link count of 1 just before optimisePath() increases
-   the link count. */
-void LocalStore::removeUnusedLinks(const GCState & state)
-{
-    AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError("opening directory '%1%'", linksDir);
-
-    int64_t actualSize = 0, unsharedSize = 0;
-
-    struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir.get())) {
-        checkInterrupt();
-        string name = dirent->d_name;
-        if (name == "." || name == "..") continue;
-        Path path = linksDir + "/" + name;
-
-        auto st = lstat(path);
-
-        if (st.st_nlink != 1) {
-            actualSize += st.st_size;
-            unsharedSize += (st.st_nlink - 1) * st.st_size;
-            continue;
-        }
-
-        printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
-
-        if (unlink(path.c_str()) == -1)
-            throw SysError("deleting '%1%'", path);
-
-        state.results.bytesFreed += st.st_size;
-    }
-
-    struct stat st;
-    if (stat(linksDir.c_str(), &st) == -1)
-        throw SysError("statting '%1%'", linksDir);
-    int64_t overhead = st.st_blocks * 512ULL;
-
-    printInfo("note: currently hard linking saves %.2f MiB",
-        ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
-}
-
-
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    GCState state(options, results);
-    state.gcKeepOutputs = settings.gcKeepOutputs;
-    state.gcKeepDerivations = settings.gcKeepDerivations;
+    bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
+    bool gcKeepOutputs = settings.gcKeepOutputs;
+    bool gcKeepDerivations = settings.gcKeepDerivations;
+
+    StorePathSet roots, dead, alive;
+
+    struct Shared
+    {
+        // The temp roots only store the hash part to make it easier to
+        // ignore suffixes like '.lock', '.chroot' and '.check'.
+        std::unordered_set<std::string> tempRoots;
+
+        // Hash part of the store path currently being deleted, if
+        // any.
+        std::optional<std::string> pending;
+    };
+
+    Sync<Shared> _shared;
+
+    std::condition_variable wakeup;
 
     /* Using `--ignore-liveness' with `--delete' can have unintended
        consequences if `keep-outputs' or `keep-derivations' are true
        (the garbage collector will recurse into deleting the outputs
        or derivers, respectively).  So disable them. */
     if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
-        state.gcKeepOutputs = false;
-        state.gcKeepDerivations = false;
+        gcKeepOutputs = false;
+        gcKeepDerivations = false;
     }
 
-    state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
-
-    if (state.shouldDelete)
+    if (shouldDelete)
         deletePath(reservedPath);
 
-    /* Acquire the global GC root.  This prevents
-       a) New roots from being added.
-       b) Processes from creating new temporary root files. */
-    AutoCloseFD fdGCLock = openGCLock(ltWrite);
+    /* Acquire the global GC root. Note: we don't use fdGCLock
+       here because then in auto-gc mode, another thread could
+       downgrade our exclusive lock. */
+    auto fdGCLock = openGCLock();
+    FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock...");
+
+    /* Start the server for receiving new roots. */
+    auto socketPath = stateDir.get() + gcSocketPath;
+    createDirs(dirOf(socketPath));
+    auto fdServer = createUnixDomainSocket(socketPath, 0666);
+
+    if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
+        throw SysError("making socket '%1%' non-blocking", socketPath);
+
+    Pipe shutdownPipe;
+    shutdownPipe.create();
+
+    std::thread serverThread([&]() {
+        Sync<std::map<int, std::thread>> connections;
+
+        Finally cleanup([&]() {
+            debug("GC roots server shutting down");
+            while (true) {
+                auto item = remove_begin(*connections.lock());
+                if (!item) break;
+                auto & [fd, thread] = *item;
+                shutdown(fd, SHUT_RDWR);
+                thread.join();
+            }
+        });
+
+        while (true) {
+            std::vector<struct pollfd> fds;
+            fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN});
+            fds.push_back({.fd = fdServer.get(), .events = POLLIN});
+            auto count = poll(fds.data(), fds.size(), -1);
+            assert(count != -1);
+
+            if (fds[0].revents)
+                /* Parent is asking us to quit. */
+                break;
+
+            if (fds[1].revents) {
+                /* Accept a new connection. */
+                assert(fds[1].revents & POLLIN);
+                AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
+                if (!fdClient) continue;
+
+                /* Process the connection in a separate thread. */
+                auto fdClient_ = fdClient.get();
+                std::thread clientThread([&, fdClient = std::move(fdClient)]() {
+                    Finally cleanup([&]() {
+                        auto conn(connections.lock());
+                        auto i = conn->find(fdClient.get());
+                        if (i != conn->end()) {
+                            i->second.detach();
+                            conn->erase(i);
+                        }
+                    });
+
+                    while (true) {
+                        try {
+                            auto path = readLine(fdClient.get());
+                            auto storePath = maybeParseStorePath(path);
+                            if (storePath) {
+                                debug("got new GC root '%s'", path);
+                                auto hashPart = std::string(storePath->hashPart());
+                                auto shared(_shared.lock());
+                                shared->tempRoots.insert(hashPart);
+                                /* If this path is currently being
+                                   deleted, then we have to wait until
+                                   deletion is finished to ensure that
+                                   the client doesn't start
+                                   re-creating it before we're
+                                   done. FIXME: ideally we would use a
+                                   FD for this so we don't block the
+                                   poll loop. */
+                                while (shared->pending == hashPart) {
+                                    debug("synchronising with deletion of path '%s'", path);
+                                    shared.wait(wakeup);
+                                }
+                            } else
+                                printError("received garbage instead of a root from client");
+                            writeFull(fdClient.get(), "1", false);
+                        } catch (Error &) { break; }
+                    }
+                });
+
+                connections.lock()->insert({fdClient_, std::move(clientThread)});
+            }
+        }
+    });
+
+    Finally stopServer([&]() {
+        writeFull(shutdownPipe.writeSide.get(), "x", false);
+        wakeup.notify_all();
+        if (serverThread.joinable()) serverThread.join();
+    });
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
@@ -722,124 +579,256 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     if (!options.ignoreLiveness)
         findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.first);
+    for (auto & i : rootMap) roots.insert(i.first);
 
-    /* Read the temporary roots.  This acquires read locks on all
-       per-process temporary root files.  So after this point no paths
-       can be added to the set of temporary roots. */
-    FDs fds;
+    /* Read the temporary roots created before we acquired the global
+       GC root. Any new roots will be sent to our socket. */
     Roots tempRoots;
-    findTempRoots(fds, tempRoots, true);
+    findTempRoots(tempRoots, true);
     for (auto & root : tempRoots) {
-        state.tempRoots.insert(root.first);
-        state.roots.insert(root.first);
+        _shared.lock()->tempRoots.insert(std::string(root.first.hashPart()));
+        roots.insert(root.first);
     }
 
-    /* After this point the set of roots or temporary roots cannot
-       increase, since we hold locks on everything.  So everything
-       that is not reachable from `roots' is garbage. */
+    /* Helper function that deletes a path from the store and throws
+       GCLimitReached if we've deleted enough garbage. */
+    auto deleteFromStore = [&](std::string_view baseName)
+    {
+        Path path = storeDir + "/" + std::string(baseName);
+        Path realPath = realStoreDir + "/" + std::string(baseName);
 
-    if (state.shouldDelete) {
-        if (pathExists(trashDir)) deleteGarbage(state, trashDir);
-        try {
-            createDirs(trashDir);
-        } catch (SysError & e) {
-            if (e.errNo == ENOSPC) {
-                printInfo("note: can't create trash directory: %s", e.msg());
-                state.moveToTrash = false;
+        printInfo("deleting '%1%'", path);
+
+        results.paths.insert(path);
+
+        uint64_t bytesFreed;
+        deletePath(realPath, bytesFreed);
+        results.bytesFreed += bytesFreed;
+
+        if (results.bytesFreed > options.maxFreed) {
+            printInfo("deleted more than %d bytes; stopping", options.maxFreed);
+            throw GCLimitReached();
+        }
+    };
+
+    std::map<StorePath, StorePathSet> referrersCache;
+
+    /* Helper function that visits all paths reachable from `start`
+       via the referrers edges and optionally derivers and derivation
+       output edges. If none of those paths are roots, then all
+       visited paths are garbage and are deleted. */
+    auto deleteReferrersClosure = [&](const StorePath & start) {
+        StorePathSet visited;
+        std::queue<StorePath> todo;
+
+        /* Wake up any GC client waiting for deletion of the paths in
+           'visited' to finish. */
+        Finally releasePending([&]() {
+            auto shared(_shared.lock());
+            shared->pending.reset();
+            wakeup.notify_all();
+        });
+
+        auto enqueue = [&](const StorePath & path) {
+            if (visited.insert(path).second)
+                todo.push(path);
+        };
+
+        enqueue(start);
+
+        while (auto path = pop(todo)) {
+            checkInterrupt();
+
+            /* Bail out if we've previously discovered that this path
+               is alive. */
+            if (alive.count(*path)) {
+                alive.insert(start);
+                return;
+            }
+
+            /* If we've previously deleted this path, we don't have to
+               handle it again. */
+            if (dead.count(*path)) continue;
+
+            auto markAlive = [&]()
+            {
+                alive.insert(*path);
+                alive.insert(start);
+                try {
+                    StorePathSet closure;
+                    computeFSClosure(*path, closure);
+                    for (auto & p : closure)
+                        alive.insert(p);
+                } catch (InvalidPath &) { }
+            };
+
+            /* If this is a root, bail out. */
+            if (roots.count(*path)) {
+                debug("cannot delete '%s' because it's a root", printStorePath(*path));
+                return markAlive();
+            }
+
+            if (options.action == GCOptions::gcDeleteSpecific
+                && !options.pathsToDelete.count(*path))
+                return;
+
+            {
+                auto hashPart = std::string(path->hashPart());
+                auto shared(_shared.lock());
+                if (shared->tempRoots.count(hashPart)) {
+                    debug("cannot delete '%s' because it's a temporary root", printStorePath(*path));
+                    return markAlive();
+                }
+                shared->pending = hashPart;
+            }
+
+            if (isValidPath(*path)) {
+
+                /* Visit the referrers of this path. */
+                auto i = referrersCache.find(*path);
+                if (i == referrersCache.end()) {
+                    StorePathSet referrers;
+                    queryReferrers(*path, referrers);
+                    referrersCache.emplace(*path, std::move(referrers));
+                    i = referrersCache.find(*path);
+                }
+                for (auto & p : i->second)
+                    enqueue(p);
+
+                /* If keep-derivations is set and this is a
+                   derivation, then visit the derivation outputs. */
+                if (gcKeepDerivations && path->isDerivation()) {
+                    for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path))
+                        if (maybeOutPath &&
+                            isValidPath(*maybeOutPath) &&
+                            queryPathInfo(*maybeOutPath)->deriver == *path)
+                            enqueue(*maybeOutPath);
+                }
+
+                /* If keep-outputs is set, then visit the derivers. */
+                if (gcKeepOutputs) {
+                    auto derivers = queryValidDerivers(*path);
+                    for (auto & i : derivers)
+                        enqueue(i);
+                }
             }
         }
-    }
 
-    /* Now either delete all garbage paths, or just the specified
+        for (auto & path : topoSortPaths(visited)) {
+            if (!dead.insert(path).second) continue;
+            if (shouldDelete) {
+                invalidatePathChecked(path);
+                deleteFromStore(path.to_string());
+                referrersCache.erase(path);
+            }
+        }
+    };
+
+    /* Synchronisation point for testing, see tests/gc-concurrent.sh. */
+    if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
+        readFile(*p);
+
+    /* Either delete all garbage paths, or just the specified
        paths (for gcDeleteSpecific). */
-
     if (options.action == GCOptions::gcDeleteSpecific) {
 
         for (auto & i : options.pathsToDelete) {
-            tryToDelete(state, printStorePath(i));
-            if (state.dead.find(i) == state.dead.end())
+            deleteReferrersClosure(i);
+            if (!dead.count(i))
                 throw Error(
-                    "cannot delete path '%1%' since it is still alive. "
-                    "To find out why use: "
+                    "Cannot delete path '%1%' since it is still alive. "
+                    "To find out why, use: "
                     "nix-store --query --roots",
                     printStorePath(i));
         }
 
     } else if (options.maxFreed > 0) {
 
-        if (state.shouldDelete)
+        if (shouldDelete)
             printInfo("deleting garbage...");
         else
             printInfo("determining live/dead paths...");
 
         try {
-
             AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
             if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
-            /* Read the store and immediately delete all paths that
-               aren't valid.  When using --max-freed etc., deleting
-               invalid paths is preferred over deleting unreachable
-               paths, since unreachable paths could become reachable
-               again.  We don't use readDirectory() here so that GCing
-               can start faster. */
+            /* Read the store and delete all paths that are invalid or
+               unreachable. We don't use readDirectory() here so that
+               GCing can start faster. */
+            auto linksName = baseNameOf(linksDir);
             Paths entries;
             struct dirent * dirent;
             while (errno = 0, dirent = readdir(dir.get())) {
                 checkInterrupt();
                 string name = dirent->d_name;
-                if (name == "." || name == "..") continue;
-                Path path = storeDir + "/" + name;
-                auto storePath = maybeParseStorePath(path);
-                if (storePath && isValidPath(*storePath))
-                    entries.push_back(path);
+                if (name == "." || name == ".." || name == linksName) continue;
+
+                if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
+                    deleteReferrersClosure(*storePath);
                 else
-                    tryToDelete(state, path);
+                    deleteFromStore(name);
+
             }
-
-            dir.reset();
-
-            /* Now delete the unreachable valid paths.  Randomise the
-               order in which we delete entries to make the collector
-               less biased towards deleting paths that come
-               alphabetically first (e.g. /nix/store/000...).  This
-               matters when using --max-freed etc. */
-            vector<Path> entries_(entries.begin(), entries.end());
-            std::mt19937 gen(1);
-            std::shuffle(entries_.begin(), entries_.end(), gen);
-
-            for (auto & i : entries_)
-                tryToDelete(state, i);
-
         } catch (GCLimitReached & e) {
         }
     }
 
-    if (state.options.action == GCOptions::gcReturnLive) {
-        for (auto & i : state.alive)
-            state.results.paths.insert(printStorePath(i));
+    if (options.action == GCOptions::gcReturnLive) {
+        for (auto & i : alive)
+            results.paths.insert(printStorePath(i));
         return;
     }
 
-    if (state.options.action == GCOptions::gcReturnDead) {
-        for (auto & i : state.dead)
-            state.results.paths.insert(printStorePath(i));
+    if (options.action == GCOptions::gcReturnDead) {
+        for (auto & i : dead)
+            results.paths.insert(printStorePath(i));
         return;
     }
 
-    /* Allow other processes to add to the store from here on. */
-    fdGCLock = -1;
-    fds.clear();
-
-    /* Delete the trash directory. */
-    printInfo(format("deleting '%1%'") % trashDir);
-    deleteGarbage(state, trashDir);
-
-    /* Clean up the links directory. */
+    /* Unlink all files in /nix/store/.links that have a link count of 1,
+       which indicates that there are no other links and so they can be
+       safely deleted.  FIXME: race condition with optimisePath(): we
+       might see a link count of 1 just before optimisePath() increases
+       the link count. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
         printInfo("deleting unused links...");
-        removeUnusedLinks(state);
+
+        AutoCloseDir dir(opendir(linksDir.c_str()));
+        if (!dir) throw SysError("opening directory '%1%'", linksDir);
+
+        int64_t actualSize = 0, unsharedSize = 0;
+
+        struct dirent * dirent;
+        while (errno = 0, dirent = readdir(dir.get())) {
+            checkInterrupt();
+            string name = dirent->d_name;
+            if (name == "." || name == "..") continue;
+            Path path = linksDir + "/" + name;
+
+            auto st = lstat(path);
+
+            if (st.st_nlink != 1) {
+                actualSize += st.st_size;
+                unsharedSize += (st.st_nlink - 1) * st.st_size;
+                continue;
+            }
+
+            printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
+
+            if (unlink(path.c_str()) == -1)
+                throw SysError("deleting '%1%'", path);
+
+            results.bytesFreed += st.st_size;
+        }
+
+        struct stat st;
+        if (stat(linksDir.c_str(), &st) == -1)
+            throw SysError("statting '%1%'", linksDir);
+        int64_t overhead = st.st_blocks * 512ULL;
+
+        printInfo("note: currently hard linking saves %.2f MiB",
+            ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
     }
 
     /* While we're at it, vacuum the database. */

src/libstore/globals.cc

@@ -148,7 +148,8 @@ StringSet Settings::getDefaultExtraPlatforms()
     // machines. Note that we can’t force processes from executing
     // x86_64 in aarch64 environments or vice versa since they can
     // always exec with their own binary preferences.
-    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
+    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") ||
+        pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
         if (std::string{SYSTEM} == "x86_64-darwin")
             extraPlatforms.insert("aarch64-darwin");
         else if (std::string{SYSTEM} == "aarch64-darwin")
@@ -159,21 +160,16 @@ StringSet Settings::getDefaultExtraPlatforms()
     return extraPlatforms;
 }
 
-bool Settings::isExperimentalFeatureEnabled(const std::string & name)
+bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature)
 {
     auto & f = experimentalFeatures.get();
-    return std::find(f.begin(), f.end(), name) != f.end();
+    return std::find(f.begin(), f.end(), feature) != f.end();
 }
 
-MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
-    : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature)
-    , missingFeature(feature)
-    {}
-
-void Settings::requireExperimentalFeature(const std::string & name)
+void Settings::requireExperimentalFeature(const ExperimentalFeature & feature)
 {
-    if (!isExperimentalFeatureEnabled(name))
-        throw MissingExperimentalFeature(name);
+    if (!isExperimentalFeatureEnabled(feature))
+        throw MissingExperimentalFeature(feature);
 }
 
 bool Settings::isWSL1()

src/libstore/globals.hh

@@ -3,6 +3,7 @@
 #include "types.hh"
 #include "config.hh"
 #include "util.hh"
+#include "experimental-features.hh"
 
 #include <map>
 #include <limits>
@@ -45,15 +46,6 @@ struct PluginFilesSetting : public BaseSetting<Paths>
     void set(const std::string & str, bool append = false) override;
 };
 
-class MissingExperimentalFeature: public Error
-{
-public:
-    std::string missingFeature;
-
-    MissingExperimentalFeature(std::string feature);
-    virtual const char* sname() const override { return "MissingExperimentalFeature"; }
-};
-
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -925,12 +917,12 @@ public:
           value.
           )"};
 
-    Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
+    Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
-    bool isExperimentalFeatureEnabled(const std::string & name);
+    bool isExperimentalFeatureEnabled(const ExperimentalFeature &);
 
-    void requireExperimentalFeature(const std::string & name);
+    void requireExperimentalFeature(const ExperimentalFeature &);
 
     Setting<bool> allowDirty{this, true, "allow-dirty",
         "Whether to allow dirty Git/Mercurial trees."};

src/libstore/http-binary-cache-store.cc

@@ -57,8 +57,8 @@ public:
     {
         // FIXME: do this lazily?
         if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
-            wantMassQuery.setDefault(cacheInfo->wantMassQuery ? "true" : "false");
-            priority.setDefault(fmt("%d", cacheInfo->priority));
+            wantMassQuery.setDefault(cacheInfo->wantMassQuery);
+            priority.setDefault(cacheInfo->priority);
         } else {
             try {
                 BinaryCacheStore::init();

src/libstore/legacy-ssh-store.cc

@@ -82,9 +82,20 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION;
             conn->to.flush();
 
-            unsigned int magic = readInt(conn->from);
-            if (magic != SERVE_MAGIC_2)
-                throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host);
+            StringSink saved;
+            try {
+                TeeSource tee(conn->from, saved);
+                unsigned int magic = readInt(tee);
+                if (magic != SERVE_MAGIC_2)
+                    throw Error("'nix-store --serve' protocol mismatch from '%s'", host);
+            } catch (SerialisationError & e) {
+                /* In case the other side is waiting for our input,
+                   close it. */
+                conn->sshConn->in.close();
+                auto msg = conn->from.drain();
+                throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'",
+                    host, chomp(*saved.s + msg));
+            }
             conn->remoteVersion = readInt(conn->from);
             if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200)
                 throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host);
@@ -237,6 +248,10 @@ private:
             conn.to
                 << settings.buildRepeat
                 << settings.enforceDeterminism;
+
+        if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) {
+            conn.to << ((int) settings.keepFailed);
+        }
     }
 
 public:
@@ -279,10 +294,10 @@ public:
         for (auto & p : drvPaths) {
             auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
             std::visit(overloaded {
-                [&](StorePathWithOutputs s) {
+                [&](const StorePathWithOutputs & s) {
                     ss.push_back(s.to_string(*this));
                 },
-                [&](StorePath drvPath) {
+                [&](const StorePath & drvPath) {
                     throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
                 },
             }, sOrDrvPath);

src/libstore/local-store.cc

@@ -145,7 +145,6 @@ LocalStore::LocalStore(const Params & params)
     , linksDir(realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
-    , trashDir(realStoreDir + "/trash")
     , tempRootsDir(stateDir + "/temproots")
     , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
     , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
@@ -309,7 +308,7 @@ LocalStore::LocalStore(const Params & params)
 
     else openDB(*state, false);
 
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
     }
 
@@ -339,7 +338,7 @@ LocalStore::LocalStore(const Params & params)
     state->stmts->QueryPathFromHashPart.create(state->db,
         "select path from ValidPaths where path >= ? limit 1;");
     state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         state->stmts->RegisterRealisedOutput.create(state->db,
             R"(
                 insert into Realisations (drvPath, outputName, outputPath, signatures)
@@ -386,6 +385,16 @@ LocalStore::LocalStore(const Params & params)
 }
 
 
+AutoCloseFD LocalStore::openGCLock()
+{
+    Path fnGCLock = stateDir + "/gc.lock";
+    auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+    if (!fdGCLock)
+        throw SysError("opening global GC lock '%1%'", fnGCLock);
+    return fdGCLock;
+}
+
+
 LocalStore::~LocalStore()
 {
     std::shared_future<void> future;
@@ -681,7 +690,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     std::optional<Hash> h;
     for (auto & i : drv.outputs) {
         std::visit(overloaded {
-            [&](DerivationOutputInputAddressed doia) {
+            [&](const DerivationOutputInputAddressed & doia) {
                 if (!h) {
                     // somewhat expensive so we do lazily
                     auto temp = hashDerivationModulo(*this, drv, true);
@@ -693,14 +702,14 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
                         printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
                 envHasRightPath(doia.path, i.first);
             },
-            [&](DerivationOutputCAFixed dof) {
+            [&](const DerivationOutputCAFixed & dof) {
                 StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
                 envHasRightPath(path, i.first);
             },
-            [&](DerivationOutputCAFloating _) {
+            [&](const DerivationOutputCAFloating &) {
                 /* Nothing to check */
             },
-            [&](DerivationOutputDeferred) {
+            [&](const DerivationOutputDeferred &) {
             },
         }, i.second.output);
     }
@@ -708,7 +717,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 
 void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
 {
-    settings.requireExperimentalFeature("ca-derivations");
+    settings.requireExperimentalFeature(Xp::CaDerivations);
     if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
         registerDrvOutput(info);
     else
@@ -717,7 +726,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
 
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
-    settings.requireExperimentalFeature("ca-derivations");
+    settings.requireExperimentalFeature(Xp::CaDerivations);
     retrySQLite<void>([&]() {
         auto state(_state.lock());
         if (auto oldR = queryRealisation_(*state, info.id)) {
@@ -825,7 +834,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
+        state_->pathInfoCache.upsert(std::string(info.path.to_string()),
             PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
@@ -1003,7 +1012,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
         return outputs;
     });
 
-    if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
+    if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
         return outputs;
 
     auto drv = readInvalidDerivation(path);
@@ -1071,14 +1080,19 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 }
 
 
+// FIXME: move this, it's not specific to LocalStore.
 void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
 {
     if (!settings.useSubstitutes) return;
     for (auto & sub : getDefaultSubstituters()) {
         for (auto & path : paths) {
+            if (infos.count(path.first))
+                // Choose first succeeding substituter.
+                continue;
+
             auto subPath(path.first);
 
-            // recompute store path so that we can use a different store root
+            // Recompute store path so that we can use a different store root.
             if (path.second) {
                 subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
                 if (sub->storeDir == storeDir)
@@ -1193,7 +1207,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(std::string(path.hashPart()));
+        state_->pathInfoCache.erase(std::string(path.to_string()));
     }
 }
 
@@ -1239,11 +1253,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             deletePath(realPath);
 
-            // text hashing has long been allowed to have non-self-references because it is used for drv files.
-            bool refersToSelf = info.references.count(info.path) > 0;
-            if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
-                settings.requireExperimentalFeature("ca-references");
-
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
             HashSink hashSink(htSHA256);
@@ -1505,7 +1514,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
     /* Acquire the global GC lock to get a consistent snapshot of
        existing and valid paths. */
-    AutoCloseFD fdGCLock = openGCLock(ltWrite);
+    auto fdGCLock = openGCLock();
+    FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
 
     StringSet store;
     for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
@@ -1516,8 +1526,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
     StorePathSet validPaths;
     PathSet done;
 
-    fdGCLock = -1;
-
     for (auto & i : queryAllValidPaths())
         verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
 

src/libstore/local-store.hh

@@ -58,9 +58,15 @@ private:
         struct Stmts;
         std::unique_ptr<Stmts> stmts;
 
+        /* The global GC lock  */
+        AutoCloseFD fdGCLock;
+
         /* The file to which we write our temporary roots. */
         AutoCloseFD fdTempRoots;
 
+        /* Connection to the garbage collector. */
+        AutoCloseFD fdRootsSocket;
+
         /* The last time we checked whether to do an auto-GC, or an
            auto-GC finished. */
         std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@@ -87,7 +93,6 @@ public:
     const Path linksDir;
     const Path reservedPath;
     const Path schemaPath;
-    const Path trashDir;
     const Path tempRootsDir;
     const Path fnTempRoots;
 
@@ -149,14 +154,11 @@ public:
 
     void addIndirectRoot(const Path & path) override;
 
-    void syncWithGC() override;
-
 private:
 
-    typedef std::shared_ptr<AutoCloseFD> FDPtr;
-    typedef list<FDPtr> FDs;
+    void findTempRoots(Roots & roots, bool censor);
 
-    void findTempRoots(FDs & fds, Roots & roots, bool censor);
+    AutoCloseFD openGCLock();
 
 public:
 
@@ -236,29 +238,12 @@ private:
     PathSet queryValidPathsOld();
     ValidPathInfo queryPathInfoOld(const Path & path);
 
-    struct GCState;
-
-    void deleteGarbage(GCState & state, const Path & path);
-
-    void tryToDelete(GCState & state, const Path & path);
-
-    bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
-
-    void deletePathRecursive(GCState & state, const Path & path);
-
-    bool isActiveTempFile(const GCState & state,
-        const Path & path, const string & suffix);
-
-    AutoCloseFD openGCLock(LockType lockType);
-
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
     void findRootsNoTemp(Roots & roots, bool censor);
 
     void findRuntimeRoots(Roots & roots, bool censor);
 
-    void removeUnusedLinks(const GCState & state);
-
     Path createTempDirInStore();
 
     void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);

src/libstore/local.mk

@@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
 libstore_LIBS = libutil
 
-libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
+libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread
 ifdef HOST_LINUX
  libstore_LDFLAGS += -ldl
 endif
@@ -60,7 +60,7 @@ $(d)/build.cc:
 
 clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
 
-$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))
 
 $(foreach i, $(wildcard src/libstore/builtins/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))

src/libstore/machines.cc

@@ -39,7 +39,8 @@ Machine::Machine(decltype(storeUri) storeUri,
     sshPublicHostKey(sshPublicHostKey)
 {}
 
-bool Machine::allSupported(const std::set<string> & features) const {
+bool Machine::allSupported(const std::set<string> & features) const
+{
     return std::all_of(features.begin(), features.end(),
         [&](const string & feature) {
             return supportedFeatures.count(feature) ||
@@ -47,14 +48,16 @@ bool Machine::allSupported(const std::set<string> & features) const {
         });
 }
 
-bool Machine::mandatoryMet(const std::set<string> & features) const {
+bool Machine::mandatoryMet(const std::set<string> & features) const
+{
     return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
         [&](const string & feature) {
             return features.count(feature);
         });
 }
 
-ref<Store> Machine::openStore() const {
+ref<Store> Machine::openStore() const
+{
     Store::Params storeParams;
     if (hasPrefix(storeUri, "ssh://")) {
         storeParams["max-connections"] = "1";
@@ -83,53 +86,86 @@ ref<Store> Machine::openStore() const {
     return nix::openStore(storeUri, storeParams);
 }
 
-void parseMachines(const std::string & s, Machines & machines)
+static std::vector<std::string> expandBuilderLines(const std::string & builders)
 {
-    for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
+    std::vector<std::string> result;
+    for (auto line : tokenizeString<std::vector<string>>(builders, "\n;")) {
         trim(line);
         line.erase(std::find(line.begin(), line.end(), '#'), line.end());
         if (line.empty()) continue;
 
         if (line[0] == '@') {
-            auto file = trim(std::string(line, 1));
+            const std::string path = trim(std::string(line, 1));
+            std::string text;
             try {
-                parseMachines(readFile(file), machines);
+                text = readFile(path);
             } catch (const SysError & e) {
                 if (e.errNo != ENOENT)
                     throw;
-                debug("cannot find machines file '%s'", file);
+                debug("cannot find machines file '%s'", path);
             }
+
+            const auto lines = expandBuilderLines(text);
+            result.insert(end(result), begin(lines), end(lines));
             continue;
         }
 
-        auto tokens = tokenizeString<std::vector<string>>(line);
-        auto sz = tokens.size();
-        if (sz < 1)
-            throw FormatError("bad machine specification '%s'", line);
-
-        auto isSet = [&](size_t n) {
-            return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
-        };
-
-        machines.emplace_back(tokens[0],
-            isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
-            isSet(2) ? tokens[2] : "",
-            isSet(3) ? std::stoull(tokens[3]) : 1LL,
-            isSet(4) ? std::stoull(tokens[4]) : 1LL,
-            isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
-            isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
-            isSet(7) ? tokens[7] : "");
+        result.emplace_back(line);
     }
+    return result;
+}
+
+static Machine parseBuilderLine(const std::string & line)
+{
+    const auto tokens = tokenizeString<std::vector<string>>(line);
+
+    auto isSet = [&](size_t fieldIndex) {
+        return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-";
+    };
+
+    auto parseUnsignedIntField = [&](size_t fieldIndex) {
+        const auto result = string2Int<unsigned int>(tokens[fieldIndex]);
+        if (!result) {
+            throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line);
+        }
+        return result.value();
+    };
+
+    auto ensureBase64 = [&](size_t fieldIndex) {
+        const auto & str = tokens[fieldIndex];
+        try {
+            base64Decode(str);
+        } catch (const Error & e) {
+            throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what());
+        }
+        return str;
+    };
+
+    if (!isSet(0))
+        throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
+
+    return {
+        tokens[0],
+        isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
+        isSet(2) ? tokens[2] : "",
+        isSet(3) ? parseUnsignedIntField(3) : 1U,
+        isSet(4) ? parseUnsignedIntField(4) : 1U,
+        isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
+        isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
+        isSet(7) ? ensureBase64(7) : ""
+    };
+}
+
+static Machines parseBuilderLines(const std::vector<std::string>& builders) {
+    Machines result;
+    std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
+    return result;
 }
 
 Machines getMachines()
 {
-    static auto machines = [&]() {
-        Machines machines;
-        parseMachines(settings.builders, machines);
-        return machines;
-    }();
-    return machines;
+    const auto builderLines = expandBuilderLines(settings.builders);
+    return parseBuilderLines(builderLines);
 }
 
 }

src/libstore/misc.cc

@@ -166,7 +166,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
         }
 
         std::visit(overloaded {
-          [&](DerivedPath::Built bfd) {
+          [&](const DerivedPath::Built & bfd) {
             if (!isValidPath(bfd.drvPath)) {
                 // FIXME: we could try to substitute the derivation.
                 auto state(state_.lock());
@@ -199,7 +199,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
                 mustBuildDrv(bfd.drvPath, *drv);
 
           },
-          [&](DerivedPath::Opaque bo) {
+          [&](const DerivedPath::Opaque & bo) {
 
             if (isValidPath(bo.path)) return;
 
@@ -239,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
 {
     return topoSort(paths,
         {[&](const StorePath & path) {
-            StorePathSet references;
             try {
-                references = queryPathInfo(path)->references;
+                return queryPathInfo(path)->references;
             } catch (InvalidPath &) {
+                return StorePathSet();
             }
-            return references;
         }},
         {[&](const StorePath & path, const StorePath & parent) {
             return BuildError(

src/libstore/path-with-outputs.cc

@@ -31,14 +31,14 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>
 std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
 {
     return std::visit(overloaded {
-        [&](DerivedPath::Opaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
+        [&](const DerivedPath::Opaque & bo) -> std::variant<StorePathWithOutputs, StorePath> {
             if (bo.path.isDerivation()) {
                 // drv path gets interpreted as "build", not "get drv file itself"
                 return bo.path;
             }
             return StorePathWithOutputs { bo.path };
         },
-        [&](DerivedPath::Built bfd) -> std::variant<StorePathWithOutputs, StorePath> {
+        [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> {
             return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
         },
     }, p.raw());

src/libstore/pathlocks.cc

@@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths)
 }
 
 
+FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg)
+    : fd(fd)
+{
+    if (wait) {
+        if (!lockFile(fd, lockType, false)) {
+            printInfo("%s", waitMsg);
+            acquired = lockFile(fd, lockType, true);
+        }
+    } else
+        acquired = lockFile(fd, lockType, false);
+}
+
+
 }

src/libstore/pathlocks.hh

@@ -35,4 +35,18 @@ public:
     void setDeletion(bool deletePaths);
 };
 
+struct FdLock
+{
+    int fd;
+    bool acquired = false;
+
+    FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg);
+
+    ~FdLock()
+    {
+        if (acquired)
+            lockFile(fd, ltNone, false);
+    }
+};
+
 }

src/libstore/profiles.cc

@@ -126,9 +126,9 @@ void deleteGeneration(const Path & profile, GenerationNumber gen)
 static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
 {
     if (dryRun)
-        printInfo(format("would remove generation %1%") % gen);
+        notice("would remove profile version %1%", gen);
     else {
-        printInfo(format("removing generation %1%") % gen);
+        notice("removing profile version %1%", gen);
         deleteGeneration(profile, gen);
     }
 }
@@ -142,7 +142,7 @@ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> &
     auto [gens, curGen] = findGenerations(profile);
 
     if (gensToDelete.count(*curGen))
-        throw Error("cannot delete current generation of profile %1%'", profile);
+        throw Error("cannot delete current version of profile %1%'", profile);
 
     for (auto & i : gens) {
         if (!gensToDelete.count(i.number)) continue;
@@ -236,6 +236,37 @@ void switchLink(Path link, Path target)
 }
 
 
+void switchGeneration(
+    const Path & profile,
+    std::optional<GenerationNumber> dstGen,
+    bool dryRun)
+{
+    PathLocks lock;
+    lockProfile(lock, profile);
+
+    auto [gens, curGen] = findGenerations(profile);
+
+    std::optional<Generation> dst;
+    for (auto & i : gens)
+        if ((!dstGen && i.number < curGen) ||
+            (dstGen && i.number == *dstGen))
+            dst = i;
+
+    if (!dst) {
+        if (dstGen)
+            throw Error("profile version %1% does not exist", *dstGen);
+        else
+            throw Error("no profile version older than the current (%1%) exists", curGen.value_or(0));
+    }
+
+    notice("switching profile from version %d to %d", curGen.value_or(0), dst->number);
+
+    if (dryRun) return;
+
+    switchLink(profile, dst->path);
+}
+
+
 void lockProfile(PathLocks & lock, const Path & profile)
 {
     lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str());

src/libstore/profiles.hh

@@ -11,7 +11,7 @@ namespace nix {
 class StorePath;
 
 
-typedef unsigned int GenerationNumber;
+typedef uint64_t GenerationNumber;
 
 struct Generation
 {
@@ -46,6 +46,13 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
 
 void switchLink(Path link, Path target);
 
+/* Roll back a profile to the specified generation, or to the most
+   recent one older than the current. */
+void switchGeneration(
+    const Path & profile,
+    std::optional<GenerationNumber> dstGen,
+    bool dryRun);
+
 /* Ensure exclusive access to a profile.  Any command that modifies
    the profile first acquires this lock. */
 void lockProfile(PathLocks & lock, const Path & profile);