Add aliases for the old flake commands

Requires reworking a bit the existing alias mechanism to allow aliases
to be composite commands themselves

.github/workflows/backport.yml

@@ -0,0 +1,26 @@
+name: Backport
+on:
+  pull_request_target:
+    types: [closed, labeled]
+jobs:
+  backport:
+    name: Backport Pull Request
+    if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          # required to find all branches
+          fetch-depth: 0
+      - name: Create backport PRs
+        # should be kept in sync with `version`
+        uses: zeebe-io/backport-action@v0.0.7
+        with:
+          # Config README: https://github.com/zeebe-io/backport-action#backport-action
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_workspace: ${{ github.workspace }}
+          pull_description: |-
+            Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}.
+          # should be kept in sync with `uses`
+          version: v0.0.5

.github/workflows/test.yml

@@ -1,20 +1,23 @@
 name: "Test"
+
 on:
   pull_request:
   push:
+
 jobs:
+
   tests:
     needs: [check_cachix]
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
-
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.4.0
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v14
+    - uses: cachix/install-nix-action@v16
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v10
       if: needs.check_cachix.outputs.secret == 'true'
@@ -23,6 +26,7 @@ jobs:
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
+
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest
@@ -34,6 +38,7 @@ jobs:
         env:
           _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
         run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}"
+
   installer:
     needs: [tests, check_cachix]
     if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
@@ -41,11 +46,11 @@ jobs:
     outputs:
       installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.4.0
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v14
+    - uses: cachix/install-nix-action@v16
     - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -53,6 +58,7 @@ jobs:
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - id: prepare-installer
       run: scripts/prepare-installer-for-github-actions
+
   installer_test:
     needs: [installer, check_cachix]
     if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
@@ -61,9 +67,9 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.4.0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v14
+    - uses: cachix/install-nix-action@v16
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"

.gitignore

@@ -26,8 +26,6 @@ perl/Makefile.config
 
 # /scripts/
 /scripts/nix-profile.sh
-/scripts/nix-reduce-build
-/scripts/nix-http-export.cgi
 /scripts/nix-profile-daemon.sh
 
 # /src/libexpr/
@@ -40,6 +38,7 @@ perl/Makefile.config
 
 # /src/libstore/
 *.gen.*
+/src/libstore/tests/libstore-tests
 
 # /src/libutil/
 /src/libutil/tests/libutil-tests

.version

@@ -1 +1 @@
-2.4
+2.5

boehmgc-coroutine-sp-fallback.diff

@@ -1,8 +1,8 @@
 diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index 1cee6a0b..46c3acd9 100644
+index 4b2c429..1fb4c52 100644
 --- a/pthread_stop_world.c
 +++ b/pthread_stop_world.c
-@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void)
      struct GC_traced_stack_sect_s *traced_stack_sect;
      pthread_t self = pthread_self();
      word total_size = 0;
@@ -11,7 +11,7 @@ index 1cee6a0b..46c3acd9 100644
  
      if (!EXPECT(GC_thr_initialized, TRUE))
        GC_thr_init();
-@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void)
            hi = p->altstack + p->altstack_size;
            /* FIXME: Need to scan the normal stack too, but how ? */
            /* FIXME: Assume stack grows down */
@@ -22,6 +22,9 @@ index 1cee6a0b..46c3acd9 100644
 +          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
 +            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
 +          }
++          if (pthread_attr_destroy(&pattr)) {
++            ABORT("GC_push_all_stacks: pthread_attr_destroy failed!");
++          }
 +          // When a thread goes into a coroutine, we lose its original sp until
 +          // control flow returns to the thread.
 +          // While in the coroutine, the sp points outside the thread stack,

doc/manual/src/SUMMARY.md.in

@@ -9,6 +9,7 @@
     - [Prerequisites](installation/prerequisites-source.md)
     - [Obtaining a Source Distribution](installation/obtaining-source.md)
     - [Building Nix from Source](installation/building-source.md)
+  - [Using Nix within Docker](installation/installing-docker.md)
   - [Security](installation/nix-security.md)
     - [Single-User Mode](installation/single-user.md)
     - [Multi-User Mode](installation/multi-user.md)
@@ -70,6 +71,7 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
+  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
   - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)

doc/manual/src/command-ref/conf-file-prefix.md

@@ -16,8 +16,9 @@ By default Nix reads settings from the following places:
     will be loaded in reverse order.
 
     Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
-    and `XDG_CONFIG_HOME`. If these are unset, it will look in
-    `$HOME/.config/nix.conf`.
+    and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to
+    `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config`
+    as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
 
   - If `NIX_CONFIG` is set, its contents is treated as the contents of
     a configuration file.

doc/manual/src/command-ref/nix-env.md

@@ -238,7 +238,16 @@ a number of possible ways:
 
 ## Examples
 
-To install a specific version of `gcc` from the active Nix expression:
+To install a package using a specific attribute path from the active Nix expression:
+
+```console
+$ nix-env -iA gcc40mips
+installing `gcc-4.0.2'
+$ nix-env -iA xorg.xorgserver
+installing `xorg-server-1.2.0'
+```
+
+To install a specific version of `gcc` using the derivation name:
 
 ```console
 $ nix-env --install gcc-3.3.2
@@ -246,6 +255,9 @@ installing `gcc-3.3.2'
 uninstalling `gcc-3.1'
 ```
 
+Using attribute path for selecting a package is preferred,
+as it is much faster and there will not be multiple matches.
+
 Note the previously installed version is removed, since
 `--preserve-installed` was not specified.
 
@@ -256,13 +268,6 @@ $ nix-env --install gcc
 installing `gcc-3.3.2'
 ```
 
-To install using a specific attribute:
-
-```console
-$ nix-env -i -A gcc40mips
-$ nix-env -i -A xorg.xorgserver
-```
-
 To install all derivations in the Nix expression `foo.nix`:
 
 ```console
@@ -374,22 +379,29 @@ For the other flags, see `--install`.
 ## Examples
 
 ```console
-$ nix-env --upgrade gcc
+$ nix-env --upgrade -A nixpkgs.gcc
 upgrading `gcc-3.3.1' to `gcc-3.4'
 ```
 
+When there are no updates available, nothing will happen:
+
 ```console
-$ nix-env -u gcc-3.3.2 --always (switch to a specific version)
+$ nix-env --upgrade -A nixpkgs.pan
+```
+
+Using `-A` is preferred when possible, as it is faster and unambiguous but
+it is also possible to upgrade to a specific version by matching the derivation name:
+
+```console
+$ nix-env -u gcc-3.3.2 --always
 upgrading `gcc-3.4' to `gcc-3.3.2'
 ```
 
-```console
-$ nix-env --upgrade pan
-(no upgrades available, so nothing happens)
-```
+To try to upgrade everything
+(matching packages based on the part of the derivation name without version):
 
 ```console
-$ nix-env -u (try to upgrade everything)
+$ nix-env -u
 upgrading `hello-2.1.2' to `hello-2.1.3'
 upgrading `mozilla-1.2' to `mozilla-1.4'
 ```
@@ -401,7 +413,7 @@ of a derivation `x` by looking at their respective `name` attributes.
 The names (e.g., `gcc-3.3.1` are split into two parts: the package name
 (`gcc`), and the version (`3.3.1`). The version part starts after the
 first dash not followed by a letter. `x` is considered an upgrade of `y`
-if their package names match, and the version of `y` is higher that that
+if their package names match, and the version of `y` is higher than that
 of `x`.
 
 The versions are compared by splitting them into contiguous components

doc/manual/src/command-ref/nix-shell.md

@@ -11,8 +11,8 @@
   [`--command` *cmd*]
   [`--run` *cmd*]
   [`--exclude` *regexp*]
-  [--pure]
-  [--keep *name*]
+  [`--pure`]
+  [`--keep` *name*]
   {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]}
 
 # Description
@@ -110,13 +110,19 @@ shell in which to build it:
 
 ```console
 $ nix-shell '<nixpkgs>' -A pan
-[nix-shell]$ unpackPhase
+[nix-shell]$ eval ${unpackPhase:-unpackPhase}
 [nix-shell]$ cd pan-*
-[nix-shell]$ configurePhase
-[nix-shell]$ buildPhase
+[nix-shell]$ eval ${configurePhase:-configurePhase}
+[nix-shell]$ eval ${buildPhase:-buildPhase}
 [nix-shell]$ ./pan/gui/pan
 ```
 
+The reason we use form `eval ${configurePhase:-configurePhase}` here is because
+those packages that override these phases do so by exporting the overridden
+values in the environment variable of the same name.
+Here bash is being told to either evaluate the contents of 'configurePhase',
+if it exists as a variable, otherwise evaluate the configurePhase function.
+
 To clear the environment first, and do some additional automatic
 initialisation of the interactive shell:
 

doc/manual/src/command-ref/nix-store.md

@@ -125,7 +125,7 @@ Special exit codes:
 
   - `104`\
     Not deterministic, the build succeeded in check mode but the
-    resulting output is not binary reproducable.
+    resulting output is not binary reproducible.
 
 With the `--keep-going` flag it's possible for multiple failures to
 occur, in this case the 1xx status codes are or combined using binary

doc/manual/src/command-ref/opt-common.md

@@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options:
     }: ...
     ```
 
-    So if you call this Nix expression (e.g., when you do `nix-env -i
+    So if you call this Nix expression (e.g., when you do `nix-env -iA
     pkgname`), the function will be called automatically using the
     value [`builtins.currentSystem`](../expressions/builtins.md) for
     the `system` argument. You can override this using `--arg`, e.g.,
-    `nix-env -i pkgname --arg system \"i686-freebsd\"`. (Note that
+    `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that
     since the argument is a Nix string literal, you have to escape the
     quotes.)
 

doc/manual/src/contributing/cli-guideline.md

@@ -3,7 +3,7 @@
 ## Goals
 
 Purpose of this document is to provide a clear direction to **help design
-delightful command line** experience. This document contain guidelines to
+delightful command line** experience. This document contains guidelines to
 follow to ensure a consistent and approachable user experience.
 
 ## Overview
@@ -103,7 +103,7 @@ impacted the most by bad user experience.
 # Help is essential
 
 Help should be built into your command line so that new users can gradually
-discover new features when they need them. 
+discover new features when they need them.
 
 ## Looking for help
 
@@ -115,7 +115,7 @@ The rules are:
 
 - Help is shown by using `--help` or `help` command (eg `nix` `--``help` or
   `nix help`).
-- For non-COMMANDs (eg. `nix` `--``help`  and `nix store` `--``help`) we **show
+- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show
   a summary** of most common use cases. Summary is presented on the STDOUT
   without any use of PAGER.
 - For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the
@@ -176,7 +176,7 @@ $ nix init --template=template#pyton
 ------------------------------------------------------------------------
 Initializing Nix project at `/path/to/here`.
       Select a template for you new project:
-          |> template#pyton
+          |> template#python
              template#python-pip
              template#python-poetry
 ```
@@ -230,17 +230,17 @@ Now **Learn** part of the output is where you educate users. You should only
 show it when you know that a build will take some time and not annoy users of
 the builds that take only few seconds.
 
-Every feature like this should go though a intensive review and testing to
-collect as much a feedback as possible and to fine tune every little detail. If
+Every feature like this should go through an intensive review and testing to
+collect as much feedback as possible and to fine tune every little detail. If
 done right this can be an awesome features beginners and advance users will
 love, but if not done perfectly it will annoy users and leave bad impression.
 
 # Input
 
-Input to a command is provided via `ARGUMENTS` and `OPTIONS`. 
+Input to a command is provided via `ARGUMENTS` and `OPTIONS`.
 
 `ARGUMENTS` represent a required input for a function. When choosing to use
-`ARGUMENT` over function please be aware of the downsides that come with it:
+`ARGUMENTS` over `OPTIONS` please be aware of the downsides that come with it:
 
 - User will need to remember the order of `ARGUMENTS`. This is not a problem if
   there is only one `ARGUMENT`.
@@ -253,7 +253,7 @@ developer consider the downsides and choose wisely.
 
 ## Naming the `OPTIONS`
 
-Then only naming convention - apart from the ones mentioned in Naming the
+The only naming convention - apart from the ones mentioned in Naming the
 `COMMANDS` section is how flags are named.
 
 Flags are a type of `OPTION` that represent an option that can be turned ON of
@@ -271,12 +271,12 @@ to improve the discoverability of possible input. A new user will most likely
 not know which `ARGUMENTS` and `OPTIONS` are required or which values are
 possible for those options.
 
-In cases, the user might not provide the input or they provide wrong input,
-rather then show the error, prompt a user with an option to find and select
+In case the user does not provide the input or they provide wrong input,
+rather than show the error, prompt a user with an option to find and select
 correct input (see examples).
 
 Prompting is of course not required when TTY is not attached to STDIN. This
-would mean that scripts wont need to handle prompt, but rather handle errors.
+would mean that scripts won't need to handle prompt, but rather handle errors.
 
 A place to use prompt and provide user with interactive select
 
@@ -300,9 +300,9 @@ going to happen.
 ```shell
 $ nix build --option substitutors https://cache.example.org
 ------------------------------------------------------------------------
-  Warning! A security related question need to be answered.
+  Warning! A security related question needs to be answered.
 ------------------------------------------------------------------------
-  The following substitutors will be used to in `my-project`: 
+  The following substitutors will be used to in `my-project`:
     - https://cache.example.org
 
   Do you allow `my-project` to use above mentioned substitutors?
@@ -311,14 +311,14 @@ $ nix build --option substitutors https://cache.example.org
 
 # Output
 
-Terminal output can be quite limiting in many ways. Which should forces us to
+Terminal output can be quite limiting in many ways. Which should force us to
 think about the experience even more. As with every design the output is a
 compromise between being terse and being verbose, between showing help to
 beginners and annoying advance users. For this it is important that we know
 what are the priorities.
 
 Nix command line should be first and foremost written with beginners in mind.
-But users wont stay beginners for long and what was once useful might quickly
+But users won't stay beginners for long and what was once useful might quickly
 become annoying. There is no golden rule that we can give in this guideline
 that would make it easier how to draw a line and find best compromise.
 
@@ -342,7 +342,7 @@ also allowing them to redirect content to a file. For example:
 ```shell
 $ nix build > build.txt
 ------------------------------------------------------------------------
-  Error! Atrribute `bin` missing at (1:94) from string.
+  Error! Attribute `bin` missing at (1:94) from string.
 ------------------------------------------------------------------------
 
   1| with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } ""
@@ -408,7 +408,7 @@ Above command clearly states that command successfully completed. And in case
 of `nix build`, which is a command that might take some time to complete, it is
 equally important to also show that a command started.
 
-## Text alignment 
+## Text alignment
 
 Text alignment is the number one design element that will present all of the
 Nix commands as a family and not as separate tools glued together.
@@ -419,7 +419,7 @@ The format we should follow is:
 $ nix COMMAND
    VERB_1 NOUN and other words
   VERB__1 NOUN and other words
-       |> Some details 
+       |> Some details
 ```
 
 Few rules that we can extract from above example:
@@ -444,13 +444,13 @@ is not even notable, therefore relying on it wouldn’t make much sense.
 
 **The bright text is much better supported** across terminals and color
 schemes. Most of the time the difference is perceived as if the bright text
-would be bold. 
+would be bold.
 
 ## Colors
 
 Humans are already conditioned by society to attach certain meaning to certain
 colors. While the meaning is not universal, a simple collection of colors is
-used to represent basic emotions. 
+used to represent basic emotions.
 
 Colors that can be used in output
 
@@ -508,7 +508,7 @@ can, with a few key strokes, be changed into and advance introspection tool.
 
 ### Progress
 
-For longer running commands we should provide and overview of the progress.
+For longer running commands we should provide and overview the progress.
 This is shown best in `nix build` example:
 
 ```shell
@@ -553,9 +553,9 @@ going to happen.
 ```shell
 $ nix build --option substitutors https://cache.example.org
 ------------------------------------------------------------------------
-  Warning! A security related question need to be answered.
+  Warning! A security related question needs to be answered.
 ------------------------------------------------------------------------
-  The following substitutors will be used to in `my-project`: 
+  The following substitutors will be used to in `my-project`:
     - https://cache.example.org
 
   Do you allow `my-project` to use above mentioned substitutors?
@@ -566,7 +566,7 @@ $ nix build --option substitutors https://cache.example.org
 
 There are many ways that you can control verbosity.
 
-Verbosity levels are: 
+Verbosity levels are:
 
 - `ERROR` (level 0)
 - `WARN` (level 1)
@@ -586,4 +586,4 @@ There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and
 
 # Appendix 1: Commands naming exceptions
 
-`nix init` and `nix repl` are well established 
+`nix init` and `nix repl` are well established

doc/manual/src/expressions/advanced-attributes.md

@@ -237,7 +237,7 @@ Derivations can declare some infrequently used optional attributes.
   - `preferLocalBuild`\
     If this attribute is set to `true` and [distributed building is
     enabled](../advanced-topics/distributed-builds.md), then, if
-    possible, the derivaton will be built locally instead of forwarded
+    possible, the derivation will be built locally instead of forwarded
     to a remote machine. This is appropriate for trivial builders
     where the cost of doing a download or remote build would exceed
     the cost of building locally.

doc/manual/src/expressions/builtins-prefix.md

@@ -12,5 +12,5 @@ For instance, `derivation` is also available as `builtins.derivation`.
 <dl>
   <dt><code>derivation <var>attrs</var></code>;
       <code>builtins.derivation <var>attrs</var></code></dt>
-  <dd><p><var>derivation</var> in described in
+  <dd><p><var>derivation</var> is described in
          <a href="derivations.md">its own section</a>.</p></dd>

doc/manual/src/expressions/expression-syntax.md

@@ -26,7 +26,7 @@ elements (referenced from the figure by number):
     called with three arguments: `stdenv`, `fetchurl`, and `perl`. They
     are needed to build Hello, but we don't know how to build them here;
     that's why they are function arguments. `stdenv` is a package that
-    is used by almost all Nix Packages packages; it provides a
+    is used by almost all Nix Packages; it provides a
     “standard” environment consisting of the things you would expect
     in a basic Unix environment: a C/C++ compiler (GCC, to be precise),
     the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`,

doc/manual/src/expressions/language-operators.md

@@ -17,12 +17,12 @@ order of precedence (from strongest to weakest binding).
 | String Concatenation     | *string1* `+` *string2*             | left          | String concatenation.                                                                                                                                                                                                         | 7          |
 | Not                      | `!` *e*                             | none          | Boolean negation.                                                                                                                                                                                                             | 8          |
 | Update                   | *e1* `//` *e2*                      | right         | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes).                                                                           | 9          |
-| Less Than                | *e1* `<` *e2*,                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
-| Less Than or Equal To    | *e1* `<=` *e2*                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
-| Greater Than             | *e1* `>` *e2*                       | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
-| Greater Than or Equal To | *e1* `>=` *e2*                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
+| Less Than                | *e1* `<` *e2*,                      | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
+| Less Than or Equal To    | *e1* `<=` *e2*                      | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
+| Greater Than             | *e1* `>` *e2*                       | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
+| Greater Than or Equal To | *e1* `>=` *e2*                      | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
 | Equality                 | *e1* `==` *e2*                      | none          | Equality.                                                                                                                                                                                                                     | 11         |
 | Inequality               | *e1* `!=` *e2*                      | none          | Inequality.                                                                                                                                                                                                                   | 11         |
 | Logical AND              | *e1* `&&` *e2*                      | left          | Logical AND.                                                                                                                                                                                                                  | 12         |
-| Logical OR               | *e1* `\|\|` *e2*                    | left          | Logical OR.                                                                                                                                                                                                                   | 13         |
-| Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to `!e1 \|\| e2`).                                                                                                                                                                            | 14         |
+| Logical OR               | *e1* <code>&#124;&#124;</code> *e2* | left          | Logical OR.                                                                                                                                                                                                                   | 13         |
+| Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to <code>!e1 &#124;&#124; e2</code>).                                                                                                                                                                            | 14         |

doc/manual/src/expressions/language-values.md

@@ -64,7 +64,7 @@ Nix has the following basic data types:
     the start of each line. To be precise, it strips from each line a
     number of spaces equal to the minimal indentation of the string as a
     whole (disregarding the indentation of empty lines). For instance,
-    the first and second line are indented two space, while the third
+    the first and second line are indented two spaces, while the third
     line is indented four spaces. Thus, two spaces are stripped from
     each line, so the resulting string is
     

doc/manual/src/expressions/simple-building-testing.md

@@ -1,6 +1,6 @@
 # Building and Testing
 
-You can now try to build Hello. Of course, you could do `nix-env -i
+You can now try to build Hello. Of course, you could do `nix-env -f . -iA
 hello`, but you may not want to install a possibly broken package just
 yet. The best way to test the package is by using the command
 `nix-build`, which builds a Nix expression and creates a symlink named

doc/manual/src/installation/building-source.md

@@ -1,9 +1,9 @@
 # Building Nix from Source
 
-After unpacking or checking out the Nix sources, issue the following
-commands:
+After cloning Nix's Git repository, issue the following commands:
 
 ```console
+$ ./bootstrap.sh
 $ ./configure options...
 $ make
 $ make install
@@ -11,13 +11,6 @@ $ make install
 
 Nix requires GNU Make so you may need to invoke `gmake` instead.
 
-When building from the Git repository, these should be preceded by the
-command:
-
-```console
-$ ./bootstrap.sh
-```
-
 The installation path can be specified by passing the `--prefix=prefix`
 to `configure`. The default installation directory is `/usr/local`. You
 can change this to any location you like. You must have write permission

doc/manual/src/installation/installing-docker.md

@@ -0,0 +1,59 @@
+# Using Nix within Docker
+
+To run the latest stable release of Nix with Docker run the following command:
+
+```console
+$ docker -ti run nixos/nix
+Unable to find image 'nixos/nix:latest' locally
+latest: Pulling from nixos/nix
+5843afab3874: Pull complete
+b52bf13f109c: Pull complete
+1e2415612aa3: Pull complete
+Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff
+Status: Downloaded newer image for nixos/nix:latest
+35ca4ada6e96:/# nix --version
+nix (Nix) 2.3.12
+35ca4ada6e96:/# exit
+```
+
+# What is included in Nix' Docker image?
+
+The official Docker image is created using `pkgs.dockerTools.buildLayeredImage`
+(and not with `Dockerfile` as it is usual with Docker images). You can still
+base your custom Docker image on it as you would do with any other Docker
+image.
+
+The Docker image is also not based on any other image and includes minimal set
+of runtime dependencies that are required to use Nix:
+
+ - pkgs.nix
+ - pkgs.bashInteractive
+ - pkgs.coreutils-full
+ - pkgs.gnutar
+ - pkgs.gzip
+ - pkgs.gnugrep
+ - pkgs.which
+ - pkgs.curl
+ - pkgs.less
+ - pkgs.wget
+ - pkgs.man
+ - pkgs.cacert.out
+ - pkgs.findutils
+
+# Docker image with the latest development version of Nix
+
+To get the latest image that was built by [Hydra](https://hydra.nixos.org) run
+the following command:
+
+```console
+$ curl -L https://hydra.nixos.org/job/nix/master/dockerImage.x86_64-linux/latest/download/1 | docker load
+$ docker run -ti nix:2.5pre20211105
+```
+
+You can also build a Docker image from source yourself:
+
+```console
+$ nix build ./\#hydraJobs.dockerImage.x86_64-linux
+$ docker load -i ./result
+$ docker run -ti nix:2.5pre20211105
+```

doc/manual/src/installation/installing-source.md

@@ -1,4 +1,4 @@
 # Installing Nix from Source
 
-If no binary package is available, you can download and compile a source
-distribution.
+If no binary package is available or if you want to hack on Nix, you
+can build Nix from its Git repository.

doc/manual/src/installation/obtaining-source.md

@@ -1,14 +1,9 @@
-# Obtaining a Source Distribution
+# Obtaining the Source
 
-The source tarball of the most recent stable release can be downloaded
-from the [Nix homepage](http://nixos.org/nix/download.html). You can
-also grab the [most recent development
-release](http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents).
-
-Alternatively, the most recent sources of Nix can be obtained from its
-[Git repository](https://github.com/NixOS/nix). For example, the
-following command will check out the latest revision into a directory
-called `nix`:
+The most recent sources of Nix can be obtained from its [Git
+repository](https://github.com/NixOS/nix). For example, the following
+command will check out the latest revision into a directory called
+`nix`:
 
 ```console
 $ git clone https://github.com/NixOS/nix

doc/manual/src/installation/prerequisites-source.md

@@ -2,9 +2,8 @@
 
   - GNU Autoconf (<https://www.gnu.org/software/autoconf/>) and the
     autoconf-archive macro collection
-    (<https://www.gnu.org/software/autoconf-archive/>). These are only
-    needed to run the bootstrap script, and are not necessary if your
-    source distribution came with a pre-built `./configure` script.
+    (<https://www.gnu.org/software/autoconf-archive/>). These are
+    needed to run the bootstrap script.
 
   - GNU Make.
 
@@ -52,8 +51,7 @@
     you need version 2.5.35, which is available on
     [SourceForge](http://lex.sourceforge.net/). Slightly older versions
     may also work, but ancient versions like the ubiquitous 2.5.4a
-    won't. Note that these are only required if you modify the parser or
-    when you are building from the Git repository.
+    won't.
 
   - The `libseccomp` is used to provide syscall filtering on Linux. This
     is an optional dependency and can be disabled passing a

doc/manual/src/introduction.md

@@ -76,7 +76,7 @@ there after an upgrade.  This means that you can _roll back_ to the
 old version:
 
 ```console
-$ nix-env --upgrade some-packages
+$ nix-env --upgrade -A nixpkgs.some-package
 $ nix-env --rollback
 ```
 
@@ -122,12 +122,12 @@ Nix expressions generally describe how to build a package from
 source, so an installation action like
 
 ```console
-$ nix-env --install firefox
+$ nix-env --install -A nixpkgs.firefox
 ```
 
 _could_ cause quite a bit of build activity, as not only Firefox but
 also all its dependencies (all the way up to the C library and the
-compiler) would have to built, at least if they are not already in the
+compiler) would have to be built, at least if they are not already in the
 Nix store.  This is a _source deployment model_.  For most users,
 building from source is not very pleasant as it takes far too long.
 However, Nix can automatically skip building from source and instead

doc/manual/src/package-management/basic-package-mgmt.md

@@ -24,7 +24,7 @@ collection; you could write your own Nix expressions based on Nixpkgs,
 or completely new ones.)
 
 You can manually download the latest version of Nixpkgs from
-<http://nixos.org/nixpkgs/download.html>. However, it’s much more
+<https://github.com/NixOS/nixpkgs>. However, it’s much more
 convenient to use the Nixpkgs [*channel*](channels.md), since it makes
 it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
 automatically added to your list of “subscribed” channels when you
@@ -47,41 +47,45 @@ $ nix-channel --update
 You can view the set of available packages in Nixpkgs:
 
 ```console
-$ nix-env -qa
-aterm-2.2
-bash-3.0
-binutils-2.15
-bison-1.875d
-blackdown-1.4.2
-bzip2-1.0.2
+$ nix-env -qaP
+nixpkgs.aterm                       aterm-2.2
+nixpkgs.bash                        bash-3.0
+nixpkgs.binutils                    binutils-2.15
+nixpkgs.bison                       bison-1.875d
+nixpkgs.blackdown                   blackdown-1.4.2
+nixpkgs.bzip2                       bzip2-1.0.2
 …
 ```
 
-The flag `-q` specifies a query operation, and `-a` means that you want
+The flag `-q` specifies a query operation, `-a` means that you want
 to show the “available” (i.e., installable) packages, as opposed to the
-installed packages. If you downloaded Nixpkgs yourself, or if you
-checked it out from GitHub, then you need to pass the path to your
-Nixpkgs tree using the `-f` flag:
+installed packages, and `-P` prints the attribute paths that can be used
+to unambiguously select a package for installation (listed in the first column).
+If you downloaded Nixpkgs yourself, or if you checked it out from GitHub,
+then you need to pass the path to your Nixpkgs tree using the `-f` flag:
 
 ```console
-$ nix-env -qaf /path/to/nixpkgs
+$ nix-env -qaPf /path/to/nixpkgs
+aterm                               aterm-2.2
+bash                                bash-3.0
+…
 ```
 
 where */path/to/nixpkgs* is where you’ve unpacked or checked out
 Nixpkgs.
 
-You can select specific packages by name:
+You can filter the packages by name:
 
 ```console
-$ nix-env -qa firefox
-firefox-34.0.5
-firefox-with-plugins-34.0.5
+$ nix-env -qaP firefox
+nixpkgs.firefox-esr                 firefox-91.3.0esr
+nixpkgs.firefox                     firefox-94.0.1
 ```
 
 and using regular expressions:
 
 ```console
-$ nix-env -qa 'firefox.*'
+$ nix-env -qaP 'firefox.*'
 ```
 
 It is also possible to see the *status* of available packages, i.e.,
@@ -89,11 +93,11 @@ whether they are installed into the user environment and/or present in
 the system:
 
 ```console
-$ nix-env -qas
+$ nix-env -qaPs
 …
--PS bash-3.0
---S binutils-2.15
-IPS bison-1.875d
+-PS  nixpkgs.bash                bash-3.0
+--S  nixpkgs.binutils            binutils-2.15
+IPS  nixpkgs.bison               bison-1.875d
 …
 ```
 
@@ -106,13 +110,13 @@ which is Nix’s mechanism for doing binary deployment. It just means that
 Nix knows that it can fetch a pre-built package from somewhere
 (typically a network server) instead of building it locally.
 
-You can install a package using `nix-env -i`. For instance,
+You can install a package using `nix-env -iA`. For instance,
 
 ```console
-$ nix-env -i subversion
+$ nix-env -iA nixpkgs.subversion
 ```
 
-will install the package called `subversion` (which is, of course, the
+will install the package called `subversion` from `nixpkgs` channel (which is, of course, the
 [Subversion version management system](http://subversion.tigris.org/)).
 
 > **Note**
@@ -122,7 +126,7 @@ will install the package called `subversion` (which is, of course, the
 > binary cache <https://cache.nixos.org>; it contains binaries for most
 > packages in Nixpkgs. Only if no binary is available in the binary
 > cache, Nix will build the package from source. So if `nix-env
-> -i subversion` results in Nix building stuff from source, then either
+> -iA nixpkgs.subversion` results in Nix building stuff from source, then either
 > the package is not built for your platform by the Nixpkgs build
 > servers, or your version of Nixpkgs is too old or too new. For
 > instance, if you have a very recent checkout of Nixpkgs, then the
@@ -133,7 +137,10 @@ will install the package called `subversion` (which is, of course, the
 > using a Git checkout of the Nixpkgs tree), you will get binaries for
 > most packages.
 
-Naturally, packages can also be uninstalled:
+Naturally, packages can also be uninstalled. Unlike when installing, you will
+need to use the derivation name (though the version part can be omitted),
+instead of the attribute path, as `nix-env` does not record which attribute
+was used for installing:
 
 ```console
 $ nix-env -e subversion
@@ -143,7 +150,7 @@ Upgrading to a new version is just as easy. If you have a new release of
 Nix Packages, you can do:
 
 ```console
-$ nix-env -u subversion
+$ nix-env -uA nixpkgs.subversion
 ```
 
 This will *only* upgrade Subversion if there is a “newer” version in the

doc/manual/src/package-management/binary-cache-substituter.md

@@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is
 not part of the Nix distribution, but you can install it from Nixpkgs:
 
 ```console
-$ nix-env -i nix-serve
+$ nix-env -iA nixpkgs.nix-serve
 ```
 
 You can then start the server, listening for HTTP connections on
@@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using
 `--option extra-binary-caches`, e.g.:
 
 ```console
-$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/
+$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/
 ```
 
 The option `extra-binary-caches` tells Nix to use this binary cache in

doc/manual/src/package-management/garbage-collection.md

@@ -44,7 +44,7 @@ collector as follows:
 $ nix-store --gc
 ```
 
-The behaviour of the gargage collector is affected by the
+The behaviour of the garbage collector is affected by the
 `keep-derivations` (default: true) and `keep-outputs` (default: false)
 options in the Nix configuration file. The defaults will ensure that all
 derivations that are build-time dependencies of garbage collector roots

doc/manual/src/package-management/profiles.md

@@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This
 would be what we would obtain if we had done
 
 ```console
-$ nix-env -i subversion
+$ nix-env -iA nixpkgs.subversion
 ```
 
 on a set of Nix expressions that contained Subversion 1.1.2.
@@ -54,7 +54,7 @@ environment is generated based on the current one. For instance,
 generation 43 was created from generation 42 when we did
 
 ```console
-$ nix-env -i subversion firefox
+$ nix-env -iA nixpkgs.subversion nixpkgs.firefox
 ```
 
 on a set of Nix expressions that contained Firefox and a new version of
@@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by
 (abbreviation `-p`):
 
 ```console
-$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion
+$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion
 ```
 
 This will *not* change the `~/.nix-profile` symlink.

doc/manual/src/package-management/ssh-substituter.md

@@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are
 available on the server `avalon`:
 
 ```console
-$ nix-env -i firefox --substituters ssh://alice@avalon
+$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon
 ```
 
 This works similar to the binary cache substituter that Nix usually

doc/manual/src/quick-start.md

@@ -19,19 +19,19 @@ to subsequent chapters.
    channel:
 
    ```console
-   $ nix-env -qa
-   docbook-xml-4.3
-   docbook-xml-4.5
-   firefox-33.0.2
-   hello-2.9
-   libxslt-1.1.28
+   $ nix-env -qaP
+   nixpkgs.docbook_xml_dtd_43                    docbook-xml-4.3
+   nixpkgs.docbook_xml_dtd_45                    docbook-xml-4.5
+   nixpkgs.firefox                               firefox-33.0.2
+   nixpkgs.hello                                 hello-2.9
+   nixpkgs.libxslt                               libxslt-1.1.28
    …
    ```
 
 1. Install some packages from the channel:
 
    ```console
-   $ nix-env -i hello
+   $ nix-env -iA nixpkgs.hello
    ```
 
    This should download pre-built packages; it should not build them

doc/manual/src/release-notes/rl-2.4.md

@@ -281,7 +281,9 @@ more than 2800 commits from 195 contributors since release 2.3.
 * The `nix` command is now marked as an experimental feature. This
   means that you need to add
 
-  > experimental-features = nix-command
+  ```
+  experimental-features = nix-command
+  ```
 
   to your `nix.conf` if you want to use it, or pass
   `--extra-experimental-features nix-command` on the command line.
@@ -289,30 +291,42 @@ more than 2800 commits from 195 contributors since release 2.3.
 * The `nix` command no longer has a syntax for referring to packages
   in a channel. This means that the following no longer works:
 
-  > nix build nixpkgs.hello # Nix 2.3
+  ```console
+  nix build nixpkgs.hello # Nix 2.3
+  ```
 
   Instead, you can either use the `#` syntax to select a package from
   a flake, e.g.
 
-  > nix build nixpkgs#hello
+  ```console
+  nix build nixpkgs#hello
+  ```
 
   Or, if you want to use the `nixpkgs` channel in the `NIX_PATH`
   environment variable:
 
-  > nix build -f '<nixpkgs>' hello
+  ```console
+  nix build -f '<nixpkgs>' hello
+  ```
 
 * The old `nix run` has been renamed to `nix shell`, while there is a
   new `nix run` that runs a default command. So instead of
 
-  > nix run nixpkgs.hello -c hello # Nix 2.3
+  ```console
+  nix run nixpkgs.hello -c hello # Nix 2.3
+  ```
 
   you should use
 
-  > nix shell nixpkgs#hello -c hello
+  ```console
+  nix shell nixpkgs#hello -c hello
+  ```
 
   or just
 
-  > nix run nixpkgs#hello
+  ```console
+  nix run nixpkgs#hello
+  ```
 
   if the command you want to run has the same name as the package.
 
@@ -381,6 +395,7 @@ dramforever,
 Dustin DeWeese,
 edef,
 Eelco Dolstra,
+Ellie Hermaszewska,
 Emilio Karakey,
 Emily,
 Eric Culp,
@@ -391,7 +406,7 @@ Federico Pellegrin,
 Finn Behrens,
 Florian Franzen,
 Félix Baylac-Jacqué,
-Gabriel Gonzalez,
+Gabriella Gonzalez,
 Geoff Reedy,
 Georges Dubus,
 Graham Christensen,
@@ -414,7 +429,6 @@ Jaroslavas Pocepko,
 Jarrett Keifer,
 Jeremy Schlatter,
 Joachim Breitner,
-Joe Hermaszewski,
 Joe Pea,
 John Ericson,
 Jonathan Ringer,

doc/manual/src/release-notes/rl-next.md

@@ -0,0 +1,7 @@
+# Release 2.5 (2021-XX-XX)
+
+* Binary cache stores now have a setting `compression-level`.
+
+* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
+
+* Lists can now be compared lexicographically using the `<` operator.

docker.nix

@@ -0,0 +1,251 @@
+{ pkgs ? import <nixpkgs> { }
+, lib ? pkgs.lib
+, name ? "nix"
+, tag ? "latest"
+, channelName ? "nixpkgs"
+, channelURL ? "https://nixos.org/channels/nixpkgs-unstable"
+}:
+let
+  defaultPkgs = with pkgs; [
+    nix
+    bashInteractive
+    coreutils-full
+    gnutar
+    gzip
+    gnugrep
+    which
+    curl
+    less
+    wget
+    man
+    cacert.out
+    findutils
+  ];
+
+  users = {
+
+    root = {
+      uid = 0;
+      shell = "/bin/bash";
+      home = "/root";
+      gid = 0;
+    };
+
+  } // lib.listToAttrs (
+    map
+      (
+        n: {
+          name = "nixbld${toString n}";
+          value = {
+            uid = 30000 + n;
+            gid = 30000;
+            groups = [ "nixbld" ];
+            description = "Nix build user ${toString n}";
+          };
+        }
+      )
+      (lib.lists.range 1 32)
+  );
+
+  groups = {
+    root.gid = 0;
+    nixbld.gid = 30000;
+  };
+
+  userToPasswd = (
+    k:
+    { uid
+    , gid ? 65534
+    , home ? "/var/empty"
+    , description ? ""
+    , shell ? "/bin/false"
+    , groups ? [ ]
+    }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}"
+  );
+  passwdContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs userToPasswd users))
+  );
+
+  userToShadow = k: { ... }: "${k}:!:1::::::";
+  shadowContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs userToShadow users))
+  );
+
+  # Map groups to members
+  # {
+  #   group = [ "user1" "user2" ];
+  # }
+  groupMemberMap = (
+    let
+      # Create a flat list of user/group mappings
+      mappings = (
+        builtins.foldl'
+          (
+            acc: user:
+              let
+                groups = users.${user}.groups or [ ];
+              in
+              acc ++ map
+                (group: {
+                  inherit user group;
+                })
+                groups
+          )
+          [ ]
+          (lib.attrNames users)
+      );
+    in
+    (
+      builtins.foldl'
+        (
+          acc: v: acc // {
+            ${v.group} = acc.${v.group} or [ ] ++ [ v.user ];
+          }
+        )
+        { }
+        mappings)
+  );
+
+  groupToGroup = k: { gid }:
+    let
+      members = groupMemberMap.${k} or [ ];
+    in
+    "${k}:x:${toString gid}:${lib.concatStringsSep "," members}";
+  groupContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs groupToGroup groups))
+  );
+
+  nixConf = {
+    sandbox = "false";
+    build-users-group = "nixbld";
+    trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
+  };
+  nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n";
+
+  baseSystem =
+    let
+      nixpkgs = pkgs.path;
+      channel = pkgs.runCommand "channel-nixos" { } ''
+        mkdir $out
+        ln -s ${nixpkgs} $out/nixpkgs
+        echo "[]" > $out/manifest.nix
+      '';
+      rootEnv = pkgs.buildPackages.buildEnv {
+        name = "root-profile-env";
+        paths = defaultPkgs;
+      };
+      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+        mkdir $out
+        cp -a ${rootEnv}/* $out/
+
+        cat > $out/manifest.nix <<EOF
+        [
+        ${lib.concatStringsSep "\n" (builtins.map (drv: let
+          outputs = drv.outputsToInstall or [ "out" ];
+        in ''
+          {
+            ${lib.concatStringsSep "\n" (builtins.map (output: ''
+              ${output} = { outPath = "${lib.getOutput output drv}"; };
+            '') outputs)}
+            outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ];
+            name = "${drv.name}";
+            outPath = "${drv}";
+            system = "${drv.system}";
+            type = "derivation";
+            meta = { };
+          }
+        '') defaultPkgs)}
+        ]
+        EOF
+      '';
+    in
+    pkgs.runCommand "base-system"
+      {
+        inherit passwdContents groupContents shadowContents nixConfContents;
+        passAsFile = [
+          "passwdContents"
+          "groupContents"
+          "shadowContents"
+          "nixConfContents"
+        ];
+        allowSubstitutes = false;
+        preferLocalBuild = true;
+      } ''
+      env
+      set -x
+      mkdir -p $out/etc
+
+      cat $passwdContentsPath > $out/etc/passwd
+      echo "" >> $out/etc/passwd
+
+      cat $groupContentsPath > $out/etc/group
+      echo "" >> $out/etc/group
+
+      cat $shadowContentsPath > $out/etc/shadow
+      echo "" >> $out/etc/shadow
+
+      mkdir -p $out/usr
+      ln -s /nix/var/nix/profiles/share $out/usr/
+
+      mkdir -p $out/nix/var/nix/gcroots
+
+      mkdir $out/tmp
+
+      mkdir -p $out/etc/nix
+      cat $nixConfContentsPath > $out/etc/nix/nix.conf
+
+      mkdir -p $out/root
+      mkdir -p $out/nix/var/nix/profiles/per-user/root
+
+      ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
+      ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+      ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
+
+      ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
+      ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
+
+      mkdir -p $out/root/.nix-defexpr
+      ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
+      echo "${channelURL} ${channelName}" > $out/root/.nix-channels
+
+      mkdir -p $out/bin $out/usr/bin
+      ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
+      ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
+    '';
+
+in
+pkgs.dockerTools.buildLayeredImageWithNixDb {
+
+  inherit name tag;
+
+  contents = [ baseSystem ];
+
+  extraCommands = ''
+    rm -rf nix-support
+    ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles
+  '';
+
+  config = {
+    Cmd = [ "/root/.nix-profile/bin/bash" ];
+    Env = [
+      "USER=root"
+      "PATH=${lib.concatStringsSep ":" [
+        "/root/.nix-profile/bin"
+        "/nix/var/nix/profiles/default/bin"
+        "/nix/var/nix/profiles/default/sbin"
+      ]}"
+      "MANPATH=${lib.concatStringsSep ":" [
+        "/root/.nix-profile/share/man"
+        "/nix/var/nix/profiles/default/share/man"
+      ]}"
+      "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels"
+    ];
+  };
+
+}

flake.nix

@@ -14,7 +14,7 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-      officialRelease = true;
+      officialRelease = false;
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
@@ -61,6 +61,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
+            "--with-boost=${boost}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
             "LDFLAGS=-fuse-ld=gold"
           ];
@@ -90,7 +91,7 @@
             libarchive
             boost
             lowdown-nix
-            gmock
+            gtest
           ]
           ++ lib.optionals stdenv.isLinux [libseccomp]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
@@ -404,6 +405,21 @@
         installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
         installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
+        # docker image with Nix inside
+        dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system:
+          let
+            pkgs = nixpkgsFor.${system};
+            image = import ./docker.nix { inherit pkgs; tag = version; };
+          in pkgs.runCommand "docker-image-tarball-${version}"
+            { meta.description = "Docker image with Nix for ${system}";
+            }
+            ''
+              mkdir -p $out/nix-support
+              image=$out/image.tar.gz
+              ln -s ${image} $image
+              echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
+            '');
+
         # Line coverage analysis.
         coverage =
           with nixpkgsFor.x86_64-linux;
@@ -444,6 +460,12 @@
           inherit (self) overlay;
         };
 
+        tests.nssPreload = (import ./tests/nss-preload.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
         tests.githubFlakes = (import ./tests/github-flakes.nix rec {
           system = "x86_64-linux";
           inherit nixpkgs;
@@ -482,12 +504,7 @@
             '';
         */
 
-      };
-
-      checks = forAllSystems (system: {
-        binaryTarball = self.hydraJobs.binaryTarball.${system};
-        perlBindings = self.hydraJobs.perlBindings.${system};
-        installTests =
+        installTests = forAllSystems (system:
           let pkgs = nixpkgsFor.${system}; in
           pkgs.runCommand "install-tests" {
             againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
@@ -499,8 +516,17 @@
             # Disabled because the latest stable version doesn't handle
             # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
             # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
-          } "touch $out";
-      });
+          } "touch $out");
+
+      };
+
+      checks = forAllSystems (system: {
+        binaryTarball = self.hydraJobs.binaryTarball.${system};
+        perlBindings = self.hydraJobs.perlBindings.${system};
+        installTests = self.hydraJobs.installTests.${system};
+      } // (if system == "x86_64-linux" then {
+        dockerImage = self.hydraJobs.dockerImage.${system};
+      } else {}));
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;

maintainers/upload-release.pl

@@ -19,6 +19,8 @@ my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
 my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
 
+my $isLatest = ($ENV{'IS_LATEST'} // "") eq "1";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -35,16 +37,18 @@ sub fetch {
 my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
 my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
 #print Dumper($evalInfo);
+my $flakeUrl = $evalInfo->{flake} or die;
+my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die);
+my $nixRev = $flakeInfo->{revision} or die;
 
-my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die;
+my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
+#print Dumper($buildInfo);
 
-my $tarballInfo = decode_json(fetch("$evalUrl/job/tarball", 'application/json'));
-
-my $releaseName = $tarballInfo->{releasename};
+my $releaseName = $buildInfo->{nixname};
 $releaseName =~ /nix-(.*)$/ or die;
 my $version = $1;
 
-print STDERR "Nix revision is $nixRev, version is $version\n";
+print STDERR "Flake URL is $flakeUrl, Nix revision is $nixRev, version is $version\n";
 
 my $releaseDir = "nix/$releaseName";
 
@@ -104,8 +108,6 @@ sub downloadFile {
     return $sha256_expected;
 }
 
-downloadFile("tarball", "2"); # .tar.bz2
-my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
@@ -134,42 +136,38 @@ for my $fn (glob "$tmpDir/*") {
     }
 }
 
-exit if $version =~ /pre/;
-
 # Update nix-fallback-paths.nix.
-system("cd $nixpkgsDir && git pull") == 0 or die;
+if ($isLatest) {
+    system("cd $nixpkgsDir && git pull") == 0 or die;
 
-sub getStorePath {
-    my ($jobName) = @_;
-    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    for my $product (values %{$buildInfo->{buildproducts}}) {
-        next unless $product->{type} eq "nix-build";
-        next if $product->{path} =~ /[a-z]+$/;
-        return $product->{path};
+    sub getStorePath {
+        my ($jobName) = @_;
+        my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+        return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
     }
-    die;
+
+    write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
+               "{\n" .
+               "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
+               "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
+               "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+               "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+               "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
+               "}\n");
+
+    system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
 }
 
-write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
-           "{\n" .
-           "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
-           "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
-           "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
-           "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
-           "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
-           "}\n");
-
-system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
-
 # Update the "latest" symlink.
 $channelsBucket->add_key(
     "nix-latest/install", "",
     { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
-    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr
+    if $isLatest;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
 system("git push --tags") == 0 or die;
-system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;

perl/lib/Nix/Config.pm.in

@@ -1,6 +1,7 @@
 package Nix::Config;
 
 use MIME::Base64;
+use Nix::Store;
 
 $version = "@PACKAGE_VERSION@";
 

scripts/create-darwin-volume.sh

@@ -742,6 +742,9 @@ setup_volume() {
 
     use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}"
 
+    _sudo "to ensure the Nix volume is not mounted" \
+        /usr/sbin/diskutil unmount force "$use_special" || true # might not be mounted
+
     use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")}
 
     setup_fstab "$use_uuid"

scripts/install-multi-user.sh

@@ -387,19 +387,28 @@ EOF
     fi
 
     for profile_target in "${PROFILE_TARGETS[@]}"; do
+        # TODO: I think it would be good to accumulate a list of all
+        #       of the copies so that people don't hit this 2 or 3x in
+        #       a row for different files.
         if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
+            # this backup process first released in Nix 2.1
             failure <<EOF
-When this script runs, it backs up the current $profile_target to
-$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
+I back up shell profile/rc scripts before I add Nix to them.
+I need to back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX,
+but the latter already exists.
 
-Please follow these instructions to clean up the old backup file:
+Here's how to clean up the old backup file:
 
-1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just
-in case.
+1. Back up (copy) $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX
+   to another location, just in case.
 
-2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
-it has anything nix-related in it. If it does, something is probably
-quite wrong. Please open an issue or get in touch immediately.
+2. Ensure $profile_target$PROFILE_BACKUP_SUFFIX does not have anything
+   Nix-related in it. If it does, something is probably quite
+   wrong. Please open an issue or get in touch immediately.
+
+3. Once you confirm $profile_target is backed up and
+   $profile_target$PROFILE_BACKUP_SUFFIX doesn't mention Nix, run:
+   mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
 EOF
         fi
     done
@@ -599,7 +608,7 @@ manager. This will happen in a few stages:
 1. Make sure your computer doesn't already have Nix. If it does, I
    will show you instructions on how to clean up your old install.
 
-2. Show you what we are going to install and where. Then we will ask
+2. Show you what I am going to install and where. Then I will ask
    if you are ready to continue.
 
 3. Create the system users and groups that the Nix daemon uses to run
@@ -614,14 +623,14 @@ manager. This will happen in a few stages:
 
 EOF
 
-    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
+    if ui_confirm "Would you like to see a more detailed list of what I will do?"; then
         cat <<EOF
 
-We will:
+I will:
 
  - make sure your computer doesn't already have Nix files
    (if it does, I will tell you how to clean them up.)
- - create local users (see the list above for the users we'll make)
+ - create local users (see the list above for the users I'll make)
  - create a local group ($NIX_BUILD_GROUP_NAME)
  - install Nix in to $NIX_ROOT
  - create a configuration file in /etc/nix
@@ -656,7 +665,7 @@ run in a headless fashion, like this:
 
   $ curl -L https://nixos.org/nix/install | sh
 
-or maybe in a CI pipeline. Because of that, we're going to skip the
+or maybe in a CI pipeline. Because of that, I'm going to skip the
 verbose output in the interest of brevity.
 
 If you would like to
@@ -670,7 +679,7 @@ EOF
     fi
 
     cat <<EOF
-This script is going to call sudo a lot. Every time we do, it'll
+This script is going to call sudo a lot. Every time I do, it'll
 output exactly what it'll do, and why.
 
 Just like this:
@@ -682,15 +691,15 @@ EOF
     cat <<EOF
 
 This might look scary, but everything can be undone by running just a
-few commands. We used to ask you to confirm each time sudo ran, but it
+few commands. I used to ask you to confirm each time sudo ran, but it
 was too many times. Instead, I'll just ask you this one time:
 
 EOF
-    if ui_confirm "Can we use sudo?"; then
+    if ui_confirm "Can I use sudo?"; then
         ok "Yay! Thanks! Let's get going!"
     else
         failure <<EOF
-That is okay, but we can't install.
+That is okay, but I can't install.
 EOF
     fi
 }
@@ -809,10 +818,10 @@ main() {
     # can fail faster in this case. Sourcing install-darwin... now runs
     # `touch /` to detect Read-only root, but it could update times on
     # pre-Catalina macOS if run as root user.
-    if [ $EUID -eq 0 ]; then
+    if [ "$EUID" -eq 0 ]; then
         failure <<EOF
-Please do not run this script with root privileges. We will call sudo
-when we need to.
+Please do not run this script with root privileges. I will call sudo
+when I need to.
 EOF
     fi
 

scripts/install-nix-from-closure.sh

@@ -134,7 +134,7 @@ fi
 
 echo "performing a single-user installation of Nix..." >&2
 
-if ! [ -e $dest ]; then
+if ! [ -e "$dest" ]; then
     cmd="mkdir -m 0755 $dest && chown $USER $dest"
     echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2
     if ! sudo sh -c "$cmd"; then
@@ -143,12 +143,12 @@ if ! [ -e $dest ]; then
     fi
 fi
 
-if ! [ -w $dest ]; then
+if ! [ -w "$dest" ]; then
     echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 
-mkdir -p $dest/store
+mkdir -p "$dest/store"
 
 printf "copying Nix to %s..." "${dest}/store" >&2
 # Insert a newline if no progress is shown.
@@ -189,17 +189,17 @@ fi
 
 # Install an SSL certificate bundle.
 if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
-    $nix/bin/nix-env -i "$cacert"
+    "$nix/bin/nix-env" -i "$cacert"
     export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt"
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
 if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
-    if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
-        $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    if ! "$nix/bin/nix-channel" --list | grep -q "^nixpkgs "; then
+        "$nix/bin/nix-channel" --add https://nixos.org/channels/nixpkgs-unstable
     fi
     if [ -z "$_NIX_INSTALLER_TEST" ]; then
-        if ! $nix/bin/nix-channel --update nixpkgs; then
+        if ! "$nix/bin/nix-channel" --update nixpkgs; then
             echo "Fetching the nixpkgs channel failed. (Are you offline?)"
             echo "To try again later, run \"nix-channel --update nixpkgs\"."
         fi
@@ -215,7 +215,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break
@@ -226,7 +226,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break

scripts/install-systemd-multi-user.sh

@@ -15,7 +15,7 @@ readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
 
 create_systemd_override() {
      header "Configuring proxy for the nix-daemon service"
-    _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)"
+    _sudo "create directory for systemd unit override" mkdir -p "$(dirname "$SERVICE_OVERRIDE")"
     cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE"
 [Service]
 $1

scripts/install.in

@@ -81,10 +81,10 @@ if [ "$(uname -s)" != "Darwin" ]; then
     require_util xz "unpack the binary tarball"
 fi
 
-if command -v wget > /dev/null 2>&1; then
-    fetch() { wget "$1" -O "$2"; }
-elif command -v curl > /dev/null 2>&1; then
+if command -v curl > /dev/null 2>&1; then
     fetch() { curl -L "$1" -o "$2"; }
+elif command -v wget > /dev/null 2>&1; then
+    fetch() { wget "$1" -O "$2"; }
 else
     oops "you don't have wget or curl installed, which I need to download the binary tarball"
 fi

scripts/local.mk

@@ -1,7 +1,5 @@
 nix_noinst_scripts := \
-  $(d)/nix-http-export.cgi \
-  $(d)/nix-profile.sh \
-  $(d)/nix-reduce-build
+  $(d)/nix-profile.sh
 
 noinst-scripts += $(nix_noinst_scripts)
 

scripts/nix-http-export.cgi.in

@@ -1,51 +0,0 @@
-#! /bin/sh
-
-export HOME=/tmp
-export NIX_REMOTE=daemon
-
-TMP_DIR="${TMP_DIR:-/tmp/nix-export}"
-
-@coreutils@/mkdir -p "$TMP_DIR" || true
-@coreutils@/chmod a+r "$TMP_DIR"
-
-needed_path="?$QUERY_STRING"
-needed_path="${needed_path#*[?&]needed_path=}"
-needed_path="${needed_path%%&*}"
-#needed_path="$(echo $needed_path  | ./unhttp)"
-needed_path="${needed_path//%2B/+}"
-needed_path="${needed_path//%3D/=}"
-
-echo needed_path: "$needed_path" >&2
-
-NIX_STORE="${NIX_STORE_DIR:-/nix/store}"
-
-echo NIX_STORE: "${NIX_STORE}" >&2
-
-full_path="${NIX_STORE}"/"$needed_path"
-
-if [ "$needed_path" != "${needed_path%.drv}" ]; then
-	echo "Status: 403 You should create the derivation file yourself"
-	echo "Content-Type: text/plain"
-	echo
-	echo "Refusing to disclose derivation contents"
-	exit
-fi
-
-if @bindir@/nix-store --check-validity "$full_path"; then
-	if ! [ -e nix-export/"$needed_path".nar.gz ]; then
-		@bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz
-		@coreutils@/ln -fs  "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz 
-	fi;
-	echo "Status: 301 Moved"
-	echo "Location: nix-export/"$needed_path".nar.gz"
-	echo
-else 
-	echo "Status: 404 No such path found"
-	echo "Content-Type: text/plain"
-	echo
-	echo "Path not found:"
-	echo "$needed_path"
-	echo "checked:"
-	echo "$full_path"
-fi
-

scripts/nix-profile-daemon.sh.in

@@ -5,7 +5,7 @@ __ETC_PROFILE_NIX_SOURCED=1
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
+if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE
 elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
     export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
@@ -18,14 +18,14 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
 else
   # Fall back to what is in the nix profiles, favouring whatever is defined last.
   check_nix_profiles() {
-    if [ "$ZSH_VERSION" ]; then
+    if [ -n "$ZSH_VERSION" ]; then
       # Zsh by default doesn't split words in unquoted parameter expansion.
       # Set local_options for these options to be reverted at the end of the function
       # and shwordsplit to force splitting words in $NIX_PROFILES below.
       setopt local_options shwordsplit
     fi
     for i in $NIX_PROFILES; do
-      if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
+      if [ -e "$i/etc/ssl/certs/ca-bundle.crt" ]; then
         export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
       fi
     done

scripts/nix-reduce-build.in

@@ -1,171 +0,0 @@
-#! @bash@
-
-WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX);
-cd "$WORKING_DIRECTORY";
-
-if test -z "$1" || test "a--help" = "a$1" ; then
-	echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2
-	echo As in: >&2
-	echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2
-	echo nix-reduce-build /etc/nixos/nixos -- \\
-	echo "   " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2
-	echo "  store path name will be added into the end of the URL" >&2
-	echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2
-	echo "  that should be a directory where gzipped 'nix-store --export' ">&2
-	echo "  files are located (they should have .nar.gz extension)"  >&2
-	echo "        Or all together: " >&2
-	echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\
-	"    ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2
-	echo "        Also supports best-effort local builds of failing expression set:" >&2
-	echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2
-	echo "  nix-daemon:// builds using daemon"
-	echo "  nix-self:// builds directly using nix-store from current installation" >&2
-	echo "  nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2;
-	echo "derivations with specified output hash (sha256, sha1 or md5)." >&2
-	echo "  nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2;
-	echo "maximum amount of paths" >&2;
-	echo "  nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2;
-	echo "maximum amount of paths" >&2;
-	echo "        If no package sources are specified, required paths are listed." >&2;
-	exit;
-fi;
-
-while ! test "$1" = "--" || test "$1" = "" ; do 
-	echo "$1" >> initial; >&2
-	shift;
-done
-shift;
-echo Will work on $(cat initial | wc -l) targets. >&2
-
-while read ; do
-	case "$REPLY" in 
-		${NIX_STORE_DIR:-/nix/store}/*)
-			echo "$REPLY" >> paths; >&2
-			;;
-		*)
-			(
-				IFS=: ;
-				nix-instantiate $REPLY >> paths;
-			);
-			;;
-	esac;
-done < initial;
-echo Proceeding $(cat paths | wc -l) paths. >&2
-
-while read; do
-	case "$REPLY" in
-		*.drv)
-			echo "$REPLY" >> derivers; >&2
-			;;
-		*)
-			nix-store --query --deriver "$REPLY" >>derivers;
-			;;
-	esac;
-done < paths;
-echo Found $(cat derivers | wc -l) derivers. >&2
-
-cat derivers | xargs nix-store --query -R > derivers-closure;
-echo Proceeding at most $(cat derivers-closure | wc -l) derivers. >&2
-
-cat derivers-closure | egrep '[.]drv$' | xargs nix-store --query --outputs > wanted-paths;
-cat derivers-closure | egrep -v '[.]drv$' >> wanted-paths;
-echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2
-
-cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
-echo We need $(cat needed-paths | wc -l) paths. >&2
-
-egrep '[.]drv$' derivers-closure > critical-derivers;
-
-if test -z "$1" ; then
-	cat needed-paths;	
-fi;
-
-refresh_critical_derivers() {
-    echo "Finding needed derivers..." >&2;
-    cat critical-derivers | while read; do
-        if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then
-            echo "$REPLY";
-        fi;
-    done > new-critical-derivers;
-    mv new-critical-derivers critical-derivers;
-    echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2
-}
-
-build_here() {
-    cat critical-derivers | while read; do 
-        echo "Realising $REPLY using nix-daemon" >&2
-        @bindir@/nix-store -r "${REPLY}"
-    done;
-}
-
-try_to_substitute(){
-    cat needed-paths | while read ; do 
-        echo "Building $REPLY using nix-daemon" >&2
-        @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}"
-    done;
-}
-
-for i in "$@"; do 
-	sshHost="${i#ssh://}";
-	httpHost="${i#http://}";
-	httpsHost="${i#https://}";
-	filePath="${i#file:/}";
-	if [ "$i" != "$sshHost" ]; then
-		cat needed-paths | while read; do 
-			echo "Getting $REPLY and its closure over ssh" >&2
-			nix-copy-closure --from "$sshHost" --gzip "$REPLY" </dev/null || true; 
-		done;
-	elif [ "$i" != "$httpHost" ] || [ "$i" != "$httpsHost" ]; then
-		cat needed-paths | while read; do
-			echo "Getting $REPLY over http/https" >&2
-			curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import;
-		done;
-	elif [ "$i" != "$filePath" ] ; then
-		cat needed-paths | while read; do 
-			echo "Installing $REPLY from file" >&2
-			gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import;
-		done;
-	elif [ "$i" = "nix-daemon://" ] ; then
-		NIX_REMOTE=daemon try_to_substitute;
-		refresh_critical_derivers;
-		NIX_REMOTE=daemon build_here;
-	elif [ "$i" = "nix-self://" ] ; then
-		NIX_REMOTE= try_to_substitute;
-		refresh_critical_derivers;
-		NIX_REMOTE= build_here;
-	elif [ "$i" = "nix-daemon-fixed://" ] ; then
-		refresh_critical_derivers;
-
-		cat critical-derivers | while read; do 
-			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
-				echo "Realising $REPLY using nix-daemon" >&2
-				NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}"
-			fi;
-		done;
-	elif [ "$i" = "nix-self-fixed://" ] ; then
-		refresh_critical_derivers;
-
-		cat critical-derivers | while read; do 
-			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
-				echo "Realising $REPLY using direct Nix build" >&2
-				NIX_REMOTE= @bindir@/nix-store -r "${REPLY}"
-			fi;
-		done;
-	elif [ "$i" = "nix-daemon-substitute://" ] ; then
-		NIX_REMOTE=daemon try_to_substitute;
-	elif [ "$i" = "nix-self-substitute://" ] ; then
-		NIX_REMOTE= try_to_substitute;
-	elif [ "$i" = "nix-daemon-build://" ] ; then
-		refresh_critical_derivers;
-		NIX_REMOTE=daemon build_here;
-	elif [ "$i" = "nix-self-build://" ] ; then
-		refresh_critical_derivers;
-		NIX_REMOTE= build_here;
-	fi;
-	mv needed-paths wanted-paths;
-	cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
-	echo We still need $(cat needed-paths | wc -l) paths. >&2
-done;
-
-cd /
-rm -r "$WORKING_DIRECTORY"

scripts/prepare-installer-for-github-actions

@@ -3,7 +3,7 @@
 set -e
 
 script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
-installerHash=$(echo $script | cut -b12-43 -)
+installerHash=$(echo "$script" | cut -b12-43 -)
 
 installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
 

src/build-remote/build-remote.cc

@@ -18,6 +18,7 @@
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"
+#include "experimental-features.hh"
 
 using namespace nix;
 using std::cin;
@@ -298,7 +299,7 @@ connected:
 
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
             for (auto & outputName : wantedOutputs) {
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
@@ -330,7 +331,7 @@ connected:
         for (auto & realisation : missingRealisations) {
             // Should hold, because if the feature isn't enabled the set
             // of missing realisations should be empty
-            settings.requireExperimentalFeature("ca-derivations");
+            settings.requireExperimentalFeature(Xp::CaDerivations);
             store->registerDrvOutput(realisation);
         }
 

src/libcmd/installables.cc

@@ -714,7 +714,7 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
                                 "the derivation '%s' doesn't have an output named '%s'",
                                 store->printStorePath(bfd.drvPath), output);
                         if (settings.isExperimentalFeatureEnabled(
-                                "ca-derivations")) {
+                                Xp::CaDerivations)) {
                             auto outputId =
                                 DrvOutput{outputHashes.at(output), output};
                             auto realisation =

src/libexpr/eval.cc

@@ -119,8 +119,8 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu
     case tList2:
     case tListN:
         str << "[ ";
-        for (unsigned int n = 0; n < v.listSize(); ++n) {
-            printValue(str, active, *v.listElems()[n]);
+        for (auto v2 : v.listItems()) {
+            printValue(str, active, *v2);
             str << " ";
         }
         str << "]";
@@ -466,7 +466,7 @@ EvalState::~EvalState()
 
 
 void EvalState::requireExperimentalFeatureOnEvaluation(
-    const std::string & feature,
+    const ExperimentalFeature & feature,
     const std::string_view fName,
     const Pos & pos)
 {
@@ -519,8 +519,12 @@ Path EvalState::checkSourcePath(const Path & path_)
         }
     }
 
-    if (!found)
-        throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath);
+    if (!found) {
+        auto modeInformation = evalSettings.pureEval
+            ? "in pure eval mode (use '--impure' to override)"
+            : "in restricted mode";
+        throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation);
+    }
 
     /* Resolve symlinks. */
     debug(format("checking access to '%s'") % abspath);
@@ -533,7 +537,7 @@ Path EvalState::checkSourcePath(const Path & path_)
         }
     }
 
-    throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path);
+    throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path);
 }
 
 
@@ -583,14 +587,20 @@ Value * EvalState::addConstant(const string & name, Value & v)
 {
     Value * v2 = allocValue();
     *v2 = v;
-    staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
-    baseEnv.values[baseEnvDispl++] = v2;
-    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2));
+    addConstant(name, v2);
     return v2;
 }
 
 
+void EvalState::addConstant(const string & name, Value * v)
+{
+    staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl);
+    baseEnv.values[baseEnvDispl++] = v;
+    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v));
+}
+
+
 Value * EvalState::addPrimOp(const string & name,
     size_t arity, PrimOpFun primOp)
 {
@@ -609,7 +619,7 @@ Value * EvalState::addPrimOp(const string & name,
 
     Value * v = allocValue();
     v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym });
-    staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
+    staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl);
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(sym, v));
     return v;
@@ -635,7 +645,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
 
     Value * v = allocValue();
     v->mkPrimOp(new PrimOp(std::move(primOp)));
-    staticBaseEnv.vars[envName] = baseEnvDispl;
+    staticBaseEnv.vars.emplace_back(envName, baseEnvDispl);
     baseEnv.values[baseEnvDispl++] = v;
     baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
     return v;
@@ -785,7 +795,7 @@ void mkPath(Value & v, const char * s)
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
-    for (size_t l = var.level; l; --l, env = env->up) ;
+    for (auto l = var.level; l; --l, env = env->up) ;
 
     if (!var.fromWith) return env->values[var.displ];
 
@@ -1058,7 +1068,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
-        size_t displ = 0;
+        Displacement displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && !i.second.inherited) {
@@ -1134,7 +1144,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
-    size_t displ = 0;
+    Displacement displ = 0;
     for (auto & i : attrs->attrs)
         env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
 
@@ -1145,8 +1155,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 void ExprList::eval(EvalState & state, Env & env, Value & v)
 {
     state.mkList(v, elems.size());
-    for (size_t n = 0; n < elems.size(); ++n)
-        v.listElems()[n] = elems[n]->maybeThunk(state, env);
+    for (auto [n, v2] : enumerate(v.listItems()))
+        const_cast<Value * &>(v2) = elems[n]->maybeThunk(state, env);
 }
 
 
@@ -1251,144 +1261,184 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
 }
 
 
-void ExprApp::eval(EvalState & state, Env & env, Value & v)
-{
-    /* FIXME: vFun prevents GCC from doing tail call optimisation. */
-    Value vFun;
-    e1->eval(state, env, vFun);
-    state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos);
-}
-
-
-void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
-{
-    /* Figure out the number of arguments still needed. */
-    size_t argsDone = 0;
-    Value * primOp = &fun;
-    while (primOp->isPrimOpApp()) {
-        argsDone++;
-        primOp = primOp->primOpApp.left;
-    }
-    assert(primOp->isPrimOp());
-    auto arity = primOp->primOp->arity;
-    auto argsLeft = arity - argsDone;
-
-    if (argsLeft == 1) {
-        /* We have all the arguments, so call the primop. */
-
-        /* Put all the arguments in an array. */
-        Value * vArgs[arity];
-        auto n = arity - 1;
-        vArgs[n--] = &arg;
-        for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left)
-            vArgs[n--] = arg->primOpApp.right;
-
-        /* And call the primop. */
-        nrPrimOpCalls++;
-        if (countCalls) primOpCalls[primOp->primOp->name]++;
-        primOp->primOp->fun(*this, pos, vArgs, v);
-    } else {
-        Value * fun2 = allocValue();
-        *fun2 = fun;
-        v.mkPrimOpApp(fun2, &arg);
-    }
-}
-
-void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos)
+void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos)
 {
     auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr;
 
     forceValue(fun, pos);
 
-    if (fun.isPrimOp() || fun.isPrimOpApp()) {
-        callPrimOp(fun, arg, v, pos);
-        return;
-    }
+    Value vCur(fun);
 
-    if (fun.type() == nAttrs) {
-      auto found = fun.attrs->find(sFunctor);
-      if (found != fun.attrs->end()) {
-        /* fun may be allocated on the stack of the calling function,
-         * but for functors we may keep a reference, so heap-allocate
-         * a copy and use that instead.
-         */
-        auto & fun2 = *allocValue();
-        fun2 = fun;
-        /* !!! Should we use the attr pos here? */
-        Value v2;
-        callFunction(*found->value, fun2, v2, pos);
-        return callFunction(v2, arg, v, pos);
-      }
-    }
+    auto makeAppChain = [&]()
+    {
+        vRes = vCur;
+        for (size_t i = 0; i < nrArgs; ++i) {
+            auto fun2 = allocValue();
+            *fun2 = vRes;
+            vRes.mkPrimOpApp(fun2, args[i]);
+        }
+    };
 
-    if (!fun.isLambda())
-        throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
+    Attr * functor;
 
-    ExprLambda & lambda(*fun.lambda.fun);
+    while (nrArgs > 0) {
 
-    auto size =
-        (lambda.arg.empty() ? 0 : 1) +
-        (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
-    Env & env2(allocEnv(size));
-    env2.up = fun.lambda.env;
+        if (vCur.isLambda()) {
 
-    size_t displ = 0;
+            ExprLambda & lambda(*vCur.lambda.fun);
 
-    if (!lambda.hasFormals())
-        env2.values[displ++] = &arg;
+            auto size =
+                (lambda.arg.empty() ? 0 : 1) +
+                (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
+            Env & env2(allocEnv(size));
+            env2.up = vCur.lambda.env;
 
-    else {
-        forceAttrs(arg, pos);
+            Displacement displ = 0;
 
-        if (!lambda.arg.empty())
-            env2.values[displ++] = &arg;
+            if (!lambda.hasFormals())
+                env2.values[displ++] = args[0];
 
-        /* For each formal argument, get the actual argument.  If
-           there is no matching actual argument but the formal
-           argument has a default, use the default. */
-        size_t attrsUsed = 0;
-        for (auto & i : lambda.formals->formals) {
-            Bindings::iterator j = arg.attrs->find(i.name);
-            if (j == arg.attrs->end()) {
-                if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
-                    lambda, i.name);
-                env2.values[displ++] = i.def->maybeThunk(*this, env2);
+            else {
+                forceAttrs(*args[0], pos);
+
+                if (!lambda.arg.empty())
+                    env2.values[displ++] = args[0];
+
+                /* For each formal argument, get the actual argument.  If
+                   there is no matching actual argument but the formal
+                   argument has a default, use the default. */
+                size_t attrsUsed = 0;
+                for (auto & i : lambda.formals->formals) {
+                    auto j = args[0]->attrs->get(i.name);
+                    if (!j) {
+                        if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'",
+                            lambda, i.name);
+                        env2.values[displ++] = i.def->maybeThunk(*this, env2);
+                    } else {
+                        attrsUsed++;
+                        env2.values[displ++] = j->value;
+                    }
+                }
+
+                /* Check that each actual argument is listed as a formal
+                   argument (unless the attribute match specifies a `...'). */
+                if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) {
+                    /* Nope, so show the first unexpected argument to the
+                       user. */
+                    for (auto & i : *args[0]->attrs)
+                        if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
+                            throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
+                    abort(); // can't happen
+                }
+            }
+
+            nrFunctionCalls++;
+            if (countCalls) incrFunctionCall(&lambda);
+
+            /* Evaluate the body. */
+            try {
+                lambda.body->eval(*this, env2, vCur);
+            } catch (Error & e) {
+                if (loggerSettings.showTrace.get()) {
+                    addErrorTrace(e, lambda.pos, "while evaluating %s",
+                        (lambda.name.set()
+                            ? "'" + (string) lambda.name + "'"
+                            : "anonymous lambda"));
+                    addErrorTrace(e, pos, "from call site%s", "");
+                }
+                throw;
+            }
+
+            nrArgs--;
+            args += 1;
+        }
+
+        else if (vCur.isPrimOp()) {
+
+            size_t argsLeft = vCur.primOp->arity;
+
+            if (nrArgs < argsLeft) {
+                /* We don't have enough arguments, so create a tPrimOpApp chain. */
+                makeAppChain();
+                return;
             } else {
-                attrsUsed++;
-                env2.values[displ++] = j->value;
+                /* We have all the arguments, so call the primop. */
+                nrPrimOpCalls++;
+                if (countCalls) primOpCalls[vCur.primOp->name]++;
+                vCur.primOp->fun(*this, pos, args, vCur);
+
+                nrArgs -= argsLeft;
+                args += argsLeft;
             }
         }
 
-        /* Check that each actual argument is listed as a formal
-           argument (unless the attribute match specifies a `...'). */
-        if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) {
-            /* Nope, so show the first unexpected argument to the
-               user. */
-            for (auto & i : *arg.attrs)
-                if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end())
-                    throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
-            abort(); // can't happen
+        else if (vCur.isPrimOpApp()) {
+            /* Figure out the number of arguments still needed. */
+            size_t argsDone = 0;
+            Value * primOp = &vCur;
+            while (primOp->isPrimOpApp()) {
+                argsDone++;
+                primOp = primOp->primOpApp.left;
+            }
+            assert(primOp->isPrimOp());
+            auto arity = primOp->primOp->arity;
+            auto argsLeft = arity - argsDone;
+
+            if (nrArgs < argsLeft) {
+                /* We still don't have enough arguments, so extend the tPrimOpApp chain. */
+                makeAppChain();
+                return;
+            } else {
+                /* We have all the arguments, so call the primop with
+                   the previous and new arguments. */
+
+                Value * vArgs[arity];
+                auto n = argsDone;
+                for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left)
+                    vArgs[--n] = arg->primOpApp.right;
+
+                for (size_t i = 0; i < argsLeft; ++i)
+                    vArgs[argsDone + i] = args[i];
+
+                nrPrimOpCalls++;
+                if (countCalls) primOpCalls[primOp->primOp->name]++;
+                primOp->primOp->fun(*this, pos, vArgs, vCur);
+
+                nrArgs -= argsLeft;
+                args += argsLeft;
+            }
         }
+
+        else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) {
+            /* 'vCur' may be allocated on the stack of the calling
+               function, but for functors we may keep a reference, so
+               heap-allocate a copy and use that instead. */
+            Value * args2[] = {allocValue(), args[0]};
+            *args2[0] = vCur;
+            /* !!! Should we use the attr pos here? */
+            callFunction(*functor->value, 2, args2, vCur, pos);
+            nrArgs--;
+            args++;
+        }
+
+        else
+            throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur);
     }
 
-    nrFunctionCalls++;
-    if (countCalls) incrFunctionCall(&lambda);
+    vRes = vCur;
+}
 
-    /* Evaluate the body.  This is conditional on showTrace, because
-       catching exceptions makes this function not tail-recursive. */
-    if (loggerSettings.showTrace.get())
-        try {
-            lambda.body->eval(*this, env2, v);
-        } catch (Error & e) {
-            addErrorTrace(e, lambda.pos, "while evaluating %s",
-              (lambda.name.set()
-                  ? "'" + (string) lambda.name + "'"
-                  : "anonymous lambda"));
-            addErrorTrace(e, pos, "from call site%s", "");
-            throw;
-        }
-    else
-        fun.lambda.fun->body->eval(*this, env2, v);
+
+void ExprCall::eval(EvalState & state, Env & env, Value & v)
+{
+    Value vFun;
+    fun->eval(state, env, vFun);
+
+    Value * vArgs[args.size()];
+    for (size_t i = 0; i < args.size(); ++i)
+        vArgs[i] = args[i]->maybeThunk(state, env);
+
+    state.callFunction(vFun, args.size(), vArgs, v, pos);
 }
 
 
@@ -1686,8 +1736,8 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            for (size_t n = 0; n < v.listSize(); ++n)
-                recurse(*v.listElems()[n]);
+            for (auto v2 : v.listItems())
+                recurse(*v2);
         }
     };
 
@@ -1871,12 +1921,12 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
 
         if (v.isList()) {
             string result;
-            for (size_t n = 0; n < v.listSize(); ++n) {
-                result += coerceToString(pos, *v.listElems()[n],
+            for (auto [n, v2] : enumerate(v.listItems())) {
+                result += coerceToString(pos, *v2,
                     context, coerceMore, copyToStore);
                 if (n < v.listSize() - 1
                     /* !!! not quite correct */
-                    && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0))
+                    && (!v2->isList() || v2->listSize() != 0))
                     result += " ";
             }
             return result;

src/libexpr/eval.hh

@@ -5,6 +5,7 @@
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "config.hh"
+#include "experimental-features.hh"
 
 #include <map>
 #include <optional>
@@ -141,7 +142,7 @@ public:
     ~EvalState();
 
     void requireExperimentalFeatureOnEvaluation(
-        const std::string & feature,
+        const ExperimentalFeature &,
         const std::string_view fName,
         const Pos & pos
     );
@@ -276,6 +277,8 @@ private:
 
     Value * addConstant(const string & name, Value & v);
 
+    void addConstant(const string & name, Value * v);
+
     Value * addPrimOp(const string & name,
         size_t arity, PrimOpFun primOp);
 
@@ -315,8 +318,14 @@ public:
 
     bool isFunctor(Value & fun);
 
-    void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos);
-    void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos);
+    // FIXME: use std::span
+    void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos);
+
+    void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos)
+    {
+        Value * args[] = {&arg};
+        callFunction(fun, 1, args, vRes, pos);
+    }
 
     /* Automatically call a function for which each argument has a
        default value or has a binding in the `args' map. */

src/libexpr/flake/config.cc

@@ -1,4 +1,5 @@
 #include "flake.hh"
+#include "globals.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -52,21 +53,19 @@ void ConfigFile::apply()
             auto trustedList = readTrustedList();
 
             bool trusted = false;
-
-            if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
+            if (nix::settings.acceptFlakeConfig){
+                trusted = true;
+            } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
                 trusted = *saved;
+                warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
             } else {
                 // FIXME: filter ANSI escapes, newlines, \r, etc.
-                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
-                    if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
-                        trustedList[name][valueS] = false;
-                        writeTrustedList(trustedList);
-                    }
-                } else {
-                    if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
-                        trustedList[name][valueS] = trusted = true;
-                        writeTrustedList(trustedList);
-                    }
+                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') {
+                    trusted = true;
+                }
+                if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?",  trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') {
+                    trustedList[name][valueS] = trusted;
+                    writeTrustedList(trustedList);
                 }
             }
 

src/libexpr/flake/flake.cc

@@ -257,8 +257,7 @@ static Flake getFlake(
                 flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
             else if (setting.value->type() == nList) {
                 std::vector<std::string> ss;
-                for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
-                    auto elem = setting.value->listElems()[n];
+                for (auto elem : setting.value->listItems()) {
                     if (elem->type() != nString)
                         throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
                             setting.name, showType(*setting.value));
@@ -297,7 +296,7 @@ LockedFlake lockFlake(
     const FlakeRef & topRef,
     const LockFlags & lockFlags)
 {
-    settings.requireExperimentalFeature("flakes");
+    settings.requireExperimentalFeature(Xp::Flakes);
 
     FlakeCache flakeCache;
 
@@ -307,7 +306,7 @@ LockedFlake lockFlake(
 
     if (lockFlags.applyNixConfig) {
         flake.config.apply();
-        // FIXME: send new config to the daemon.
+        state.store->setOptions();
     }
 
     try {
@@ -446,22 +445,18 @@ LockedFlake lockFlake(
                            update it. */
                         auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
-                        auto hasChildUpdate =
+                        auto mustRefetch =
                             lb != lockFlags.inputUpdates.end()
                             && lb->size() > inputPath.size()
                             && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
 
-                        if (hasChildUpdate) {
-                            auto inputFlake = getFlake(
-                                state, oldLock->lockedRef, false, flakeCache);
-                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath);
-                        } else {
+                        FlakeInputs fakeInputs;
+
+                        if (!mustRefetch) {
                             /* No need to fetch this flake, we can be
                                lazy. However there may be new overrides on the
                                inputs of this flake, so we need to check
                                those. */
-                            FlakeInputs fakeInputs;
-
                             for (auto & i : oldLock->inputs) {
                                 if (auto lockedNode = std::get_if<0>(&i.second)) {
                                     fakeInputs.emplace(i.first, FlakeInput {
@@ -469,15 +464,28 @@ LockedFlake lockFlake(
                                         .isFlake = (*lockedNode)->isFlake,
                                     });
                                 } else if (auto follows = std::get_if<1>(&i.second)) {
+                                    auto o = input.overrides.find(i.first);
+                                    // If the override disappeared, we have to refetch the flake,
+                                    // since some of the inputs may not be present in the lockfile.
+                                    if (o == input.overrides.end()) {
+                                        mustRefetch = true;
+                                        // There's no point populating the rest of the fake inputs,
+                                        // since we'll refetch the flake anyways.
+                                        break;
+                                    }
                                     fakeInputs.emplace(i.first, FlakeInput {
                                         .follows = *follows,
                                     });
                                 }
                             }
-
-                            computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath);
                         }
 
+                        computeLocks(
+                            mustRefetch
+                            ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs
+                            : fakeInputs,
+                            childNode, inputPath, oldLock, parent, parentPath);
+
                     } else {
                         /* We need to create a new lock file entry. So fetch
                            this input. */
@@ -687,7 +695,7 @@ void callFlake(EvalState & state,
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.requireExperimentalFeatureOnEvaluation("flakes", "builtins.getFlake", pos);
+    state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
 
     auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);

src/libexpr/get-drvs.cc

@@ -102,9 +102,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
             state->forceList(*i->value, *i->pos);
 
             /* For each output... */
-            for (unsigned int j = 0; j < i->value->listSize(); ++j) {
+            for (auto elem : i->value->listItems()) {
                 /* Evaluate the corresponding set. */
-                string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
+                string name = state->forceStringNoCtx(*elem, *i->pos);
                 Bindings::iterator out = attrs->find(state->symbols.create(name));
                 if (out == attrs->end()) continue; // FIXME: throw error?
                 state->forceAttrs(*out->value);
@@ -128,9 +128,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
-    for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if ((*i)->type() != nString) throw errMsg;
-        auto out = outputs.find((*i)->string.s);
+    for (auto elem : outTI->listItems()) {
+        if (elem->type() != nString) throw errMsg;
+        auto out = outputs.find(elem->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
     }
@@ -174,8 +174,8 @@ bool DrvInfo::checkMeta(Value & v)
 {
     state->forceValue(v);
     if (v.type() == nList) {
-        for (unsigned int n = 0; n < v.listSize(); ++n)
-            if (!checkMeta(*v.listElems()[n])) return false;
+        for (auto elem : v.listItems())
+            if (!checkMeta(*elem)) return false;
         return true;
     }
     else if (v.type() == nAttrs) {
@@ -364,10 +364,10 @@ static void getDerivations(EvalState & state, Value & vIn,
     }
 
     else if (v.type() == nList) {
-        for (unsigned int n = 0; n < v.listSize(); ++n) {
-            string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
-                getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+        for (auto [n, elem] : enumerate(v.listItems())) {
+            string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
+            if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures))
+                getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
     }
 

src/libexpr/lexer.l

@@ -64,6 +64,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
+// FIXME: optimize
 static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
 {
     string t;

src/libexpr/nixexpr.cc

@@ -143,6 +143,16 @@ void ExprLambda::show(std::ostream & str) const
     str << ": " << *body << ")";
 }
 
+void ExprCall::show(std::ostream & str) const
+{
+    str << '(' << *fun;
+    for (auto e : args) {
+        str <<  ' ';
+        str << *e;
+    }
+    str << ')';
+}
+
 void ExprLet::show(std::ostream & str) const
 {
     str << "(let ";
@@ -263,13 +273,13 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
     const StaticEnv * curEnv;
-    unsigned int level;
+    Level level;
     int withLevel = -1;
     for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) {
         if (curEnv->isWith) {
             if (withLevel == -1) withLevel = level;
         } else {
-            StaticEnv::Vars::const_iterator i = curEnv->vars.find(name);
+            auto i = curEnv->find(name);
             if (i != curEnv->vars.end()) {
                 fromWith = false;
                 this->level = level;
@@ -311,14 +321,16 @@ void ExprOpHasAttr::bindVars(const StaticEnv & env)
 void ExprAttrs::bindVars(const StaticEnv & env)
 {
     const StaticEnv * dynamicEnv = &env;
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0);
 
     if (recursive) {
         dynamicEnv = &newEnv;
 
-        unsigned int displ = 0;
+        Displacement displ = 0;
         for (auto & i : attrs)
-            newEnv.vars[i.first] = i.second.displ = displ++;
+            newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
+
+        // No need to sort newEnv since attrs is in sorted order.
 
         for (auto & i : attrs)
             i.second.e->bindVars(i.second.inherited ? env : newEnv);
@@ -342,15 +354,20 @@ void ExprList::bindVars(const StaticEnv & env)
 
 void ExprLambda::bindVars(const StaticEnv & env)
 {
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(
+        false, &env,
+        (hasFormals() ? formals->formals.size() : 0) +
+        (arg.empty() ? 0 : 1));
 
-    unsigned int displ = 0;
+    Displacement displ = 0;
 
-    if (!arg.empty()) newEnv.vars[arg] = displ++;
+    if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++);
 
     if (hasFormals()) {
         for (auto & i : formals->formals)
-            newEnv.vars[i.name] = displ++;
+            newEnv.vars.emplace_back(i.name, displ++);
+
+        newEnv.sort();
 
         for (auto & i : formals->formals)
             if (i.def) i.def->bindVars(newEnv);
@@ -359,13 +376,22 @@ void ExprLambda::bindVars(const StaticEnv & env)
     body->bindVars(newEnv);
 }
 
+void ExprCall::bindVars(const StaticEnv & env)
+{
+    fun->bindVars(env);
+    for (auto e : args)
+        e->bindVars(env);
+}
+
 void ExprLet::bindVars(const StaticEnv & env)
 {
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(false, &env, attrs->attrs.size());
 
-    unsigned int displ = 0;
+    Displacement displ = 0;
     for (auto & i : attrs->attrs)
-        newEnv.vars[i.first] = i.second.displ = displ++;
+        newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
+
+    // No need to sort newEnv since attrs->attrs is in sorted order.
 
     for (auto & i : attrs->attrs)
         i.second.e->bindVars(i.second.inherited ? env : newEnv);
@@ -379,7 +405,7 @@ void ExprWith::bindVars(const StaticEnv & env)
        level so that `lookupVar' can look up variables in the previous
        `with' if this one doesn't contain the desired attribute. */
     const StaticEnv * curEnv;
-    unsigned int level;
+    Level level;
     prevWith = 0;
     for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++)
         if (curEnv->isWith) {
@@ -452,5 +478,4 @@ size_t SymbolTable::totalSize() const
     return n;
 }
 
-
 }

src/libexpr/nixexpr.hh

@@ -4,8 +4,6 @@
 #include "symbol-table.hh"
 #include "error.hh"
 
-#include <map>
-
 
 namespace nix {
 
@@ -135,6 +133,9 @@ struct ExprPath : Expr
     Value * maybeThunk(EvalState & state, Env & env);
 };
 
+typedef uint32_t Level;
+typedef uint32_t Displacement;
+
 struct ExprVar : Expr
 {
     Pos pos;
@@ -150,8 +151,8 @@ struct ExprVar : Expr
        value is obtained by getting the attribute named `name' from
        the set stored in the environment that is `level' levels up
        from the current one.*/
-    unsigned int level;
-    unsigned int displ;
+    Level level;
+    Displacement displ;
 
     ExprVar(const Symbol & name) : name(name) { };
     ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
@@ -185,7 +186,7 @@ struct ExprAttrs : Expr
         bool inherited;
         Expr * e;
         Pos pos;
-        unsigned int displ; // displacement
+        Displacement displ; // displacement
         AttrDef(Expr * e, const Pos & pos, bool inherited=false)
             : inherited(inherited), e(e), pos(pos) { };
         AttrDef() { };
@@ -250,6 +251,17 @@ struct ExprLambda : Expr
     COMMON_METHODS
 };
 
+struct ExprCall : Expr
+{
+    Expr * fun;
+    std::vector<Expr *> args;
+    Pos pos;
+    ExprCall(const Pos & pos, Expr * fun, std::vector<Expr *> && args)
+        : fun(fun), args(args), pos(pos)
+    { }
+    COMMON_METHODS
+};
+
 struct ExprLet : Expr
 {
     ExprAttrs * attrs;
@@ -308,7 +320,6 @@ struct ExprOpNot : Expr
         void eval(EvalState & state, Env & env, Value & v); \
     };
 
-MakeBinOp(ExprApp, "")
 MakeBinOp(ExprOpEq, "==")
 MakeBinOp(ExprOpNEq, "!=")
 MakeBinOp(ExprOpAnd, "&&")
@@ -342,9 +353,28 @@ struct StaticEnv
 {
     bool isWith;
     const StaticEnv * up;
-    typedef std::map<Symbol, unsigned int> Vars;
+
+    // Note: these must be in sorted order.
+    typedef std::vector<std::pair<Symbol, Displacement>> Vars;
     Vars vars;
-    StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { };
+
+    StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
+        vars.reserve(expectedSize);
+    };
+
+    void sort()
+    {
+        std::sort(vars.begin(), vars.end(),
+            [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
+    }
+
+    Vars::const_iterator find(const Symbol & name) const
+    {
+        Vars::value_type key(name, 0);
+        auto i = std::lower_bound(vars.begin(), vars.end(), key);
+        if (i != vars.end() && i->first == name) return i;
+        return vars.end();
+    }
 };
 
 

src/libexpr/parser.y

@@ -33,11 +33,9 @@ namespace nix {
         Symbol file;
         FileOrigin origin;
         std::optional<ErrorInfo> error;
-        Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
             , symbols(state.symbols)
-            , sLetBody(symbols.create("<let-body>"))
             { };
     };
 
@@ -126,14 +124,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
                     auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
                         dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs[ad.first] = ad.second;
+                    jAttrs->attrs.emplace(ad.first, ad.second);
                 }
             } else {
                 dupAttr(attrPath, pos, j->second.pos);
             }
         } else {
             // This attr path is not defined. Let's create it.
-            attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos);
+            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
             e->setName(i->symbol);
         }
     } else {
@@ -283,7 +281,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 }
 
 %type <e> start expr expr_function expr_if expr_op
-%type <e> expr_app expr_select expr_simple
+%type <e> expr_select expr_simple expr_app
 %type <list> expr_list
 %type <attrs> binds
 %type <formals> formals
@@ -353,13 +351,13 @@ expr_if
 
 expr_op
   : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); }
+  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); }
   | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
   | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
-  | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); }
-  | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); }
+  | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); }
+  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); }
+  | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); }
+  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); }
   | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); }
   | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); }
   | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); }
@@ -367,17 +365,22 @@ expr_op
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
     { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
-  | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); }
-  | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); }
-  | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); }
+  | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
+  | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
+  | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
   | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); }
   | expr_app
   ;
 
 expr_app
-  : expr_app expr_select
-    { $$ = new ExprApp(CUR_POS, $1, $2); }
-  | expr_select { $$ = $1; }
+  : expr_app expr_select {
+      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
+          e2->args.push_back($2);
+          $$ = $1;
+      } else
+          $$ = new ExprCall(CUR_POS, $1, {$2});
+  }
+  | expr_select
   ;
 
 expr_select
@@ -388,7 +391,7 @@ expr_select
   | /* Backwards compatibility: because Nixpkgs has a rarely used
        function named ‘or’, allow stuff like ‘map or [...]’. */
     expr_simple OR_KW
-    { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); }
+    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); }
   | expr_simple { $$ = $1; }
   ;
 
@@ -412,13 +415,13 @@ expr_simple
   }
   | SPATH {
       string path($1 + 1, strlen($1) - 2);
-      $$ = new ExprApp(CUR_POS,
-          new ExprApp(new ExprVar(data->symbols.create("__findFile")),
-              new ExprVar(data->symbols.create("__nixPath"))),
-          new ExprString(data->symbols.create(path)));
+      $$ = new ExprCall(CUR_POS,
+          new ExprVar(data->symbols.create("__findFile")),
+          {new ExprVar(data->symbols.create("__nixPath")),
+           new ExprString(data->symbols.create(path))});
   }
   | URI {
-      static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
+      static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals);
       if (noURLLiterals)
           throw ParseError({
               .msg = hintfmt("URL literals are disabled"),
@@ -483,7 +486,7 @@ binds
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos);
           Pos pos = makeCurPos(@3, data);
-          $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true);
+          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
       }
     }
   | binds INHERIT '(' expr ')' attrs ';'
@@ -492,7 +495,7 @@ binds
       for (auto & i : *$6) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos);
-          $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data));
+          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)));
       }
     }
   | { $$ = new ExprAttrs(makeCurPos(@0, data)); }

src/libexpr/primops.cc

@@ -70,7 +70,7 @@ void EvalState::realiseContext(const PathSet & context)
                 if (outputPaths.count(outputName) == 0)
                     throw Error("derivation '%s' does not have an output named '%s'",
                             store->printStorePath(drvPath), outputName);
-                allowedPaths->insert(store->printStorePath(outputPaths.at(outputName)));
+                allowPath(outputPaths.at(outputName));
             }
         }
     }
@@ -184,14 +184,17 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
             Env * env = &state.allocEnv(vScope->attrs->size());
             env->up = &state.baseEnv;
 
-            StaticEnv staticEnv(false, &state.staticBaseEnv);
+            StaticEnv staticEnv(false, &state.staticBaseEnv, vScope->attrs->size());
 
             unsigned int displ = 0;
             for (auto & attr : *vScope->attrs) {
-                staticEnv.vars[attr.name] = displ;
+                staticEnv.vars.emplace_back(attr.name, displ);
                 env->values[displ++] = attr.value;
             }
 
+            // No need to call staticEnv.sort(), because
+            // args[0]->attrs is already sorted.
+
             printTalkative("evaluating file '%1%'", realPath);
             Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv);
 
@@ -332,9 +335,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     PathSet context;
     auto program = state.coerceToString(pos, *elems[0], context, false, false);
     Strings commandArgs;
-    for (unsigned int i = 1; i < args[0]->listSize(); ++i) {
+    for (unsigned int i = 1; i < args[0]->listSize(); ++i)
         commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false));
-    }
     try {
         state.realiseContext(context);
     } catch (InvalidPathError & e) {
@@ -514,7 +516,11 @@ static RegisterPrimOp primop_isPath({
 
 struct CompareValues
 {
-    bool operator () (const Value * v1, const Value * v2) const
+    EvalState & state;
+
+    CompareValues(EvalState & state) : state(state) { };
+
+    bool operator () (Value * v1, Value * v2) const
     {
         if (v1->type() == nFloat && v2->type() == nInt)
             return v1->fpoint < v2->integer;
@@ -531,6 +537,17 @@ struct CompareValues
                 return strcmp(v1->string.s, v2->string.s) < 0;
             case nPath:
                 return strcmp(v1->path, v2->path) < 0;
+            case nList:
+                // Lexicographic comparison
+                for (size_t i = 0;; i++) {
+                    if (i == v2->listSize()) {
+                        return false;
+                    } else if (i == v1->listSize()) {
+                        return true;
+                    } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) {
+                        return (*this)(v1->listElems()[i], v2->listElems()[i]);
+                    }
+                }
             default:
                 throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
         }
@@ -598,8 +615,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     state.forceList(*startSet->value, pos);
 
     ValueList workSet;
-    for (unsigned int n = 0; n < startSet->value->listSize(); ++n)
-        workSet.push_back(startSet->value->listElems()[n]);
+    for (auto elem : startSet->value->listItems())
+        workSet.push_back(elem);
 
     /* Get the operator. */
     Bindings::iterator op = getAttr(
@@ -618,7 +635,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     ValueList res;
     // `doneKeys' doesn't need to be a GC root, because its values are
     // reachable from res.
-    set<Value *, CompareValues> doneKeys;
+    auto cmp = CompareValues(state);
+    set<Value *, decltype(cmp)> doneKeys(cmp);
     while (!workSet.empty()) {
         Value * e = *(workSet.begin());
         workSet.pop_front();
@@ -643,9 +661,9 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         state.forceList(call, pos);
 
         /* Add the values returned by the operator to the work set. */
-        for (unsigned int n = 0; n < call.listSize(); ++n) {
-            state.forceValue(*call.listElems()[n], pos);
-            workSet.push_back(call.listElems()[n]);
+        for (auto elem : call.listItems()) {
+            state.forceValue(*elem, pos);
+            workSet.push_back(elem);
         }
     }
 
@@ -985,16 +1003,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
             if (i->name == state.sContentAddressed) {
-                settings.requireExperimentalFeature("ca-derivations");
                 contentAddressed = state.forceBool(*i->value, pos);
+                if (contentAddressed)
+                    settings.requireExperimentalFeature(Xp::CaDerivations);
             }
 
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
             else if (i->name == state.sArgs) {
                 state.forceList(*i->value, pos);
-                for (unsigned int n = 0; n < i->value->listSize(); ++n) {
-                    string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
+                for (auto elem : i->value->listItems()) {
+                    string s = state.coerceToString(posDrvName, *elem, context, true);
                     drv.args.push_back(s);
                 }
             }
@@ -1008,7 +1027,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                     if (i->name == state.sStructuredAttrs) continue;
 
                     auto placeholder(jsonObject->placeholder(key));
-                    printValueAsJSON(state, true, *i->value, placeholder, context);
+                    printValueAsJSON(state, true, *i->value, pos, placeholder, context);
 
                     if (i->name == state.sBuilder)
                         drv.builder = state.forceString(*i->value, context, posDrvName);
@@ -1024,8 +1043,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                         /* Require ‘outputs’ to be a list of strings. */
                         state.forceList(*i->value, posDrvName);
                         Strings ss;
-                        for (unsigned int n = 0; n < i->value->listSize(); ++n)
-                            ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName));
+                        for (auto elem : i->value->listItems())
+                            ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName));
                         handleOutputs(ss);
                     }
 
@@ -1440,20 +1459,19 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 
     SearchPath searchPath;
 
-    for (unsigned int n = 0; n < args[0]->listSize(); ++n) {
-        Value & v2(*args[0]->listElems()[n]);
-        state.forceAttrs(v2, pos);
+    for (auto v2 : args[0]->listItems()) {
+        state.forceAttrs(*v2, pos);
 
         string prefix;
-        Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix"));
-        if (i != v2.attrs->end())
+        Bindings::iterator i = v2->attrs->find(state.symbols.create("prefix"));
+        if (i != v2->attrs->end())
             prefix = state.forceStringNoCtx(*i->value, pos);
 
         i = getAttr(
             state,
             "findFile",
             "path",
-            v2.attrs,
+            v2->attrs,
             pos
         );
 
@@ -1579,7 +1597,7 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value
 {
     std::ostringstream out;
     PathSet context;
-    printValueAsXML(state, true, false, *args[0], out, context);
+    printValueAsXML(state, true, false, *args[0], out, context, pos);
     mkString(v, out.str(), context);
 }
 
@@ -1687,7 +1705,7 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu
 {
     std::ostringstream out;
     PathSet context;
-    printValueAsJSON(state, true, *args[0], out, context);
+    printValueAsJSON(state, true, *args[0], pos, out, context);
     mkString(v, out.str(), context);
 }
 
@@ -1859,12 +1877,12 @@ static void addPath(
         // be rewritten to the actual output).
         state.realiseContext(context);
 
+        StorePathSet refs;
+
         if (state.store->isInStore(path)) {
             auto [storePath, subPath] = state.store->toStorePath(path);
-            auto info = state.store->queryPathInfo(storePath);
-            if (!info->references.empty())
-                throw EvalError("store path '%s' is not allowed to have references",
-                    state.store->printStorePath(storePath));
+            // FIXME: we should scanForReferences on the path before adding it
+            refs = state.store->queryPathInfo(storePath)->references;
             path = state.store->toRealPath(storePath) + subPath;
         }
 
@@ -1880,9 +1898,6 @@ static void addPath(
             Value arg1;
             mkString(arg1, path);
 
-            Value fun2;
-            state.callFunction(*filterFun, arg1, fun2, noPos);
-
             Value arg2;
             mkString(arg2,
                 S_ISREG(st.st_mode) ? "regular" :
@@ -1890,8 +1905,9 @@ static void addPath(
                 S_ISLNK(st.st_mode) ? "symlink" :
                 "unknown" /* not supported, will fail! */);
 
+            Value * args []{&arg1, &arg2};
             Value res;
-            state.callFunction(fun2, arg2, res, noPos);
+            state.callFunction(*filterFun, 2, args, res, pos);
 
             return state.forceBool(res, pos);
         }) : defaultPathFilter;
@@ -1904,7 +1920,7 @@ static void addPath(
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
             dstPath = state.store->printStorePath(settings.readOnlyMode
                 ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
-                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
+                : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs));
             if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
                 throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
         } else
@@ -2221,9 +2237,9 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
 
     /* Get the attribute names to be removed. */
     std::set<Symbol> names;
-    for (unsigned int i = 0; i < args[1]->listSize(); ++i) {
-        state.forceStringNoCtx(*args[1]->listElems()[i], pos);
-        names.insert(state.symbols.create(args[1]->listElems()[i]->string.s));
+    for (auto elem : args[1]->listItems()) {
+        state.forceStringNoCtx(*elem, pos);
+        names.insert(state.symbols.create(elem->string.s));
     }
 
     /* Copy all attributes not in that set.  Note that we don't need
@@ -2231,7 +2247,7 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args,
        vector. */
     state.mkAttrs(v, args[0]->attrs->size());
     for (auto & i : *args[0]->attrs) {
-        if (names.find(i.name) == names.end())
+        if (!names.count(i.name))
             v.attrs->push_back(i);
     }
 }
@@ -2265,15 +2281,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
 
     std::set<Symbol> seen;
 
-    for (unsigned int i = 0; i < args[0]->listSize(); ++i) {
-        Value & v2(*args[0]->listElems()[i]);
-        state.forceAttrs(v2, pos);
+    for (auto v2 : args[0]->listItems()) {
+        state.forceAttrs(*v2, pos);
 
         Bindings::iterator j = getAttr(
             state,
             "listToAttrs",
             state.sName,
-            v2.attrs,
+            v2->attrs,
             pos
         );
 
@@ -2285,7 +2300,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
                 state,
                 "listToAttrs",
                 state.sValue,
-                v2.attrs,
+                v2->attrs,
                 pos
             );
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
@@ -2352,11 +2367,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va
     Value * res[args[1]->listSize()];
     unsigned int found = 0;
 
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
-        Value & v2(*args[1]->listElems()[n]);
-        state.forceAttrs(v2, pos);
-        Bindings::iterator i = v2.attrs->find(attrName);
-        if (i != v2.attrs->end())
+    for (auto v2 : args[1]->listItems()) {
+        state.forceAttrs(*v2, pos);
+        Bindings::iterator i = v2->attrs->find(attrName);
+        if (i != v2->attrs->end())
             res[found++] = i->value;
     }
 
@@ -2631,8 +2645,8 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value
 {
     bool res = false;
     state.forceList(*args[1], pos);
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n)
-        if (state.eqValues(*args[0], *args[1]->listElems()[n])) {
+    for (auto elem : args[1]->listItems())
+        if (state.eqValues(*args[0], *elem)) {
             res = true;
             break;
         }
@@ -2691,11 +2705,10 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args,
     if (args[2]->listSize()) {
         Value * vCur = args[1];
 
-        for (unsigned int n = 0; n < args[2]->listSize(); ++n) {
-            Value vTmp;
-            state.callFunction(*args[0], *vCur, vTmp, pos);
+        for (auto [n, elem] : enumerate(args[2]->listItems())) {
+            Value * vs []{vCur, elem};
             vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue();
-            state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos);
+            state.callFunction(*args[0], 2, vs, *vCur, pos);
         }
         state.forceValue(v, pos);
     } else {
@@ -2709,9 +2722,9 @@ static RegisterPrimOp primop_foldlStrict({
     .args = {"op", "nul", "list"},
     .doc = R"(
       Reduce a list by applying a binary operator, from left to right,
-      e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
+      e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2)
       ...`. The operator is applied strictly, i.e., its arguments are
-      evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]`
+      evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]`
       evaluates to 6.
     )",
     .fun = prim_foldlStrict,
@@ -2723,8 +2736,8 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg
     state.forceList(*args[1], pos);
 
     Value vTmp;
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
-        state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos);
+    for (auto elem : args[1]->listItems()) {
+        state.callFunction(*args[0], *elem, vTmp, pos);
         bool res = state.forceBool(vTmp, pos);
         if (res == any) {
             mkBool(v, any);
@@ -2816,17 +2829,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
         v.listElems()[n] = args[1]->listElems()[n];
     }
 
-
     auto comparator = [&](Value * a, Value * b) {
         /* Optimization: if the comparator is lessThan, bypass
            callFunction. */
         if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan)
-            return CompareValues()(a, b);
+            return CompareValues(state)(a, b);
 
-        Value vTmp1, vTmp2;
-        state.callFunction(*args[0], *a, vTmp1, pos);
-        state.callFunction(vTmp1, *b, vTmp2, pos);
-        return state.forceBool(vTmp2, pos);
+        Value * vs[] = {a, b};
+        Value vBool;
+        state.callFunction(*args[0], 2, vs, vBool, pos);
+        return state.forceBool(vBool, pos);
     };
 
     /* FIXME: std::sort can segfault if the comparator is not a strict
@@ -3103,7 +3115,7 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va
 {
     state.forceValue(*args[0], pos);
     state.forceValue(*args[1], pos);
-    CompareValues comp;
+    CompareValues comp{state};
     mkBool(v, comp(args[0], args[1]));
 }
 
@@ -3454,9 +3466,9 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * *
     res.reserve((args[1]->listSize() + 32) * sep.size());
     bool first = true;
 
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
+    for (auto elem : args[1]->listItems()) {
         if (first) first = false; else res += sep;
-        res += state.coerceToString(pos, *args[1]->listElems()[n], context);
+        res += state.coerceToString(pos, *elem, context);
     }
 
     mkString(v, res, context);
@@ -3485,14 +3497,14 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
 
     vector<string> from;
     from.reserve(args[0]->listSize());
-    for (unsigned int n = 0; n < args[0]->listSize(); ++n)
-        from.push_back(state.forceString(*args[0]->listElems()[n], pos));
+    for (auto elem : args[0]->listItems())
+        from.push_back(state.forceString(*elem, pos));
 
     vector<std::pair<string, PathSet>> to;
     to.reserve(args[1]->listSize());
-    for (unsigned int n = 0; n < args[1]->listSize(); ++n) {
+    for (auto elem : args[1]->listItems()) {
         PathSet ctx;
-        auto s = state.forceString(*args[1]->listElems()[n], ctx, pos);
+        auto s = state.forceString(*elem, ctx, pos);
         to.push_back(std::make_pair(std::move(s), std::move(ctx)));
     }
 
@@ -3693,7 +3705,7 @@ void EvalState::createBaseEnv()
        language feature gets added.  It's not necessary to increase it
        when primops get added, because you can just use `builtins ?
        primOp' to check. */
-    mkInt(v, 5);
+    mkInt(v, 6);
     addConstant("__langVersion", v);
 
     // Miscellaneous
@@ -3727,14 +3739,20 @@ void EvalState::createBaseEnv()
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
     sDerivationNix = symbols.create("//builtin/derivation.nix");
-    eval(parse(
-        #include "primops/derivation.nix.gen.hh"
-        , foFile, sDerivationNix, "/", staticBaseEnv), v);
-    addConstant("derivation", v);
+    auto vDerivation = allocValue();
+    addConstant("derivation", vDerivation);
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */
     baseEnv.values[0]->attrs->sort();
+
+    staticBaseEnv.sort();
+
+    /* Note: we have to initialize the 'derivation' constant *after*
+       building baseEnv/staticBaseEnv because it uses 'builtins'. */
+    eval(parse(
+        #include "primops/derivation.nix.gen.hh"
+        , foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/primops/context.cc

@@ -118,9 +118,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
             auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
             state.mkList(outputsVal, info.second.outputs.size());
             size_t i = 0;
-            for (const auto & output : info.second.outputs) {
+            for (const auto & output : info.second.outputs)
                 mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
-            }
         }
         infoVal.attrs->sort();
     }
@@ -181,8 +180,8 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                     .errPos = *i.pos
                 });
             }
-            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
-                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+            for (auto elem : iter->value->listItems()) {
+                auto name = state.forceStringNoCtx(*elem, *iter->pos);
                 context.insert("!" + name + "!" + string(i.name));
             }
         }

src/libexpr/primops/fetchMercurial.cc

@@ -15,7 +15,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
     std::string name = "source";
     PathSet context;
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
 
     if (args[0]->type() == nAttrs) {
 

src/libexpr/primops/fetchTree.cc

@@ -74,7 +74,10 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defau
 
 std::string fixURIForGit(std::string uri, EvalState & state)
 {
-    static std::regex scp_uri("([^/].*)@(.*):(.*)");
+    /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes
+     * them by removing the `:` and assuming a scheme of `ssh://`
+     * */
+    static std::regex scp_uri("([^/]*)@(.*):(.*)");
     if (uri[0] != '/' && std::regex_match(uri, scp_uri))
         return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
     else
@@ -97,7 +100,7 @@ static void fetchTree(
     fetchers::Input input;
     PathSet context;
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
 
     if (args[0]->type() == nAttrs) {
         state.forceAttrs(*args[0], pos);
@@ -121,7 +124,7 @@ static void fetchTree(
 
         for (auto & attr : *args[0]->attrs) {
             if (attr.name == state.sType) continue;
-            state.forceValue(*attr.value);
+            state.forceValue(*attr.value, *attr.pos);
             if (attr.value->type() == nPath || attr.value->type() == nString) {
                 auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false);
                 attrs.emplace(attr.name,
@@ -176,7 +179,7 @@ static void fetchTree(
 
 static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    settings.requireExperimentalFeature("flakes");
+    settings.requireExperimentalFeature(Xp::Flakes);
     fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false });
 }
 
@@ -189,7 +192,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     std::optional<std::string> url;
     std::optional<Hash> expectedHash;
 
-    state.forceValue(*args[0]);
+    state.forceValue(*args[0], pos);
 
     if (args[0]->type() == nAttrs) {
 
@@ -287,13 +290,13 @@ static RegisterPrimOp primop_fetchTarball({
       stdenv.mkDerivation { … }
       ```
 
-      The fetched tarball is cached for a certain amount of time (1 hour
-      by default) in `~/.cache/nix/tarballs/`. You can change the cache
-      timeout either on the command line with `--option tarball-ttl number
-      of seconds` or in the Nix configuration file with this option: ` 
-      number of seconds to cache `.
+      The fetched tarball is cached for a certain amount of time (1
+      hour by default) in `~/.cache/nix/tarballs/`. You can change the
+      cache timeout either on the command line with `--tarball-ttl`
+      *number-of-seconds* or in the Nix configuration file by adding
+      the line `tarball-ttl = ` *number-of-seconds*.
 
-      Note that when obtaining the hash with ` nix-prefetch-url ` the
+      Note that when obtaining the hash with `nix-prefetch-url` the
       option `--unpack` is required.
 
       This function can also verify the contents against a hash. In that
@@ -393,7 +396,7 @@ static RegisterPrimOp primop_fetchGit({
           ```
 
           > **Note**
-          > 
+          >
           > It is nice to always specify the branch which a revision
           > belongs to. Without the branch being specified, the fetcher
           > might fail if the default branch changes. Additionally, it can
@@ -430,12 +433,12 @@ static RegisterPrimOp primop_fetchGit({
           ```
 
           > **Note**
-          > 
+          >
           > Nix will refetch the branch in accordance with
           > the option `tarball-ttl`.
 
           > **Note**
-          > 
+          >
           > This behavior is disabled in *Pure evaluation mode*.
     )",
     .fun = prim_fetchGit,

src/libexpr/value-to-json.cc

@@ -10,11 +10,11 @@
 namespace nix {
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, JSONPlaceholder & out, PathSet & context)
+    Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context)
 {
     checkInterrupt();
 
-    if (strict) state.forceValue(v);
+    if (strict) state.forceValue(v, pos);
 
     switch (v.type()) {
 
@@ -40,7 +40,7 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case nAttrs: {
-            auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
+            auto maybeString = state.tryAttrsToString(pos, v, context, false, false);
             if (maybeString) {
                 out.write(*maybeString);
                 break;
@@ -54,18 +54,18 @@ void printValueAsJSON(EvalState & state, bool strict,
                 for (auto & j : names) {
                     Attr & a(*v.attrs->find(state.symbols.create(j)));
                     auto placeholder(obj.placeholder(j));
-                    printValueAsJSON(state, strict, *a.value, placeholder, context);
+                    printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context);
                 }
             } else
-                printValueAsJSON(state, strict, *i->value, out, context);
+                printValueAsJSON(state, strict, *i->value, *i->pos, out, context);
             break;
         }
 
         case nList: {
             auto list(out.list());
-            for (unsigned int n = 0; n < v.listSize(); ++n) {
+            for (auto elem : v.listItems()) {
                 auto placeholder(list.placeholder());
-                printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context);
+                printValueAsJSON(state, strict, *elem, pos, placeholder, context);
             }
             break;
         }
@@ -79,18 +79,20 @@ void printValueAsJSON(EvalState & state, bool strict,
             break;
 
         case nThunk:
-            throw TypeError("cannot convert %1% to JSON", showType(v));
-
         case nFunction:
-            throw TypeError("cannot convert %1% to JSON", showType(v));
+            auto e = TypeError({
+                .msg = hintfmt("cannot convert %1% to JSON", showType(v)),
+                .errPos = v.determinePos(pos)
+            });
+            throw e.addTrace(pos, hintfmt("message for the trace"));
     }
 }
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, std::ostream & str, PathSet & context)
+    Value & v, const Pos & pos, std::ostream & str, PathSet & context)
 {
     JSONPlaceholder out(str);
-    printValueAsJSON(state, strict, v, out, context);
+    printValueAsJSON(state, strict, v, pos, out, context);
 }
 
 void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,

src/libexpr/value-to-json.hh

@@ -11,9 +11,9 @@ namespace nix {
 class JSONPlaceholder;
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, JSONPlaceholder & out, PathSet & context);
+    Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context);
 
 void printValueAsJSON(EvalState & state, bool strict,
-    Value & v, std::ostream & str, PathSet & context);
+    Value & v, const Pos & pos, std::ostream & str, PathSet & context);
 
 }

src/libexpr/value-to-xml.cc

@@ -18,7 +18,8 @@ static XMLAttrs singletonAttrs(const string & name, const string & value)
 
 
 static void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen);
+    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+    const Pos & pos);
 
 
 static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos)
@@ -46,17 +47,18 @@ static void showAttrs(EvalState & state, bool strict, bool location,
 
         XMLOpenElement _(doc, "attr", xmlAttrs);
         printValueAsXML(state, strict, location,
-            *a.value, doc, context, drvsSeen);
+            *a.value, doc, context, drvsSeen, *a.pos);
     }
 }
 
 
 static void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen)
+    Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+    const Pos & pos)
 {
     checkInterrupt();
 
-    if (strict) state.forceValue(v);
+    if (strict) state.forceValue(v, pos);
 
     switch (v.type()) {
 
@@ -91,14 +93,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
                 Path drvPath;
                 a = v.attrs->find(state.sDrvPath);
                 if (a != v.attrs->end()) {
-                    if (strict) state.forceValue(*a->value);
+                    if (strict) state.forceValue(*a->value, *a->pos);
                     if (a->value->type() == nString)
                         xmlAttrs["drvPath"] = drvPath = a->value->string.s;
                 }
 
                 a = v.attrs->find(state.sOutPath);
                 if (a != v.attrs->end()) {
-                    if (strict) state.forceValue(*a->value);
+                    if (strict) state.forceValue(*a->value, *a->pos);
                     if (a->value->type() == nString)
                         xmlAttrs["outPath"] = a->value->string.s;
                 }
@@ -120,8 +122,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
         case nList: {
             XMLOpenElement _(doc, "list");
-            for (unsigned int n = 0; n < v.listSize(); ++n)
-                printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
+            for (auto v2 : v.listItems())
+                printValueAsXML(state, strict, location, *v2, doc, context, drvsSeen, pos);
             break;
         }
 
@@ -149,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
         }
 
         case nExternal:
-            v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
+            v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
             break;
 
         case nFloat:
@@ -163,19 +165,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
 
 void ExternalValueBase::printValueAsXML(EvalState & state, bool strict,
-    bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const
+    bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+    const Pos & pos) const
 {
     doc.writeEmptyElement("unevaluated");
 }
 
 
 void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, std::ostream & out, PathSet & context)
+    Value & v, std::ostream & out, PathSet & context, const Pos & pos)
 {
     XMLWriter doc(true, out);
     XMLOpenElement root(doc, "expr");
     PathSet drvsSeen;
-    printValueAsXML(state, strict, location, v, doc, context, drvsSeen);
+    printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos);
 }
 
 

src/libexpr/value-to-xml.hh

@@ -9,6 +9,6 @@
 namespace nix {
 
 void printValueAsXML(EvalState & state, bool strict, bool location,
-    Value & v, std::ostream & out, PathSet & context);
-    
+    Value & v, std::ostream & out, PathSet & context, const Pos & pos);
+
 }

src/libexpr/value.hh

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <cassert>
+
 #include "symbol-table.hh"
 
 #if HAVE_BOEHMGC
@@ -94,7 +96,8 @@ class ExternalValueBase
 
     /* Print the value as XML. Defaults to unevaluated */
     virtual void printValueAsXML(EvalState & state, bool strict, bool location,
-        XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const;
+        XMLWriter & doc, PathSet & context, PathSet & drvsSeen,
+        const Pos & pos) const;
 
     virtual ~ExternalValueBase()
     {
@@ -349,6 +352,34 @@ public:
     bool isTrivial() const;
 
     std::vector<std::pair<Path, std::string>> getContext();
+
+    auto listItems()
+    {
+        struct ListIterable
+        {
+            typedef Value * const * iterator;
+            iterator _begin, _end;
+            iterator begin() const { return _begin; }
+            iterator end() const { return _end; }
+        };
+        assert(isList());
+        auto begin = listElems();
+        return ListIterable { begin, begin + listSize() };
+    }
+
+    auto listItems() const
+    {
+        struct ConstListIterable
+        {
+            typedef const Value * const * iterator;
+            iterator _begin, _end;
+            iterator begin() const { return _begin; }
+            iterator end() const { return _end; }
+        };
+        assert(isList());
+        auto begin = listElems();
+        return ConstListIterable { begin, begin + listSize() };
+    }
 };
 
 

src/libfetchers/git.cc

@@ -51,7 +51,7 @@ struct GitInputScheme : InputScheme
         for (auto &[name, value] : url.query) {
             if (name == "rev" || name == "ref")
                 attrs.emplace(name, value);
-            else if (name == "shallow")
+            else if (name == "shallow" || name == "submodules")
                 attrs.emplace(name, Explicit<bool> { value == "1" });
             else
                 url2.query.emplace(name, value);
@@ -324,17 +324,13 @@ struct GitInputScheme : InputScheme
             Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false);
             repoDir = cacheDir;
 
-            Path cacheDirLock = cacheDir + ".lock";
             createDirs(dirOf(cacheDir));
-            AutoCloseFD lock = openLockFile(cacheDirLock, true);
-            lockFile(lock.get(), ltWrite, true);
+            PathLocks cacheDirLock({cacheDir + ".lock"});
 
             if (!pathExists(cacheDir)) {
                 runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
             }
 
-            deleteLockFile(cacheDirLock, lock.get());
-
             Path localRefFile =
                 input.getRef()->compare(0, 5, "refs/") == 0
                 ? cacheDir + "/" + *input.getRef()
@@ -399,6 +395,8 @@ struct GitInputScheme : InputScheme
 
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
+
+            // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
         }
 
         bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";

src/libfetchers/github.cc

@@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         if ("PAT" == token.substr(0, fldsplit))
             return std::make_pair("Private-token", token.substr(fldsplit+1));
         warn("Unrecognized GitLab token type %s",  token.substr(0, fldsplit));
-        return std::nullopt;
+        return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1));
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override

src/libmain/progress-bar.cc

@@ -103,17 +103,19 @@ public:
     ~ProgressBar()
     {
         stop();
-        updateThread.join();
     }
 
     void stop() override
     {
-        auto state(state_.lock());
-        if (!state->active) return;
-        state->active = false;
-        writeToStderr("\r\e[K");
-        updateCV.notify_one();
-        quitCV.notify_one();
+        {
+            auto state(state_.lock());
+            if (!state->active) return;
+            state->active = false;
+            writeToStderr("\r\e[K");
+            updateCV.notify_one();
+            quitCV.notify_one();
+        }
+        updateThread.join();
     }
 
     bool isVerbose() override {

src/libmain/shared.cc

@@ -15,9 +15,14 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <signal.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
+#ifdef __linux__
+#include <features.h>
+#endif
+#ifdef __GLIBC__
+#include <gnu/lib-names.h>
+#include <nss.h>
+#include <dlfcn.h>
+#endif
 
 #include <openssl/crypto.h>
 
@@ -121,21 +126,30 @@ static void preloadNSS() {
        been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
        load its lookup libraries in the parent before any child gets a chance to. */
     std::call_once(dns_resolve_flag, []() {
-        struct addrinfo *res = NULL;
-
-        /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN.
-           We need the resolution to be done locally, as nscd socket will not be accessible in the
-           sandbox. */
-        char * previous_env = getenv("LOCALDOMAIN");
-        setenv("LOCALDOMAIN", "invalid", 1);
-        if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) {
-            if (res) freeaddrinfo(res);
-        }
-        if (previous_env) {
-             setenv("LOCALDOMAIN", previous_env, 1);
-        } else {
-             unsetenv("LOCALDOMAIN");
+#ifdef __GLIBC__
+        /* On linux, glibc will run every lookup through the nss layer.
+         * That means every lookup goes, by default, through nscd, which acts as a local
+         * cache.
+         * Because we run builds in a sandbox, we also remove access to nscd otherwise
+         * lookups would leak into the sandbox.
+         *
+         * But now we have a new problem, we need to make sure the nss_dns backend that
+         * does the dns lookups when nscd is not available is loaded or available.
+         *
+         * We can't make it available without leaking nix's environment, so instead we'll
+         * load the backend, and configure nss so it does not try to run dns lookups
+         * through nscd.
+         *
+         * This is technically only used for builtins:fetch* functions so we only care
+         * about dns.
+         *
+         * All other platforms are unaffected.
+         */
+        if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) {
+            printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend"));
         }
+        __nss_configure_lookup ("hosts", "dns");
+#endif
     });
 }
 

src/libstore/binary-cache-store.cc

@@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    std::string hashPart(narInfo->path.hashPart());
-
     {
         auto state_(state.lock());
-        state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
+        state_->pathInfoCache.upsert(
+            std::string(narInfo->path.to_string()),
+            PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
     }
 
     if (diskCache)
-        diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+        diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
 }
 
 AutoCloseFD openFile(const Path & path)
@@ -149,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     {
     FdSink fileSink(fdTemp.get());
     TeeSink teeSinkCompressed { fileSink, fileHashSink };
-    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed);
+    auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel);
     TeeSink teeSinkUncompressed { *compressionSink, narHashSink };
     TeeSource teeSource { narSource, teeSinkUncompressed };
     narAccessor = makeNarAccessor(teeSource);
@@ -308,16 +308,17 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 }
 
 StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
     if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
         unsupported("addToStoreFromDump");
     return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
-            makeFixedOutputPath(method, nar.first, name),
+            makeFixedOutputPath(method, nar.first, name, references),
             nar.first,
         };
         info.narSize = nar.second;
+        info.references = references;
         return info;
     })->path;
 }
@@ -385,7 +386,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 }
 
 StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
@@ -404,10 +405,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
     });
     return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
-            makeFixedOutputPath(method, h, name),
+            makeFixedOutputPath(method, h, name, references),
             nar.first,
         };
         info.narSize = nar.second;
+        info.references = references;
         info.ca = FixedOutputHash {
             .method = method,
             .hash = h,
@@ -437,40 +439,29 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
     })->path;
 }
 
-std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
+void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id,
+    Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
-    if (diskCache) {
-        auto [cacheOutcome, maybeCachedRealisation] =
-            diskCache->lookupRealisation(getUri(), id);
-        switch (cacheOutcome) {
-            case NarInfoDiskCache::oValid:
-                debug("Returning a cached realisation for %s", id.to_string());
-                return *maybeCachedRealisation;
-            case NarInfoDiskCache::oInvalid:
-                debug("Returning a cached missing realisation for %s", id.to_string());
-                return {};
-            case NarInfoDiskCache::oUnknown:
-                break;
-        }
-    }
-
     auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
-    auto rawOutputInfo = getFile(outputInfoFilePath);
 
-    if (rawOutputInfo) {
-        auto realisation = Realisation::fromJSON(
-            nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath);
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
-        if (diskCache)
-            diskCache->upsertRealisation(
-                getUri(), realisation);
+    Callback<std::shared_ptr<std::string>> newCallback = {
+        [=](std::future<std::shared_ptr<std::string>> fut) {
+            try {
+                auto data = fut.get();
+                if (!data) return (*callbackPtr)(nullptr);
 
-        return {realisation};
-    } else {
-        if (diskCache)
-            diskCache->upsertAbsentRealisation(getUri(), id);
-        return std::nullopt;
-    }
+                auto realisation = Realisation::fromJSON(
+                    nlohmann::json::parse(*data), outputInfoFilePath);
+                return (*callbackPtr)(std::make_shared<const Realisation>(realisation));
+            } catch (...) {
+                callbackPtr->rethrow();
+            }
+        }
+    };
+
+    getFile(outputInfoFilePath, std::move(newCallback));
 }
 
 void BinaryCacheStore::registerDrvOutput(const Realisation& info) {

src/libstore/binary-cache-store.hh

@@ -15,13 +15,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
 {
     using StoreConfig::StoreConfig;
 
-    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
+    const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"};
     const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
     const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
     const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"};
     const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression",
-        "enable multi-threading compression, available for xz only currently"};
+        "enable multi-threading compression for NARs, available for xz and zstd only currently"};
+    const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level",
+        "specify 'preset level' of compression to be used with NARs: "
+        "meaning and accepted range of values depends on compression method selected, "
+        "other than -1 which we reserve to indicate Nix defaults should be used"};
 };
 
 class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
@@ -93,18 +97,19 @@ public:
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
+        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override;
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,
-        PathFilter & filter, RepairFlag repair) override;
+        PathFilter & filter, RepairFlag repair, const StorePathSet & references) override;
 
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
 
     void registerDrvOutput(const Realisation & info) override;
 
-    std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
+    void queryRealisationUncached(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 

src/libstore/build/derivation-goal.cc

@@ -204,7 +204,7 @@ void DerivationGoal::haveDerivation()
     trace("have derivation");
 
     if (drv->type() == DerivationType::CAFloating)
-        settings.requireExperimentalFeature("ca-derivations");
+        settings.requireExperimentalFeature(Xp::CaDerivations);
 
     retrySubstitution = false;
 
@@ -453,7 +453,7 @@ void DerivationGoal::inputsRealised()
     if (useDerivation) {
         auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
 
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
             ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
             || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
             /* We are be able to resolve this derivation based on the
@@ -1275,7 +1275,7 @@ void DerivationGoal::checkPathValidity()
                     : PathStatus::Corrupt,
             };
         }
-        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
             auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
             if (auto real = worker.store.queryRealisation(drvOutput)) {
                 info.known = {

src/libstore/build/drv-output-substitution-goal.cc

@@ -1,6 +1,8 @@
 #include "drv-output-substitution-goal.hh"
+#include "finally.hh"
 #include "worker.hh"
 #include "substitution-goal.hh"
+#include "callback.hh"
 
 namespace nix {
 
@@ -50,14 +52,42 @@ void DrvOutputSubstitutionGoal::tryNext()
         return;
     }
 
-    auto sub = subs.front();
+    sub = subs.front();
     subs.pop_front();
 
     // FIXME: Make async
-    outputInfo = sub->queryRealisation(id);
+    // outputInfo = sub->queryRealisation(id);
+    outPipe.create();
+    promise = decltype(promise)();
+
+    sub->queryRealisation(
+        id, { [&](std::future<std::shared_ptr<const Realisation>> res) {
+            try {
+                Finally updateStats([this]() { outPipe.writeSide.close(); });
+                promise.set_value(res.get());
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+        } });
+
+    worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
+
+    state = &DrvOutputSubstitutionGoal::realisationFetched;
+}
+
+void DrvOutputSubstitutionGoal::realisationFetched()
+{
+    worker.childTerminated(this);
+
+    try {
+        outputInfo = promise.get_future().get();
+    } catch (std::exception & e) {
+        printError(e.what());
+        substituterFailed = true;
+    }
+
     if (!outputInfo) {
-        tryNext();
-        return;
+        return tryNext();
     }
 
     for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
@@ -119,4 +149,10 @@ void DrvOutputSubstitutionGoal::work()
     (this->*state)();
 }
 
+void DrvOutputSubstitutionGoal::handleEOF(int fd)
+{
+    if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
+}
+
+
 }

src/libstore/build/drv-output-substitution-goal.hh

@@ -3,6 +3,8 @@
 #include "store-api.hh"
 #include "goal.hh"
 #include "realisation.hh"
+#include <thread>
+#include <future>
 
 namespace nix {
 
@@ -20,11 +22,18 @@ private:
 
     // The realisation corresponding to the given output id.
     // Will be filled once we can get it.
-    std::optional<Realisation> outputInfo;
+    std::shared_ptr<const Realisation> outputInfo;
 
     /* The remaining substituters. */
     std::list<ref<Store>> subs;
 
+    /* The current substituter. */
+    std::shared_ptr<Store> sub;
+
+    Pipe outPipe;
+    std::thread thr;
+    std::promise<std::shared_ptr<const Realisation>> promise;
+
     /* Whether a substituter failed. */
     bool substituterFailed = false;
 
@@ -36,6 +45,7 @@ public:
 
     void init();
     void tryNext();
+    void realisationFetched();
     void outPathValid();
     void finished();
 
@@ -44,7 +54,7 @@ public:
     string key() override;
 
     void work() override;
-
+    void handleEOF(int fd) override;
 };
 
 }

src/libstore/build/entry-points.cc

@@ -1,4 +1,3 @@
-#include "machines.hh"
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "derivation-goal.hh"
@@ -74,7 +73,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
                     outputId,
                     Realisation{ outputId, *staticOutput.second}
                     );
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
             auto realisation = this->queryRealisation(outputId);
             if (realisation)
                 result.builtOutputs.insert_or_assign(

src/libstore/build/local-derivation-goal.cc

@@ -342,7 +342,7 @@ int childEntry(void * arg)
     return 1;
 }
 
-
+#if __linux__
 static void linkOrCopy(const Path & from, const Path & to)
 {
     if (link(from.c_str(), to.c_str()) == -1) {
@@ -358,6 +358,7 @@ static void linkOrCopy(const Path & from, const Path & to)
         copyPath(from, to);
     }
 }
+#endif
 
 
 void LocalDerivationGoal::startBuilder()
@@ -917,7 +918,9 @@ void LocalDerivationGoal::startBuilder()
     } else
 #endif
     {
+#if __linux__
     fallback:
+#endif
         pid = startProcess([&]() {
             runChild();
         });
@@ -1179,7 +1182,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair,
+        const StorePathSet & references = StorePathSet()) override
     { throw Error("addToStore"); }
 
     void addToStore(const ValidPathInfo & info, Source & narSource,
@@ -1198,9 +1202,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     }
 
     StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
+        const StorePathSet & references = StorePathSet()) override
     {
-        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
+        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references);
         goal.addDependency(path);
         return path;
     }
@@ -1224,13 +1229,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
     // corresponds to an allowed derivation
     { throw Error("registerDrvOutput"); }
 
-    std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
+    void queryRealisationUncached(const DrvOutput & id,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override
     // XXX: This should probably be allowed if the realisation corresponds to
     // an allowed derivation
     {
         if (!goal.isAllowed(id))
-            throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string());
-        return next->queryRealisation(id);
+            callback(nullptr);
+        next->queryRealisation(id, std::move(callback));
     }
 
     void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
@@ -1259,7 +1265,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
             for (auto & [outputName, outputPath] : outputs)
                 if (wantOutput(outputName, bfd.outputs)) {
                     newPaths.insert(outputPath);
-                    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+                    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
                         auto thisRealisation = next->queryRealisation(
                             DrvOutput{drvHashes.at(outputName), outputName}
                         );
@@ -1320,7 +1326,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
 
 void LocalDerivationGoal::startDaemon()
 {
-    settings.requireExperimentalFeature("recursive-nix");
+    settings.requireExperimentalFeature(Xp::RecursiveNix);
 
     Store::Params params;
     params["path-info-cache-size"] = "0";
@@ -1353,7 +1359,7 @@ void LocalDerivationGoal::startDaemon()
             AutoCloseFD remote = accept(daemonSocket.get(),
                 (struct sockaddr *) &remoteAddr, &remoteAddrLen);
             if (!remote) {
-                if (errno == EINTR) continue;
+                if (errno == EINTR || errno == EAGAIN) continue;
                 if (errno == EINVAL) break;
                 throw SysError("accepting connection");
             }
@@ -1991,7 +1997,7 @@ void LocalDerivationGoal::runChild()
                 else if (drv->builder == "builtin:unpack-channel")
                     builtinUnpackChannel(drv2);
                 else
-                    throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
+                    throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8));
                 _exit(0);
             } catch (std::exception & e) {
                 writeFull(STDERR_FILENO, e.what() + std::string("\n"));
@@ -2561,7 +2567,7 @@ void LocalDerivationGoal::registerOutputs()
        that for floating CA derivations, which otherwise couldn't be cached,
        but it's fine to do in all cases. */
 
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         for (auto& [outputName, newInfo] : infos) {
             auto thisRealisation = Realisation{
                 .id = DrvOutput{initialOutputs.at(outputName).outputHash,

src/libstore/content-address.cc

@@ -120,8 +120,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
 
 ContentAddressMethod parseContentAddressMethod(std::string_view caMethod)
 {
-    std::string_view asPrefix {std::string{caMethod} + ":"};
-    return parseContentAddressMethodPrefix(asPrefix);
+    std::string asPrefix = std::string{caMethod} + ":";
+    // parseContentAddressMethodPrefix takes its argument by reference
+    std::string_view asPrefixView = asPrefix;
+    return parseContentAddressMethodPrefix(asPrefixView);
 }
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt)

src/libstore/daemon.cc

@@ -230,7 +230,7 @@ struct ClientSettings
                 else if (name == settings.experimentalFeatures.name) {
                     // We don’t want to forward the experimental features to
                     // the daemon, as that could cause some pretty weird stuff
-                    if (tokenizeString<Strings>(value) != settings.experimentalFeatures.get())
+                    if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get())
                         debug("Ignoring the client-specified experimental features");
                 }
                 else if (trusted
@@ -403,9 +403,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                         return store->queryPathInfo(path);
                     },
                     [&](FixedOutputHashMethod & fohm) {
-                        if (!refs.empty())
-                            throw UnimplementedError("cannot yet have refs with flat or nar-hashed data");
-                        auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair);
+                        auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
                         return store->queryPathInfo(path);
                     },
                 }, contentAddressMethod);
@@ -625,9 +623,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    // Obsolete.
     case wopSyncWithGC: {
         logger->startWork();
-        store->syncWithGC();
         logger->stopWork();
         to << 1;
         break;

src/libstore/derivations.cc

@@ -187,7 +187,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
                 },
             };
         } else {
-            settings.requireExperimentalFeature("ca-derivations");
+            settings.requireExperimentalFeature(Xp::CaDerivations);
             assert(pathS == "");
             return DerivationOutput {
                 .output = DerivationOutputCAFloating {

src/libstore/derived-path.cc

@@ -100,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
                     staticOutputHashes(store, store.readDerivation(p.drvPath));
                 for (auto& [outputName, outputPath] : p.outputs) {
                     if (settings.isExperimentalFeatureEnabled(
-                            "ca-derivations")) {
+                                Xp::CaDerivations)) {
                         auto thisRealisation = store.queryRealisation(
                             DrvOutput{drvHashes.at(outputName), outputName});
                         assert(thisRealisation);  // We’ve built it, so we must h

src/libstore/dummy-store.cc

@@ -50,8 +50,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
     void narFromPath(const StorePath & path, Sink & sink) override
     { unsupported("narFromPath"); }
 
-    std::optional<const Realisation> queryRealisation(const DrvOutput&) override
-    { unsupported("queryRealisation"); }
+    void queryRealisationUncached(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override
+    { callback(nullptr); }
 };
 
 static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;

src/libstore/filetransfer.cc

@@ -544,6 +544,14 @@ struct curlFileTransfer : public FileTransfer
             stopWorkerThread();
         });
 
+#ifdef __linux__
+        /* Cause this thread to not share any FS attributes with the main thread,
+           because this causes setns() in restoreMountNamespace() to fail.
+           Ideally, this would happen in the std::thread() constructor. */
+        if (unshare(CLONE_FS) != 0)
+            throw SysError("unsharing filesystem state in download thread");
+#endif
+
         std::map<CURL *, std::shared_ptr<TransferItem>> items;
 
         bool quit = false;
@@ -725,16 +733,8 @@ ref<FileTransfer> getFileTransfer()
 {
     static ref<curlFileTransfer> fileTransfer = makeCurlFileTransfer();
 
-    // this has to be done in its own scope to make sure that the lock is released
-    // before creating a new fileTransfer instance.
-    auto needsRecreation = [&]() -> bool {
-        auto state = fileTransfer->state_.lock();
-        return state->quit;
-    };
-
-    if (needsRecreation()) {
+    if (fileTransfer->state_.lock()->quit)
         fileTransfer = makeCurlFileTransfer();
-    }
 
     return fileTransfer;
 }

src/libstore/gc.cc

@@ -10,48 +10,22 @@
 #include <regex>
 #include <random>
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
+#include <climits>
 #include <errno.h>
 #include <fcntl.h>
+#include <poll.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/types.h>
+#include <sys/un.h>
 #include <unistd.h>
-#include <climits>
 
 namespace nix {
 
 
-static string gcLockName = "gc.lock";
-static string gcRootsDir = "gcroots";
-
-
-/* Acquire the global GC lock.  This is used to prevent new Nix
-   processes from starting after the temporary root files have been
-   read.  To be precise: when they try to create a new temporary root
-   file, they will block until the garbage collector has finished /
-   yielded the GC lock. */
-AutoCloseFD LocalStore::openGCLock(LockType lockType)
-{
-    Path fnGCLock = (format("%1%/%2%")
-        % stateDir % gcLockName).str();
-
-    debug(format("acquiring global GC lock '%1%'") % fnGCLock);
-
-    AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
-    if (!fdGCLock)
-        throw SysError("opening global GC lock '%1%'", fnGCLock);
-
-    if (!lockFile(fdGCLock.get(), lockType, false)) {
-        printInfo("waiting for the big garbage collector lock...");
-        lockFile(fdGCLock.get(), lockType, true);
-    }
-
-    /* !!! Restrict read permission on the GC root.  Otherwise any
-       process that can open the file for reading can DoS the
-       collector. */
-
-    return fdGCLock;
-}
+static std::string gcSocketPath = "/gc-socket/socket";
+static std::string gcRootsDir = "gcroots";
 
 
 static void makeSymlink(const Path & link, const Path & target)
@@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target)
 }
 
 
-void LocalStore::syncWithGC()
-{
-    AutoCloseFD fdGCLock = openGCLock(ltRead);
-}
-
-
 void LocalStore::addIndirectRoot(const Path & path)
 {
     string hash = hashString(htSHA1, path).to_string(Base32, false);
@@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
                 "creating a garbage collector root (%1%) in the Nix store is forbidden "
                 "(are you running nix-build inside the store?)", gcRoot);
 
+    /* Register this root with the garbage collector, if it's
+       running. This should be superfluous since the caller should
+       have registered this root yet, but let's be on the safe
+       side. */
+    addTempRoot(storePath);
+
     /* Don't clobber the link if it already exists and doesn't
        point to the Nix store. */
     if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
@@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
     makeSymlink(gcRoot, printStorePath(storePath));
     addIndirectRoot(gcRoot);
 
-    /* Grab the global GC root, causing us to block while a GC is in
-       progress.  This prevents the set of permanent roots from
-       increasing while a GC is in progress. */
-    syncWithGC();
-
     return gcRoot;
 }
 
@@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path)
     if (!state->fdTempRoots) {
 
         while (1) {
-            AutoCloseFD fdGCLock = openGCLock(ltRead);
-
             if (pathExists(fnTempRoots))
                 /* It *must* be stale, since there can be no two
                    processes with the same pid. */
@@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path)
 
             state->fdTempRoots = openLockFile(fnTempRoots, true);
 
-            fdGCLock = -1;
-
-            debug(format("acquiring read lock on '%1%'") % fnTempRoots);
-            lockFile(state->fdTempRoots.get(), ltRead, true);
+            debug("acquiring write lock on '%s'", fnTempRoots);
+            lockFile(state->fdTempRoots.get(), ltWrite, true);
 
             /* Check whether the garbage collector didn't get in our
                way. */
@@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path)
 
     }
 
-    /* Upgrade the lock to a write lock.  This will cause us to block
-       if the garbage collector is holding our lock. */
-    debug(format("acquiring write lock on '%1%'") % fnTempRoots);
-    lockFile(state->fdTempRoots.get(), ltWrite, true);
+    if (!state->fdGCLock)
+        state->fdGCLock = openGCLock();
 
+ restart:
+    FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
+
+    if (!gcLock.acquired) {
+        /* We couldn't get a shared global GC lock, so the garbage
+           collector is running. So we have to connect to the garbage
+           collector and inform it about our root. */
+        if (!state->fdRootsSocket) {
+            auto socketPath = stateDir.get() + gcSocketPath;
+            debug("connecting to '%s'", socketPath);
+            state->fdRootsSocket = createUnixDomainSocket();
+            nix::connect(state->fdRootsSocket.get(), socketPath);
+        }
+
+        try {
+            debug("sending GC root '%s'", printStorePath(path));
+            writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
+            char c;
+            readFull(state->fdRootsSocket.get(), &c, 1);
+            assert(c == '1');
+            debug("got ack for GC root '%s'", printStorePath(path));
+        } catch (SysError & e) {
+            /* The garbage collector may have exited, so we need to
+               restart. */
+            if (e.errNo == EPIPE) {
+                debug("GC socket disconnected");
+                state->fdRootsSocket.close();
+                goto restart;
+            }
+        } catch (EndOfFile & e) {
+            debug("GC socket disconnected");
+            state->fdRootsSocket.close();
+            goto restart;
+        }
+    }
+
+    /* Append the store path to the temporary roots file. */
     string s = printStorePath(path) + '\0';
     writeFull(state->fdTempRoots.get(), s);
-
-    /* Downgrade to a read lock. */
-    debug(format("downgrading to read lock on '%1%'") % fnTempRoots);
-    lockFile(state->fdTempRoots.get(), ltRead, true);
 }
 
 
 static std::string censored = "{censored}";
 
 
-void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
+void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
 {
     /* Read the `temproots' directory for per-process temporary root
        files. */
@@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
         pid_t pid = std::stoi(i.name);
 
         debug(format("reading temporary root file '%1%'") % path);
-        FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
-        if (!*fd) {
+        AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666));
+        if (!fd) {
             /* It's okay if the file has disappeared. */
             if (errno == ENOENT) continue;
             throw SysError("opening temporary roots file '%1%'", path);
         }
 
-        /* This should work, but doesn't, for some reason. */
-        //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
-        //if (*fd == -1) continue;
-
         /* Try to acquire a write lock without blocking.  This can
            only succeed if the owning process has died.  In that case
            we don't care about its temporary roots. */
-        if (lockFile(fd->get(), ltWrite, false)) {
+        if (lockFile(fd.get(), ltWrite, false)) {
             printInfo("removing stale temporary roots file '%1%'", path);
             unlink(path.c_str());
-            writeFull(fd->get(), "d");
+            writeFull(fd.get(), "d");
             continue;
         }
 
-        /* Acquire a read lock.  This will prevent the owning process
-           from upgrading to a write lock, therefore it will block in
-           addTempRoot(). */
-        debug(format("waiting for read lock on '%1%'") % path);
-        lockFile(fd->get(), ltRead, true);
-
         /* Read the entire file. */
-        string contents = readFile(fd->get());
+        string contents = readFile(fd.get());
 
         /* Extract the roots. */
         string::size_type pos = 0, end;
@@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
             tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
             pos = end + 1;
         }
-
-        fds.push_back(fd); /* keep open */
     }
 }
 
@@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor)
     Roots roots;
     findRootsNoTemp(roots, censor);
 
-    FDs fds;
-    findTempRoots(fds, roots, censor);
+    findTempRoots(roots, censor);
 
     return roots;
 }
@@ -341,6 +324,7 @@ static string quoteRegexChars(const string & raw)
     return std::regex_replace(raw, specialRegex, R"(\$&)");
 }
 
+#if __linux__
 static void readFileRoots(const char * path, UncheckedRoots & roots)
 {
     try {
@@ -350,6 +334,7 @@ static void readFileRoots(const char * path, UncheckedRoots & roots)
             throw;
     }
 }
+#endif
 
 void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 {
@@ -431,7 +416,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
     }
 #endif
 
-#if defined(__linux__)
+#if __linux__
     readFileRoots("/proc/sys/kernel/modprobe", unchecked);
     readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
     readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
@@ -455,265 +440,139 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 struct GCLimitReached { };
 
 
-struct LocalStore::GCState
-{
-    const GCOptions & options;
-    GCResults & results;
-    StorePathSet roots;
-    StorePathSet tempRoots;
-    StorePathSet dead;
-    StorePathSet alive;
-    bool gcKeepOutputs;
-    bool gcKeepDerivations;
-    uint64_t bytesInvalidated;
-    bool moveToTrash = true;
-    bool shouldDelete;
-    GCState(const GCOptions & options, GCResults & results)
-        : options(options), results(results), bytesInvalidated(0) { }
-};
-
-
-bool LocalStore::isActiveTempFile(const GCState & state,
-    const Path & path, const string & suffix)
-{
-    return hasSuffix(path, suffix)
-        && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size())));
-}
-
-
-void LocalStore::deleteGarbage(GCState & state, const Path & path)
-{
-    uint64_t bytesFreed;
-    deletePath(path, bytesFreed);
-    state.results.bytesFreed += bytesFreed;
-}
-
-
-void LocalStore::deletePathRecursive(GCState & state, const Path & path)
-{
-    checkInterrupt();
-
-    uint64_t size = 0;
-
-    auto storePath = maybeParseStorePath(path);
-    if (storePath && isValidPath(*storePath)) {
-        StorePathSet referrers;
-        queryReferrers(*storePath, referrers);
-        for (auto & i : referrers)
-            if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
-        size = queryPathInfo(*storePath)->narSize;
-        invalidatePathChecked(*storePath);
-    }
-
-    Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
-
-    struct stat st;
-    if (lstat(realPath.c_str(), &st)) {
-        if (errno == ENOENT) return;
-        throw SysError("getting status of %1%", realPath);
-    }
-
-    printInfo(format("deleting '%1%'") % path);
-
-    state.results.paths.insert(path);
-
-    /* If the path is not a regular file or symlink, move it to the
-       trash directory.  The move is to ensure that later (when we're
-       not holding the global GC lock) we can delete the path without
-       being afraid that the path has become alive again.  Otherwise
-       delete it right away. */
-    if (state.moveToTrash && S_ISDIR(st.st_mode)) {
-        // Estimate the amount freed using the narSize field.  FIXME:
-        // if the path was not valid, need to determine the actual
-        // size.
-        try {
-            if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
-                throw SysError("making '%1%' writable", realPath);
-            Path tmp = trashDir + "/" + std::string(baseNameOf(path));
-            if (rename(realPath.c_str(), tmp.c_str()))
-                throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
-            state.bytesInvalidated += size;
-        } catch (SysError & e) {
-            if (e.errNo == ENOSPC) {
-                printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
-                deleteGarbage(state, realPath);
-            }
-        }
-    } else
-        deleteGarbage(state, realPath);
-
-    if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
-        printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
-        throw GCLimitReached();
-    }
-}
-
-
-bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path)
-{
-    if (visited.count(path)) return false;
-
-    if (state.alive.count(path)) return true;
-
-    if (state.dead.count(path)) return false;
-
-    if (state.roots.count(path)) {
-        debug("cannot delete '%1%' because it's a root", printStorePath(path));
-        state.alive.insert(path);
-        return true;
-    }
-
-    visited.insert(path);
-
-    if (!isValidPath(path)) return false;
-
-    StorePathSet incoming;
-
-    /* Don't delete this path if any of its referrers are alive. */
-    queryReferrers(path, incoming);
-
-    /* If keep-derivations is set and this is a derivation, then
-       don't delete the derivation if any of the outputs are alive. */
-    if (state.gcKeepDerivations && path.isDerivation()) {
-        for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
-            if (maybeOutPath &&
-                isValidPath(*maybeOutPath) &&
-                queryPathInfo(*maybeOutPath)->deriver == path
-                )
-                incoming.insert(*maybeOutPath);
-    }
-
-    /* If keep-outputs is set, then don't delete this path if there
-       are derivers of this path that are not garbage. */
-    if (state.gcKeepOutputs) {
-        auto derivers = queryValidDerivers(path);
-        for (auto & i : derivers)
-            incoming.insert(i);
-    }
-
-    for (auto & i : incoming)
-        if (i != path)
-            if (canReachRoot(state, visited, i)) {
-                state.alive.insert(path);
-                return true;
-            }
-
-    return false;
-}
-
-
-void LocalStore::tryToDelete(GCState & state, const Path & path)
-{
-    checkInterrupt();
-
-    auto realPath = realStoreDir + "/" + std::string(baseNameOf(path));
-    if (realPath == linksDir || realPath == trashDir) return;
-
-    //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
-
-    auto storePath = maybeParseStorePath(path);
-
-    if (!storePath || !isValidPath(*storePath)) {
-        /* A lock file belonging to a path that we're building right
-           now isn't garbage. */
-        if (isActiveTempFile(state, path, ".lock")) return;
-
-        /* Don't delete .chroot directories for derivations that are
-           currently being built. */
-        if (isActiveTempFile(state, path, ".chroot")) return;
-
-        /* Don't delete .check directories for derivations that are
-           currently being built, because we may need to run
-           diff-hook. */
-        if (isActiveTempFile(state, path, ".check")) return;
-    }
-
-    StorePathSet visited;
-
-    if (storePath && canReachRoot(state, visited, *storePath)) {
-        debug("cannot delete '%s' because it's still reachable", path);
-    } else {
-        /* No path we visited was a root, so everything is garbage.
-           But we only delete ‘path’ and its referrers here so that
-           ‘nix-store --delete’ doesn't have the unexpected effect of
-           recursing into derivations and outputs. */
-        for (auto & i : visited)
-            state.dead.insert(i);
-        if (state.shouldDelete)
-            deletePathRecursive(state, path);
-    }
-}
-
-
-/* Unlink all files in /nix/store/.links that have a link count of 1,
-   which indicates that there are no other links and so they can be
-   safely deleted.  FIXME: race condition with optimisePath(): we
-   might see a link count of 1 just before optimisePath() increases
-   the link count. */
-void LocalStore::removeUnusedLinks(const GCState & state)
-{
-    AutoCloseDir dir(opendir(linksDir.c_str()));
-    if (!dir) throw SysError("opening directory '%1%'", linksDir);
-
-    int64_t actualSize = 0, unsharedSize = 0;
-
-    struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir.get())) {
-        checkInterrupt();
-        string name = dirent->d_name;
-        if (name == "." || name == "..") continue;
-        Path path = linksDir + "/" + name;
-
-        auto st = lstat(path);
-
-        if (st.st_nlink != 1) {
-            actualSize += st.st_size;
-            unsharedSize += (st.st_nlink - 1) * st.st_size;
-            continue;
-        }
-
-        printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
-
-        if (unlink(path.c_str()) == -1)
-            throw SysError("deleting '%1%'", path);
-
-        state.results.bytesFreed += st.st_size;
-    }
-
-    struct stat st;
-    if (stat(linksDir.c_str(), &st) == -1)
-        throw SysError("statting '%1%'", linksDir);
-    int64_t overhead = st.st_blocks * 512ULL;
-
-    printInfo("note: currently hard linking saves %.2f MiB",
-        ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
-}
-
-
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
-    GCState state(options, results);
-    state.gcKeepOutputs = settings.gcKeepOutputs;
-    state.gcKeepDerivations = settings.gcKeepDerivations;
+    bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
+    bool gcKeepOutputs = settings.gcKeepOutputs;
+    bool gcKeepDerivations = settings.gcKeepDerivations;
+
+    StorePathSet roots, dead, alive;
+
+    struct Shared
+    {
+        // The temp roots only store the hash part to make it easier to
+        // ignore suffixes like '.lock', '.chroot' and '.check'.
+        std::unordered_set<std::string> tempRoots;
+
+        // Hash part of the store path currently being deleted, if
+        // any.
+        std::optional<std::string> pending;
+    };
+
+    Sync<Shared> _shared;
+
+    std::condition_variable wakeup;
 
     /* Using `--ignore-liveness' with `--delete' can have unintended
        consequences if `keep-outputs' or `keep-derivations' are true
        (the garbage collector will recurse into deleting the outputs
        or derivers, respectively).  So disable them. */
     if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
-        state.gcKeepOutputs = false;
-        state.gcKeepDerivations = false;
+        gcKeepOutputs = false;
+        gcKeepDerivations = false;
     }
 
-    state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
-
-    if (state.shouldDelete)
+    if (shouldDelete)
         deletePath(reservedPath);
 
-    /* Acquire the global GC root.  This prevents
-       a) New roots from being added.
-       b) Processes from creating new temporary root files. */
-    AutoCloseFD fdGCLock = openGCLock(ltWrite);
+    /* Acquire the global GC root. Note: we don't use fdGCLock
+       here because then in auto-gc mode, another thread could
+       downgrade our exclusive lock. */
+    auto fdGCLock = openGCLock();
+    FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock...");
+
+    /* Start the server for receiving new roots. */
+    auto socketPath = stateDir.get() + gcSocketPath;
+    createDirs(dirOf(socketPath));
+    auto fdServer = createUnixDomainSocket(socketPath, 0666);
+
+    if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
+        throw SysError("making socket '%1%' non-blocking", socketPath);
+
+    Pipe shutdownPipe;
+    shutdownPipe.create();
+
+    std::thread serverThread([&]() {
+        Sync<std::map<int, std::thread>> connections;
+
+        Finally cleanup([&]() {
+            debug("GC roots server shutting down");
+            while (true) {
+                auto item = remove_begin(*connections.lock());
+                if (!item) break;
+                auto & [fd, thread] = *item;
+                shutdown(fd, SHUT_RDWR);
+                thread.join();
+            }
+        });
+
+        while (true) {
+            std::vector<struct pollfd> fds;
+            fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN});
+            fds.push_back({.fd = fdServer.get(), .events = POLLIN});
+            auto count = poll(fds.data(), fds.size(), -1);
+            assert(count != -1);
+
+            if (fds[0].revents)
+                /* Parent is asking us to quit. */
+                break;
+
+            if (fds[1].revents) {
+                /* Accept a new connection. */
+                assert(fds[1].revents & POLLIN);
+                AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
+                if (!fdClient) continue;
+
+                /* Process the connection in a separate thread. */
+                auto fdClient_ = fdClient.get();
+                std::thread clientThread([&, fdClient = std::move(fdClient)]() {
+                    Finally cleanup([&]() {
+                        auto conn(connections.lock());
+                        auto i = conn->find(fdClient.get());
+                        if (i != conn->end()) {
+                            i->second.detach();
+                            conn->erase(i);
+                        }
+                    });
+
+                    while (true) {
+                        try {
+                            auto path = readLine(fdClient.get());
+                            auto storePath = maybeParseStorePath(path);
+                            if (storePath) {
+                                debug("got new GC root '%s'", path);
+                                auto hashPart = std::string(storePath->hashPart());
+                                auto shared(_shared.lock());
+                                shared->tempRoots.insert(hashPart);
+                                /* If this path is currently being
+                                   deleted, then we have to wait until
+                                   deletion is finished to ensure that
+                                   the client doesn't start
+                                   re-creating it before we're
+                                   done. FIXME: ideally we would use a
+                                   FD for this so we don't block the
+                                   poll loop. */
+                                while (shared->pending == hashPart) {
+                                    debug("synchronising with deletion of path '%s'", path);
+                                    shared.wait(wakeup);
+                                }
+                            } else
+                                printError("received garbage instead of a root from client");
+                            writeFull(fdClient.get(), "1", false);
+                        } catch (Error &) { break; }
+                    }
+                });
+
+                connections.lock()->insert({fdClient_, std::move(clientThread)});
+            }
+        }
+    });
+
+    Finally stopServer([&]() {
+        writeFull(shutdownPipe.writeSide.get(), "x", false);
+        wakeup.notify_all();
+        if (serverThread.joinable()) serverThread.join();
+    });
 
     /* Find the roots.  Since we've grabbed the GC lock, the set of
        permanent roots cannot increase now. */
@@ -722,124 +581,256 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     if (!options.ignoreLiveness)
         findRootsNoTemp(rootMap, true);
 
-    for (auto & i : rootMap) state.roots.insert(i.first);
+    for (auto & i : rootMap) roots.insert(i.first);
 
-    /* Read the temporary roots.  This acquires read locks on all
-       per-process temporary root files.  So after this point no paths
-       can be added to the set of temporary roots. */
-    FDs fds;
+    /* Read the temporary roots created before we acquired the global
+       GC root. Any new roots will be sent to our socket. */
     Roots tempRoots;
-    findTempRoots(fds, tempRoots, true);
+    findTempRoots(tempRoots, true);
     for (auto & root : tempRoots) {
-        state.tempRoots.insert(root.first);
-        state.roots.insert(root.first);
+        _shared.lock()->tempRoots.insert(std::string(root.first.hashPart()));
+        roots.insert(root.first);
     }
 
-    /* After this point the set of roots or temporary roots cannot
-       increase, since we hold locks on everything.  So everything
-       that is not reachable from `roots' is garbage. */
+    /* Helper function that deletes a path from the store and throws
+       GCLimitReached if we've deleted enough garbage. */
+    auto deleteFromStore = [&](std::string_view baseName)
+    {
+        Path path = storeDir + "/" + std::string(baseName);
+        Path realPath = realStoreDir + "/" + std::string(baseName);
 
-    if (state.shouldDelete) {
-        if (pathExists(trashDir)) deleteGarbage(state, trashDir);
-        try {
-            createDirs(trashDir);
-        } catch (SysError & e) {
-            if (e.errNo == ENOSPC) {
-                printInfo("note: can't create trash directory: %s", e.msg());
-                state.moveToTrash = false;
+        printInfo("deleting '%1%'", path);
+
+        results.paths.insert(path);
+
+        uint64_t bytesFreed;
+        deletePath(realPath, bytesFreed);
+        results.bytesFreed += bytesFreed;
+
+        if (results.bytesFreed > options.maxFreed) {
+            printInfo("deleted more than %d bytes; stopping", options.maxFreed);
+            throw GCLimitReached();
+        }
+    };
+
+    std::map<StorePath, StorePathSet> referrersCache;
+
+    /* Helper function that visits all paths reachable from `start`
+       via the referrers edges and optionally derivers and derivation
+       output edges. If none of those paths are roots, then all
+       visited paths are garbage and are deleted. */
+    auto deleteReferrersClosure = [&](const StorePath & start) {
+        StorePathSet visited;
+        std::queue<StorePath> todo;
+
+        /* Wake up any GC client waiting for deletion of the paths in
+           'visited' to finish. */
+        Finally releasePending([&]() {
+            auto shared(_shared.lock());
+            shared->pending.reset();
+            wakeup.notify_all();
+        });
+
+        auto enqueue = [&](const StorePath & path) {
+            if (visited.insert(path).second)
+                todo.push(path);
+        };
+
+        enqueue(start);
+
+        while (auto path = pop(todo)) {
+            checkInterrupt();
+
+            /* Bail out if we've previously discovered that this path
+               is alive. */
+            if (alive.count(*path)) {
+                alive.insert(start);
+                return;
+            }
+
+            /* If we've previously deleted this path, we don't have to
+               handle it again. */
+            if (dead.count(*path)) continue;
+
+            auto markAlive = [&]()
+            {
+                alive.insert(*path);
+                alive.insert(start);
+                try {
+                    StorePathSet closure;
+                    computeFSClosure(*path, closure);
+                    for (auto & p : closure)
+                        alive.insert(p);
+                } catch (InvalidPath &) { }
+            };
+
+            /* If this is a root, bail out. */
+            if (roots.count(*path)) {
+                debug("cannot delete '%s' because it's a root", printStorePath(*path));
+                return markAlive();
+            }
+
+            if (options.action == GCOptions::gcDeleteSpecific
+                && !options.pathsToDelete.count(*path))
+                return;
+
+            {
+                auto hashPart = std::string(path->hashPart());
+                auto shared(_shared.lock());
+                if (shared->tempRoots.count(hashPart)) {
+                    debug("cannot delete '%s' because it's a temporary root", printStorePath(*path));
+                    return markAlive();
+                }
+                shared->pending = hashPart;
+            }
+
+            if (isValidPath(*path)) {
+
+                /* Visit the referrers of this path. */
+                auto i = referrersCache.find(*path);
+                if (i == referrersCache.end()) {
+                    StorePathSet referrers;
+                    queryReferrers(*path, referrers);
+                    referrersCache.emplace(*path, std::move(referrers));
+                    i = referrersCache.find(*path);
+                }
+                for (auto & p : i->second)
+                    enqueue(p);
+
+                /* If keep-derivations is set and this is a
+                   derivation, then visit the derivation outputs. */
+                if (gcKeepDerivations && path->isDerivation()) {
+                    for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path))
+                        if (maybeOutPath &&
+                            isValidPath(*maybeOutPath) &&
+                            queryPathInfo(*maybeOutPath)->deriver == *path)
+                            enqueue(*maybeOutPath);
+                }
+
+                /* If keep-outputs is set, then visit the derivers. */
+                if (gcKeepOutputs) {
+                    auto derivers = queryValidDerivers(*path);
+                    for (auto & i : derivers)
+                        enqueue(i);
+                }
             }
         }
-    }
 
-    /* Now either delete all garbage paths, or just the specified
+        for (auto & path : topoSortPaths(visited)) {
+            if (!dead.insert(path).second) continue;
+            if (shouldDelete) {
+                invalidatePathChecked(path);
+                deleteFromStore(path.to_string());
+                referrersCache.erase(path);
+            }
+        }
+    };
+
+    /* Synchronisation point for testing, see tests/gc-concurrent.sh. */
+    if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
+        readFile(*p);
+
+    /* Either delete all garbage paths, or just the specified
        paths (for gcDeleteSpecific). */
-
     if (options.action == GCOptions::gcDeleteSpecific) {
 
         for (auto & i : options.pathsToDelete) {
-            tryToDelete(state, printStorePath(i));
-            if (state.dead.find(i) == state.dead.end())
+            deleteReferrersClosure(i);
+            if (!dead.count(i))
                 throw Error(
-                    "cannot delete path '%1%' since it is still alive. "
-                    "To find out why use: "
+                    "Cannot delete path '%1%' since it is still alive. "
+                    "To find out why, use: "
                     "nix-store --query --roots",
                     printStorePath(i));
         }
 
     } else if (options.maxFreed > 0) {
 
-        if (state.shouldDelete)
+        if (shouldDelete)
             printInfo("deleting garbage...");
         else
             printInfo("determining live/dead paths...");
 
         try {
-
             AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
             if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
 
-            /* Read the store and immediately delete all paths that
-               aren't valid.  When using --max-freed etc., deleting
-               invalid paths is preferred over deleting unreachable
-               paths, since unreachable paths could become reachable
-               again.  We don't use readDirectory() here so that GCing
-               can start faster. */
+            /* Read the store and delete all paths that are invalid or
+               unreachable. We don't use readDirectory() here so that
+               GCing can start faster. */
+            auto linksName = baseNameOf(linksDir);
             Paths entries;
             struct dirent * dirent;
             while (errno = 0, dirent = readdir(dir.get())) {
                 checkInterrupt();
                 string name = dirent->d_name;
-                if (name == "." || name == "..") continue;
-                Path path = storeDir + "/" + name;
-                auto storePath = maybeParseStorePath(path);
-                if (storePath && isValidPath(*storePath))
-                    entries.push_back(path);
+                if (name == "." || name == ".." || name == linksName) continue;
+
+                if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
+                    deleteReferrersClosure(*storePath);
                 else
-                    tryToDelete(state, path);
+                    deleteFromStore(name);
+
             }
-
-            dir.reset();
-
-            /* Now delete the unreachable valid paths.  Randomise the
-               order in which we delete entries to make the collector
-               less biased towards deleting paths that come
-               alphabetically first (e.g. /nix/store/000...).  This
-               matters when using --max-freed etc. */
-            vector<Path> entries_(entries.begin(), entries.end());
-            std::mt19937 gen(1);
-            std::shuffle(entries_.begin(), entries_.end(), gen);
-
-            for (auto & i : entries_)
-                tryToDelete(state, i);
-
         } catch (GCLimitReached & e) {
         }
     }
 
-    if (state.options.action == GCOptions::gcReturnLive) {
-        for (auto & i : state.alive)
-            state.results.paths.insert(printStorePath(i));
+    if (options.action == GCOptions::gcReturnLive) {
+        for (auto & i : alive)
+            results.paths.insert(printStorePath(i));
         return;
     }
 
-    if (state.options.action == GCOptions::gcReturnDead) {
-        for (auto & i : state.dead)
-            state.results.paths.insert(printStorePath(i));
+    if (options.action == GCOptions::gcReturnDead) {
+        for (auto & i : dead)
+            results.paths.insert(printStorePath(i));
         return;
     }
 
-    /* Allow other processes to add to the store from here on. */
-    fdGCLock = -1;
-    fds.clear();
-
-    /* Delete the trash directory. */
-    printInfo(format("deleting '%1%'") % trashDir);
-    deleteGarbage(state, trashDir);
-
-    /* Clean up the links directory. */
+    /* Unlink all files in /nix/store/.links that have a link count of 1,
+       which indicates that there are no other links and so they can be
+       safely deleted.  FIXME: race condition with optimisePath(): we
+       might see a link count of 1 just before optimisePath() increases
+       the link count. */
     if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
         printInfo("deleting unused links...");
-        removeUnusedLinks(state);
+
+        AutoCloseDir dir(opendir(linksDir.c_str()));
+        if (!dir) throw SysError("opening directory '%1%'", linksDir);
+
+        int64_t actualSize = 0, unsharedSize = 0;
+
+        struct dirent * dirent;
+        while (errno = 0, dirent = readdir(dir.get())) {
+            checkInterrupt();
+            string name = dirent->d_name;
+            if (name == "." || name == "..") continue;
+            Path path = linksDir + "/" + name;
+
+            auto st = lstat(path);
+
+            if (st.st_nlink != 1) {
+                actualSize += st.st_size;
+                unsharedSize += (st.st_nlink - 1) * st.st_size;
+                continue;
+            }
+
+            printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
+
+            if (unlink(path.c_str()) == -1)
+                throw SysError("deleting '%1%'", path);
+
+            results.bytesFreed += st.st_size;
+        }
+
+        struct stat st;
+        if (stat(linksDir.c_str(), &st) == -1)
+            throw SysError("statting '%1%'", linksDir);
+        int64_t overhead = st.st_blocks * 512ULL;
+
+        printInfo("note: currently hard linking saves %.2f MiB",
+            ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
     }
 
     /* While we're at it, vacuum the database. */

src/libstore/globals.cc

@@ -122,7 +122,7 @@ StringSet Settings::getDefaultSystemFeatures()
     /* For backwards compatibility, accept some "features" that are
        used in Nixpkgs to route builds to certain machines but don't
        actually require anything special on the machines. */
-    StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"};
+    StringSet features{"nixos-test", "benchmark", "big-parallel"};
 
     #if __linux__
     if (access("/dev/kvm", R_OK | W_OK) == 0)
@@ -148,7 +148,8 @@ StringSet Settings::getDefaultExtraPlatforms()
     // machines. Note that we can’t force processes from executing
     // x86_64 in aarch64 environments or vice versa since they can
     // always exec with their own binary preferences.
-    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
+    if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") ||
+        pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
         if (std::string{SYSTEM} == "x86_64-darwin")
             extraPlatforms.insert("aarch64-darwin");
         else if (std::string{SYSTEM} == "aarch64-darwin")
@@ -159,21 +160,16 @@ StringSet Settings::getDefaultExtraPlatforms()
     return extraPlatforms;
 }
 
-bool Settings::isExperimentalFeatureEnabled(const std::string & name)
+bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature)
 {
     auto & f = experimentalFeatures.get();
-    return std::find(f.begin(), f.end(), name) != f.end();
+    return std::find(f.begin(), f.end(), feature) != f.end();
 }
 
-MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
-    : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", feature)
-    , missingFeature(feature)
-    {}
-
-void Settings::requireExperimentalFeature(const std::string & name)
+void Settings::requireExperimentalFeature(const ExperimentalFeature & feature)
 {
-    if (!isExperimentalFeatureEnabled(name))
-        throw MissingExperimentalFeature(name);
+    if (!isExperimentalFeatureEnabled(feature))
+        throw MissingExperimentalFeature(feature);
 }
 
 bool Settings::isWSL1()

src/libstore/globals.hh

@@ -3,6 +3,7 @@
 #include "types.hh"
 #include "config.hh"
 #include "util.hh"
+#include "experimental-features.hh"
 
 #include <map>
 #include <limits>
@@ -45,15 +46,6 @@ struct PluginFilesSetting : public BaseSetting<Paths>
     void set(const std::string & str, bool append = false) override;
 };
 
-class MissingExperimentalFeature: public Error
-{
-public:
-    std::string missingFeature;
-
-    MissingExperimentalFeature(std::string feature);
-    virtual const char* sname() const override { return "MissingExperimentalFeature"; }
-};
-
 class Settings : public Config {
 
     unsigned int getDefaultCores();
@@ -925,12 +917,12 @@ public:
           value.
           )"};
 
-    Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
+    Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
-    bool isExperimentalFeatureEnabled(const std::string & name);
+    bool isExperimentalFeatureEnabled(const ExperimentalFeature &);
 
-    void requireExperimentalFeature(const std::string & name);
+    void requireExperimentalFeature(const ExperimentalFeature &);
 
     Setting<bool> allowDirty{this, true, "allow-dirty",
         "Whether to allow dirty Git/Mercurial trees."};
@@ -959,6 +951,9 @@ public:
 
     Setting<bool> useRegistries{this, true, "use-registries",
         "Whether to use flake registries to resolve flake references."};
+
+    Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config",
+        "Whether to accept nix configuration from a flake without prompting."};
 };
 
 

src/libstore/legacy-ssh-store.cc

@@ -227,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
 
     StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method, HashType hashAlgo,
-        PathFilter & filter, RepairFlag repair) override
+        PathFilter & filter, RepairFlag repair, const StorePathSet & references) override
     { unsupported("addToStore"); }
 
     StorePath addTextToStore(const string & name, const string & s,
@@ -367,7 +367,8 @@ public:
         return conn->remoteVersion;
     }
 
-    std::optional<const Realisation> queryRealisation(const DrvOutput&) override
+    void queryRealisationUncached(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override
     // TODO: Implement
     { unsupported("queryRealisation"); }
 };

src/libstore/local-store.cc

@@ -145,7 +145,6 @@ LocalStore::LocalStore(const Params & params)
     , linksDir(realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
     , schemaPath(dbDir + "/schema")
-    , trashDir(realStoreDir + "/trash")
     , tempRootsDir(stateDir + "/temproots")
     , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
     , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
@@ -309,7 +308,7 @@ LocalStore::LocalStore(const Params & params)
 
     else openDB(*state, false);
 
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
     }
 
@@ -339,7 +338,7 @@ LocalStore::LocalStore(const Params & params)
     state->stmts->QueryPathFromHashPart.create(state->db,
         "select path from ValidPaths where path >= ? limit 1;");
     state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
-    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         state->stmts->RegisterRealisedOutput.create(state->db,
             R"(
                 insert into Realisations (drvPath, outputName, outputPath, signatures)
@@ -386,6 +385,16 @@ LocalStore::LocalStore(const Params & params)
 }
 
 
+AutoCloseFD LocalStore::openGCLock()
+{
+    Path fnGCLock = stateDir + "/gc.lock";
+    auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+    if (!fdGCLock)
+        throw SysError("opening global GC lock '%1%'", fnGCLock);
+    return fdGCLock;
+}
+
+
 LocalStore::~LocalStore()
 {
     std::shared_future<void> future;
@@ -495,9 +504,6 @@ void LocalStore::makeStoreWritable()
         throw SysError("getting info about the Nix store mount point");
 
     if (stat.f_flag & ST_RDONLY) {
-        if (unshare(CLONE_NEWNS) == -1)
-            throw SysError("setting up a private mount namespace");
-
         if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
             throw SysError("remounting %1% writable", realStoreDir);
     }
@@ -708,7 +714,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 
 void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
 {
-    settings.requireExperimentalFeature("ca-derivations");
+    settings.requireExperimentalFeature(Xp::CaDerivations);
     if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
         registerDrvOutput(info);
     else
@@ -717,7 +723,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
 
 void LocalStore::registerDrvOutput(const Realisation & info)
 {
-    settings.requireExperimentalFeature("ca-derivations");
+    settings.requireExperimentalFeature(Xp::CaDerivations);
     retrySQLite<void>([&]() {
         auto state(_state.lock());
         if (auto oldR = queryRealisation_(*state, info.id)) {
@@ -825,7 +831,7 @@ uint64_t LocalStore::addValidPath(State & state,
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
+        state_->pathInfoCache.upsert(std::string(info.path.to_string()),
             PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
     }
 
@@ -1003,7 +1009,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
         return outputs;
     });
 
-    if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
+    if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
         return outputs;
 
     auto drv = readInvalidDerivation(path);
@@ -1198,7 +1204,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
 
     {
         auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(std::string(path.hashPart()));
+        state_->pathInfoCache.erase(std::string(path.to_string()));
     }
 }
 
@@ -1302,7 +1308,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
 {
     /* For computing the store path. */
     auto hashSink = std::make_unique<HashSink>(hashAlgo);
@@ -1358,7 +1364,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
 
     auto [hash, size] = hashSink->finish();
 
-    auto dstPath = makeFixedOutputPath(method, hash, name);
+    auto dstPath = makeFixedOutputPath(method, hash, name, references);
 
     addTempRoot(dstPath);
 
@@ -1405,6 +1411,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
 
             ValidPathInfo info { dstPath, narHash.first };
             info.narSize = narHash.second;
+            info.references = references;
             info.ca = FixedOutputHash { .method = method, .hash = hash };
             registerValidPath(info);
         }
@@ -1505,7 +1512,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
     /* Acquire the global GC lock to get a consistent snapshot of
        existing and valid paths. */
-    AutoCloseFD fdGCLock = openGCLock(ltWrite);
+    auto fdGCLock = openGCLock();
+    FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
 
     StringSet store;
     for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
@@ -1516,8 +1524,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
     StorePathSet validPaths;
     PathSet done;
 
-    fdGCLock = -1;
-
     for (auto & i : queryAllValidPaths())
         verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
 
@@ -1830,13 +1836,24 @@ std::optional<const Realisation> LocalStore::queryRealisation_(
     return { res };
 }
 
-std::optional<const Realisation>
-LocalStore::queryRealisation(const DrvOutput & id)
+void LocalStore::queryRealisationUncached(const DrvOutput & id,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
-    return retrySQLite<std::optional<const Realisation>>([&]() {
-        auto state(_state.lock());
-        return queryRealisation_(*state, id);
-    });
+    try {
+        auto maybeRealisation
+            = retrySQLite<std::optional<const Realisation>>([&]() {
+                  auto state(_state.lock());
+                  return queryRealisation_(*state, id);
+              });
+        if (maybeRealisation)
+            callback(
+                std::make_shared<const Realisation>(maybeRealisation.value()));
+        else
+            callback(nullptr);
+
+    } catch (...) {
+        callback.rethrow();
+    }
 }
 
 FixedOutputHash LocalStore::hashCAPath(

src/libstore/local-store.hh

@@ -58,9 +58,15 @@ private:
         struct Stmts;
         std::unique_ptr<Stmts> stmts;
 
+        /* The global GC lock  */
+        AutoCloseFD fdGCLock;
+
         /* The file to which we write our temporary roots. */
         AutoCloseFD fdTempRoots;
 
+        /* Connection to the garbage collector. */
+        AutoCloseFD fdRootsSocket;
+
         /* The last time we checked whether to do an auto-GC, or an
            auto-GC finished. */
         std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@@ -87,7 +93,6 @@ public:
     const Path linksDir;
     const Path reservedPath;
     const Path schemaPath;
-    const Path trashDir;
     const Path tempRootsDir;
     const Path fnTempRoots;
 
@@ -140,7 +145,7 @@ public:
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
+        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
 
     StorePath addTextToStore(const string & name, const string & s,
         const StorePathSet & references, RepairFlag repair) override;
@@ -149,14 +154,11 @@ public:
 
     void addIndirectRoot(const Path & path) override;
 
-    void syncWithGC() override;
-
 private:
 
-    typedef std::shared_ptr<AutoCloseFD> FDPtr;
-    typedef list<FDPtr> FDs;
+    void findTempRoots(Roots & roots, bool censor);
 
-    void findTempRoots(FDs & fds, Roots & roots, bool censor);
+    AutoCloseFD openGCLock();
 
 public:
 
@@ -205,7 +207,8 @@ public:
 
     std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id);
     std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id);
-    std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
+    void queryRealisationUncached(const DrvOutput&,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
 
 private:
 
@@ -236,29 +239,12 @@ private:
     PathSet queryValidPathsOld();
     ValidPathInfo queryPathInfoOld(const Path & path);
 
-    struct GCState;
-
-    void deleteGarbage(GCState & state, const Path & path);
-
-    void tryToDelete(GCState & state, const Path & path);
-
-    bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
-
-    void deletePathRecursive(GCState & state, const Path & path);
-
-    bool isActiveTempFile(const GCState & state,
-        const Path & path, const string & suffix);
-
-    AutoCloseFD openGCLock(LockType lockType);
-
     void findRoots(const Path & path, unsigned char type, Roots & roots);
 
     void findRootsNoTemp(Roots & roots, bool censor);
 
     void findRuntimeRoots(Roots & roots, bool censor);
 
-    void removeUnusedLinks(const GCState & state);
-
     Path createTempDirInStore();
 
     void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);

src/libstore/machines.cc

@@ -39,7 +39,8 @@ Machine::Machine(decltype(storeUri) storeUri,
     sshPublicHostKey(sshPublicHostKey)
 {}
 
-bool Machine::allSupported(const std::set<string> & features) const {
+bool Machine::allSupported(const std::set<string> & features) const
+{
     return std::all_of(features.begin(), features.end(),
         [&](const string & feature) {
             return supportedFeatures.count(feature) ||
@@ -47,14 +48,16 @@ bool Machine::allSupported(const std::set<string> & features) const {
         });
 }
 
-bool Machine::mandatoryMet(const std::set<string> & features) const {
+bool Machine::mandatoryMet(const std::set<string> & features) const
+{
     return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(),
         [&](const string & feature) {
             return features.count(feature);
         });
 }
 
-ref<Store> Machine::openStore() const {
+ref<Store> Machine::openStore() const
+{
     Store::Params storeParams;
     if (hasPrefix(storeUri, "ssh://")) {
         storeParams["max-connections"] = "1";
@@ -83,53 +86,86 @@ ref<Store> Machine::openStore() const {
     return nix::openStore(storeUri, storeParams);
 }
 
-void parseMachines(const std::string & s, Machines & machines)
+static std::vector<std::string> expandBuilderLines(const std::string & builders)
 {
-    for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) {
+    std::vector<std::string> result;
+    for (auto line : tokenizeString<std::vector<string>>(builders, "\n;")) {
         trim(line);
         line.erase(std::find(line.begin(), line.end(), '#'), line.end());
         if (line.empty()) continue;
 
         if (line[0] == '@') {
-            auto file = trim(std::string(line, 1));
+            const std::string path = trim(std::string(line, 1));
+            std::string text;
             try {
-                parseMachines(readFile(file), machines);
+                text = readFile(path);
             } catch (const SysError & e) {
                 if (e.errNo != ENOENT)
                     throw;
-                debug("cannot find machines file '%s'", file);
+                debug("cannot find machines file '%s'", path);
             }
+
+            const auto lines = expandBuilderLines(text);
+            result.insert(end(result), begin(lines), end(lines));
             continue;
         }
 
-        auto tokens = tokenizeString<std::vector<string>>(line);
-        auto sz = tokens.size();
-        if (sz < 1)
-            throw FormatError("bad machine specification '%s'", line);
-
-        auto isSet = [&](size_t n) {
-            return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";
-        };
-
-        machines.emplace_back(tokens[0],
-            isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
-            isSet(2) ? tokens[2] : "",
-            isSet(3) ? std::stoull(tokens[3]) : 1LL,
-            isSet(4) ? std::stoull(tokens[4]) : 1LL,
-            isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
-            isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
-            isSet(7) ? tokens[7] : "");
+        result.emplace_back(line);
     }
+    return result;
+}
+
+static Machine parseBuilderLine(const std::string & line)
+{
+    const auto tokens = tokenizeString<std::vector<string>>(line);
+
+    auto isSet = [&](size_t fieldIndex) {
+        return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-";
+    };
+
+    auto parseUnsignedIntField = [&](size_t fieldIndex) {
+        const auto result = string2Int<unsigned int>(tokens[fieldIndex]);
+        if (!result) {
+            throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line);
+        }
+        return result.value();
+    };
+
+    auto ensureBase64 = [&](size_t fieldIndex) {
+        const auto & str = tokens[fieldIndex];
+        try {
+            base64Decode(str);
+        } catch (const Error & e) {
+            throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what());
+        }
+        return str;
+    };
+
+    if (!isSet(0))
+        throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line);
+
+    return {
+        tokens[0],
+        isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem},
+        isSet(2) ? tokens[2] : "",
+        isSet(3) ? parseUnsignedIntField(3) : 1U,
+        isSet(4) ? parseUnsignedIntField(4) : 1U,
+        isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{},
+        isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{},
+        isSet(7) ? ensureBase64(7) : ""
+    };
+}
+
+static Machines parseBuilderLines(const std::vector<std::string>& builders) {
+    Machines result;
+    std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine);
+    return result;
 }
 
 Machines getMachines()
 {
-    static auto machines = [&]() {
-        Machines machines;
-        parseMachines(settings.builders, machines);
-        return machines;
-    }();
-    return machines;
+    const auto builderLines = expandBuilderLines(settings.builders);
+    return parseBuilderLines(builderLines);
 }
 
 }

src/libstore/misc.cc

@@ -239,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
 {
     return topoSort(paths,
         {[&](const StorePath & path) {
-            StorePathSet references;
             try {
-                references = queryPathInfo(path)->references;
+                return queryPathInfo(path)->references;
             } catch (InvalidPath &) {
+                return StorePathSet();
             }
-            return references;
         }},
         {[&](const StorePath & path, const StorePath & parent) {
             return BuildError(

src/libstore/pathlocks.cc

@@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths)
 }
 
 
+FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg)
+    : fd(fd)
+{
+    if (wait) {
+        if (!lockFile(fd, lockType, false)) {
+            printInfo("%s", waitMsg);
+            acquired = lockFile(fd, lockType, true);
+        }
+    } else
+        acquired = lockFile(fd, lockType, false);
+}
+
+
 }

src/libstore/pathlocks.hh

@@ -35,4 +35,18 @@ public:
     void setDeletion(bool deletePaths);
 };
 
+struct FdLock
+{
+    int fd;
+    bool acquired = false;
+
+    FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg);
+
+    ~FdLock()
+    {
+        if (acquired)
+            lockFile(fd, ltNone, false);
+    }
+};
+
 }

src/libstore/references.cc

@@ -54,12 +54,12 @@ void RefScanSink::operator () (std::string_view data)
        fragment, so search in the concatenation of the tail of the
        previous fragment and the start of the current fragment. */
     auto s = tail;
-    s.append(data.data(), refLength);
+    auto tailLen = std::min(data.size(), refLength);
+    s.append(data.data(), tailLen);
     search(s, hashes, seen);
 
     search(data, hashes, seen);
 
-    auto tailLen = std::min(data.size(), refLength);
     auto rest = refLength - tailLen;
     if (rest < tail.size())
         tail = tail.substr(tail.size() - rest);

src/libstore/remote-store.cc

@@ -290,6 +290,10 @@ ConnectionHandle RemoteStore::getConnection()
     return ConnectionHandle(connections->get());
 }
 
+void RemoteStore::setOptions()
+{
+    setOptions(*(getConnection().handle));
+}
 
 bool RemoteStore::isValidPathUncached(const StorePath & path)
 {
@@ -578,9 +582,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 
 
 StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method, HashType hashType, RepairFlag repair)
+      FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
 {
-    StorePathSet references;
     return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path;
 }
 
@@ -677,23 +680,33 @@ void RemoteStore::registerDrvOutput(const Realisation & info)
     conn.processStderr();
 }
 
-std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
+void RemoteStore::queryRealisationUncached(const DrvOutput & id,
+    Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
     auto conn(getConnection());
     conn->to << wopQueryRealisation;
     conn->to << id.to_string();
     conn.processStderr();
-    if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
-        auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
-        if (outPaths.empty())
-            return std::nullopt;
-        return {Realisation{.id = id, .outPath = *outPaths.begin()}};
-    } else {
-        auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{});
-        if (realisations.empty())
-            return std::nullopt;
-        return *realisations.begin();
-    }
+
+    auto real = [&]() -> std::shared_ptr<const Realisation> {
+        if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) {
+            auto outPaths = worker_proto::read(
+                *this, conn->from, Phantom<std::set<StorePath>> {});
+            if (outPaths.empty())
+                return nullptr;
+            return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() });
+        } else {
+            auto realisations = worker_proto::read(
+                *this, conn->from, Phantom<std::set<Realisation>> {});
+            if (realisations.empty())
+                return nullptr;
+            return std::make_shared<const Realisation>(*realisations.begin());
+        }
+    }();
+
+    try {
+        callback(std::shared_ptr<const Realisation>(real));
+    } catch (...) { return callback.rethrow(); }
 }
 
 static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
@@ -797,15 +810,6 @@ void RemoteStore::addIndirectRoot(const Path & path)
 }
 
 
-void RemoteStore::syncWithGC()
-{
-    auto conn(getConnection());
-    conn->to << wopSyncWithGC;
-    conn.processStderr();
-    readInt(conn->from);
-}
-
-
 Roots RemoteStore::findRoots(bool censor)
 {
     auto conn(getConnection());

src/libstore/remote-store.hh

@@ -73,7 +73,7 @@ public:
 
     /* Add a content-addressable store path. Does not support references. `dump` will be drained. */
     StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
@@ -88,7 +88,8 @@ public:
 
     void registerDrvOutput(const Realisation & info) override;
 
-    std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
+    void queryRealisationUncached(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
 
     void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
 
@@ -101,8 +102,6 @@ public:
 
     void addIndirectRoot(const Path & path) override;
 
-    void syncWithGC() override;
-
     Roots findRoots(bool censor) override;
 
     void collectGarbage(const GCOptions & options, GCResults & results) override;
@@ -149,6 +148,8 @@ protected:
 
     virtual void setOptions(Connection & conn);
 
+    void setOptions() override;
+
     ConnectionHandle getConnection();
 
     friend struct ConnectionHandle;

src/libstore/sandbox-defaults.sb

@@ -100,4 +100,5 @@
 
 ; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin.
 (allow file-read*
-       (subpath "/Library/Apple/usr/libexec/oah"))
+       (subpath "/Library/Apple/usr/libexec/oah")
+       (subpath "/System/Library/Apple/usr/libexec/oah"))

src/libstore/sqlite.cc

@@ -1,4 +1,5 @@
 #include "sqlite.hh"
+#include "globals.hh"
 #include "util.hh"
 
 #include <sqlite3.h>
@@ -27,8 +28,12 @@ namespace nix {
 
 SQLite::SQLite(const Path & path, bool create)
 {
+    // useSQLiteWAL also indicates what virtual file system we need.  Using
+    // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
+    // for Linux (WSL) where useSQLiteWAL should be false by default.
+    const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile";
     if (sqlite3_open_v2(path.c_str(), &db,
-            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
+            SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK)
         throw Error("cannot open SQLite database '%s'", path);
 
     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)

src/libstore/store-api.cc

@@ -237,7 +237,7 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
 
 
 StorePath Store::addToStore(const string & name, const Path & _srcPath,
-    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+    FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references)
 {
     Path srcPath(absPath(_srcPath));
     auto source = sinkToSource([&](Sink & sink) {
@@ -246,7 +246,7 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath,
         else
             readFile(srcPath, sink);
     });
-    return addToStoreFromDump(*source, name, method, hashAlgo, repair);
+    return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
 }
 
 
@@ -355,8 +355,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
 StringSet StoreConfig::getDefaultSystemFeatures()
 {
     auto res = settings.systemFeatures.get();
-    if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
         res.insert("ca-derivations");
+
+    if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix))
+        res.insert("recursive-nix");
+
     return res;
 }
 
@@ -414,11 +419,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path)
 
 bool Store::isValidPath(const StorePath & storePath)
 {
-    std::string hashPart(storePath.hashPart());
-
     {
         auto state_(state.lock());
-        auto res = state_->pathInfoCache.get(hashPart);
+        auto res = state_->pathInfoCache.get(std::string(storePath.to_string()));
         if (res && res->isKnownNow()) {
             stats.narInfoReadAverted++;
             return res->didExist();
@@ -426,11 +429,11 @@ bool Store::isValidPath(const StorePath & storePath)
     }
 
     if (diskCache) {
-        auto res = diskCache->lookupNarInfo(getUri(), hashPart);
+        auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart()));
         if (res.first != NarInfoDiskCache::oUnknown) {
             stats.narInfoReadAverted++;
             auto state_(state.lock());
-            state_->pathInfoCache.upsert(hashPart,
+            state_->pathInfoCache.upsert(std::string(storePath.to_string()),
                 res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second });
             return res.first == NarInfoDiskCache::oValid;
         }
@@ -440,7 +443,7 @@ bool Store::isValidPath(const StorePath & storePath)
 
     if (diskCache && !valid)
         // FIXME: handle valid = true case.
-        diskCache->upsertNarInfo(getUri(), hashPart, 0);
+        diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0);
 
     return valid;
 }
@@ -487,13 +490,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
 void Store::queryPathInfo(const StorePath & storePath,
     Callback<ref<const ValidPathInfo>> callback) noexcept
 {
-    std::string hashPart;
+    auto hashPart = std::string(storePath.hashPart());
 
     try {
-        hashPart = storePath.hashPart();
-
         {
-            auto res = state.lock()->pathInfoCache.get(hashPart);
+            auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
             if (res && res->isKnownNow()) {
                 stats.narInfoReadAverted++;
                 if (!res->didExist())
@@ -508,7 +509,7 @@ void Store::queryPathInfo(const StorePath & storePath,
                 stats.narInfoReadAverted++;
                 {
                     auto state_(state.lock());
-                    state_->pathInfoCache.upsert(hashPart,
+                    state_->pathInfoCache.upsert(std::string(storePath.to_string()),
                         res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
                     if (res.first == NarInfoDiskCache::oInvalid ||
                         !goodStorePath(storePath, res.second->path))
@@ -523,7 +524,7 @@ void Store::queryPathInfo(const StorePath & storePath,
     auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
     queryPathInfoUncached(storePath,
-        {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+        {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
 
             try {
                 auto info = fut.get();
@@ -533,14 +534,12 @@ void Store::queryPathInfo(const StorePath & storePath,
 
                 {
                     auto state_(state.lock());
-                    state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
+                    state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info });
                 }
 
-                auto storePath = parseStorePath(storePathS);
-
                 if (!info || !goodStorePath(storePath, info->path)) {
                     stats.narInfoMissing++;
-                    throw InvalidPath("path '%s' is not valid", storePathS);
+                    throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
                 }
 
                 (*callbackPtr)(ref<const ValidPathInfo>(info));
@@ -548,6 +547,74 @@ void Store::queryPathInfo(const StorePath & storePath,
         }});
 }
 
+void Store::queryRealisation(const DrvOutput & id,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept
+{
+
+    try {
+        if (diskCache) {
+            auto [cacheOutcome, maybeCachedRealisation]
+                = diskCache->lookupRealisation(getUri(), id);
+            switch (cacheOutcome) {
+            case NarInfoDiskCache::oValid:
+                debug("Returning a cached realisation for %s", id.to_string());
+                callback(maybeCachedRealisation);
+                return;
+            case NarInfoDiskCache::oInvalid:
+                debug(
+                    "Returning a cached missing realisation for %s",
+                    id.to_string());
+                callback(nullptr);
+                return;
+            case NarInfoDiskCache::oUnknown:
+                break;
+            }
+        }
+    } catch (...) {
+        return callback.rethrow();
+    }
+
+    auto callbackPtr
+        = std::make_shared<decltype(callback)>(std::move(callback));
+
+    queryRealisationUncached(
+        id,
+        { [this, id, callbackPtr](
+              std::future<std::shared_ptr<const Realisation>> fut) {
+            try {
+                auto info = fut.get();
+
+                if (diskCache) {
+                    if (info)
+                        diskCache->upsertRealisation(getUri(), *info);
+                    else
+                        diskCache->upsertAbsentRealisation(getUri(), id);
+                }
+
+                (*callbackPtr)(std::shared_ptr<const Realisation>(info));
+
+            } catch (...) {
+                callbackPtr->rethrow();
+            }
+        } });
+}
+
+std::shared_ptr<const Realisation> Store::queryRealisation(const DrvOutput & id)
+{
+    using RealPtr = std::shared_ptr<const Realisation>;
+    std::promise<RealPtr> promise;
+
+    queryRealisation(id,
+        {[&](std::future<RealPtr> result) {
+            try {
+                promise.set_value(result.get());
+            } catch (...) {
+                promise.set_exception(std::current_exception());
+            }
+        }});
+
+    return promise.get_future().get();
+}
 
 void Store::substitutePaths(const StorePathSet & paths)
 {
@@ -860,7 +927,7 @@ std::map<StorePath, StorePath> copyPaths(
     for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto realisation = std::get_if<Realisation>(&path.raw)) {
-            settings.requireExperimentalFeature("ca-derivations");
+            settings.requireExperimentalFeature(Xp::CaDerivations);
             toplevelRealisations.insert(*realisation);
         }
     }
@@ -892,7 +959,7 @@ std::map<StorePath, StorePath> copyPaths(
         // Don't fail if the remote doesn't support CA derivations is it might
         // not be within our control to change that, and we might still want
         // to at least copy the output paths.
-        if (e.missingFeature == "ca-derivations")
+        if (e.missingFeature == Xp::CaDerivations)
             ignoreException();
         else
             throw;

src/libstore/store-api.hh

@@ -232,7 +232,6 @@ protected:
 
     struct State
     {
-        // FIXME: fix key
         LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
     };
 
@@ -370,6 +369,14 @@ public:
     void queryPathInfo(const StorePath & path,
         Callback<ref<const ValidPathInfo>> callback) noexcept;
 
+    /* Query the information about a realisation. */
+    std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &);
+
+    /* Asynchronous version of queryRealisation(). */
+    void queryRealisation(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept;
+
+
     /* Check whether the given valid path info is sufficiently attested, by
        either being signed by a trusted public key or content-addressed, in
        order to be included in the given store.
@@ -394,11 +401,11 @@ protected:
 
     virtual void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0;
+    virtual void queryRealisationUncached(const DrvOutput &,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept = 0;
 
 public:
 
-    virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0;
-
     /* Queries the set of incoming FS references for a store path.
        The result is not cleared. */
     virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
@@ -453,7 +460,7 @@ public:
        libutil/archive.hh). */
     virtual StorePath addToStore(const string & name, const Path & srcPath,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
-        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair);
+        PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet());
 
     /* Copy the contents of a path to the store and register the
        validity the resulting path, using a constant amount of
@@ -469,7 +476,8 @@ public:
        `dump` may be drained */
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(Source & dump, const string & name,
-        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair,
+        const StorePathSet & references = StorePathSet())
     { unsupported("addToStoreFromDump"); }
 
     /* Like addToStore, but the contents written to the output path is
@@ -561,26 +569,6 @@ public:
     virtual void addIndirectRoot(const Path & path)
     { unsupported("addIndirectRoot"); }
 
-    /* Acquire the global GC lock, then immediately release it.  This
-       function must be called after registering a new permanent root,
-       but before exiting.  Otherwise, it is possible that a running
-       garbage collector doesn't see the new root and deletes the
-       stuff we've just built.  By acquiring the lock briefly, we
-       ensure that either:
-
-       - The collector is already running, and so we block until the
-         collector is finished.  The collector will know about our
-         *temporary* locks, which should include whatever it is we
-         want to register as a permanent lock.
-
-       - The collector isn't running, or it's just started but hasn't
-         acquired the GC lock yet.  In that case we get and release
-         the lock right away, then exit.  The collector scans the
-         permanent root and sees ours.
-
-       In either case the permanent root is seen by the collector. */
-    virtual void syncWithGC() { };
-
     /* Find the roots of the garbage collector.  Each root is a pair
        (link, storepath) where `link' is the path of the symlink
        outside of the Nix store that point to `storePath'. If
@@ -745,6 +733,11 @@ public:
     virtual void createUser(const std::string & userName, uid_t userId)
     { }
 
+    /*
+     * Synchronises the options of the client with those of the daemon
+     * (a no-op when there’s no daemon)
+     */
+    virtual void setOptions() { }
 protected:
 
     Stats stats;