Merge pull request #6112 from NixOS/backport-6110-to-2.6-maintenance

[Backport 2.6-maintenance] Create daemon-socket folder during install

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,7 @@
+**Release Notes**
+Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed.
+
+
+**Testing**
+
+If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master).

.github/workflows/backport.yml

@@ -0,0 +1,26 @@
+name: Backport
+on:
+  pull_request_target:
+    types: [closed, labeled]
+jobs:
+  backport:
+    name: Backport Pull Request
+    if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name))
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          # required to find all branches
+          fetch-depth: 0
+      - name: Create backport PRs
+        # should be kept in sync with `version`
+        uses: zeebe-io/backport-action@v0.0.7
+        with:
+          # Config README: https://github.com/zeebe-io/backport-action#backport-action
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_workspace: ${{ github.workspace }}
+          pull_description: |-
+            Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}.
+          # should be kept in sync with `uses`
+          version: v0.0.5

.github/workflows/ci.yml

@@ -1,26 +1,32 @@
-name: "Test"
+name: "CI"
+
 on:
   pull_request:
   push:
+
 jobs:
+
   tests:
+    needs: [check_cachix]
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
-
+    timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.4.0
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v16
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v10
+      if: needs.check_cachix.outputs.secret == 'true'
       with:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
+
   check_cachix:
     name: Cachix secret present for installer tests
     runs-on: ubuntu-latest
@@ -32,6 +38,7 @@ jobs:
         env:
           _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
         run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}"
+
   installer:
     needs: [tests, check_cachix]
     if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
@@ -39,11 +46,11 @@ jobs:
     outputs:
       installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.4.0
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v16
     - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -51,6 +58,7 @@ jobs:
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - id: prepare-installer
       run: scripts/prepare-installer-for-github-actions
+
   installer_test:
     needs: [installer, check_cachix]
     if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true'
@@ -59,10 +67,42 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2.3.4
+    - uses: actions/checkout@v2.4.0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v13
+    - uses: cachix/install-nix-action@v16
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
     - run: nix-instantiate -E 'builtins.currentTime' --eval
+
+  docker_push_image:
+    needs: [check_cachix, tests]
+    if: >-
+      github.event_name == 'push' &&
+      github.ref_name == 'master' &&
+      needs.check_cachix.outputs.secret == 'true'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v16
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - run: echo NIX_VERSION="$(nix-instantiate --eval -E '(import ./default.nix).defaultPackage.${builtins.currentSystem}.version' | tr -d \")" >> $GITHUB_ENV
+    - uses: cachix/cachix-action@v10
+      if: needs.check_cachix.outputs.secret == 'true'
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - run: nix-build -A checks.$(nix-instantiate --eval -E 'builtins.currentSystem' --json).dockerImage
+    - run: docker load -i ./result/image.tar.gz
+    - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
+    - run: docker tag nix:$NIX_VERSION nixos/nix:master
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - run: docker push nixos/nix:$NIX_VERSION
+    - run: docker push nixos/nix:master

.github/workflows/hydra_status.yml

@@ -0,0 +1,16 @@
+name: Hydra status
+on:
+  schedule:
+    - cron: "12,42 * * * *"
+  workflow_dispatch:
+jobs:
+  check_hydra_status:
+    name: Check Hydra status
+    if: github.repository_owner == 'NixOS'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2.4.0
+      with:
+        fetch-depth: 0
+    - run: bash scripts/check-hydra-status.sh
+

.gitignore

@@ -26,8 +26,6 @@ perl/Makefile.config
 
 # /scripts/
 /scripts/nix-profile.sh
-/scripts/nix-reduce-build
-/scripts/nix-http-export.cgi
 /scripts/nix-profile-daemon.sh
 
 # /src/libexpr/
@@ -40,6 +38,7 @@ perl/Makefile.config
 
 # /src/libstore/
 *.gen.*
+/src/libstore/tests/libstore-tests
 
 # /src/libutil/
 /src/libutil/tests/libutil-tests

.version

@@ -1 +1 @@
-2.4
+2.6.1

Makefile

@@ -4,11 +4,13 @@ makefiles = \
   src/libutil/local.mk \
   src/libutil/tests/local.mk \
   src/libstore/local.mk \
+  src/libstore/tests/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
   src/libexpr/local.mk \
   src/libcmd/local.mk \
   src/nix/local.mk \
+  src/nlohmann/local.mk \
   src/resolve-system-dependencies/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \

boehmgc-coroutine-sp-fallback.diff

@@ -1,8 +1,8 @@
 diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index 1cee6a0b..46c3acd9 100644
+index 4b2c429..1fb4c52 100644
 --- a/pthread_stop_world.c
 +++ b/pthread_stop_world.c
-@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void)
      struct GC_traced_stack_sect_s *traced_stack_sect;
      pthread_t self = pthread_self();
      word total_size = 0;
@@ -11,7 +11,7 @@ index 1cee6a0b..46c3acd9 100644
  
      if (!EXPECT(GC_thr_initialized, TRUE))
        GC_thr_init();
-@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void)
+@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void)
            hi = p->altstack + p->altstack_size;
            /* FIXME: Need to scan the normal stack too, but how ? */
            /* FIXME: Assume stack grows down */
@@ -22,6 +22,9 @@ index 1cee6a0b..46c3acd9 100644
 +          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
 +            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
 +          }
++          if (pthread_attr_destroy(&pattr)) {
++            ABORT("GC_push_all_stacks: pthread_attr_destroy failed!");
++          }
 +          // When a thread goes into a coroutine, we lose its original sp until
 +          // control flow returns to the thread.
 +          // While in the coroutine, the sp points outside the thread stack,

configure.ac

@@ -188,17 +188,24 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
     [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
 ])
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
 # Look for libcpuid.
+have_libcpuid=
 if test "$machine_name" = "x86_64"; then
-  PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"])
-  have_libcpuid=1
-  AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])
+    AC_ARG_ENABLE([cpuid],
+                  AS_HELP_STRING([--disable-cpuid], [Do not determine microarchitecture levels with libcpuid (relevant to x86_64 only)]))
+    if test "x$enable_cpuid" != "xno"; then
+      PKG_CHECK_MODULES([LIBCPUID], [libcpuid],
+        [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"
+         have_libcpuid=1
+         AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])]
+      )
+    fi
 fi
 AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
 
@@ -260,6 +267,8 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
+# Look for lowdown library.
+PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.8.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])

doc/manual/generate-manpage.nix

@@ -1,4 +1,4 @@
-command:
+{ command, renderLinks ? false }:
 
 with builtins;
 with import ./utils.nix;
@@ -20,7 +20,11 @@ let
            categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues def.commands)));
            listCommands = cmds:
              concatStrings (map (name:
-               "* [`${command} ${name}`](./${appendName filename name}.md) - ${cmds.${name}.description}\n")
+               "* "
+               + (if renderLinks
+                  then "[`${command} ${name}`](./${appendName filename name}.md)"
+                  else "`${command} ${name}`")
+               + " - ${cmds.${name}.description}\n")
                (attrNames cmds));
          in
          "where *subcommand* is one of the following:\n\n"
@@ -89,7 +93,7 @@ let
 in
 
 let
-  manpages = processCommand { filename = "nix"; command = "nix"; def = command; };
+  manpages = processCommand { filename = "nix"; command = "nix"; def = builtins.fromJSON command; };
   summary = concatStrings (map (manpage: "    - [${manpage.command}](command-ref/new-cli/${manpage.name})\n") manpages);
 in
 (listToAttrs manpages) // { "SUMMARY.md" = summary; }

doc/manual/generate-options.nix

@@ -8,17 +8,19 @@ concatStrings (map
     let option = options.${name}; in
     "  - `${name}`  \n\n"
     + concatStrings (map (s: "    ${s}\n") (splitLines option.description)) + "\n\n"
-    + "    **Default:** " + (
-      if option.value == "" || option.value == []
-      then "*empty*"
-      else if isBool option.value
-      then (if option.value then "`true`" else "`false`")
-      else
-        # n.b. a StringMap value type is specified as a string, but
-        # this shows the value type.  The empty stringmap is "null" in
-        # JSON, but that converts to "{ }" here.
-        (if isAttrs option.value then "`\"\"`"
-         else "`" + toString option.value + "`")) + "\n\n"
+    + (if option.documentDefault
+       then "    **Default:** " + (
+       if option.value == "" || option.value == []
+       then "*empty*"
+       else if isBool option.value
+       then (if option.value then "`true`" else "`false`")
+       else
+         # n.b. a StringMap value type is specified as a string, but
+         # this shows the value type.  The empty stringmap is "null" in
+         # JSON, but that converts to "{ }" here.
+         (if isAttrs option.value then "`\"\"`"
+          else "`" + toString option.value + "`")) + "\n\n"
+       else "    **Default:** *machine-specific*")
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
        else "")

doc/manual/local.mk

@@ -12,11 +12,13 @@ man-pages := $(foreach n, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
+# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
 	NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
-	NIX_STATE_DIR=/dummy
+	NIX_STATE_DIR=/dummy \
+	NIX_CONFIG='cores = 0'
 
 nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 
@@ -44,7 +46,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli
 
 $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix
 	@rm -rf $@
-	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))'
+	$(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }'
 
 $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix
 	@cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp
@@ -78,23 +80,23 @@ man: doc/manual/generated/man1/nix3-manpages
 all: doc/manual/generated/man1/nix3-manpages
 
 $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages
-	@mkdir -p $$(dirname $@)
-	$(trace-install) install -m 0644 $$(dirname $<)/* $$(dirname $@)
+	@mkdir -p $(DESTDIR)$$(dirname $@)
+	$(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@)
 
 doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
-	@mkdir -p $$(dirname $@)
+	@mkdir -p $(DESTDIR)$$(dirname $@)
 	$(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \
 	  name=$$(basename $$i .md); \
 	  tmpFile=$$(mktemp); \
 	  if [[ $$name = SUMMARY ]]; then continue; fi; \
 	  printf "Title: %s\n\n" "$$name" > $$tmpFile; \
 	  cat $$i >> $$tmpFile; \
-	  lowdown -sT man -M section=1 $$tmpFile -o $$(dirname $@)/$$name.1; \
+	  lowdown -sT man -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \
 	  rm $$tmpFile; \
 	done
-	touch $@
+	@touch $@
 
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md
-	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(docdir)/manual
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md)
+	$(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual
 
 endif

doc/manual/src/SUMMARY.md.in

@@ -9,6 +9,7 @@
     - [Prerequisites](installation/prerequisites-source.md)
     - [Obtaining a Source Distribution](installation/obtaining-source.md)
     - [Building Nix from Source](installation/building-source.md)
+  - [Using Nix within Docker](installation/installing-docker.md)
   - [Security](installation/nix-security.md)
     - [Single-User Mode](installation/single-user.md)
     - [Multi-User Mode](installation/multi-user.md)
@@ -70,6 +71,9 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
+  - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md)
+  - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md)
+  - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)
   - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md)

doc/manual/src/advanced-topics/distributed-builds.md

@@ -53,8 +53,8 @@ example, the following command allows you to build a derivation for
 $ uname
 Linux
 
-$ nix build \
-  '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+$ nix build --impure \
+  --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
 [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
 

doc/manual/src/command-ref/conf-file-prefix.md

@@ -16,8 +16,9 @@ By default Nix reads settings from the following places:
     will be loaded in reverse order.
 
     Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS`
-    and `XDG_CONFIG_HOME`. If these are unset, it will look in
-    `$HOME/.config/nix.conf`.
+    and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to
+    `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config`
+    as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
 
   - If `NIX_CONFIG` is set, its contents is treated as the contents of
     a configuration file.

doc/manual/src/command-ref/nix-env.md

@@ -238,7 +238,16 @@ a number of possible ways:
 
 ## Examples
 
-To install a specific version of `gcc` from the active Nix expression:
+To install a package using a specific attribute path from the active Nix expression:
+
+```console
+$ nix-env -iA gcc40mips
+installing `gcc-4.0.2'
+$ nix-env -iA xorg.xorgserver
+installing `xorg-server-1.2.0'
+```
+
+To install a specific version of `gcc` using the derivation name:
 
 ```console
 $ nix-env --install gcc-3.3.2
@@ -246,6 +255,9 @@ installing `gcc-3.3.2'
 uninstalling `gcc-3.1'
 ```
 
+Using attribute path for selecting a package is preferred,
+as it is much faster and there will not be multiple matches.
+
 Note the previously installed version is removed, since
 `--preserve-installed` was not specified.
 
@@ -256,13 +268,6 @@ $ nix-env --install gcc
 installing `gcc-3.3.2'
 ```
 
-To install using a specific attribute:
-
-```console
-$ nix-env -i -A gcc40mips
-$ nix-env -i -A xorg.xorgserver
-```
-
 To install all derivations in the Nix expression `foo.nix`:
 
 ```console
@@ -374,22 +379,29 @@ For the other flags, see `--install`.
 ## Examples
 
 ```console
-$ nix-env --upgrade gcc
+$ nix-env --upgrade -A nixpkgs.gcc
 upgrading `gcc-3.3.1' to `gcc-3.4'
 ```
 
+When there are no updates available, nothing will happen:
+
 ```console
-$ nix-env -u gcc-3.3.2 --always (switch to a specific version)
+$ nix-env --upgrade -A nixpkgs.pan
+```
+
+Using `-A` is preferred when possible, as it is faster and unambiguous but
+it is also possible to upgrade to a specific version by matching the derivation name:
+
+```console
+$ nix-env -u gcc-3.3.2 --always
 upgrading `gcc-3.4' to `gcc-3.3.2'
 ```
 
-```console
-$ nix-env --upgrade pan
-(no upgrades available, so nothing happens)
-```
+To try to upgrade everything
+(matching packages based on the part of the derivation name without version):
 
 ```console
-$ nix-env -u (try to upgrade everything)
+$ nix-env -u
 upgrading `hello-2.1.2' to `hello-2.1.3'
 upgrading `mozilla-1.2' to `mozilla-1.4'
 ```
@@ -401,7 +413,7 @@ of a derivation `x` by looking at their respective `name` attributes.
 The names (e.g., `gcc-3.3.1` are split into two parts: the package name
 (`gcc`), and the version (`3.3.1`). The version part starts after the
 first dash not followed by a letter. `x` is considered an upgrade of `y`
-if their package names match, and the version of `y` is higher that that
+if their package names match, and the version of `y` is higher than that
 of `x`.
 
 The versions are compared by splitting them into contiguous components

doc/manual/src/command-ref/nix-shell.md

@@ -11,8 +11,8 @@
   [`--command` *cmd*]
   [`--run` *cmd*]
   [`--exclude` *regexp*]
-  [--pure]
-  [--keep *name*]
+  [`--pure`]
+  [`--keep` *name*]
   {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]}
 
 # Description
@@ -101,7 +101,8 @@ The following common options are supported:
 
   - `NIX_BUILD_SHELL`\
     Shell used to start the interactive environment. Defaults to the
-    `bash` found in `PATH`.
+    `bash` found in `<nixpkgs>`, falling back to the `bash` found in
+    `PATH` if not found.
 
 # Examples
 
@@ -110,13 +111,19 @@ shell in which to build it:
 
 ```console
 $ nix-shell '<nixpkgs>' -A pan
-[nix-shell]$ unpackPhase
+[nix-shell]$ eval ${unpackPhase:-unpackPhase}
 [nix-shell]$ cd pan-*
-[nix-shell]$ configurePhase
-[nix-shell]$ buildPhase
+[nix-shell]$ eval ${configurePhase:-configurePhase}
+[nix-shell]$ eval ${buildPhase:-buildPhase}
 [nix-shell]$ ./pan/gui/pan
 ```
 
+The reason we use form `eval ${configurePhase:-configurePhase}` here is because
+those packages that override these phases do so by exporting the overridden
+values in the environment variable of the same name.
+Here bash is being told to either evaluate the contents of 'configurePhase',
+if it exists as a variable, otherwise evaluate the configurePhase function.
+
 To clear the environment first, and do some additional automatic
 initialisation of the interactive shell:
 

doc/manual/src/command-ref/nix-store.md

@@ -125,7 +125,7 @@ Special exit codes:
 
   - `104`\
     Not deterministic, the build succeeded in check mode but the
-    resulting output is not binary reproducable.
+    resulting output is not binary reproducible.
 
 With the `--keep-going` flag it's possible for multiple failures to
 occur, in this case the 1xx status codes are or combined using binary

doc/manual/src/command-ref/opt-common.md

@@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options:
     }: ...
     ```
 
-    So if you call this Nix expression (e.g., when you do `nix-env -i
+    So if you call this Nix expression (e.g., when you do `nix-env -iA
     pkgname`), the function will be called automatically using the
     value [`builtins.currentSystem`](../expressions/builtins.md) for
     the `system` argument. You can override this using `--arg`, e.g.,
-    `nix-env -i pkgname --arg system \"i686-freebsd\"`. (Note that
+    `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that
     since the argument is a Nix string literal, you have to escape the
     quotes.)
 

doc/manual/src/contributing/cli-guideline.md

@@ -3,7 +3,7 @@
 ## Goals
 
 Purpose of this document is to provide a clear direction to **help design
-delightful command line** experience. This document contain guidelines to
+delightful command line** experience. This document contains guidelines to
 follow to ensure a consistent and approachable user experience.
 
 ## Overview
@@ -103,7 +103,7 @@ impacted the most by bad user experience.
 # Help is essential
 
 Help should be built into your command line so that new users can gradually
-discover new features when they need them. 
+discover new features when they need them.
 
 ## Looking for help
 
@@ -115,7 +115,7 @@ The rules are:
 
 - Help is shown by using `--help` or `help` command (eg `nix` `--``help` or
   `nix help`).
-- For non-COMMANDs (eg. `nix` `--``help`  and `nix store` `--``help`) we **show
+- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show
   a summary** of most common use cases. Summary is presented on the STDOUT
   without any use of PAGER.
 - For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the
@@ -176,7 +176,7 @@ $ nix init --template=template#pyton
 ------------------------------------------------------------------------
 Initializing Nix project at `/path/to/here`.
       Select a template for you new project:
-          |> template#pyton
+          |> template#python
              template#python-pip
              template#python-poetry
 ```
@@ -230,17 +230,17 @@ Now **Learn** part of the output is where you educate users. You should only
 show it when you know that a build will take some time and not annoy users of
 the builds that take only few seconds.
 
-Every feature like this should go though a intensive review and testing to
-collect as much a feedback as possible and to fine tune every little detail. If
+Every feature like this should go through an intensive review and testing to
+collect as much feedback as possible and to fine tune every little detail. If
 done right this can be an awesome features beginners and advance users will
 love, but if not done perfectly it will annoy users and leave bad impression.
 
 # Input
 
-Input to a command is provided via `ARGUMENTS` and `OPTIONS`. 
+Input to a command is provided via `ARGUMENTS` and `OPTIONS`.
 
 `ARGUMENTS` represent a required input for a function. When choosing to use
-`ARGUMENT` over function please be aware of the downsides that come with it:
+`ARGUMENTS` over `OPTIONS` please be aware of the downsides that come with it:
 
 - User will need to remember the order of `ARGUMENTS`. This is not a problem if
   there is only one `ARGUMENT`.
@@ -253,7 +253,7 @@ developer consider the downsides and choose wisely.
 
 ## Naming the `OPTIONS`
 
-Then only naming convention - apart from the ones mentioned in Naming the
+The only naming convention - apart from the ones mentioned in Naming the
 `COMMANDS` section is how flags are named.
 
 Flags are a type of `OPTION` that represent an option that can be turned ON of
@@ -271,12 +271,12 @@ to improve the discoverability of possible input. A new user will most likely
 not know which `ARGUMENTS` and `OPTIONS` are required or which values are
 possible for those options.
 
-In cases, the user might not provide the input or they provide wrong input,
-rather then show the error, prompt a user with an option to find and select
+In case the user does not provide the input or they provide wrong input,
+rather than show the error, prompt a user with an option to find and select
 correct input (see examples).
 
 Prompting is of course not required when TTY is not attached to STDIN. This
-would mean that scripts wont need to handle prompt, but rather handle errors.
+would mean that scripts won't need to handle prompt, but rather handle errors.
 
 A place to use prompt and provide user with interactive select
 
@@ -300,9 +300,9 @@ going to happen.
 ```shell
 $ nix build --option substitutors https://cache.example.org
 ------------------------------------------------------------------------
-  Warning! A security related question need to be answered.
+  Warning! A security related question needs to be answered.
 ------------------------------------------------------------------------
-  The following substitutors will be used to in `my-project`: 
+  The following substitutors will be used to in `my-project`:
     - https://cache.example.org
 
   Do you allow `my-project` to use above mentioned substitutors?
@@ -311,14 +311,14 @@ $ nix build --option substitutors https://cache.example.org
 
 # Output
 
-Terminal output can be quite limiting in many ways. Which should forces us to
+Terminal output can be quite limiting in many ways. Which should force us to
 think about the experience even more. As with every design the output is a
 compromise between being terse and being verbose, between showing help to
 beginners and annoying advance users. For this it is important that we know
 what are the priorities.
 
 Nix command line should be first and foremost written with beginners in mind.
-But users wont stay beginners for long and what was once useful might quickly
+But users won't stay beginners for long and what was once useful might quickly
 become annoying. There is no golden rule that we can give in this guideline
 that would make it easier how to draw a line and find best compromise.
 
@@ -342,7 +342,7 @@ also allowing them to redirect content to a file. For example:
 ```shell
 $ nix build > build.txt
 ------------------------------------------------------------------------
-  Error! Atrribute `bin` missing at (1:94) from string.
+  Error! Attribute `bin` missing at (1:94) from string.
 ------------------------------------------------------------------------
 
   1| with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } ""
@@ -408,7 +408,7 @@ Above command clearly states that command successfully completed. And in case
 of `nix build`, which is a command that might take some time to complete, it is
 equally important to also show that a command started.
 
-## Text alignment 
+## Text alignment
 
 Text alignment is the number one design element that will present all of the
 Nix commands as a family and not as separate tools glued together.
@@ -419,7 +419,7 @@ The format we should follow is:
 $ nix COMMAND
    VERB_1 NOUN and other words
   VERB__1 NOUN and other words
-       |> Some details 
+       |> Some details
 ```
 
 Few rules that we can extract from above example:
@@ -444,13 +444,13 @@ is not even notable, therefore relying on it wouldn’t make much sense.
 
 **The bright text is much better supported** across terminals and color
 schemes. Most of the time the difference is perceived as if the bright text
-would be bold. 
+would be bold.
 
 ## Colors
 
 Humans are already conditioned by society to attach certain meaning to certain
 colors. While the meaning is not universal, a simple collection of colors is
-used to represent basic emotions. 
+used to represent basic emotions.
 
 Colors that can be used in output
 
@@ -508,7 +508,7 @@ can, with a few key strokes, be changed into and advance introspection tool.
 
 ### Progress
 
-For longer running commands we should provide and overview of the progress.
+For longer running commands we should provide and overview the progress.
 This is shown best in `nix build` example:
 
 ```shell
@@ -553,9 +553,9 @@ going to happen.
 ```shell
 $ nix build --option substitutors https://cache.example.org
 ------------------------------------------------------------------------
-  Warning! A security related question need to be answered.
+  Warning! A security related question needs to be answered.
 ------------------------------------------------------------------------
-  The following substitutors will be used to in `my-project`: 
+  The following substitutors will be used to in `my-project`:
     - https://cache.example.org
 
   Do you allow `my-project` to use above mentioned substitutors?
@@ -566,7 +566,7 @@ $ nix build --option substitutors https://cache.example.org
 
 There are many ways that you can control verbosity.
 
-Verbosity levels are: 
+Verbosity levels are:
 
 - `ERROR` (level 0)
 - `WARN` (level 1)
@@ -586,4 +586,4 @@ There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and
 
 # Appendix 1: Commands naming exceptions
 
-`nix init` and `nix repl` are well established 
+`nix init` and `nix repl` are well established

doc/manual/src/contributing/hacking.md

@@ -35,6 +35,25 @@ variables are set up so that those dependencies can be found:
 $ nix-shell
 ```
 
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop
+```
+
+To get a shell with a different compilation environment (e.g. stdenv,
+gccStdenv, clangStdenv, clang11Stdenv):
+
+```console
+$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages
+```
+
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop .#clang11StdenvPackages
+```
+
 To build Nix itself in this shell:
 
 ```console

doc/manual/src/expressions/advanced-attributes.md

@@ -237,7 +237,7 @@ Derivations can declare some infrequently used optional attributes.
   - `preferLocalBuild`\
     If this attribute is set to `true` and [distributed building is
     enabled](../advanced-topics/distributed-builds.md), then, if
-    possible, the derivaton will be built locally instead of forwarded
+    possible, the derivation will be built locally instead of forwarded
     to a remote machine. This is appropriate for trivial builders
     where the cost of doing a download or remote build would exceed
     the cost of building locally.

doc/manual/src/expressions/builtins-prefix.md

@@ -12,5 +12,5 @@ For instance, `derivation` is also available as `builtins.derivation`.
 <dl>
   <dt><code>derivation <var>attrs</var></code>;
       <code>builtins.derivation <var>attrs</var></code></dt>
-  <dd><p><var>derivation</var> in described in
+  <dd><p><var>derivation</var> is described in
          <a href="derivations.md">its own section</a>.</p></dd>

doc/manual/src/expressions/expression-syntax.md

@@ -26,7 +26,7 @@ elements (referenced from the figure by number):
     called with three arguments: `stdenv`, `fetchurl`, and `perl`. They
     are needed to build Hello, but we don't know how to build them here;
     that's why they are function arguments. `stdenv` is a package that
-    is used by almost all Nix Packages packages; it provides a
+    is used by almost all Nix Packages; it provides a
     “standard” environment consisting of the things you would expect
     in a basic Unix environment: a C/C++ compiler (GCC, to be precise),
     the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`,

doc/manual/src/expressions/language-constructs.md

@@ -284,6 +284,10 @@ The points of interest are:
     function is called with the `localServer` argument set to `true` but
     the `db4` argument set to `null`, then the evaluation fails.
 
+    Note that `->` is the [logical
+    implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication)
+    Boolean operation.
+
 2.  This is a more subtle condition: if Subversion is built with Apache
     (`httpServer`) support, then the Expat library (an XML library) used
     by Subversion should be same as the one used by Apache. This is

doc/manual/src/expressions/language-operators.md

@@ -17,12 +17,12 @@ order of precedence (from strongest to weakest binding).
 | String Concatenation     | *string1* `+` *string2*             | left          | String concatenation.                                                                                                                                                                                                         | 7          |
 | Not                      | `!` *e*                             | none          | Boolean negation.                                                                                                                                                                                                             | 8          |
 | Update                   | *e1* `//` *e2*                      | right         | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes).                                                                           | 9          |
-| Less Than                | *e1* `<` *e2*,                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
-| Less Than or Equal To    | *e1* `<=` *e2*                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
-| Greater Than             | *e1* `>` *e2*                       | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
-| Greater Than or Equal To | *e1* `>=` *e2*                      | none          | Arithmetic comparison.                                                                                                                                                                                                        | 10         |
+| Less Than                | *e1* `<` *e2*,                      | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
+| Less Than or Equal To    | *e1* `<=` *e2*                      | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
+| Greater Than             | *e1* `>` *e2*                       | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
+| Greater Than or Equal To | *e1* `>=` *e2*                      | none          | Arithmetic/lexicographic comparison.                                                                                                                                                                                                        | 10         |
 | Equality                 | *e1* `==` *e2*                      | none          | Equality.                                                                                                                                                                                                                     | 11         |
 | Inequality               | *e1* `!=` *e2*                      | none          | Inequality.                                                                                                                                                                                                                   | 11         |
 | Logical AND              | *e1* `&&` *e2*                      | left          | Logical AND.                                                                                                                                                                                                                  | 12         |
-| Logical OR               | *e1* `\|\|` *e2*                    | left          | Logical OR.                                                                                                                                                                                                                   | 13         |
-| Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to `!e1 \|\| e2`).                                                                                                                                                                            | 14         |
+| Logical OR               | *e1* <code>&#124;&#124;</code> *e2* | left          | Logical OR.                                                                                                                                                                                                                   | 13         |
+| Logical Implication      | *e1* `->` *e2*                      | none          | Logical implication (equivalent to <code>!e1 &#124;&#124; e2</code>).                                                                                                                                                                            | 14         |

doc/manual/src/expressions/language-values.md

@@ -64,7 +64,7 @@ Nix has the following basic data types:
     the start of each line. To be precise, it strips from each line a
     number of spaces equal to the minimal indentation of the string as a
     whole (disregarding the indentation of empty lines). For instance,
-    the first and second line are indented two space, while the third
+    the first and second line are indented two spaces, while the third
     line is indented four spaces. Thus, two spaces are stripped from
     each line, so the resulting string is
     
@@ -139,6 +139,13 @@ Nix has the following basic data types:
     environment variable `NIX_PATH` will be searched for the given file
     or directory name.
 
+    Antiquotation is supported in any paths except those in angle brackets.
+    `./${foo}-${bar}.nix` is a more convenient way of writing 
+    `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At
+    least one slash must appear *before* any antiquotations for this to be
+    recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division
+    operation. `./a.${foo}/b.${bar}` is a path.
+
   - *Booleans* with values `true` and `false`.
 
   - The null value, denoted as `null`.

doc/manual/src/expressions/simple-building-testing.md

@@ -1,6 +1,6 @@
 # Building and Testing
 
-You can now try to build Hello. Of course, you could do `nix-env -i
+You can now try to build Hello. Of course, you could do `nix-env -f . -iA
 hello`, but you may not want to install a possibly broken package just
 yet. The best way to test the package is by using the command
 `nix-build`, which builds a Nix expression and creates a symlink named

doc/manual/src/glossary.md

@@ -47,7 +47,7 @@
     the store object at `P` contains the path `Q` somewhere. The
     *references* of a store path are the set of store paths to which it
     has a reference.
-    
+
     A derivation can reference other derivations and sources (but not
     output paths), whereas an output path only references other output
     paths.
@@ -66,7 +66,7 @@
     is necessary to deploy whole closures, since otherwise at runtime
     files could be missing. The command `nix-store -qR` prints out
     closures of store paths.
-    
+
     As an example, if the store object at path `P` contains a reference
     to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
     references `R` then `R` is also in the closure of `P`.
@@ -98,3 +98,7 @@
     store. It can contain regular files, directories and symbolic
     links.  NARs are generated and unpacked using `nix-store --dump`
     and `nix-store --restore`.
+  - `∅` \
+    The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.
+  - `ε` \
+    The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.

doc/manual/src/installation/building-source.md

@@ -1,9 +1,9 @@
 # Building Nix from Source
 
-After unpacking or checking out the Nix sources, issue the following
-commands:
+After cloning Nix's Git repository, issue the following commands:
 
 ```console
+$ ./bootstrap.sh
 $ ./configure options...
 $ make
 $ make install
@@ -11,13 +11,6 @@ $ make install
 
 Nix requires GNU Make so you may need to invoke `gmake` instead.
 
-When building from the Git repository, these should be preceded by the
-command:
-
-```console
-$ ./bootstrap.sh
-```
-
 The installation path can be specified by passing the `--prefix=prefix`
 to `configure`. The default installation directory is `/usr/local`. You
 can change this to any location you like. You must have write permission

doc/manual/src/installation/env-variables.md

@@ -40,7 +40,7 @@ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
 > **Note**
 > 
 > You must not add the export and then do the install, as the Nix
-> installer will detect the presense of Nix configuration, and abort.
+> installer will detect the presence of Nix configuration, and abort.
 
 ## `NIX_SSL_CERT_FILE` with macOS and the Nix daemon
 

doc/manual/src/installation/installing-binary.md

@@ -119,6 +119,30 @@ this to run the installer, but it may help if you run into trouble:
 - update `/etc/synthetic.conf` to direct macOS to create a "synthetic"
   empty root directory to mount your volume
 - specify mount options for the volume in `/etc/fstab`
+  - `rw`: read-write
+  - `noauto`: prevent the system from auto-mounting the volume (so the
+    LaunchDaemon mentioned below can control mounting it, and to avoid
+    masking problems with that mounting service).
+  - `nobrowse`: prevent the Nix Store volume from showing up on your
+    desktop; also keeps Spotlight from spending resources to index
+    this volume
+  <!-- TODO:
+  - `suid`: honor setuid? surely not? ...
+  - `owners`: honor file ownership on the volume
+
+    For now I'll avoid pretending to understand suid/owners more
+    than I do. There've been some vague reports of file-ownership
+    and permission issues, particularly in cloud/VM/headless setups.
+    My pet theory is that this has something to do with these setups
+    not having a token that gets delegated to initial/admin accounts
+    on macOS. See scripts/create-darwin-volume.sh for a little more.
+
+    In any case, by Dec 4 2021, it _seems_ like some combination of
+    suid, owners, and calling diskutil enableOwnership have stopped
+    new reports from coming in. But I hesitate to celebrate because we
+    haven't really named and catalogued the behavior, understood what
+    we're fixing, and validated that all 3 components are essential.
+  -->
 - if you have FileVault enabled
     - generate an encryption password
     - put it in your system Keychain

doc/manual/src/installation/installing-docker.md

@@ -0,0 +1,59 @@
+# Using Nix within Docker
+
+To run the latest stable release of Nix with Docker run the following command:
+
+```console
+$ docker run -ti nixos/nix
+Unable to find image 'nixos/nix:latest' locally
+latest: Pulling from nixos/nix
+5843afab3874: Pull complete
+b52bf13f109c: Pull complete
+1e2415612aa3: Pull complete
+Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff
+Status: Downloaded newer image for nixos/nix:latest
+35ca4ada6e96:/# nix --version
+nix (Nix) 2.3.12
+35ca4ada6e96:/# exit
+```
+
+# What is included in Nix's Docker image?
+
+The official Docker image is created using `pkgs.dockerTools.buildLayeredImage`
+(and not with `Dockerfile` as it is usual with Docker images). You can still
+base your custom Docker image on it as you would do with any other Docker
+image.
+
+The Docker image is also not based on any other image and includes minimal set
+of runtime dependencies that are required to use Nix:
+
+ - pkgs.nix
+ - pkgs.bashInteractive
+ - pkgs.coreutils-full
+ - pkgs.gnutar
+ - pkgs.gzip
+ - pkgs.gnugrep
+ - pkgs.which
+ - pkgs.curl
+ - pkgs.less
+ - pkgs.wget
+ - pkgs.man
+ - pkgs.cacert.out
+ - pkgs.findutils
+
+# Docker image with the latest development version of Nix
+
+To get the latest image that was built by [Hydra](https://hydra.nixos.org) run
+the following command:
+
+```console
+$ curl -L https://hydra.nixos.org/job/nix/master/dockerImage.x86_64-linux/latest/download/1 | docker load
+$ docker run -ti nix:2.5pre20211105
+```
+
+You can also build a Docker image from source yourself:
+
+```console
+$ nix build ./\#hydraJobs.dockerImage.x86_64-linux
+$ docker load -i ./result/image.tar.gz
+$ docker run -ti nix:2.5pre20211105
+```

doc/manual/src/installation/installing-source.md

@@ -1,4 +1,4 @@
 # Installing Nix from Source
 
-If no binary package is available, you can download and compile a source
-distribution.
+If no binary package is available or if you want to hack on Nix, you
+can build Nix from its Git repository.

doc/manual/src/installation/obtaining-source.md

@@ -1,14 +1,9 @@
-# Obtaining a Source Distribution
+# Obtaining the Source
 
-The source tarball of the most recent stable release can be downloaded
-from the [Nix homepage](http://nixos.org/nix/download.html). You can
-also grab the [most recent development
-release](http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents).
-
-Alternatively, the most recent sources of Nix can be obtained from its
-[Git repository](https://github.com/NixOS/nix). For example, the
-following command will check out the latest revision into a directory
-called `nix`:
+The most recent sources of Nix can be obtained from its [Git
+repository](https://github.com/NixOS/nix). For example, the following
+command will check out the latest revision into a directory called
+`nix`:
 
 ```console
 $ git clone https://github.com/NixOS/nix

doc/manual/src/installation/prerequisites-source.md

@@ -2,9 +2,8 @@
 
   - GNU Autoconf (<https://www.gnu.org/software/autoconf/>) and the
     autoconf-archive macro collection
-    (<https://www.gnu.org/software/autoconf-archive/>). These are only
-    needed to run the bootstrap script, and are not necessary if your
-    source distribution came with a pre-built `./configure` script.
+    (<https://www.gnu.org/software/autoconf-archive/>). These are
+    needed to run the bootstrap script.
 
   - GNU Make.
 
@@ -26,15 +25,6 @@
     available for download from the official repository
     <https://github.com/google/brotli>.
 
-  - The bzip2 compressor program and the `libbz2` library. Thus you must
-    have bzip2 installed, including development headers and libraries.
-    If your distribution does not provide these, you can obtain bzip2
-    from
-    <https://sourceware.org/bzip2/>.
-
-  - `liblzma`, which is provided by XZ Utils. If your distribution does
-    not provide this, you can get it from <https://tukaani.org/xz/>.
-
   - cURL and its library. If your distribution does not provide it, you
     can get it from <https://curl.haxx.se/>.
 
@@ -54,6 +44,11 @@
     obtained from the its repository
     <https://github.com/troglobit/editline>.
 
+  - The `libsodium` library for verifying cryptographic signatures
+    of contents fetched from binary caches.
+    It can be obtained from the official web site
+    <https://libsodium.org>.
+
   - Recent versions of Bison and Flex to build the parser. (This is
     because Nix needs GLR support in Bison and reentrancy support in
     Flex.) For Bison, you need version 2.6, which can be obtained from
@@ -61,11 +56,18 @@
     you need version 2.5.35, which is available on
     [SourceForge](http://lex.sourceforge.net/). Slightly older versions
     may also work, but ancient versions like the ubiquitous 2.5.4a
-    won't. Note that these are only required if you modify the parser or
-    when you are building from the Git repository.
+    won't.
 
   - The `libseccomp` is used to provide syscall filtering on Linux. This
     is an optional dependency and can be disabled passing a
     `--disable-seccomp-sandboxing` option to the `configure` script (Not
     recommended unless your system doesn't support `libseccomp`). To get
     the library, visit <https://github.com/seccomp/libseccomp>.
+
+  - On 64-bit x86 machines only, `libcpuid` library
+    is used to determine which microarchitecture levels are supported
+    (e.g., as whether to have `x86_64-v2-linux` among additional system types).
+    The library is available from its homepage
+    <http://libcpuid.sourceforge.net>.
+    This is an optional dependency and can be disabled
+    by providing a `--disable-cpuid` to the `configure` script.

doc/manual/src/installation/supported-platforms.md

@@ -4,4 +4,4 @@ Nix is currently supported on the following platforms:
 
   - Linux (i686, x86\_64, aarch64).
 
-  - macOS (x86\_64).
+  - macOS (x86\_64, aarch64).

doc/manual/src/introduction.md

@@ -76,7 +76,7 @@ there after an upgrade.  This means that you can _roll back_ to the
 old version:
 
 ```console
-$ nix-env --upgrade some-packages
+$ nix-env --upgrade -A nixpkgs.some-package
 $ nix-env --rollback
 ```
 
@@ -122,12 +122,12 @@ Nix expressions generally describe how to build a package from
 source, so an installation action like
 
 ```console
-$ nix-env --install firefox
+$ nix-env --install -A nixpkgs.firefox
 ```
 
 _could_ cause quite a bit of build activity, as not only Firefox but
 also all its dependencies (all the way up to the C library and the
-compiler) would have to built, at least if they are not already in the
+compiler) would have to be built, at least if they are not already in the
 Nix store.  This is a _source deployment model_.  For most users,
 building from source is not very pleasant as it takes far too long.
 However, Nix can automatically skip building from source and instead

doc/manual/src/package-management/basic-package-mgmt.md

@@ -24,7 +24,7 @@ collection; you could write your own Nix expressions based on Nixpkgs,
 or completely new ones.)
 
 You can manually download the latest version of Nixpkgs from
-<http://nixos.org/nixpkgs/download.html>. However, it’s much more
+<https://github.com/NixOS/nixpkgs>. However, it’s much more
 convenient to use the Nixpkgs [*channel*](channels.md), since it makes
 it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
 automatically added to your list of “subscribed” channels when you
@@ -40,48 +40,52 @@ $ nix-channel --update
 > 
 > On NixOS, you’re automatically subscribed to a NixOS channel
 > corresponding to your NixOS major release (e.g.
-> <http://nixos.org/channels/nixos-14.12>). A NixOS channel is identical
+> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical
 > to the Nixpkgs channel, except that it contains only Linux binaries
 > and is updated only if a set of regression tests succeed.
 
 You can view the set of available packages in Nixpkgs:
 
 ```console
-$ nix-env -qa
-aterm-2.2
-bash-3.0
-binutils-2.15
-bison-1.875d
-blackdown-1.4.2
-bzip2-1.0.2
+$ nix-env -qaP
+nixpkgs.aterm                       aterm-2.2
+nixpkgs.bash                        bash-3.0
+nixpkgs.binutils                    binutils-2.15
+nixpkgs.bison                       bison-1.875d
+nixpkgs.blackdown                   blackdown-1.4.2
+nixpkgs.bzip2                       bzip2-1.0.2
 …
 ```
 
-The flag `-q` specifies a query operation, and `-a` means that you want
+The flag `-q` specifies a query operation, `-a` means that you want
 to show the “available” (i.e., installable) packages, as opposed to the
-installed packages. If you downloaded Nixpkgs yourself, or if you
-checked it out from GitHub, then you need to pass the path to your
-Nixpkgs tree using the `-f` flag:
+installed packages, and `-P` prints the attribute paths that can be used
+to unambiguously select a package for installation (listed in the first column).
+If you downloaded Nixpkgs yourself, or if you checked it out from GitHub,
+then you need to pass the path to your Nixpkgs tree using the `-f` flag:
 
 ```console
-$ nix-env -qaf /path/to/nixpkgs
+$ nix-env -qaPf /path/to/nixpkgs
+aterm                               aterm-2.2
+bash                                bash-3.0
+…
 ```
 
 where */path/to/nixpkgs* is where you’ve unpacked or checked out
 Nixpkgs.
 
-You can select specific packages by name:
+You can filter the packages by name:
 
 ```console
-$ nix-env -qa firefox
-firefox-34.0.5
-firefox-with-plugins-34.0.5
+$ nix-env -qaP firefox
+nixpkgs.firefox-esr                 firefox-91.3.0esr
+nixpkgs.firefox                     firefox-94.0.1
 ```
 
 and using regular expressions:
 
 ```console
-$ nix-env -qa 'firefox.*'
+$ nix-env -qaP 'firefox.*'
 ```
 
 It is also possible to see the *status* of available packages, i.e.,
@@ -89,11 +93,11 @@ whether they are installed into the user environment and/or present in
 the system:
 
 ```console
-$ nix-env -qas
+$ nix-env -qaPs
 …
--PS bash-3.0
---S binutils-2.15
-IPS bison-1.875d
+-PS  nixpkgs.bash                bash-3.0
+--S  nixpkgs.binutils            binutils-2.15
+IPS  nixpkgs.bison               bison-1.875d
 …
 ```
 
@@ -106,13 +110,13 @@ which is Nix’s mechanism for doing binary deployment. It just means that
 Nix knows that it can fetch a pre-built package from somewhere
 (typically a network server) instead of building it locally.
 
-You can install a package using `nix-env -i`. For instance,
+You can install a package using `nix-env -iA`. For instance,
 
 ```console
-$ nix-env -i subversion
+$ nix-env -iA nixpkgs.subversion
 ```
 
-will install the package called `subversion` (which is, of course, the
+will install the package called `subversion` from `nixpkgs` channel (which is, of course, the
 [Subversion version management system](http://subversion.tigris.org/)).
 
 > **Note**
@@ -122,7 +126,7 @@ will install the package called `subversion` (which is, of course, the
 > binary cache <https://cache.nixos.org>; it contains binaries for most
 > packages in Nixpkgs. Only if no binary is available in the binary
 > cache, Nix will build the package from source. So if `nix-env
-> -i subversion` results in Nix building stuff from source, then either
+> -iA nixpkgs.subversion` results in Nix building stuff from source, then either
 > the package is not built for your platform by the Nixpkgs build
 > servers, or your version of Nixpkgs is too old or too new. For
 > instance, if you have a very recent checkout of Nixpkgs, then the
@@ -133,7 +137,10 @@ will install the package called `subversion` (which is, of course, the
 > using a Git checkout of the Nixpkgs tree), you will get binaries for
 > most packages.
 
-Naturally, packages can also be uninstalled:
+Naturally, packages can also be uninstalled. Unlike when installing, you will
+need to use the derivation name (though the version part can be omitted),
+instead of the attribute path, as `nix-env` does not record which attribute
+was used for installing:
 
 ```console
 $ nix-env -e subversion
@@ -143,7 +150,7 @@ Upgrading to a new version is just as easy. If you have a new release of
 Nix Packages, you can do:
 
 ```console
-$ nix-env -u subversion
+$ nix-env -uA nixpkgs.subversion
 ```
 
 This will *only* upgrade Subversion if there is a “newer” version in the

doc/manual/src/package-management/binary-cache-substituter.md

@@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is
 not part of the Nix distribution, but you can install it from Nixpkgs:
 
 ```console
-$ nix-env -i nix-serve
+$ nix-env -iA nixpkgs.nix-serve
 ```
 
 You can then start the server, listening for HTTP connections on
@@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using
 `--option extra-binary-caches`, e.g.:
 
 ```console
-$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/
+$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/
 ```
 
 The option `extra-binary-caches` tells Nix to use this binary cache in

doc/manual/src/package-management/garbage-collection.md

@@ -44,7 +44,7 @@ collector as follows:
 $ nix-store --gc
 ```
 
-The behaviour of the gargage collector is affected by the
+The behaviour of the garbage collector is affected by the
 `keep-derivations` (default: true) and `keep-outputs` (default: false)
 options in the Nix configuration file. The defaults will ensure that all
 derivations that are build-time dependencies of garbage collector roots

doc/manual/src/package-management/profiles.md

@@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This
 would be what we would obtain if we had done
 
 ```console
-$ nix-env -i subversion
+$ nix-env -iA nixpkgs.subversion
 ```
 
 on a set of Nix expressions that contained Subversion 1.1.2.
@@ -54,7 +54,7 @@ environment is generated based on the current one. For instance,
 generation 43 was created from generation 42 when we did
 
 ```console
-$ nix-env -i subversion firefox
+$ nix-env -iA nixpkgs.subversion nixpkgs.firefox
 ```
 
 on a set of Nix expressions that contained Firefox and a new version of
@@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by
 (abbreviation `-p`):
 
 ```console
-$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion
+$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion
 ```
 
 This will *not* change the `~/.nix-profile` symlink.

doc/manual/src/package-management/ssh-substituter.md

@@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are
 available on the server `avalon`:
 
 ```console
-$ nix-env -i firefox --substituters ssh://alice@avalon
+$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon
 ```
 
 This works similar to the binary cache substituter that Nix usually

doc/manual/src/quick-start.md

@@ -19,19 +19,19 @@ to subsequent chapters.
    channel:
 
    ```console
-   $ nix-env -qa
-   docbook-xml-4.3
-   docbook-xml-4.5
-   firefox-33.0.2
-   hello-2.9
-   libxslt-1.1.28
+   $ nix-env -qaP
+   nixpkgs.docbook_xml_dtd_43                    docbook-xml-4.3
+   nixpkgs.docbook_xml_dtd_45                    docbook-xml-4.5
+   nixpkgs.firefox                               firefox-33.0.2
+   nixpkgs.hello                                 hello-2.9
+   nixpkgs.libxslt                               libxslt-1.1.28
    …
    ```
 
 1. Install some packages from the channel:
 
    ```console
-   $ nix-env -i hello
+   $ nix-env -iA nixpkgs.hello
    ```
 
    This should download pre-built packages; it should not build them

doc/manual/src/release-notes/rl-2.4.md

@@ -1,8 +1,542 @@
-# Release 2.4 (202X-XX-XX)
+# Release 2.4 (2021-11-01)
 
-  - It is now an error to modify the `plugin-files` setting via a
-    command-line flag that appears after the first non-flag argument
-    to any command, including a subcommand to `nix`. For example,
-    `nix-instantiate default.nix --plugin-files ""` must now become
-    `nix-instantiate --plugin-files "" default.nix`.
-  - Plugins that add new `nix` subcommands are now actually respected.
+This is the first release in more than two years and is the result of
+more than 2800 commits from 195 contributors since release 2.3.
+
+## Highlights
+
+* Nix's **error messages** have been improved a lot. For instance,
+  evaluation errors now point out the location of the error:
+
+  ```
+  $ nix build
+  error: undefined variable 'bzip3'
+
+         at /nix/store/449lv242z0zsgwv95a8124xi11sp419f-source/flake.nix:88:13:
+
+             87|           [ curl
+             88|             bzip3 xz brotli editline
+               |             ^
+             89|             openssl sqlite
+  ```
+
+* The **`nix` command** has seen a lot of work and is now almost at
+  feature parity with the old command-line interface (the `nix-*`
+  commands). It aims to be [more modern, consistent and pleasant to
+  use](../contributing/cli-guideline.md) than the old CLI. It is still
+  marked as experimental but its interface should not change much
+  anymore in future releases.
+
+* **Flakes** are a new format to package Nix-based projects in a more
+  discoverable, composable, consistent and reproducible way. A flake
+  is just a repository or tarball containing a file named `flake.nix`
+  that specifies dependencies on other flakes and returns any Nix
+  assets such as packages, Nixpkgs overlays, NixOS modules or CI
+  tests. The new `nix` CLI is primarily based around flakes; for
+  example, a command like `nix run nixpkgs#hello` runs the `hello`
+  application from the `nixpkgs` flake.
+
+  Flakes are currently marked as experimental. For an introduction,
+  see [this blog
+  post](https://www.tweag.io/blog/2020-05-25-flakes/). For detailed
+  information about flake syntax and semantics, see the [`nix flake`
+  manual page](../command-ref/new-cli/nix3-flake.md).
+
+* Nix's store can now be **content-addressed**, meaning that the hash
+  component of a store path is the hash of the path's
+  contents. Previously Nix could only build **input-addressed** store
+  paths, where the hash is computed from the derivation dependency
+  graph. Content-addressing allows deduplication, early cutoff in
+  build systems, and unprivileged closure copying. This is still [an
+  experimental
+  feature](https://discourse.nixos.org/t/content-addressed-nix-call-for-testers/12881).
+
+* The Nix manual has been converted into Markdown, making it easier to
+  contribute. In addition, every `nix` subcommand now has a manual
+  page, documenting every option.
+
+* A new setting that allows **experimental features** to be enabled
+  selectively. This allows us to merge unstable features into Nix more
+  quickly and do more frequent releases.
+
+## Other features
+
+* There are many new `nix` subcommands:
+
+  - `nix develop` is intended to replace `nix-shell`. It has a number
+    of new features:
+
+    * It automatically sets the output environment variables (such as
+      `$out`) to writable locations (such as `./outputs/out`).
+
+    * It can store the environment in a profile. This is useful for
+      offline work.
+
+    * It can run specific phases directly. For instance, `nix develop
+      --build` runs `buildPhase`.
+
+    - It allows dependencies in the Nix store to be "redirected" to
+      arbitrary directories using the `--redirect` flag. This is
+      useful if you want to hack on a package *and* some of its
+      dependencies at the same time.
+
+  - `nix print-dev-env` prints the environment variables and bash
+    functions defined by a derivation. This is useful for users of
+    other shells than bash (especially with `--json`).
+
+  - `nix shell` was previously named `nix run` and is intended to
+    replace `nix-shell -p`, but without the `stdenv` overhead. It
+    simply starts a shell where some packages have been added to
+    `$PATH`.
+
+  - `nix run` (not to be confused with the old subcommand that has
+    been renamed to `nix shell`) runs an "app", a flake output that
+    specifies a command to run, or an eponymous program from a
+    package. For example, `nix run nixpkgs#hello` runs the `hello`
+    program from the `hello` package in `nixpkgs`.
+
+  - `nix flake` is the container for flake-related operations, such as
+    creating a new flake, querying the contents of a flake or updating
+    flake lock files.
+
+  - `nix registry` allows you to query and update the flake registry,
+    which maps identifiers such as `nixpkgs` to concrete flake URLs.
+
+  - `nix profile` is intended to replace `nix-env`. Its main advantage
+    is that it keeps track of the provenance of installed packages
+    (e.g. exactly which flake version a package came from). It also
+    has some helpful subcommands:
+
+    * `nix profile history` shows what packages were added, upgraded
+      or removed between each version of a profile.
+
+    * `nix profile diff-closures` shows the changes between the
+      closures of each version of a profile. This allows you to
+      discover the addition or removal of dependencies or size
+      changes.
+
+    **Warning**: after a profile has been updated using `nix profile`,
+    it is no longer usable with `nix-env`.
+
+  - `nix store diff-closures` shows the differences between the
+    closures of two store paths in terms of the versions and sizes of
+    dependencies in the closures.
+
+  - `nix store make-content-addressable` rewrites an arbitrary closure
+    to make it content-addressed. Such paths can be copied into other
+    stores without requiring signatures.
+
+  - `nix bundle` uses the [`nix-bundle`
+    program](https://github.com/matthewbauer/nix-bundle) to convert a
+    closure into a self-extracting executable.
+
+  - Various other replacements for the old CLI, e.g. `nix store gc`,
+    `nix store delete`, `nix store repair`, `nix nar dump-path`, `nix
+    store prefetch-file`, `nix store prefetch-tarball`, `nix key` and
+    `nix daemon`.
+
+* Nix now has an **evaluation cache** for flake outputs. For example,
+  a second invocation of the command `nix run nixpkgs#firefox` will
+  not need to evaluate the `firefox` attribute because it's already in
+  the evaluation cache. This is made possible by the hermetic
+  evaluation model of flakes.
+
+* The new `--offline` flag disables substituters and causes all
+  locally cached tarballs and repositories to be considered
+  up-to-date.
+
+* The new `--refresh` flag causes all locally cached tarballs and
+  repositories to be considered out-of-date.
+
+* Many `nix` subcommands now have a `--json` option to produce
+  machine-readable output.
+
+* `nix repl` has a new `:doc` command to show documentation about
+  builtin functions (e.g. `:doc builtins.map`).
+
+* Binary cache stores now have an option `index-debug-info` to create
+  an index of DWARF debuginfo files for use by
+  [`dwarffs`](https://github.com/edolstra/dwarffs).
+
+* To support flakes, Nix now has an extensible mechanism for fetching
+  source trees. Currently it has the following backends:
+
+  * Git repositories
+
+  * Mercurial repositories
+
+  * GitHub and GitLab repositories (an optimisation for faster
+    fetching than Git)
+
+  * Tarballs
+
+  * Arbitrary directories
+
+  The fetcher infrastructure is exposed via flake input specifications
+  and via the `fetchTree` built-in.
+
+* **Languages changes**: the only new language feature is that you can
+  now have antiquotations in paths, e.g. `./${foo}` instead of `./. +
+  foo`.
+
+* **New built-in functions**:
+
+  - `builtins.fetchTree` allows fetching a source tree using any
+    backends supported by the fetcher infrastructure. It subsumes the
+    functionality of existing built-ins like `fetchGit`,
+    `fetchMercurial` and `fetchTarball`.
+
+  - `builtins.getFlake` fetches a flake and returns its output
+    attributes. This function should not be used inside flakes! Use
+    flake inputs instead.
+
+  - `builtins.floor` and `builtins.ceil` round a floating-point number
+    down and up, respectively.
+
+* Experimental support for recursive Nix. This means that Nix
+  derivations can now call Nix to build other derivations. This is not
+  in a stable state yet and not well
+  [documented](https://github.com/NixOS/nix/commit/c4d7c76b641d82b2696fef73ce0ac160043c18da).
+
+* The new experimental feature `no-url-literals` disables URL
+  literals. This helps to implement [RFC
+  45](https://github.com/NixOS/rfcs/pull/45).
+
+* Nix now uses `libarchive` to decompress and unpack tarballs and zip
+  files, so `tar` is no longer required.
+
+* The priority of substituters can now be overridden using the
+  `priority` substituter setting (e.g. `--substituters
+  'http://cache.nixos.org?priority=100 daemon?priority=10'`).
+
+* `nix edit` now supports non-derivation attributes, e.g. `nix edit
+  .#nixosConfigurations.bla`.
+
+* The `nix` command now provides command line completion for `bash`,
+  `zsh` and `fish`. Since the support for getting completions is built
+  into `nix`, it's easy to add support for other shells.
+
+* The new `--log-format` flag selects what Nix's output looks like. It
+  defaults to a terse progress indicator. There is a new
+  `internal-json` output format for use by other programs.
+
+* `nix eval` has a new `--apply` flag that applies a function to the
+  evaluation result.
+
+* `nix eval` has a new `--write-to` flag that allows it to write a
+  nested attribute set of string leaves to a corresponding directory
+  tree.
+
+* Memory improvements: many operations that add paths to the store or
+  copy paths between stores now run in constant memory.
+
+* Many `nix` commands now support the flag `--derivation` to operate
+  on a `.drv` file itself instead of its outputs.
+
+* There is a new store called `dummy://` that does not support
+  building or adding paths. This is useful if you want to use the Nix
+  evaluator but don't have a Nix store.
+
+* The `ssh-ng://` store now allows substituting paths on the remote,
+  as `ssh://` already did.
+
+* When auto-calling a function with an ellipsis, all arguments are now
+  passed.
+
+* New `nix-shell` features:
+
+  - It preserves the `PS1` environment variable if
+    `NIX_SHELL_PRESERVE_PROMPT` is set.
+
+  - With `-p`, it passes any `--arg`s as Nixpkgs arguments.
+
+  - Support for structured attributes.
+
+* `nix-prefetch-url` has a new `--executable` flag.
+
+* On `x86_64` systems, [`x86_64` microarchitecture
+  levels](https://lwn.net/Articles/844831/) are mapped to additional
+  system types (e.g. `x86_64-v1-linux`).
+
+* The new `--eval-store` flag allows you to use a different store for
+  evaluation than for building or storing the build result. This is
+  primarily useful when you want to query whether something exists in
+  a read-only store, such as a binary cache:
+
+  ```
+  # nix path-info --json --store https://cache.nixos.org \
+    --eval-store auto nixpkgs#hello
+  ```
+
+  (Here `auto` indicates the local store.)
+
+* The Nix daemon has a new low-latency mechanism for copying
+  closures. This is useful when building on remote stores such as
+  `ssh-ng://`.
+
+* Plugins can now register `nix` subcommands.
+
+* The `--indirect` flag to `nix-store --add-root` has become a no-op.
+  `--add-root` will always generate indirect GC roots from now on.
+
+## Incompatible changes
+
+* The `nix` command is now marked as an experimental feature. This
+  means that you need to add
+
+  ```
+  experimental-features = nix-command
+  ```
+
+  to your `nix.conf` if you want to use it, or pass
+  `--extra-experimental-features nix-command` on the command line.
+
+* The `nix` command no longer has a syntax for referring to packages
+  in a channel. This means that the following no longer works:
+
+  ```console
+  nix build nixpkgs.hello # Nix 2.3
+  ```
+
+  Instead, you can either use the `#` syntax to select a package from
+  a flake, e.g.
+
+  ```console
+  nix build nixpkgs#hello
+  ```
+
+  Or, if you want to use the `nixpkgs` channel in the `NIX_PATH`
+  environment variable:
+
+  ```console
+  nix build -f '<nixpkgs>' hello
+  ```
+
+* The old `nix run` has been renamed to `nix shell`, while there is a
+  new `nix run` that runs a default command. So instead of
+
+  ```console
+  nix run nixpkgs.hello -c hello # Nix 2.3
+  ```
+
+  you should use
+
+  ```console
+  nix shell nixpkgs#hello -c hello
+  ```
+
+  or just
+
+  ```console
+  nix run nixpkgs#hello
+  ```
+
+  if the command you want to run has the same name as the package.
+
+* It is now an error to modify the `plugin-files` setting via a
+  command-line flag that appears after the first non-flag argument to
+  any command, including a subcommand to `nix`. For example,
+  `nix-instantiate default.nix --plugin-files ""` must now become
+  `nix-instantiate --plugin-files "" default.nix`.
+
+* We no longer release source tarballs. If you want to build from
+  source, please build from the tags in the Git repository.
+
+## Contributors
+
+This release has contributions from
+Adam Höse,
+Albert Safin,
+Alex Kovar,
+Alex Zero,
+Alexander Bantyev,
+Alexandre Esteves,
+Alyssa Ross,
+Anatole Lucet,
+Anders Kaseorg,
+Andreas Rammhold,
+Antoine Eiche,
+Antoine Martin,
+Arnout Engelen,
+Arthur Gautier,
+aszlig,
+Ben Burdette,
+Benjamin Hipple,
+Bernardo Meurer,
+Björn Gohla,
+Bjørn Forsman,
+Bob van der Linden,
+Brian Leung,
+Brian McKenna,
+Brian Wignall,
+Bruce Toll,
+Bryan Richter,
+Calle Rosenquist,
+Calvin Loncaric,
+Carlo Nucera,
+Carlos D'Agostino,
+Chaz Schlarp,
+Christian Höppner,
+Christian Kampka,
+Chua Hou,
+Chuck,
+Cole Helbling,
+Daiderd Jordan,
+Dan Callahan,
+Dani,
+Daniel Fitzpatrick,
+Danila Fedorin,
+Daniël de Kok,
+Danny Bautista,
+DavHau,
+David McFarland,
+Dima,
+Domen Kožar,
+Dominik Schrempf,
+Dominique Martinet,
+dramforever,
+Dustin DeWeese,
+edef,
+Eelco Dolstra,
+Ellie Hermaszewska,
+Emilio Karakey,
+Emily,
+Eric Culp,
+Ersin Akinci,
+Fabian Möller,
+Farid Zakaria,
+Federico Pellegrin,
+Finn Behrens,
+Florian Franzen,
+Félix Baylac-Jacqué,
+Gabriella Gonzalez,
+Geoff Reedy,
+Georges Dubus,
+Graham Christensen,
+Greg Hale,
+Greg Price,
+Gregor Kleen,
+Gregory Hale,
+Griffin Smith,
+Guillaume Bouchard,
+Harald van Dijk,
+illustris,
+Ivan Zvonimir Horvat,
+Jade,
+Jake Waksbaum,
+jakobrs,
+James Ottaway,
+Jan Tojnar,
+Janne Heß,
+Jaroslavas Pocepko,
+Jarrett Keifer,
+Jeremy Schlatter,
+Joachim Breitner,
+Joe Pea,
+John Ericson,
+Jonathan Ringer,
+Josef Kemetmüller,
+Joseph Lucas,
+Jude Taylor,
+Julian Stecklina,
+Julien Tanguy,
+Jörg Thalheim,
+Kai Wohlfahrt,
+keke,
+Keshav Kini,
+Kevin Quick,
+Kevin Stock,
+Kjetil Orbekk,
+Krzysztof Gogolewski,
+kvtb,
+Lars Mühmel,
+Leonhard Markert,
+Lily Ballard,
+Linus Heckemann,
+Lorenzo Manacorda,
+Lucas Desgouilles,
+Lucas Franceschino,
+Lucas Hoffmann,
+Luke Granger-Brown,
+Madeline Haraj,
+Marwan Aljubeh,
+Mat Marini,
+Mateusz Piotrowski,
+Matthew Bauer,
+Matthew Kenigsberg,
+Mauricio Scheffer,
+Maximilian Bosch,
+Michael Adler,
+Michael Bishop,
+Michael Fellinger,
+Michael Forney,
+Michael Reilly,
+mlatus,
+Mykola Orliuk,
+Nathan van Doorn,
+Naïm Favier,
+ng0,
+Nick Van den Broeck,
+Nicolas Stig124 Formichella,
+Niels Egberts,
+Niklas Hambüchen,
+Nikola Knezevic,
+oxalica,
+p01arst0rm,
+Pamplemousse,
+Patrick Hilhorst,
+Paul Opiyo,
+Pavol Rusnak,
+Peter Kolloch,
+Philipp Bartsch,
+Philipp Middendorf,
+Piotr Szubiakowski,
+Profpatsch,
+Puck Meerburg,
+Ricardo M. Correia,
+Rickard Nilsson,
+Robert Hensing,
+Robin Gloster,
+Rodrigo,
+Rok Garbas,
+Ronnie Ebrin,
+Rovanion Luckey,
+Ryan Burns,
+Ryan Mulligan,
+Ryne Everett,
+Sam Doshi,
+Sam Lidder,
+Samir Talwar,
+Samuel Dionne-Riel,
+Sebastian Ullrich,
+Sergei Trofimovich,
+Sevan Janiyan,
+Shao Cheng,
+Shea Levy,
+Silvan Mosberger,
+Stefan Frijters,
+Stefan Jaax,
+sternenseemann,
+Steven Shaw,
+Stéphan Kochen,
+SuperSandro2000,
+Suraj Barkale,
+Taeer Bar-Yam,
+Thomas Churchman,
+Théophane Hufschmitt,
+Timothy DeHerrera,
+Timothy Klim,
+Tobias Möst,
+Tobias Pflug,
+Tom Bereknyei,
+Travis A. Everett,
+Ujjwal Jain,
+Vladimír Čunát,
+Wil Taylor,
+Will Dietz,
+Yaroslav Bolyukin,
+Yestin L. Harrison,
+YI,
+Yorick van Pelt,
+Yuriy Taraday and
+zimbatm.

doc/manual/src/release-notes/rl-2.5.md

@@ -0,0 +1,16 @@
+# Release 2.5 (2021-12-13)
+
+* The garbage collector no longer blocks new builds, so the message
+  `waiting for the big garbage collector lock...` is a thing of the
+  past.
+
+* Binary cache stores now have a setting `compression-level`.
+
+* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
+
+* Lists can now be compared lexicographically using the `<` operator.
+
+* New built-in function: `builtins.groupBy`, with the same functionality as
+  Nixpkgs' `lib.groupBy`, but faster.
+
+* `nix repl` now has a `:log` command.

doc/manual/src/release-notes/rl-2.6.md

@@ -0,0 +1,21 @@
+# Release 2.6 (2022-01-24)
+
+* The Nix CLI now searches for a `flake.nix` up until the root of the current
+  Git repository or a filesystem boundary rather than just in the current
+  directory.
+* The TOML parser used by `builtins.fromTOML` has been replaced by [a
+  more compliant one](https://github.com/ToruNiina/toml11).
+* Added `:st`/`:show-trace` commands to `nix repl`, which are used to
+  set or toggle display of error traces.
+* New builtin function `builtins.zipAttrsWith` with the same
+  functionality as `lib.zipAttrsWith` from Nixpkgs, but much more
+  efficient.
+* New command `nix store copy-log` to copy build logs from one store
+  to another.
+* The `commit-lockfile-summary` option can be set to a non-empty
+  string to override the commit summary used when commiting an updated
+  lockfile.  This may be used in conjunction with the `nixConfig`
+  attribute in `flake.nix` to better conform to repository
+  conventions.
+* `docker run -ti nixos/nix:master` will place you in the Docker
+  container with the latest version of Nix from the `master` branch.

doc/manual/src/release-notes/rl-next.md

@@ -0,0 +1,2 @@
+# Release X.Y (202?-??-??)
+

docker.nix

@@ -0,0 +1,264 @@
+{ pkgs ? import <nixpkgs> { }
+, lib ? pkgs.lib
+, name ? "nix"
+, tag ? "latest"
+, channelName ? "nixpkgs"
+, channelURL ? "https://nixos.org/channels/nixpkgs-unstable"
+}:
+let
+  defaultPkgs = with pkgs; [
+    nix
+    bashInteractive
+    coreutils-full
+    gnutar
+    gzip
+    gnugrep
+    which
+    curl
+    less
+    wget
+    man
+    cacert.out
+    findutils
+    iana-etc
+    git
+  ];
+
+  users = {
+
+    root = {
+      uid = 0;
+      shell = "/bin/bash";
+      home = "/root";
+      gid = 0;
+    };
+
+  } // lib.listToAttrs (
+    map
+      (
+        n: {
+          name = "nixbld${toString n}";
+          value = {
+            uid = 30000 + n;
+            gid = 30000;
+            groups = [ "nixbld" ];
+            description = "Nix build user ${toString n}";
+          };
+        }
+      )
+      (lib.lists.range 1 32)
+  );
+
+  groups = {
+    root.gid = 0;
+    nixbld.gid = 30000;
+  };
+
+  userToPasswd = (
+    k:
+    { uid
+    , gid ? 65534
+    , home ? "/var/empty"
+    , description ? ""
+    , shell ? "/bin/false"
+    , groups ? [ ]
+    }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}"
+  );
+  passwdContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs userToPasswd users))
+  );
+
+  userToShadow = k: { ... }: "${k}:!:1::::::";
+  shadowContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs userToShadow users))
+  );
+
+  # Map groups to members
+  # {
+  #   group = [ "user1" "user2" ];
+  # }
+  groupMemberMap = (
+    let
+      # Create a flat list of user/group mappings
+      mappings = (
+        builtins.foldl'
+          (
+            acc: user:
+              let
+                groups = users.${user}.groups or [ ];
+              in
+              acc ++ map
+                (group: {
+                  inherit user group;
+                })
+                groups
+          )
+          [ ]
+          (lib.attrNames users)
+      );
+    in
+    (
+      builtins.foldl'
+        (
+          acc: v: acc // {
+            ${v.group} = acc.${v.group} or [ ] ++ [ v.user ];
+          }
+        )
+        { }
+        mappings)
+  );
+
+  groupToGroup = k: { gid }:
+    let
+      members = groupMemberMap.${k} or [ ];
+    in
+    "${k}:x:${toString gid}:${lib.concatStringsSep "," members}";
+  groupContents = (
+    lib.concatStringsSep "\n"
+      (lib.attrValues (lib.mapAttrs groupToGroup groups))
+  );
+
+  nixConf = {
+    sandbox = "false";
+    build-users-group = "nixbld";
+    trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
+  };
+  nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n";
+
+  baseSystem =
+    let
+      nixpkgs = pkgs.path;
+      channel = pkgs.runCommand "channel-nixos" { } ''
+        mkdir $out
+        ln -s ${nixpkgs} $out/nixpkgs
+        echo "[]" > $out/manifest.nix
+      '';
+      rootEnv = pkgs.buildPackages.buildEnv {
+        name = "root-profile-env";
+        paths = defaultPkgs;
+      };
+      manifest = pkgs.buildPackages.runCommand "manifest.nix" { } ''
+        cat > $out <<EOF
+        [
+        ${lib.concatStringsSep "\n" (builtins.map (drv: let
+          outputs = drv.outputsToInstall or [ "out" ];
+        in ''
+          {
+            ${lib.concatStringsSep "\n" (builtins.map (output: ''
+              ${output} = { outPath = "${lib.getOutput output drv}"; };
+            '') outputs)}
+            outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ];
+            name = "${drv.name}";
+            outPath = "${drv}";
+            system = "${drv.system}";
+            type = "derivation";
+            meta = { };
+          }
+        '') defaultPkgs)}
+        ]
+        EOF
+      '';
+      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+        mkdir $out
+        cp -a ${rootEnv}/* $out/
+        ln -s ${manifest} $out/manifest.nix
+      '';
+    in
+    pkgs.runCommand "base-system"
+      {
+        inherit passwdContents groupContents shadowContents nixConfContents;
+        passAsFile = [
+          "passwdContents"
+          "groupContents"
+          "shadowContents"
+          "nixConfContents"
+        ];
+        allowSubstitutes = false;
+        preferLocalBuild = true;
+      } ''
+      env
+      set -x
+      mkdir -p $out/etc
+
+      mkdir -p $out/etc/ssl/certs
+      ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs
+
+      cat $passwdContentsPath > $out/etc/passwd
+      echo "" >> $out/etc/passwd
+
+      cat $groupContentsPath > $out/etc/group
+      echo "" >> $out/etc/group
+
+      cat $shadowContentsPath > $out/etc/shadow
+      echo "" >> $out/etc/shadow
+
+      mkdir -p $out/usr
+      ln -s /nix/var/nix/profiles/share $out/usr/
+
+      mkdir -p $out/nix/var/nix/gcroots
+
+      mkdir $out/tmp
+
+      mkdir -p $out/var/tmp
+
+      mkdir -p $out/etc/nix
+      cat $nixConfContentsPath > $out/etc/nix/nix.conf
+
+      mkdir -p $out/root
+      mkdir -p $out/nix/var/nix/profiles/per-user/root
+
+      ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
+      ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+      ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
+
+      ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
+      ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
+
+      mkdir -p $out/root/.nix-defexpr
+      ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
+      echo "${channelURL} ${channelName}" > $out/root/.nix-channels
+
+      mkdir -p $out/bin $out/usr/bin
+      ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
+      ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
+    '';
+
+in
+pkgs.dockerTools.buildLayeredImageWithNixDb {
+
+  inherit name tag;
+
+  contents = [ baseSystem ];
+
+  extraCommands = ''
+    rm -rf nix-support
+    ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles
+  '';
+  fakeRootCommands = ''
+    chmod 1777 tmp
+    chmod 1777 var/tmp
+  '';
+
+  config = {
+    Cmd = [ "/root/.nix-profile/bin/bash" ];
+    Env = [
+      "USER=root"
+      "PATH=${lib.concatStringsSep ":" [
+        "/root/.nix-profile/bin"
+        "/nix/var/nix/profiles/default/bin"
+        "/nix/var/nix/profiles/default/sbin"
+      ]}"
+      "MANPATH=${lib.concatStringsSep ":" [
+        "/root/.nix-profile/share/man"
+        "/nix/var/nix/profiles/default/share/man"
+      ]}"
+      "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+      "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels"
+    ];
+  };
+
+}

flake.lock

@@ -3,27 +3,26 @@
     "lowdown-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1617481909,
-        "narHash": "sha256-SqnfOFuLuVRRNeVJr1yeEPJue/qWoCp5N6o5Kr///p4=",
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
         "owner": "kristapsdz",
         "repo": "lowdown",
-        "rev": "148f9b2f586c41b7e36e73009db43ea68c7a1a4d",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
         "type": "github"
       },
       "original": {
         "owner": "kristapsdz",
-        "ref": "VERSION_0_8_4",
         "repo": "lowdown",
         "type": "github"
       }
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1628689438,
-        "narHash": "sha256-YMINW6YmubHZVdliGsAJpnnMYXRrvppv59LgwtnyYhs=",
+        "lastModified": 1632864508,
+        "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "f6551e1efa261568c82b76c3a582b2c2ceb1f53f",
+        "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
         "type": "github"
       },
       "original": {
@@ -32,10 +31,26 @@
         "type": "indirect"
       }
     },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "indirect"
+      }
+    },
     "root": {
       "inputs": {
         "lowdown-src": "lowdown-src",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-regression": "nixpkgs-regression"
       }
     }
   },

flake.nix

@@ -2,9 +2,10 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small";
-  inputs.lowdown-src = { url = "github:kristapsdz/lowdown/VERSION_0_8_4"; flake = false; };
+  inputs.nixpkgs-regression.url = "nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
+  inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
-  outputs = { self, nixpkgs, lowdown-src }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }:
 
     let
 
@@ -14,7 +15,7 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
@@ -22,15 +23,36 @@
 
       crossSystems = [ "armv6l-linux" "armv7l-linux" ];
 
+      stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ];
+
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+      forAllSystemsAndStdenvs = f: forAllSystems (system:
+        nixpkgs.lib.listToAttrs
+          (map
+            (n:
+            nixpkgs.lib.nameValuePair "${n}Packages" (
+              f system n
+            )) stdenvs
+          )
+      );
+
+      forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv);
 
       # Memoize nixpkgs for different platforms for efficiency.
-      nixpkgsFor = forAllSystems (system:
-        import nixpkgs {
-          inherit system;
-          overlays = [ self.overlay ];
-        }
-      );
+      nixpkgsFor =
+        let stdenvsPackages = forAllSystemsAndStdenvs
+          (system: stdenv:
+            import nixpkgs {
+              inherit system;
+              overlays = [
+                (overlayFor (p: p.${stdenv}))
+              ];
+            }
+          );
+        in
+        # Add the `stdenvPackages` at toplevel, both because these are the ones
+        # we want most of the time and for backwards compatibility
+        forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages);
 
       commonDeps = pkgs: with pkgs; rec {
         # Use "busybox-sandbox-shell" if present,
@@ -61,6 +83,7 @@
 
         configureFlags =
           lib.optionals stdenv.isLinux [
+            "--with-boost=${boost}/lib"
             "--with-sandbox-shell=${sh}/bin/busybox"
             "LDFLAGS=-fuse-ld=gold"
           ];
@@ -70,15 +93,15 @@
           [
             buildPackages.bison
             buildPackages.flex
-            (lib.getBin buildPackages.lowdown)
+            (lib.getBin buildPackages.lowdown-nix)
             buildPackages.mdbook
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
-            buildPackages.pkgconfig
+            buildPackages.pkg-config
 
             # Tests
             buildPackages.git
-            buildPackages.mercurial
+            buildPackages.mercurial # FIXME: remove? only needed for tests
             buildPackages.jq
           ]
           ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)];
@@ -89,8 +112,8 @@
             openssl sqlite
             libarchive
             boost
-            lowdown
-            gmock
+            lowdown-nix
+            gtest
           ]
           ++ lib.optionals stdenv.isLinux [libseccomp]
           ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
@@ -126,8 +149,7 @@
           ''
             mkdir -p $out/nix-support
 
-            # Converts /nix/store/50p3qk8kka9dl6wyq40vydq945k0j3kv-nix-2.4pre20201102_550e11f/bin/nix
-            # To 50p3qk8kka9dl6wyq40vydq945k0j3kv/bin/nix
+            # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
             tarballPath() {
               # Remove the store prefix
               local path=''${1#${builtins.storeDir}/}
@@ -153,13 +175,15 @@
             echo "file installer $out/install" >> $out/nix-support/hydra-build-products
           '';
 
-      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; pkgs.stdenv.mkDerivation {
+      testNixVersions = pkgs: client: daemon: with commonDeps pkgs; with pkgs.lib; pkgs.stdenv.mkDerivation {
         NIX_DAEMON_PACKAGE = daemon;
         NIX_CLIENT_PACKAGE = client;
-        # Must keep this name short as OSX has a rather strict limit on the
-        # socket path length, and this name appears in the path of the
-        # nix-daemon socket used in the tests
-        name = "nix-tests";
+        name =
+          "nix-tests"
+          + optionalString
+            (versionAtLeast daemon.version "2.4pre20211005" &&
+             versionAtLeast client.version "2.4pre20211005")
+            "-${client.version}-against-${daemon.version}";
         inherit version;
 
         src = self;
@@ -178,8 +202,8 @@
         installPhase = ''
           mkdir -p $out
         '';
-        installCheckPhase = "make installcheck";
 
+        installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES";
       };
 
       binaryTarball = buildPackages: nix: pkgs: let
@@ -253,18 +277,15 @@
             $(cat ${installerClosureInfo}/store-paths)
         '';
 
-    in {
-
-      # A Nixpkgs overlay that overrides the 'nix' and
-      # 'nix.perl-bindings' packages.
-      overlay = final: prev: {
-
-        # An older version of Nix to test against when using the daemon.
-        # Currently using `nixUnstable` as the stable one doesn't respect
-        # `NIX_DAEMON_SOCKET_PATH` which is needed for the tests.
+      overlayFor = getStdenv: final: prev:
+      let currentStdenv = getStdenv final; in
+      {
         nixStable = prev.nix;
 
-        nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
+        # Forward from the previous stage as we don’t want it to pick the lowdown override
+        nixUnstable = prev.nixUnstable;
+
+        nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation {
           name = "nix-${version}";
           inherit version;
 
@@ -279,6 +300,8 @@
 
           propagatedBuildInputs = propagatedDeps;
 
+          disallowedReferences = [ boost ];
+
           preConfigure =
             ''
               # Copy libboost_context so we don't get all of Boost in our closure.
@@ -286,9 +309,16 @@
               mkdir -p $out/lib
               cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
               rm -f $out/lib/*.a
-              ${lib.optionalString stdenv.isLinux ''
+              ${lib.optionalString currentStdenv.isLinux ''
                 chmod u+w $out/lib/*.so.*
-                patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+                patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+              ''}
+              ${lib.optionalString currentStdenv.isDarwin ''
+                for LIB in $out/lib/*.dylib; do
+                  chmod u+w $LIB
+                  install_name_tool -id $LIB $LIB
+                done
+                install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
               ''}
             '';
 
@@ -306,6 +336,12 @@
           postInstall = ''
             mkdir -p $doc/nix-support
             echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+            ${lib.optionalString currentStdenv.isDarwin ''
+            install_name_tool \
+              -change ${boost}/lib/libboost_context.dylib \
+              $out/lib/libboost_context.dylib \
+              $out/lib/libnixutil.dylib
+            ''}
           '';
 
           doInstallCheck = true;
@@ -315,7 +351,7 @@
 
           strictDeps = true;
 
-          passthru.perl-bindings = with final; stdenv.mkDerivation {
+          passthru.perl-bindings = with final; currentStdenv.mkDerivation {
             name = "nix-perl-${version}";
 
             src = self;
@@ -323,7 +359,7 @@
             nativeBuildInputs =
               [ buildPackages.autoconf-archive
                 buildPackages.autoreconfHook
-                buildPackages.pkgconfig
+                buildPackages.pkg-config
               ];
 
             buildInputs =
@@ -334,8 +370,8 @@
                 pkgs.perl
                 boost
               ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
+              ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium
+              ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security;
 
             configureFlags = ''
               --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
@@ -349,15 +385,8 @@
 
         };
 
-        lowdown = with final; stdenv.mkDerivation rec {
-          name = "lowdown-0.8.4";
-
-          /*
-          src = fetchurl {
-            url = "https://kristaps.bsd.lv/lowdown/snapshots/${name}.tar.gz";
-            hash = "sha512-U9WeGoInT9vrawwa57t6u9dEdRge4/P+0wLxmQyOL9nhzOEUU2FRz2Be9H0dCjYE7p2v3vCXIYk40M+jjULATw==";
-          };
-          */
+        lowdown-nix = with final; currentStdenv.mkDerivation rec {
+          name = "lowdown-0.9.0";
 
           src = lowdown-src;
 
@@ -366,15 +395,20 @@
           nativeBuildInputs = [ buildPackages.which ];
 
           configurePhase = ''
-              ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
+              ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
               ./configure \
                 PREFIX=${placeholder "dev"} \
                 BINDIR=${placeholder "bin"}/bin
-            '';
+          '';
         };
-
       };
 
+    in {
+
+      # A Nixpkgs overlay that overrides the 'nix' and
+      # 'nix.perl-bindings' packages.
+      overlay = overlayFor (p: p.stdenv);
+
       hydraJobs = {
 
         # Binary package for various platforms.
@@ -410,6 +444,21 @@
         installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ];
         installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
+        # docker image with Nix inside
+        dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system:
+          let
+            pkgs = nixpkgsFor.${system};
+            image = import ./docker.nix { inherit pkgs; tag = version; };
+          in pkgs.runCommand "docker-image-tarball-${version}"
+            { meta.description = "Docker image with Nix for ${system}";
+            }
+            ''
+              mkdir -p $out/nix-support
+              image=$out/image.tar.gz
+              ln -s ${image} $image
+              echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
+            '');
+
         # Line coverage analysis.
         coverage =
           with nixpkgsFor.x86_64-linux;
@@ -450,6 +499,12 @@
           inherit (self) overlay;
         };
 
+        tests.nssPreload = (import ./tests/nss-preload.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
         tests.githubFlakes = (import ./tests/github-flakes.nix rec {
           system = "x86_64-linux";
           inherit nixpkgs;
@@ -464,48 +519,47 @@
               inherit (self) overlay;
             });
 
-        /*
-        # Check whether we can still evaluate all of Nixpkgs.
+        # Make sure that nix-env still produces the exact same result
+        # on a particular version of Nixpkgs.
         tests.evalNixpkgs =
-          import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
-            # FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
-            inherit nixpkgs;
-            pkgs = nixpkgsFor.x86_64-linux;
-            officialRelease = false;
-          };
-
-        # Check whether we can still evaluate NixOS.
-        tests.evalNixOS =
           with nixpkgsFor.x86_64-linux;
           runCommand "eval-nixos" { buildInputs = [ nix ]; }
             ''
-              export NIX_STATE_DIR=$TMPDIR
-
-              nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
-                --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
-
-              touch $out
+              type -p nix-env
+              # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
+              time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+              [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]]
+              mkdir $out
             '';
-        */
+
+        metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
+          pkgs = nixpkgsFor.x86_64-linux;
+          nixpkgs = nixpkgs-regression;
+        };
+
+        installTests = forAllSystems (system:
+          let pkgs = nixpkgsFor.${system}; in
+          pkgs.runCommand "install-tests" {
+            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
+            againstCurrentUnstable =
+              # FIXME: temporarily disable this on macOS because of #3605.
+              if system == "x86_64-linux"
+              then testNixVersions pkgs pkgs.nix pkgs.nixUnstable
+              else null;
+            # Disabled because the latest stable version doesn't handle
+            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
+            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
+          } "touch $out");
 
       };
 
       checks = forAllSystems (system: {
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
-        installTests =
-          let pkgs = nixpkgsFor.${system}; in
-          pkgs.runCommand "install-tests" {
-            againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix;
-            againstCurrentUnstable = testNixVersions pkgs pkgs.nix pkgs.nixUnstable;
-            # Disabled because the latest stable version doesn't handle
-            # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
-            # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable;
-          } "touch $out";
-      } // (if system == "x86_64-linux" then (builtins.listToAttrs (map (crossSystem: {
-        name = "binaryTarball-${crossSystem}";
-        value = self.hydraJobs.binaryTarballCross.${system}.${crossSystem};
-      }) crossSystems)) else {}));
+        installTests = self.hydraJobs.installTests.${system};
+      } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
+        dockerImage = self.hydraJobs.dockerImage.${system};
+      });
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;
@@ -589,15 +643,22 @@
           doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
         };
-      }) crossSystems)));
+      }) crossSystems)) // (builtins.listToAttrs (map (stdenvName:
+      nixpkgsFor.${system}.lib.nameValuePair
+        "nix-${stdenvName}"
+        nixpkgsFor.${system}."${stdenvName}Packages".nix
+      ) stdenvs))
+      );
 
       defaultPackage = forAllSystems (system: self.packages.${system}.nix);
 
-      devShell = forAllSystems (system:
+      devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages);
+
+      devShells = forAllSystemsAndStdenvs (system: stdenv:
         with nixpkgsFor.${system};
         with commonDeps pkgs;
 
-        stdenv.mkDerivation {
+        nixpkgsFor.${system}.${stdenv}.mkDerivation {
           name = "nix";
 
           outputs = [ "out" "dev" "doc" ];
@@ -616,6 +677,9 @@
               PATH=$prefix/bin:$PATH
               unset PYTHONPATH
               export MANPATH=$out/share/man:$MANPATH
+
+              # Make bash completion work.
+              XDG_DATA_DIRS+=:$out/share
             '';
         });
 

maintainers/upload-release.pl

@@ -19,6 +19,8 @@ my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine";
 
 my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
 
+my $isLatest = ($ENV{'IS_LATEST'} // "") eq "1";
+
 # FIXME: cut&paste from nixos-channel-scripts.
 sub fetch {
     my ($url, $type) = @_;
@@ -35,16 +37,18 @@ sub fetch {
 my $evalUrl = "https://hydra.nixos.org/eval/$evalId";
 my $evalInfo = decode_json(fetch($evalUrl, 'application/json'));
 #print Dumper($evalInfo);
+my $flakeUrl = $evalInfo->{flake} or die;
+my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die);
+my $nixRev = $flakeInfo->{revision} or die;
 
-my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die;
+my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
+#print Dumper($buildInfo);
 
-my $tarballInfo = decode_json(fetch("$evalUrl/job/tarball", 'application/json'));
-
-my $releaseName = $tarballInfo->{releasename};
+my $releaseName = $buildInfo->{nixname};
 $releaseName =~ /nix-(.*)$/ or die;
 my $version = $1;
 
-print STDERR "Nix revision is $nixRev, version is $version\n";
+print STDERR "Flake URL is $flakeUrl, Nix revision is $nixRev, version is $version\n";
 
 my $releaseDir = "nix/$releaseName";
 
@@ -104,8 +108,6 @@ sub downloadFile {
     return $sha256_expected;
 }
 
-downloadFile("tarball", "2"); # .tar.bz2
-my $tarballHash = downloadFile("tarball", "3"); # .tar.xz
 downloadFile("binaryTarball.i686-linux", "1");
 downloadFile("binaryTarball.x86_64-linux", "1");
 downloadFile("binaryTarball.aarch64-linux", "1");
@@ -134,42 +136,38 @@ for my $fn (glob "$tmpDir/*") {
     }
 }
 
-exit if $version =~ /pre/;
-
 # Update nix-fallback-paths.nix.
-system("cd $nixpkgsDir && git pull") == 0 or die;
+if ($isLatest) {
+    system("cd $nixpkgsDir && git pull") == 0 or die;
 
-sub getStorePath {
-    my ($jobName) = @_;
-    my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
-    for my $product (values %{$buildInfo->{buildproducts}}) {
-        next unless $product->{type} eq "nix-build";
-        next if $product->{path} =~ /[a-z]+$/;
-        return $product->{path};
+    sub getStorePath {
+        my ($jobName) = @_;
+        my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+        return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'";
     }
-    die;
+
+    write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
+               "{\n" .
+               "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
+               "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
+               "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
+               "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
+               "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
+               "}\n");
+
+    system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
 }
 
-write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix",
-           "{\n" .
-           "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
-           "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
-           "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
-           "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
-           "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
-           "}\n");
-
-system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die;
-
 # Update the "latest" symlink.
 $channelsBucket->add_key(
     "nix-latest/install", "",
     { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
-    or die $channelsBucket->err . ": " . $channelsBucket->errstr;
+    or die $channelsBucket->err . ": " . $channelsBucket->errstr
+    if $isLatest;
 
 # Tag the release in Git.
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
 system("git push --tags") == 0 or die;
-system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;

misc/bash/completion.sh

@@ -7,13 +7,15 @@ function _complete_nix {
         local completion=${line%%	*}
         if [[ -z $have_type ]]; then
             have_type=1
-            if [[ $completion = filenames ]]; then
+            if [[ $completion == filenames ]]; then
                 compopt -o filenames
+            elif [[ $completion == attrs ]]; then
+                compopt -o nospace
             fi
         else
             COMPREPLY+=("$completion")
         fi
-    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
+    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}")
     __ltrim_colon_completions "$cur"
 }
 

misc/fish/completion.fish

@@ -19,7 +19,6 @@ end
 
 function _nix_accepts_files
   set -l response (_nix_complete)
-  # First line is either filenames or no-filenames.
   test $response[1] = 'filenames'
 end
 

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -25,5 +25,10 @@
     <string>/var/log/nix-daemon.log</string>
     <key>StandardOutPath</key>
     <string>/dev/null</string>
+    <key>SoftResourceLimits</key>
+    <dict>
+      <key>NumberOfFiles</key>
+      <integer>4096</integer>
+    </dict>
   </dict>
 </plist>

mk/libraries.mk

@@ -125,8 +125,8 @@ define build-library
     $(1)_PATH := $$(_d)/$$($(1)_NAME).a
 
     $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/
-	$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
-	$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
+	$$(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$?
+	$$(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o
 
     $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS)
 

mk/tests.mk

@@ -13,3 +13,7 @@ define run-install-test
 endef
 
 .PHONY: check installcheck
+
+print-top-help += \
+  echo "  check: Run unit tests"; \
+  echo "  installcheck: Run functional tests";

perl/configure.ac

@@ -41,7 +41,7 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};')
 AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
 AC_MSG_RESULT($perllibdir)
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Check for the required Perl dependencies (DBI and DBD::SQLite).

perl/lib/Nix/Config.pm.in

@@ -1,6 +1,7 @@
 package Nix::Config;
 
 use MIME::Base64;
+use Nix::Store;
 
 $version = "@PACKAGE_VERSION@";
 

scripts/check-hydra-status.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+# set -x
+
+
+# mapfile BUILDS_FOR_LATEST_EVAL < <(
+# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+#   jq -r '.evals[0].builds[] | @sh')
+BUILDS_FOR_LATEST_EVAL=$(
+curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+  jq -r '.evals[0].builds[]')
+
+someBuildFailed=0
+
+for buildId in $BUILDS_FOR_LATEST_EVAL; do
+  buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
+
+  buildStatus=$(echo "$buildInfo" | \
+    jq -r '.buildstatus')
+
+  if [[ "$buildStatus" -ne 0 ]]; then
+    someBuildFailed=1
+    echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra"
+  fi
+done
+
+exit "$someBuildFailed"

scripts/create-darwin-volume.sh

@@ -440,7 +440,22 @@ add_nix_vol_fstab_line() {
     # shellcheck disable=SC1003,SC2026
     local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}"
     shift
-    EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <<EOF
+
+    # wrap `ex` to work around a problem with vim plugins breaking exit codes;
+    # (see https://github.com/NixOS/nix/issues/5468)
+    # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split
+    # the EDITOR env.
+    #
+    # TODO: at some point we should switch to `--clean`, but it wasn't added
+    # until https://github.com/vim/vim/releases/tag/v8.0.1554 while the macOS
+    # minver 10.12.6 seems to have released with vim 7.4
+    cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF
+#!/bin/sh
+/usr/bin/ex --noplugin "\$@"
+EOF
+    chmod 755 "$SCRATCH/ex_cleanroom_wrapper"
+
+    EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" <<EOF
 :a
 UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners
 .
@@ -631,7 +646,7 @@ EOF
         # technically /etc/synthetic.d/nix is supported in Big Sur+
         # but handling both takes even more code...
         _sudo "to add Nix to /etc/synthetic.conf" \
-            /usr/bin/ex /etc/synthetic.conf <<EOF
+            /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF
 :a
 ${NIX_ROOT:1}
 .
@@ -715,7 +730,8 @@ create_volume() {
     # 6) getting special w/ awk may be fragile, but doing it to:
     #    - save time over running slow diskutil commands
     #    - skirt risk we grab wrong volume if multiple match
-    /usr/sbin/diskutil apfs addVolume "$NIX_VOLUME_USE_DISK" "$NIX_VOLUME_FS" "$NIX_VOLUME_LABEL" -nomount | /usr/bin/awk '/Created new APFS Volume/ {print $5}'
+    _sudo "to create a new APFS volume '$NIX_VOLUME_LABEL' on $NIX_VOLUME_USE_DISK" \
+        /usr/sbin/diskutil apfs addVolume "$NIX_VOLUME_USE_DISK" "$NIX_VOLUME_FS" "$NIX_VOLUME_LABEL" -nomount | /usr/bin/awk '/Created new APFS Volume/ {print $5}'
 }
 
 volume_uuid_from_special() {
@@ -738,10 +754,12 @@ await_volume() {
 setup_volume() {
     local use_special use_uuid profile_packages
     task "Creating a Nix volume" >&2
-    # DOING: I'm tempted to wrap this call in a grep to get the new disk special without doing anything too complex, but this sudo wrapper *is* a little complex, so it'll be a PITA unless maybe we can skip sudo on this. Let's just try it without.
 
     use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}"
 
+    _sudo "to ensure the Nix volume is not mounted" \
+        /usr/sbin/diskutil unmount force "$use_special" || true # might not be mounted
+
     use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")}
 
     setup_fstab "$use_uuid"
@@ -759,6 +777,11 @@ setup_volume() {
 
     await_volume
 
+    if [ "$(/usr/sbin/diskutil info -plist "$NIX_ROOT" | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        _sudo "to set enableOwnership (enabling users to own files)" \
+            /usr/sbin/diskutil enableOwnership "$NIX_ROOT"
+    fi
+
     # TODO: below is a vague kludge for now; I just don't know
     # what if any safe action there is to take here. Also, the
     # reminder isn't very helpful.
@@ -786,7 +809,7 @@ setup_volume_daemon() {
     local volume_uuid="$2"
     if ! test_voldaemon; then
         task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2
-        _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <<EOF
+        _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF
 :a
 $(generate_mount_daemon "$cmd_type" "$volume_uuid")
 .

scripts/install-darwin-multi-user.sh

@@ -13,11 +13,22 @@ NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 read_only_root() {
     # this touch command ~should~ always produce an error
     # as of this change I confirmed /usr/bin/touch emits:
+    # "touch: /: Operation not permitted" Monterey
     # "touch: /: Read-only file system" Catalina+ and Big Sur
     # "touch: /: Permission denied" Mojave
     # (not matching prefix for compat w/ coreutils touch in case using
     # an explicit path causes problems; its prefix differs)
-    [[ "$(/usr/bin/touch / 2>&1)" = *"Read-only file system" ]]
+    case "$(/usr/bin/touch / 2>&1)" in
+        *"Read-only file system") # Catalina, Big Sur
+            return 0
+            ;;
+        *"Operation not permitted") # Monterey
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
 
     # Avoiding the slow semantic way to get this information (~330ms vs ~8ms)
     # unless using touch causes problems. Just in case, that approach is:
@@ -67,7 +78,7 @@ poly_service_installed_check() {
 poly_service_uninstall_directions() {
     echo "$1. Remove macOS-specific components:"
     if should_create_volume && test_nix_volume_mountd_installed; then
-        darwin_volume_uninstall_directions
+        nix_volume_mountd_uninstall_directions
     fi
     if test_nix_daemon_installed; then
         nix_daemon_uninstall_directions
@@ -206,4 +217,8 @@ poly_prepare_to_install() {
 EOF
         setup_darwin_volume
     fi
+
+    if [ "$(/usr/sbin/diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        failure "This script needs a /nix volume with global permissions! This may require running sudo /usr/sbin/diskutil enableOwnership /nix."
+    fi
 }

scripts/install-multi-user.sh

@@ -33,7 +33,7 @@ NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 readonly NIX_ROOT="/nix"
 readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-}
 
-readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv" "/etc/bash.bashrc" "/etc/zsh/zshenv")
+readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/etc/bash.bashrc" "/etc/zsh/zshrc")
 readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix"
 readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh"
 
@@ -377,6 +377,11 @@ cure_artifacts() {
 }
 
 validate_starting_assumptions() {
+    task "Checking for artifacts of previous installs"
+    cat <<EOF
+Before I try to install, I'll check for signs Nix already is or has
+been installed on this system.
+EOF
     if type nix-env 2> /dev/null >&2; then
         warning <<EOF
 Nix already appears to be installed. This installer may run into issues.
@@ -386,20 +391,34 @@ $(uninstall_directions)
 EOF
     fi
 
+    # TODO: I think it would be good for this step to accumulate more
+    #       knowledge of older obsolete artifacts, if there are any.
+    #       We could issue a "reminder" here that the user might want
+    #       to clean them up?
+
     for profile_target in "${PROFILE_TARGETS[@]}"; do
+        # TODO: I think it would be good to accumulate a list of all
+        #       of the copies so that people don't hit this 2 or 3x in
+        #       a row for different files.
         if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then
+            # this backup process first released in Nix 2.1
             failure <<EOF
-When this script runs, it backs up the current $profile_target to
-$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though.
+I back up shell profile/rc scripts before I add Nix to them.
+I need to back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX,
+but the latter already exists.
 
-Please follow these instructions to clean up the old backup file:
+Here's how to clean up the old backup file:
 
-1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just
-in case.
+1. Back up (copy) $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX
+   to another location, just in case.
 
-2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like
-it has anything nix-related in it. If it does, something is probably
-quite wrong. Please open an issue or get in touch immediately.
+2. Ensure $profile_target$PROFILE_BACKUP_SUFFIX does not have anything
+   Nix-related in it. If it does, something is probably quite
+   wrong. Please open an issue or get in touch immediately.
+
+3. Once you confirm $profile_target is backed up and
+   $profile_target$PROFILE_BACKUP_SUFFIX doesn't mention Nix, run:
+   mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target
 EOF
         fi
     done
@@ -571,7 +590,7 @@ create_directories() {
               "$get_chr_own" -R "root:$NIX_BUILD_GROUP_NAME" "$NIX_ROOT" || true
     fi
     _sudo "to make the basic directory structure of Nix (part 1)" \
-          install -dv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
+          install -dv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool,/daemon-socket} /nix/var/nix/{gcroots,profiles}/per-user
 
     _sudo "to make the basic directory structure of Nix (part 2)" \
           install -dv -g "$NIX_BUILD_GROUP_NAME" -m 1775 /nix/store
@@ -599,7 +618,7 @@ manager. This will happen in a few stages:
 1. Make sure your computer doesn't already have Nix. If it does, I
    will show you instructions on how to clean up your old install.
 
-2. Show you what we are going to install and where. Then we will ask
+2. Show you what I am going to install and where. Then I will ask
    if you are ready to continue.
 
 3. Create the system users and groups that the Nix daemon uses to run
@@ -614,14 +633,14 @@ manager. This will happen in a few stages:
 
 EOF
 
-    if ui_confirm "Would you like to see a more detailed list of what we will do?"; then
+    if ui_confirm "Would you like to see a more detailed list of what I will do?"; then
         cat <<EOF
 
-We will:
+I will:
 
  - make sure your computer doesn't already have Nix files
    (if it does, I will tell you how to clean them up.)
- - create local users (see the list above for the users we'll make)
+ - create local users (see the list above for the users I'll make)
  - create a local group ($NIX_BUILD_GROUP_NAME)
  - install Nix in to $NIX_ROOT
  - create a configuration file in /etc/nix
@@ -656,7 +675,7 @@ run in a headless fashion, like this:
 
   $ curl -L https://nixos.org/nix/install | sh
 
-or maybe in a CI pipeline. Because of that, we're going to skip the
+or maybe in a CI pipeline. Because of that, I'm going to skip the
 verbose output in the interest of brevity.
 
 If you would like to
@@ -670,7 +689,7 @@ EOF
     fi
 
     cat <<EOF
-This script is going to call sudo a lot. Every time we do, it'll
+This script is going to call sudo a lot. Every time I do, it'll
 output exactly what it'll do, and why.
 
 Just like this:
@@ -682,15 +701,15 @@ EOF
     cat <<EOF
 
 This might look scary, but everything can be undone by running just a
-few commands. We used to ask you to confirm each time sudo ran, but it
+few commands. I used to ask you to confirm each time sudo ran, but it
 was too many times. Instead, I'll just ask you this one time:
 
 EOF
-    if ui_confirm "Can we use sudo?"; then
+    if ui_confirm "Can I use sudo?"; then
         ok "Yay! Thanks! Let's get going!"
     else
         failure <<EOF
-That is okay, but we can't install.
+That is okay, but I can't install.
 EOF
     fi
 }
@@ -809,10 +828,10 @@ main() {
     # can fail faster in this case. Sourcing install-darwin... now runs
     # `touch /` to detect Read-only root, but it could update times on
     # pre-Catalina macOS if run as root user.
-    if [ $EUID -eq 0 ]; then
+    if [ "$EUID" -eq 0 ]; then
         failure <<EOF
-Please do not run this script with root privileges. We will call sudo
-when we need to.
+Please do not run this script with root privileges. I will call sudo
+when I need to.
 EOF
     fi
 

scripts/install-nix-from-closure.sh

@@ -38,7 +38,7 @@ fi
 
 # Determine if we could use the multi-user installer or not
 if [ "$(uname -s)" = "Linux" ]; then
-    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
+    echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2
 fi
 
 case "$(uname -s)" in
@@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do
                 echo "              providing multi-user support and better isolation for local builds."
                 echo "              Both for security and reproducibility, this method is recommended if"
                 echo "              supported on your platform."
-                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo "              See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation"
                 echo ""
                 echo " --no-daemon: Simple, single-user installation that does not require root and is"
                 echo "              trivial to uninstall."
@@ -134,7 +134,7 @@ fi
 
 echo "performing a single-user installation of Nix..." >&2
 
-if ! [ -e $dest ]; then
+if ! [ -e "$dest" ]; then
     cmd="mkdir -m 0755 $dest && chown $USER $dest"
     echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2
     if ! sudo sh -c "$cmd"; then
@@ -143,12 +143,12 @@ if ! [ -e $dest ]; then
     fi
 fi
 
-if ! [ -w $dest ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+if ! [ -w "$dest" ]; then
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 
-mkdir -p $dest/store
+mkdir -p "$dest/store"
 
 printf "copying Nix to %s..." "${dest}/store" >&2
 # Insert a newline if no progress is shown.
@@ -189,17 +189,17 @@ fi
 
 # Install an SSL certificate bundle.
 if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
-    $nix/bin/nix-env -i "$cacert"
+    "$nix/bin/nix-env" -i "$cacert"
     export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt"
 fi
 
 # Subscribe the user to the Nixpkgs channel and fetch it.
 if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then
-    if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
-        $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+    if ! "$nix/bin/nix-channel" --list | grep -q "^nixpkgs "; then
+        "$nix/bin/nix-channel" --add https://nixos.org/channels/nixpkgs-unstable
     fi
     if [ -z "$_NIX_INSTALLER_TEST" ]; then
-        if ! $nix/bin/nix-channel --update nixpkgs; then
+        if ! "$nix/bin/nix-channel" --update nixpkgs; then
             echo "Fetching the nixpkgs channel failed. (Are you offline?)"
             echo "To try again later, run \"nix-channel --update nixpkgs\"."
         fi
@@ -215,7 +215,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break
@@ -226,7 +226,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
         if [ -w "$fn" ]; then
             if ! grep -q "$p" "$fn"; then
                 echo "modifying $fn..." >&2
-                echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+                printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn"
             fi
             added=1
             break

scripts/install-systemd-multi-user.sh

@@ -15,7 +15,7 @@ readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
 
 create_systemd_override() {
      header "Configuring proxy for the nix-daemon service"
-    _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)"
+    _sudo "create directory for systemd unit override" mkdir -p "$(dirname "$SERVICE_OVERRIDE")"
     cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE"
 [Service]
 $1

scripts/install.in

@@ -76,14 +76,21 @@ fi
 
 tarball=$tmpDir/nix-@nixVersion@-$system.tar.xz
 
-require_util curl "download the binary tarball"
 require_util tar "unpack the binary tarball"
 if [ "$(uname -s)" != "Darwin" ]; then
     require_util xz "unpack the binary tarball"
 fi
 
+if command -v curl > /dev/null 2>&1; then
+    fetch() { curl -L "$1" -o "$2"; }
+elif command -v wget > /dev/null 2>&1; then
+    fetch() { wget "$1" -O "$2"; }
+else
+    oops "you don't have wget or curl installed, which I need to download the binary tarball"
+fi
+
 echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
-curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
+fetch "$url" "$tarball" || oops "failed to download '$url'"
 
 if command -v sha256sum > /dev/null 2>&1; then
     hash2="$(sha256sum -b "$tarball" | cut -c1-64)"

scripts/local.mk

@@ -1,7 +1,5 @@
 nix_noinst_scripts := \
-  $(d)/nix-http-export.cgi \
-  $(d)/nix-profile.sh \
-  $(d)/nix-reduce-build
+  $(d)/nix-profile.sh
 
 noinst-scripts += $(nix_noinst_scripts)
 

scripts/nix-http-export.cgi.in

@@ -1,51 +0,0 @@
-#! /bin/sh
-
-export HOME=/tmp
-export NIX_REMOTE=daemon
-
-TMP_DIR="${TMP_DIR:-/tmp/nix-export}"
-
-@coreutils@/mkdir -p "$TMP_DIR" || true
-@coreutils@/chmod a+r "$TMP_DIR"
-
-needed_path="?$QUERY_STRING"
-needed_path="${needed_path#*[?&]needed_path=}"
-needed_path="${needed_path%%&*}"
-#needed_path="$(echo $needed_path  | ./unhttp)"
-needed_path="${needed_path//%2B/+}"
-needed_path="${needed_path//%3D/=}"
-
-echo needed_path: "$needed_path" >&2
-
-NIX_STORE="${NIX_STORE_DIR:-/nix/store}"
-
-echo NIX_STORE: "${NIX_STORE}" >&2
-
-full_path="${NIX_STORE}"/"$needed_path"
-
-if [ "$needed_path" != "${needed_path%.drv}" ]; then
-	echo "Status: 403 You should create the derivation file yourself"
-	echo "Content-Type: text/plain"
-	echo
-	echo "Refusing to disclose derivation contents"
-	exit
-fi
-
-if @bindir@/nix-store --check-validity "$full_path"; then
-	if ! [ -e nix-export/"$needed_path".nar.gz ]; then
-		@bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz
-		@coreutils@/ln -fs  "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz 
-	fi;
-	echo "Status: 301 Moved"
-	echo "Location: nix-export/"$needed_path".nar.gz"
-	echo
-else 
-	echo "Status: 404 No such path found"
-	echo "Content-Type: text/plain"
-	echo
-	echo "Path not found:"
-	echo "$needed_path"
-	echo "checked:"
-	echo "$full_path"
-fi
-

scripts/nix-profile-daemon.sh.in

@@ -5,7 +5,7 @@ __ETC_PROFILE_NIX_SOURCED=1
 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile"
 
 # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
+if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then
     : # Allow users to override the NIX_SSL_CERT_FILE
 elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
     export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
@@ -18,14 +18,14 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS
 else
   # Fall back to what is in the nix profiles, favouring whatever is defined last.
   check_nix_profiles() {
-    if [ "$ZSH_VERSION" ]; then
+    if [ -n "$ZSH_VERSION" ]; then
       # Zsh by default doesn't split words in unquoted parameter expansion.
       # Set local_options for these options to be reverted at the end of the function
       # and shwordsplit to force splitting words in $NIX_PROFILES below.
       setopt local_options shwordsplit
     fi
     for i in $NIX_PROFILES; do
-      if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then
+      if [ -e "$i/etc/ssl/certs/ca-bundle.crt" ]; then
         export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt
       fi
     done

scripts/nix-reduce-build.in

@@ -1,171 +0,0 @@
-#! @bash@
-
-WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX);
-cd "$WORKING_DIRECTORY";
-
-if test -z "$1" || test "a--help" = "a$1" ; then
-	echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2
-	echo As in: >&2
-	echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2
-	echo nix-reduce-build /etc/nixos/nixos -- \\
-	echo "   " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2
-	echo "  store path name will be added into the end of the URL" >&2
-	echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2
-	echo "  that should be a directory where gzipped 'nix-store --export' ">&2
-	echo "  files are located (they should have .nar.gz extension)"  >&2
-	echo "        Or all together: " >&2
-	echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\
-	"    ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2
-	echo "        Also supports best-effort local builds of failing expression set:" >&2
-	echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2
-	echo "  nix-daemon:// builds using daemon"
-	echo "  nix-self:// builds directly using nix-store from current installation" >&2
-	echo "  nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2;
-	echo "derivations with specified output hash (sha256, sha1 or md5)." >&2
-	echo "  nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2;
-	echo "maximum amount of paths" >&2;
-	echo "  nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2;
-	echo "maximum amount of paths" >&2;
-	echo "        If no package sources are specified, required paths are listed." >&2;
-	exit;
-fi;
-
-while ! test "$1" = "--" || test "$1" = "" ; do 
-	echo "$1" >> initial; >&2
-	shift;
-done
-shift;
-echo Will work on $(cat initial | wc -l) targets. >&2
-
-while read ; do
-	case "$REPLY" in 
-		${NIX_STORE_DIR:-/nix/store}/*)
-			echo "$REPLY" >> paths; >&2
-			;;
-		*)
-			(
-				IFS=: ;
-				nix-instantiate $REPLY >> paths;
-			);
-			;;
-	esac;
-done < initial;
-echo Proceeding $(cat paths | wc -l) paths. >&2
-
-while read; do
-	case "$REPLY" in
-		*.drv)
-			echo "$REPLY" >> derivers; >&2
-			;;
-		*)
-			nix-store --query --deriver "$REPLY" >>derivers;
-			;;
-	esac;
-done < paths;
-echo Found $(cat derivers | wc -l) derivers. >&2
-
-cat derivers | xargs nix-store --query -R > derivers-closure;
-echo Proceeding at most $(cat derivers-closure | wc -l) derivers. >&2
-
-cat derivers-closure | egrep '[.]drv$' | xargs nix-store --query --outputs > wanted-paths;
-cat derivers-closure | egrep -v '[.]drv$' >> wanted-paths;
-echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2
-
-cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
-echo We need $(cat needed-paths | wc -l) paths. >&2
-
-egrep '[.]drv$' derivers-closure > critical-derivers;
-
-if test -z "$1" ; then
-	cat needed-paths;	
-fi;
-
-refresh_critical_derivers() {
-    echo "Finding needed derivers..." >&2;
-    cat critical-derivers | while read; do
-        if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then
-            echo "$REPLY";
-        fi;
-    done > new-critical-derivers;
-    mv new-critical-derivers critical-derivers;
-    echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2
-}
-
-build_here() {
-    cat critical-derivers | while read; do 
-        echo "Realising $REPLY using nix-daemon" >&2
-        @bindir@/nix-store -r "${REPLY}"
-    done;
-}
-
-try_to_substitute(){
-    cat needed-paths | while read ; do 
-        echo "Building $REPLY using nix-daemon" >&2
-        @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}"
-    done;
-}
-
-for i in "$@"; do 
-	sshHost="${i#ssh://}";
-	httpHost="${i#http://}";
-	httpsHost="${i#https://}";
-	filePath="${i#file:/}";
-	if [ "$i" != "$sshHost" ]; then
-		cat needed-paths | while read; do 
-			echo "Getting $REPLY and its closure over ssh" >&2
-			nix-copy-closure --from "$sshHost" --gzip "$REPLY" </dev/null || true; 
-		done;
-	elif [ "$i" != "$httpHost" ] || [ "$i" != "$httpsHost" ]; then
-		cat needed-paths | while read; do
-			echo "Getting $REPLY over http/https" >&2
-			curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import;
-		done;
-	elif [ "$i" != "$filePath" ] ; then
-		cat needed-paths | while read; do 
-			echo "Installing $REPLY from file" >&2
-			gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import;
-		done;
-	elif [ "$i" = "nix-daemon://" ] ; then
-		NIX_REMOTE=daemon try_to_substitute;
-		refresh_critical_derivers;
-		NIX_REMOTE=daemon build_here;
-	elif [ "$i" = "nix-self://" ] ; then
-		NIX_REMOTE= try_to_substitute;
-		refresh_critical_derivers;
-		NIX_REMOTE= build_here;
-	elif [ "$i" = "nix-daemon-fixed://" ] ; then
-		refresh_critical_derivers;
-
-		cat critical-derivers | while read; do 
-			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
-				echo "Realising $REPLY using nix-daemon" >&2
-				NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}"
-			fi;
-		done;
-	elif [ "$i" = "nix-self-fixed://" ] ; then
-		refresh_critical_derivers;
-
-		cat critical-derivers | while read; do 
-			if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then
-				echo "Realising $REPLY using direct Nix build" >&2
-				NIX_REMOTE= @bindir@/nix-store -r "${REPLY}"
-			fi;
-		done;
-	elif [ "$i" = "nix-daemon-substitute://" ] ; then
-		NIX_REMOTE=daemon try_to_substitute;
-	elif [ "$i" = "nix-self-substitute://" ] ; then
-		NIX_REMOTE= try_to_substitute;
-	elif [ "$i" = "nix-daemon-build://" ] ; then
-		refresh_critical_derivers;
-		NIX_REMOTE=daemon build_here;
-	elif [ "$i" = "nix-self-build://" ] ; then
-		refresh_critical_derivers;
-		NIX_REMOTE= build_here;
-	fi;
-	mv needed-paths wanted-paths;
-	cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths;
-	echo We still need $(cat needed-paths | wc -l) paths. >&2
-done;
-
-cd /
-rm -r "$WORKING_DIRECTORY"

scripts/prepare-installer-for-github-actions

@@ -3,7 +3,7 @@
 set -e
 
 script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
-installerHash=$(echo $script | cut -b12-43 -)
+installerHash=$(echo "$script" | cut -b12-43 -)
 
 installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
 

src/build-remote/build-remote.cc

@@ -18,6 +18,7 @@
 #include "derivations.hh"
 #include "local-store.hh"
 #include "legacy.hh"
+#include "experimental-features.hh"
 
 using namespace nix;
 using std::cin;
@@ -130,11 +131,14 @@ static int main_build_remote(int argc, char * * argv)
                 for (auto & m : machines) {
                     debug("considering building on remote machine '%s'", m.storeUri);
 
-                    if (m.enabled && std::find(m.systemTypes.begin(),
-                            m.systemTypes.end(),
-                            neededSystem) != m.systemTypes.end() &&
+                    if (m.enabled
+                        && (neededSystem == "builtin"
+                            || std::find(m.systemTypes.begin(),
+                                m.systemTypes.end(),
+                                neededSystem) != m.systemTypes.end()) &&
                         m.allSupported(requiredFeatures) &&
-                        m.mandatoryMet(requiredFeatures)) {
+                        m.mandatoryMet(requiredFeatures))
+                    {
                         rightType = true;
                         AutoCloseFD free;
                         uint64_t load = 0;
@@ -295,7 +299,7 @@ connected:
 
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
             for (auto & outputName : wantedOutputs) {
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
@@ -327,7 +331,7 @@ connected:
         for (auto & realisation : missingRealisations) {
             // Should hold, because if the feature isn't enabled the set
             // of missing realisations should be empty
-            settings.requireExperimentalFeature("ca-derivations");
+            settings.requireExperimentalFeature(Xp::CaDerivations);
             store->registerDrvOutput(realisation);
         }
 

src/cpptoml/LICENSE

@@ -1,18 +0,0 @@
-Copyright (c) 2014 Chase Geigle
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

src/libcmd/command.cc

@@ -54,6 +54,36 @@ void StoreCommand::run()
     run(getStore());
 }
 
+CopyCommand::CopyCommand()
+{
+    addFlag({
+        .longName = "from",
+        .description = "URL of the source Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&srcUri},
+    });
+
+    addFlag({
+        .longName = "to",
+        .description = "URL of the destination Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&dstUri},
+    });
+}
+
+ref<Store> CopyCommand::createStore()
+{
+    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+}
+
+ref<Store> CopyCommand::getDstStore()
+{
+    if (srcUri.empty() && dstUri.empty())
+        throw UsageError("you must pass '--from' and/or '--to'");
+
+    return dstUri.empty() ? openStore() : openStore(dstUri);
+}
+
 EvalCommand::EvalCommand()
 {
 }
@@ -74,7 +104,15 @@ ref<Store> EvalCommand::getEvalStore()
 ref<EvalState> EvalCommand::getEvalState()
 {
     if (!evalState)
-        evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
+        evalState =
+            #if HAVE_BOEHMGC
+            std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
+                searchPath, getEvalStore(), getStore())
+            #else
+            std::make_shared<EvalState>(
+                searchPath, getEvalStore(), getStore())
+            #endif
+            ;
     return ref<EvalState>(evalState);
 }
 
@@ -120,7 +158,7 @@ void BuiltPathsCommand::run(ref<Store> store)
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations
             StorePathSet pathsRoots, pathsClosure;
-            for (auto & root: paths) {
+            for (auto & root : paths) {
                 auto rootFromThis = root.outPaths();
                 pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
             }
@@ -138,17 +176,20 @@ StorePathsCommand::StorePathsCommand(bool recursive)
 {
 }
 
-void StorePathsCommand::run(ref<Store> store, BuiltPaths paths)
+void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
 {
-    StorePaths storePaths;
-    for (auto& builtPath : paths)
-        for (auto& p : builtPath.outPaths())
-            storePaths.push_back(p);
+    StorePathSet storePaths;
+    for (auto & builtPath : paths)
+        for (auto & p : builtPath.outPaths())
+            storePaths.insert(p);
 
-    run(store, std::move(storePaths));
+    auto sorted = store->topoSortPaths(storePaths);
+    std::reverse(sorted.begin(), sorted.end());
+
+    run(store, std::move(sorted));
 }
 
-void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
 {
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
@@ -200,10 +241,10 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
-            [&](BuiltPath::Opaque bo) {
+            [&](const BuiltPath::Opaque & bo) {
                 result.push_back(bo.path);
             },
-            [&](BuiltPath::Built bfd) {
+            [&](const BuiltPath::Built & bfd) {
                 for (auto & output : bfd.outputs) {
                     result.push_back(output.second);
                 }

src/libcmd/command.hh

@@ -43,6 +43,19 @@ private:
     std::shared_ptr<Store> _store;
 };
 
+/* A command that copies something between `--from` and `--to`
+   stores. */
+struct CopyCommand : virtual StoreCommand
+{
+    std::string srcUri, dstUri;
+
+    CopyCommand();
+
+    ref<Store> createStore() override;
+
+    ref<Store> getDstStore();
+};
+
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     EvalCommand();
@@ -108,6 +121,8 @@ enum class Realise {
        exists. */
     Derivation,
     /* Evaluate in dry-run mode. Postcondition: nothing. */
+    // FIXME: currently unused, but could be revived if we can
+    // evaluate derivations in-memory.
     Nothing
 };
 
@@ -167,7 +182,7 @@ public:
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, BuiltPaths paths) = 0;
+    virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
 
     void run(ref<Store> store) override;
 
@@ -180,9 +195,9 @@ struct StorePathsCommand : public BuiltPathsCommand
 
     using BuiltPathsCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+    virtual void run(ref<Store> store, std::vector<StorePath> && storePaths) = 0;
 
-    void run(ref<Store> store, BuiltPaths paths) override;
+    void run(ref<Store> store, BuiltPaths && paths) override;
 };
 
 /* A command that operates on exactly one store path. */
@@ -192,7 +207,7 @@ struct StorePathCommand : public StorePathsCommand
 
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store, std::vector<StorePath> storePaths) override;
+    void run(ref<Store> store, std::vector<StorePath> && storePaths) override;
 };
 
 /* A helper class for registering commands globally. */
@@ -223,15 +238,18 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store, Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode = bmNormal);
 
 std::set<StorePath> toStorePaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
+    const std::vector<std::shared_ptr<Installable>> & installables);
 
 StorePath toStorePath(
     ref<Store> evalStore,
@@ -240,8 +258,9 @@ StorePath toStorePath(
     OperateOn operateOn,
     std::shared_ptr<Installable> installable);
 
-std::set<StorePath> toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables,
+std::set<StorePath> toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
     bool useDeriver = false);
 
 BuiltPaths toBuiltPaths(
@@ -249,7 +268,7 @@ BuiltPaths toBuiltPaths(
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
+    const std::vector<std::shared_ptr<Installable>> & installables);
 
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */

src/libcmd/installables.cc

@@ -191,7 +191,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
         auto sep = prefix_.rfind('.');
         std::string searchWord;
         if (sep != std::string::npos) {
-            searchWord = prefix_.substr(sep, std::string::npos);
+            searchWord = prefix_.substr(sep + 1, std::string::npos);
             prefix_ = prefix_.substr(0, sep);
         } else {
             searchWord = prefix_;
@@ -203,6 +203,8 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
         Value v2;
         state->autoCallFunction(*autoArgs, v1, v2);
 
+        completionType = ctAttrs;
+
         if (v2.type() == nAttrs) {
             for (auto & i : *v2.attrs) {
                 std::string name = i.name;
@@ -232,7 +234,9 @@ void completeFlakeRefWithFragment(
        prefix. */
     try {
         auto hash = prefix.find('#');
-        if (hash != std::string::npos) {
+        if (hash == std::string::npos) {
+            completeFlakeRef(evalState->store, prefix);
+        } else {
             auto fragment = prefix.substr(hash + 1);
             auto flakeRefS = std::string(prefix.substr(0, hash));
             // FIXME: do tilde expansion.
@@ -248,6 +252,8 @@ void completeFlakeRefWithFragment(
                flake. */
             attrPathPrefixes.push_back("");
 
+            completionType = ctAttrs;
+
             for (auto & attrPathPrefixS : attrPathPrefixes) {
                 auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS);
                 auto attrPathS = attrPathPrefixS + std::string(fragment);
@@ -285,12 +291,13 @@ void completeFlakeRefWithFragment(
     } catch (Error & e) {
         warn(e.msg());
     }
-
-    completeFlakeRef(evalState->store, prefix);
 }
 
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
+    if (!settings.isExperimentalFeatureEnabled(Xp::Flakes))
+        return;
+
     if (prefix == "")
         completions->add(".");
 
@@ -338,6 +345,18 @@ Installable::getCursor(EvalState & state)
     return cursors[0];
 }
 
+static StorePath getDeriver(
+    ref<Store> store,
+    const Installable & i,
+    const StorePath & drvPath)
+{
+    auto derivers = store->queryValidDerivers(drvPath);
+    if (derivers.empty())
+        throw Error("'%s' does not have a known deriver", i.what());
+    // FIXME: use all derivers?
+    return *derivers.begin();
+}
+
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
@@ -346,7 +365,7 @@ struct InstallableStorePath : Installable
     InstallableStorePath(ref<Store> store, StorePath && storePath)
         : store(store), storePath(std::move(storePath)) { }
 
-    std::string what() override { return store->printStorePath(storePath); }
+    std::string what() const override { return store->printStorePath(storePath); }
 
     DerivedPaths toDerivedPaths() override
     {
@@ -367,6 +386,15 @@ struct InstallableStorePath : Installable
         }
     }
 
+    StorePathSet toDrvPaths(ref<Store> store) override
+    {
+        if (storePath.isDerivation()) {
+            return {storePath};
+        } else {
+            return {getDeriver(store, *this, storePath)};
+        }
+    }
+
     std::optional<StorePath> getStorePath() override
     {
         return storePath;
@@ -395,6 +423,14 @@ DerivedPaths InstallableValue::toDerivedPaths()
     return res;
 }
 
+StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
+{
+    StorePathSet res;
+    for (auto & drv : toDerivations())
+        res.insert(drv.drvPath);
+    return res;
+}
+
 struct InstallableAttrPath : InstallableValue
 {
     SourceExprCommand & cmd;
@@ -405,7 +441,7 @@ struct InstallableAttrPath : InstallableValue
         : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
     { }
 
-    std::string what() override { return attrPath; }
+    std::string what() const override { return attrPath; }
 
     std::pair<Value *, Pos> toValue(EvalState & state) override
     {
@@ -654,6 +690,17 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         for (auto & s : ss) {
             std::exception_ptr ex;
 
+            if (s.find('/') != std::string::npos) {
+                try {
+                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
+                    continue;
+                } catch (BadStorePath &) {
+                } catch (...) {
+                    if (!ex)
+                        ex = std::current_exception();
+                }
+            }
+
             try {
                 auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
@@ -668,25 +715,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
                 ex = std::current_exception();
             }
 
-            if (s.find('/') != std::string::npos) {
-                try {
-                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
-                    continue;
-                } catch (BadStorePath &) {
-                } catch (...) {
-                    if (!ex)
-                        ex = std::current_exception();
-                }
-            }
-
             std::rethrow_exception(ex);
-
-            /*
-            throw Error(
-                pathExists(s)
-                ? "path '%s' is not a flake or a store path"
-                : "don't know how to handle argument '%s'", s);
-            */
         }
     }
 
@@ -704,13 +733,13 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
 BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
 {
     BuiltPaths res;
-    for (auto & b : hopefullyBuiltPaths)
+    for (const auto & b : hopefullyBuiltPaths)
         std::visit(
             overloaded{
-                [&](DerivedPath::Opaque bo) {
+                [&](const DerivedPath::Opaque & bo) {
                     res.push_back(BuiltPath::Opaque{bo.path});
                 },
-                [&](DerivedPath::Built bfd) {
+                [&](const DerivedPath::Built & bfd) {
                     OutputPathMap outputs;
                     auto drv = evalStore->readDerivation(bfd.drvPath);
                     auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
@@ -721,7 +750,7 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
                                 "the derivation '%s' doesn't have an output named '%s'",
                                 store->printStorePath(bfd.drvPath), output);
                         if (settings.isExperimentalFeatureEnabled(
-                                "ca-derivations")) {
+                                Xp::CaDerivations)) {
                             auto outputId =
                                 DrvOutput{outputHashes.at(output), output};
                             auto realisation =
@@ -750,8 +779,12 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
     return res;
 }
 
-BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode)
 {
     if (mode == Realise::Nothing)
         settings.readOnlyMode = true;
@@ -763,7 +796,7 @@ BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
         pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
     }
 
-    if (mode == Realise::Nothing)
+    if (mode == Realise::Nothing || mode == Realise::Derivation)
         printMissing(store, pathsToBuild, lvlError);
     else if (mode == Realise::Outputs)
         store->buildPaths(pathsToBuild, bMode, evalStore);
@@ -776,7 +809,7 @@ BuiltPaths toBuiltPaths(
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
     if (operateOn == OperateOn::Output)
         return build(evalStore, store, mode, installables);
@@ -795,7 +828,7 @@ StorePathSet toStorePaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
     StorePathSet outPaths;
     for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
@@ -819,24 +852,22 @@ StorePath toStorePath(
     return *paths.begin();
 }
 
-StorePathSet toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables, bool useDeriver)
+StorePathSet toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    bool useDeriver)
 {
     StorePathSet drvPaths;
 
-    for (auto & i : installables)
-        for (auto & b : i->toDerivedPaths())
+    for (const auto & i : installables)
+        for (const auto & b : i->toDerivedPaths())
             std::visit(overloaded {
-                [&](DerivedPath::Opaque bo) {
+                [&](const DerivedPath::Opaque & bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
-                    auto derivers = store->queryValidDerivers(bo.path);
-                    if (derivers.empty())
-                        throw Error("'%s' does not have a known deriver", i->what());
-                    // FIXME: use all derivers?
-                    drvPaths.insert(*derivers.begin());
+                    drvPaths.insert(getDeriver(store, *i, bo.path));
                 },
-                [&](DerivedPath::Built bfd) {
+                [&](const DerivedPath::Built & bfd) {
                     drvPaths.insert(bfd.drvPath);
                 },
             }, b.raw());

src/libcmd/installables.hh

@@ -33,10 +33,15 @@ struct Installable
 {
     virtual ~Installable() { }
 
-    virtual std::string what() = 0;
+    virtual std::string what() const = 0;
 
     virtual DerivedPaths toDerivedPaths() = 0;
 
+    virtual StorePathSet toDrvPaths(ref<Store> store)
+    {
+        throw Error("'%s' cannot be converted to a derivation path", what());
+    }
+
     DerivedPath toDerivedPath();
 
     UnresolvedApp toApp(EvalState & state);
@@ -81,6 +86,8 @@ struct InstallableValue : Installable
     virtual std::vector<DerivationInfo> toDerivations() = 0;
 
     DerivedPaths toDerivedPaths() override;
+
+    StorePathSet toDrvPaths(ref<Store> store) override;
 };
 
 struct InstallableFlake : InstallableValue
@@ -99,7 +106,7 @@ struct InstallableFlake : InstallableValue
         Strings && prefixes,
         const flake::LockFlags & lockFlags);
 
-    std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
+    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
 
     std::vector<std::string> getActualAttrPaths();
 

src/libcmd/local.mk

@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS = -llowdown -pthread
+libcmd_LDFLAGS += -llowdown -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 

src/libcmd/markdown.cc

@@ -12,7 +12,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     struct lowdown_opts opts {
         .type = LOWDOWN_TERM,
         .maxdepth = 20,
-        .cols = std::min(getWindowSize().second, (unsigned short) 80),
+        .cols = std::max(getWindowSize().second, (unsigned short) 80),
         .hmargin = 0,
         .vmargin = 0,
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
@@ -25,7 +25,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     Finally freeDoc([&]() { lowdown_doc_free(doc); });
 
     size_t maxn = 0;
-    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size(), nullptr);
     if (!node)
         throw Error("cannot parse Markdown document");
     Finally freeNode([&]() { lowdown_node_free(node); });
@@ -40,11 +40,11 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot allocate Markdown output buffer");
     Finally freeBuffer([&]() { lowdown_buf_free(buf); });
 
-    int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
+    int rndr_res = lowdown_term_rndr(buf, renderer, node);
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return std::string(buf->data, buf->size);
+    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
 }
 
 }

src/libexpr/attr-set.cc

@@ -7,26 +7,19 @@
 namespace nix {
 
 
+
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
 Bindings * EvalState::allocBindings(size_t capacity)
 {
+    if (capacity == 0)
+        return &emptyBindings;
     if (capacity > std::numeric_limits<Bindings::size_t>::max())
         throw Error("attribute set of size %d is too big", capacity);
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
-}
-
-
-void EvalState::mkAttrs(Value & v, size_t capacity)
-{
-    if (capacity == 0) {
-        v = vEmptySet;
-        return;
-    }
-    v.mkAttrs(allocBindings(capacity));
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
@@ -41,15 +34,36 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 }
 
 
-Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+Value * EvalState::allocAttr(Value & vAttrs, std::string_view name)
 {
     return allocAttr(vAttrs, symbols.create(name));
 }
 
 
+Value & BindingsBuilder::alloc(const Symbol & name, ptr<Pos> pos)
+{
+    auto value = state.allocValue();
+    bindings->push_back(Attr(name, value, pos));
+    return *value;
+}
+
+
+Value & BindingsBuilder::alloc(std::string_view name, ptr<Pos> pos)
+{
+    return alloc(state.symbols.create(name), pos);
+}
+
+
 void Bindings::sort()
 {
-    std::sort(begin(), end());
+    if (size_) std::sort(begin(), end());
+}
+
+
+Value & Value::mkAttrs(BindingsBuilder & bindings)
+{
+    mkAttrs(bindings.finish());
+    return *this;
 }
 
 

src/libexpr/attr-set.hh

@@ -17,8 +17,8 @@ struct Attr
 {
     Symbol name;
     Value * value;
-    Pos * pos;
-    Attr(Symbol name, Value * value, Pos * pos = &noPos)
+    ptr<Pos> pos;
+    Attr(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos))
         : name(name), value(value), pos(pos) { };
     Attr() : pos(&noPos) { };
     bool operator < (const Attr & a) const
@@ -35,13 +35,13 @@ class Bindings
 {
 public:
     typedef uint32_t size_t;
-    Pos *pos;
+    ptr<Pos> pos;
 
 private:
     size_t size_, capacity_;
     Attr attrs[0];
 
-    Bindings(size_t capacity) : size_(0), capacity_(capacity) { }
+    Bindings(size_t capacity) : pos(&noPos), size_(0), capacity_(capacity) { }
     Bindings(const Bindings & bindings) = delete;
 
 public:
@@ -113,5 +113,52 @@ public:
     friend class EvalState;
 };
 
+/* A wrapper around Bindings that ensures that its always in sorted
+   order at the end. The only way to consume a BindingsBuilder is to
+   call finish(), which sorts the bindings. */
+class BindingsBuilder
+{
+    Bindings * bindings;
+
+public:
+    // needed by std::back_inserter
+    using value_type = Attr;
+
+    EvalState & state;
+
+    BindingsBuilder(EvalState & state, Bindings * bindings)
+        : bindings(bindings), state(state)
+    { }
+
+    void insert(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos))
+    {
+        insert(Attr(name, value, pos));
+    }
+
+    void insert(const Attr & attr)
+    {
+        push_back(attr);
+    }
+
+    void push_back(const Attr & attr)
+    {
+        bindings->push_back(attr);
+    }
+
+    Value & alloc(const Symbol & name, ptr<Pos> pos = ptr(&noPos));
+
+    Value & alloc(std::string_view name, ptr<Pos> pos = ptr(&noPos));
+
+    Bindings * finish()
+    {
+        bindings->sort();
+        return bindings;
+    }
+
+    Bindings * alreadySorted()
+    {
+        return bindings;
+    }
+};
 
 }

src/libexpr/common-eval-args.cc

@@ -73,17 +73,16 @@ MixEvalArgs::MixEvalArgs()
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
-    Bindings * res = state.allocBindings(autoArgs.size());
+    auto res = state.buildBindings(autoArgs.size());
     for (auto & i : autoArgs) {
-        Value * v = state.allocValue();
+        auto v = state.allocValue();
         if (i.second[0] == 'E')
             state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
         else
-            mkString(*v, string(i.second, 1));
-        res->push_back(Attr(state.symbols.create(i.first), v));
+            v->mkString(((std::string_view) i.second).substr(1));
+        res.insert(state.symbols.create(i.first), v);
     }
-    res->sort();
-    return res;
+    return res.finish();
 }
 
 Path lookupFileArg(EvalState & state, string s)

src/libexpr/eval.hh

@@ -5,6 +5,7 @@
 #include "nixexpr.hh"
 #include "symbol-table.hh"
 #include "config.hh"
+#include "experimental-features.hh"
 
 #include <map>
 #include <optional>
@@ -43,8 +44,6 @@ struct Env
 };
 
 
-Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
-
 void copyContext(const Value & v, PathSet & context);
 
 
@@ -81,7 +80,8 @@ public:
         sContentAddressed,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
-        sDescription, sSelf, sEpsilon;
+        sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
+        sPrefix;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -92,7 +92,7 @@ public:
        mode. */
     std::optional<PathSet> allowedPaths;
 
-    Value vEmptySet;
+    Bindings emptyBindings;
 
     /* Store used to materialise .drv files. */
     const ref<Store> store;
@@ -100,6 +100,8 @@ public:
     /* Store used to build stuff. */
     const ref<Store> buildStore;
 
+    RootValue vCallFlake = nullptr;
+    RootValue vImportedDrvToDerivation = nullptr;
 
 private:
     SrcToStore srcToStore;
@@ -130,6 +132,9 @@ private:
     /* Cache used by prim_match(). */
     std::shared_ptr<RegexCache> regexCache;
 
+    /* Allocation cache for GC'd Value objects. */
+    std::shared_ptr<void *> valueAllocCache;
+
 public:
 
     EvalState(
@@ -138,10 +143,25 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
+    void requireExperimentalFeatureOnEvaluation(
+        const ExperimentalFeature &,
+        const std::string_view fName,
+        const Pos & pos
+    );
+
     void addToSearchPath(const string & s);
 
     SearchPath getSearchPath() { return searchPath; }
 
+    /* Allow access to a path. */
+    void allowPath(const Path & path);
+
+    /* Allow access to a store path. Note that this gets remapped to
+       the real store path if `store` is a chroot store. */
+    void allowPath(const StorePath & storePath);
+
+    /* Check whether access to a path is allowed and throw an error if
+       not. Otherwise return the canonicalised path. */
     Path checkSourcePath(const Path & path);
 
     void checkURI(const std::string & uri);
@@ -160,8 +180,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(std::string_view s, const Path & basePath);
+    Expr * parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string s, const Path & basePath);
 
     Expr * parseStdin();
 
@@ -170,6 +190,14 @@ public:
        trivial (i.e. doesn't require arbitrary computation). */
     void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
 
+    /* Like `cacheFile`, but with an already parsed expression. */
+    void cacheFile(
+        const Path & path,
+        const Path & resolvedPath,
+        Expr * e,
+        Value & v,
+        bool mustBeTrivial = false);
+
     void resetFileCache();
 
     /* Look up a file in the search path. */
@@ -224,7 +252,8 @@ public:
        booleans and lists to a string.  If `copyToStore' is set,
        referenced paths are copied to the Nix store as a side effect. */
     string coerceToString(const Pos & pos, Value & v, PathSet & context,
-        bool coerceMore = false, bool copyToStore = true);
+        bool coerceMore = false, bool copyToStore = true,
+        bool canonicalizePath = true);
 
     string copyPathToStore(PathSet & context, const Path & path);
 
@@ -250,6 +279,8 @@ private:
 
     Value * addConstant(const string & name, Value & v);
 
+    void addConstant(const string & name, Value * v);
+
     Value * addPrimOp(const string & name,
         size_t arity, PrimOpFun primOp);
 
@@ -278,7 +309,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, FileOrigin origin, const Path & path,
+    Expr * parse(char * text, size_t length, FileOrigin origin, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:
@@ -289,8 +320,14 @@ public:
 
     bool isFunctor(Value & fun);
 
-    void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos);
-    void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos);
+    // FIXME: use std::span
+    void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos);
+
+    void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos)
+    {
+        Value * args[] = {&arg};
+        callFunction(fun, 1, args, vRes, pos);
+    }
 
     /* Automatically call a function for which each argument has a
        default value or has a binding in the `args' map. */
@@ -301,21 +338,28 @@ public:
     Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
-    Value * allocAttr(Value & vAttrs, const std::string & name);
+    Value * allocAttr(Value & vAttrs, std::string_view name);
 
     Bindings * allocBindings(size_t capacity);
 
+    BindingsBuilder buildBindings(size_t capacity)
+    {
+        return BindingsBuilder(*this, allocBindings(capacity));
+    }
+
     void mkList(Value & v, size_t length);
-    void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
-    void mkPos(Value & v, Pos * pos);
+    void mkPos(Value & v, ptr<Pos> pos);
 
     void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos);
 
     /* Print statistics. */
     void printStats();
 
-    void realiseContext(const PathSet & context);
+    /* Realise the given context, and return a mapping from the placeholders
+     * used to construct the associated value to their final store path
+     */
+    [[nodiscard]] StringMap realiseContext(const PathSet & context);
 
 private:
 
@@ -356,6 +400,9 @@ private:
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
     friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
+    friend struct Value;
 };
 
 

src/libexpr/flake/config.cc

@@ -1,4 +1,5 @@
 #include "flake.hh"
+#include "globals.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -37,11 +38,11 @@ void ConfigFile::apply()
 
         // FIXME: Move into libutil/config.cc.
         std::string valueS;
-        if (auto s = std::get_if<std::string>(&value))
+        if (auto* s = std::get_if<std::string>(&value))
             valueS = *s;
-        else if (auto n = std::get_if<int64_t>(&value))
-            valueS = fmt("%d", n);
-        else if (auto b = std::get_if<Explicit<bool>>(&value))
+        else if (auto* n = std::get_if<int64_t>(&value))
+            valueS = fmt("%d", *n);
+        else if (auto* b = std::get_if<Explicit<bool>>(&value))
             valueS = b->t ? "true" : "false";
         else if (auto ss = std::get_if<std::vector<std::string>>(&value))
             valueS = concatStringsSep(" ", *ss); // FIXME: evil
@@ -52,21 +53,19 @@ void ConfigFile::apply()
             auto trustedList = readTrustedList();
 
             bool trusted = false;
-
-            if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
+            if (nix::settings.acceptFlakeConfig){
+                trusted = true;
+            } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
                 trusted = *saved;
+                warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
             } else {
                 // FIXME: filter ANSI escapes, newlines, \r, etc.
-                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
-                    if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
-                        trustedList[name][valueS] = false;
-                        writeTrustedList(trustedList);
-                    }
-                } else {
-                    if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
-                        trustedList[name][valueS] = trusted = true;
-                        writeTrustedList(trustedList);
-                    }
+                if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') {
+                    trusted = true;
+                }
+                if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?",  trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') {
+                    trustedList[name][valueS] = trusted;
+                    writeTrustedList(trustedList);
                 }
             }
 

src/libexpr/flake/flake.cc

@@ -64,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
     debug("got tree '%s' from '%s'",
         state.store->printStorePath(tree.storePath), lockedRef);
 
-    if (state.allowedPaths)
-        state.allowedPaths->insert(tree.actualPath);
+    state.allowPath(tree.storePath);
 
     assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
 
@@ -89,10 +88,12 @@ static void expectType(EvalState & state, ValueType type,
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos);
+    EvalState & state, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir);
 
 static FlakeInput parseFlakeInput(EvalState & state,
-    const std::string & inputName, Value * value, const Pos & pos)
+    const std::string & inputName, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir)
 {
     expectType(state, nAttrs, *value, pos);
 
@@ -116,7 +117,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nBool, *attr.value, *attr.pos);
                 input.isFlake = attr.value->boolean;
             } else if (attr.name == sInputs) {
-                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
+                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir);
             } else if (attr.name == sFollows) {
                 expectType(state, nString, *attr.value, *attr.pos);
                 input.follows = parseInputPath(attr.value->string.s);
@@ -154,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         if (!attrs.empty())
             throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
         if (url)
-            input.ref = parseFlakeRef(*url, {}, true);
+            input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake);
     }
 
     if (!input.follows && !input.ref)
@@ -164,7 +165,8 @@ static FlakeInput parseFlakeInput(EvalState & state,
 }
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
-    EvalState & state, Value * value, const Pos & pos)
+    EvalState & state, Value * value, const Pos & pos,
+    const std::optional<Path> & baseDir)
 {
     std::map<FlakeId, FlakeInput> inputs;
 
@@ -175,7 +177,8 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
             parseFlakeInput(state,
                 inputAttr.name,
                 inputAttr.value,
-                *inputAttr.pos));
+                *inputAttr.pos,
+                baseDir));
     }
 
     return inputs;
@@ -191,7 +194,8 @@ static Flake getFlake(
         state, originalRef, allowLookup, flakeCache);
 
     // Guard against symlink attacks.
-    auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
+    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true);
+    auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
             lockedRef, state.store->printStorePath(sourceInfo.storePath));
@@ -219,14 +223,14 @@ static Flake getFlake(
     auto sInputs = state.symbols.create("inputs");
 
     if (auto inputs = vInfo.attrs->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
+        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir);
 
     auto sOutputs = state.symbols.create("outputs");
 
     if (auto outputs = vInfo.attrs->get(sOutputs)) {
         expectType(state, nFunction, *outputs->value, *outputs->pos);
 
-        if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
+        if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) {
             for (auto & formal : outputs->value->lambda.fun->formals->formals) {
                 if (formal.name != state.sSelf)
                     flake.inputs.emplace(formal.name, FlakeInput {
@@ -247,14 +251,17 @@ static Flake getFlake(
             forceTrivialValue(state, *setting.value, *setting.pos);
             if (setting.value->type() == nString)
                 flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
+            else if (setting.value->type() == nPath) {
+                PathSet emptyContext = {};
+                flake.config.settings.insert({setting.name, state.coerceToString(*setting.pos, *setting.value, emptyContext, false, true, true)});
+            }
             else if (setting.value->type() == nInt)
                 flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
             else if (setting.value->type() == nBool)
-                flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
+                flake.config.settings.insert({setting.name, Explicit<bool> { state.forceBool(*setting.value, *setting.pos) }});
             else if (setting.value->type() == nList) {
                 std::vector<std::string> ss;
-                for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
-                    auto elem = setting.value->listElems()[n];
+                for (auto elem : setting.value->listItems()) {
                     if (elem->type() != nString)
                         throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
                             setting.name, showType(*setting.value));
@@ -293,7 +300,7 @@ LockedFlake lockFlake(
     const FlakeRef & topRef,
     const LockFlags & lockFlags)
 {
-    settings.requireExperimentalFeature("flakes");
+    settings.requireExperimentalFeature(Xp::Flakes);
 
     FlakeCache flakeCache;
 
@@ -303,7 +310,7 @@ LockedFlake lockFlake(
 
     if (lockFlags.applyNixConfig) {
         flake.config.apply();
-        // FIXME: send new config to the daemon.
+        state.store->setOptions();
     }
 
     try {
@@ -325,25 +332,40 @@ LockedFlake lockFlake(
 
         std::vector<FlakeRef> parents;
 
+        struct LockParent {
+            /* The path to this parent. */
+            InputPath path;
+
+            /* Whether we are currently inside a top-level lockfile
+               (inputs absolute) or subordinate lockfile (inputs
+               relative). */
+            bool absolute;
+        };
+
         std::function<void(
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
-            std::shared_ptr<const Node> oldNode)>
+            std::shared_ptr<const Node> oldNode,
+            const LockParent & parent,
+            const Path & parentPath,
+            bool trustLock)>
             computeLocks;
 
         computeLocks = [&](
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
-            std::shared_ptr<const Node> oldNode)
+            std::shared_ptr<const Node> oldNode,
+            const LockParent & parent,
+            const Path & parentPath,
+            bool trustLock)
         {
             debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
             /* Get the overrides (i.e. attributes of the form
                'inputs.nixops.inputs.nixpkgs.url = ...'). */
-            // FIXME: check this
-            for (auto & [id, input] : flake.inputs) {
+            for (auto & [id, input] : flakeInputs) {
                 for (auto & [idOverride, inputOverride] : input.overrides) {
                     auto inputPath(inputPathPrefix);
                     inputPath.push_back(id);
@@ -379,15 +401,19 @@ LockedFlake lockFlake(
                        path we haven't processed yet. */
                     if (input.follows) {
                         InputPath target;
-                        if (hasOverride || input.absolute)
-                            /* 'follows' from an override is relative to the
-                               root of the graph. */
+
+                        if (parent.absolute && !hasOverride) {
                             target = *input.follows;
-                        else {
-                            /* Otherwise, it's relative to the current flake. */
-                            target = inputPathPrefix;
+                        } else {
+                            if (hasOverride) {
+                                target = inputPathPrefix;
+                                target.pop_back();
+                            } else
+                                target = parent.path;
+
                             for (auto & i : *input.follows) target.push_back(i);
                         }
+
                         debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
                         node->inputs.insert_or_assign(id, target);
                         continue;
@@ -425,22 +451,18 @@ LockedFlake lockFlake(
                            update it. */
                         auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
 
-                        auto hasChildUpdate =
+                        auto mustRefetch =
                             lb != lockFlags.inputUpdates.end()
                             && lb->size() > inputPath.size()
                             && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
 
-                        if (hasChildUpdate) {
-                            auto inputFlake = getFlake(
-                                state, oldLock->lockedRef, false, flakeCache);
-                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
-                        } else {
+                        FlakeInputs fakeInputs;
+
+                        if (!mustRefetch) {
                             /* No need to fetch this flake, we can be
                                lazy. However there may be new overrides on the
                                inputs of this flake, so we need to check
                                those. */
-                            FlakeInputs fakeInputs;
-
                             for (auto & i : oldLock->inputs) {
                                 if (auto lockedNode = std::get_if<0>(&i.second)) {
                                     fakeInputs.emplace(i.first, FlakeInput {
@@ -448,16 +470,39 @@ LockedFlake lockFlake(
                                         .isFlake = (*lockedNode)->isFlake,
                                     });
                                 } else if (auto follows = std::get_if<1>(&i.second)) {
+                                    if (! trustLock) {
+                                        // It is possible that the flake has changed,
+                                        // so we must confirm all the follows that are in the lockfile are also in the flake.
+                                        auto overridePath(inputPath);
+                                        overridePath.push_back(i.first);
+                                        auto o = overrides.find(overridePath);
+                                        // If the override disappeared, we have to refetch the flake,
+                                        // since some of the inputs may not be present in the lockfile.
+                                        if (o == overrides.end()) {
+                                            mustRefetch = true;
+                                            // There's no point populating the rest of the fake inputs,
+                                            // since we'll refetch the flake anyways.
+                                            break;
+                                        }
+                                    }
                                     fakeInputs.emplace(i.first, FlakeInput {
                                         .follows = *follows,
-                                        .absolute = true
                                     });
                                 }
                             }
-
-                            computeLocks(fakeInputs, childNode, inputPath, oldLock);
                         }
 
+                        LockParent newParent {
+                            .path = inputPath,
+                            .absolute = true
+                        };
+
+                        computeLocks(
+                            mustRefetch
+                            ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs
+                            : fakeInputs,
+                            childNode, inputPath, oldLock, newParent, parentPath, !mustRefetch);
+
                     } else {
                         /* We need to create a new lock file entry. So fetch
                            this input. */
@@ -467,7 +512,15 @@ LockedFlake lockFlake(
                             throw Error("cannot update flake input '%s' in pure mode", inputPathS);
 
                         if (input.isFlake) {
-                            auto inputFlake = getFlake(state, *input.ref, useRegistries, flakeCache);
+                            Path localPath = parentPath;
+                            FlakeRef localRef = *input.ref;
+
+                            // If this input is a path, recurse it down.
+                            // This allows us to resolve path inputs relative to the current flake.
+                            if (localRef.input.getType() == "path")
+                                localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
+
+                            auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache);
 
                             /* Note: in case of an --override-input, we use
                                the *original* ref (input2.ref) for the
@@ -488,6 +541,13 @@ LockedFlake lockFlake(
                             parents.push_back(*input.ref);
                             Finally cleanup([&]() { parents.pop_back(); });
 
+                            // Follows paths from existing inputs in the top-level lockfile are absolute,
+                            // whereas paths in subordinate lockfiles are relative to those lockfiles.
+                            LockParent newParent {
+                                .path = inputPath,
+                                .absolute = oldLock ? true : false
+                            };
+
                             /* Recursively process the inputs of this
                                flake. Also, unless we already have this flake
                                in the top-level lock file, use this flake's
@@ -497,7 +557,8 @@ LockedFlake lockFlake(
                                 oldLock
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
                                 : LockFile::read(
-                                    inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+                                    inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root,
+                                newParent, localPath, false);
                         }
 
                         else {
@@ -515,9 +576,17 @@ LockedFlake lockFlake(
             }
         };
 
+        LockParent parent {
+            .path = {},
+            .absolute = true
+        };
+
+        // Bring in the current ref for relative path resolution if we have it
+        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true);
+
         computeLocks(
             flake.inputs, newLockFile.root, {},
-            lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath, false);
 
         for (auto & i : lockFlags.inputOverrides)
             if (!overridesUsed.count(i.first))
@@ -564,12 +633,24 @@ LockedFlake lockFlake(
 
                         newLockFile.write(path);
 
+                        std::optional<std::string> commitMessage = std::nullopt;
+                        if (lockFlags.commitLockFile) {
+                            std::string cm;
+
+                            cm = settings.commitLockFileSummary.get();
+
+                            if (cm == "") {
+                                cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
+                            }
+
+                            cm += "\n\nFlake lock file updates:\n\n";
+                            cm += filterANSIEscapes(diff, true);
+                            commitMessage = cm;
+                        }
+
                         topRef.input.markChangedFile(
                             (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
-                            lockFlags.commitLockFile
-                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s",
-                                    relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true)))
-                            : std::nullopt);
+                            commitMessage);
 
                         /* Rewriting the lockfile changed the top-level
                            repo, so we should re-read it. FIXME: we could
@@ -593,8 +674,10 @@ LockedFlake lockFlake(
                     }
                 } else
                     throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
-            } else
+            } else {
                 warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
+                flake.forceDirty = true;
+            }
         }
 
         return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
@@ -615,28 +698,34 @@ void callFlake(EvalState & state,
     auto vTmp1 = state.allocValue();
     auto vTmp2 = state.allocValue();
 
-    mkString(*vLocks, lockedFlake.lockFile.to_string());
+    vLocks->mkString(lockedFlake.lockFile.to_string());
 
-    emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
+    emitTreeAttrs(
+        state,
+        *lockedFlake.flake.sourceInfo,
+        lockedFlake.flake.lockedRef.input,
+        *vRootSrc,
+        false,
+        lockedFlake.flake.forceDirty);
 
-    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+    vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
 
-    static RootValue vCallFlake = nullptr;
-
-    if (!vCallFlake) {
-        vCallFlake = allocRootValue(state.allocValue());
+    if (!state.vCallFlake) {
+        state.vCallFlake = allocRootValue(state.allocValue());
         state.eval(state.parseExprFromString(
             #include "call-flake.nix.gen.hh"
-            , "/"), **vCallFlake);
+            , "/"), **state.vCallFlake);
     }
 
-    state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+    state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
     state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
     state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
 }
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
+    state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
+
     auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
     if (evalSettings.pureEval && !flakeRef.input.isImmutable())
@@ -646,13 +735,13 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
         lockFlake(state, flakeRef,
             LockFlags {
                 .updateLockFile = false,
-                .useRegistries = !evalSettings.pureEval && !settings.useRegistries,
+                .useRegistries = !evalSettings.pureEval && settings.useRegistries,
                 .allowMutable  = !evalSettings.pureEval,
             }),
         v);
 }
 
-static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
+static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
 
 }
 
@@ -662,8 +751,9 @@ Fingerprint LockedFlake::getFingerprint() const
     // and we haven't changed it, then it's sufficient to use
     // flake.sourceInfo.storePath for the fingerprint.
     return hashString(htSHA256,
-        fmt("%s;%d;%d;%s",
+        fmt("%s;%s;%d;%d;%s",
             flake.sourceInfo->storePath.to_string(),
+            flake.lockedRef.subdir,
             flake.lockedRef.input.getRevCount().value_or(0),
             flake.lockedRef.input.getLastModified().value_or(0),
             lockFile));

src/libexpr/flake/flake.hh

@@ -43,7 +43,6 @@ struct FlakeInput
     std::optional<FlakeRef> ref;
     bool isFlake = true;  // true = process flake to get outputs, false = (fetched) static source path
     std::optional<InputPath> follows;
-    bool absolute = false; // whether 'follows' is relative to the flake root
     FlakeInputs overrides;
 };
 
@@ -59,9 +58,10 @@ struct ConfigFile
 /* The contents of a flake.nix file. */
 struct Flake
 {
-    FlakeRef originalRef;   // the original flake specification (by the user)
-    FlakeRef resolvedRef;   // registry references and caching resolved to the specific underlying flake
-    FlakeRef lockedRef;     // the specific local store result of invoking the fetcher
+    FlakeRef originalRef; // the original flake specification (by the user)
+    FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
+    FlakeRef lockedRef; // the specific local store result of invoking the fetcher
+    bool forceDirty = false; // pretend that 'lockedRef' is dirty
     std::optional<std::string> description;
     std::shared_ptr<const fetchers::Tree> sourceInfo;
     FlakeInputs inputs;
@@ -141,6 +141,8 @@ void emitTreeAttrs(
     EvalState & state,
     const fetchers::Tree & tree,
     const fetchers::Input & input,
-    Value & v, bool emptyRevFallback = false);
+    Value & v,
+    bool emptyRevFallback = false,
+    bool forceDirty = false);
 
 }

src/libexpr/flake/flakeref.cc

@@ -48,9 +48,12 @@ FlakeRef FlakeRef::resolve(ref<Store> store) const
 }
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
     if (fragment != "")
         throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
     return flakeRef;
@@ -67,7 +70,10 @@ std::optional<FlakeRef> maybeParseFlakeRef(
 }
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
     using namespace fetchers;
 
@@ -112,46 +118,71 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
                to 'baseDir'). If so, search upward to the root of the
                repo (i.e. the directory containing .git). */
 
-            path = absPath(path, baseDir, true);
+            path = absPath(path, baseDir);
 
-            if (!S_ISDIR(lstat(path).st_mode))
-                throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+            if (isFlake) {
 
-            if (!allowMissing && !pathExists(path + "/flake.nix"))
-                throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+                if (!allowMissing && !pathExists(path + "/flake.nix")){
+                    notice("path '%s' does not contain a 'flake.nix', searching up",path);
 
-            auto flakeRoot = path;
-            std::string subdir;
-
-            while (flakeRoot != "/") {
-                if (pathExists(flakeRoot + "/.git")) {
-                    auto base = std::string("git+file://") + flakeRoot;
-
-                    auto parsedURL = ParsedURL{
-                        .url = base, // FIXME
-                        .base = base,
-                        .scheme = "git+file",
-                        .authority = "",
-                        .path = flakeRoot,
-                        .query = decodeQuery(match[2]),
-                    };
-
-                    if (subdir != "") {
-                        if (parsedURL.query.count("dir"))
-                            throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
-                        parsedURL.query.insert_or_assign("dir", subdir);
+                    // Save device to detect filesystem boundary
+                    dev_t device = lstat(path).st_dev;
+                    bool found = false;
+                    while (path != "/") {
+                        if (pathExists(path + "/flake.nix")) {
+                            found = true;
+                            break;
+                        } else if (pathExists(path + "/.git"))
+                            throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
+                        else {
+                            if (lstat(path).st_dev != device)
+                                throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
+                        }
+                        path = dirOf(path);
                     }
-
-                    if (pathExists(flakeRoot + "/.git/shallow"))
-                        parsedURL.query.insert_or_assign("shallow", "1");
-
-                    return std::make_pair(
-                        FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
-                        fragment);
+                    if (!found)
+                        throw BadURL("could not find a flake.nix file");
                 }
 
-                subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-                flakeRoot = dirOf(flakeRoot);
+                if (!S_ISDIR(lstat(path).st_mode))
+                    throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+
+                if (!allowMissing && !pathExists(path + "/flake.nix"))
+                    throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+
+                auto flakeRoot = path;
+                std::string subdir;
+
+                while (flakeRoot != "/") {
+                    if (pathExists(flakeRoot + "/.git")) {
+                        auto base = std::string("git+file://") + flakeRoot;
+
+                        auto parsedURL = ParsedURL{
+                            .url = base, // FIXME
+                            .base = base,
+                            .scheme = "git+file",
+                            .authority = "",
+                            .path = flakeRoot,
+                            .query = decodeQuery(match[2]),
+                        };
+
+                        if (subdir != "") {
+                            if (parsedURL.query.count("dir"))
+                                throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                            parsedURL.query.insert_or_assign("dir", subdir);
+                        }
+
+                        if (pathExists(flakeRoot + "/.git/shallow"))
+                            parsedURL.query.insert_or_assign("shallow", "1");
+
+                        return std::make_pair(
+                            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                            fragment);
+                    }
+
+                    subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+                    flakeRoot = dirOf(flakeRoot);
+                }
             }
 
         } else {
@@ -172,8 +203,12 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
         auto parsedURL = parseURL(url);
         std::string fragment;
         std::swap(fragment, parsedURL.fragment);
+
+        auto input = Input::fromURL(parsedURL);
+        input.parent = baseDir;
+
         return std::make_pair(
-            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+            FlakeRef(std::move(input), get(parsedURL.query, "dir").value_or("")),
             fragment);
     }
 }

src/libexpr/flake/flakeref.hh

@@ -62,13 +62,19 @@ struct FlakeRef
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<FlakeRef> maybeParseFlake(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});

src/libexpr/get-drvs.cc

@@ -102,9 +102,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
             state->forceList(*i->value, *i->pos);
 
             /* For each output... */
-            for (unsigned int j = 0; j < i->value->listSize(); ++j) {
+            for (auto elem : i->value->listItems()) {
                 /* Evaluate the corresponding set. */
-                string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos);
+                string name = state->forceStringNoCtx(*elem, *i->pos);
                 Bindings::iterator out = attrs->find(state->symbols.create(name));
                 if (out == attrs->end()) continue; // FIXME: throw error?
                 state->forceAttrs(*out->value);
@@ -128,9 +128,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
         /* ^ this shows during `nix-env -i` right under the bad derivation */
     if (!outTI->isList()) throw errMsg;
     Outputs result;
-    for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
-        if ((*i)->type() != nString) throw errMsg;
-        auto out = outputs.find((*i)->string.s);
+    for (auto elem : outTI->listItems()) {
+        if (elem->type() != nString) throw errMsg;
+        auto out = outputs.find(elem->string.s);
         if (out == outputs.end()) throw errMsg;
         result.insert(*out);
     }
@@ -174,8 +174,8 @@ bool DrvInfo::checkMeta(Value & v)
 {
     state->forceValue(v);
     if (v.type() == nList) {
-        for (unsigned int n = 0; n < v.listSize(); ++n)
-            if (!checkMeta(*v.listElems()[n])) return false;
+        for (auto elem : v.listItems())
+            if (!checkMeta(*elem)) return false;
         return true;
     }
     else if (v.type() == nAttrs) {
@@ -254,15 +254,14 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 void DrvInfo::setMeta(const string & name, Value * v)
 {
     getMeta();
-    Bindings * old = meta;
-    meta = state->allocBindings(1 + (old ? old->size() : 0));
+    auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0));
     Symbol sym = state->symbols.create(name);
-    if (old)
-        for (auto i : *old)
+    if (meta)
+        for (auto i : *meta)
             if (i.name != sym)
-                meta->push_back(i);
-    if (v) meta->push_back(Attr(sym, v));
-    meta->sort();
+                attrs.insert(i);
+    if (v) attrs.insert(sym, v);
+    meta = attrs.finish();
 }
 
 
@@ -364,10 +363,10 @@ static void getDerivations(EvalState & state, Value & vIn,
     }
 
     else if (v.type() == nList) {
-        for (unsigned int n = 0; n < v.listSize(); ++n) {
-            string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
-            if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
-                getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
+        for (auto [n, elem] : enumerate(v.listItems())) {
+            string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
+            if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures))
+                getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
     }
 

src/libexpr/json-to-value.cc

@@ -37,10 +37,10 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueMap attrs;
         std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
-            state.mkAttrs(v, attrs.size());
+            auto attrs2 = state.buildBindings(attrs.size());
             for (auto & i : attrs)
-                v.attrs->push_back(Attr(i.first, i.second));
+                attrs2.insert(i.first, i.second);
+            parent->value(state).mkAttrs(attrs2.alreadySorted());
             return std::move(parent);
         }
         void add() override { v = nullptr; }
@@ -76,45 +76,51 @@ class JSONSax : nlohmann::json_sax<json> {
     EvalState & state;
     std::unique_ptr<JSONState> rs;
 
-    template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
-    {
-        f(rs->value(state), args...);
-        rs->add();
-        return true;
-    }
-
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
 
     bool null()
     {
-        return handle_value(mkNull);
+        rs->value(state).mkNull();
+        rs->add();
+        return true;
     }
 
     bool boolean(bool val)
     {
-        return handle_value(mkBool, val);
+        rs->value(state).mkBool(val);
+        rs->add();
+        return true;
     }
 
     bool number_integer(number_integer_t val)
     {
-        return handle_value(mkInt, val);
+        rs->value(state).mkInt(val);
+        rs->add();
+        return true;
     }
 
     bool number_unsigned(number_unsigned_t val)
     {
-        return handle_value(mkInt, val);
+        rs->value(state).mkInt(val);
+        rs->add();
+        return true;
     }
 
     bool number_float(number_float_t val, const string_t & s)
     {
-        return handle_value(mkFloat, val);
+        rs->value(state).mkFloat(val);
+        rs->add();
+        return true;
     }
 
     bool string(string_t & val)
     {
-        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
+        rs->value(state).mkString(val);
+        rs->add();
+        return true;
     }
+
 #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
     bool binary(binary_t&)
     {

src/libexpr/lexer.l

@@ -9,6 +9,9 @@
 %s DEFAULT
 %x STRING
 %x IND_STRING
+%x INPATH
+%x INPATH_SLASH
+%x PATH_START
 
 
 %{
@@ -25,6 +28,8 @@ using namespace nix;
 
 namespace nix {
 
+// backup to recover from yyless(0)
+YYLTYPE prev_yylloc;
 
 static void initLoc(YYLTYPE * loc)
 {
@@ -35,6 +40,8 @@ static void initLoc(YYLTYPE * loc)
 
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
+    prev_yylloc = *loc;
+
     loc->first_line = loc->last_line;
     loc->first_column = loc->last_column;
 
@@ -57,28 +64,32 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
-static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
+// we make use of the fact that the parser receives a private copy of the input
+// string and can munge around in it.
+static Expr * unescapeStr(SymbolTable & symbols, char * s, size_t length)
 {
-    string t;
-    t.reserve(length);
+    char * result = s;
+    char * t = s;
     char c;
+    // the input string is terminated with *two* NULs, so we can safely take
+    // *one* character after the one being checked against.
     while ((c = *s++)) {
         if (c == '\\') {
-            assert(*s);
             c = *s++;
-            if (c == 'n') t += '\n';
-            else if (c == 'r') t += '\r';
-            else if (c == 't') t += '\t';
-            else t += c;
+            if (c == 'n') *t = '\n';
+            else if (c == 'r') *t = '\r';
+            else if (c == 't') *t = '\t';
+            else *t = c;
         }
         else if (c == '\r') {
             /* Normalise CR and CR/LF into LF. */
-            t += '\n';
+            *t = '\n';
             if (*s == '\n') s++; /* cr/lf */
         }
-        else t += c;
+        else *t = c;
+        t++;
     }
-    return new ExprString(symbols.create(t));
+    return new ExprString(symbols.create({result, size_t(t - result)}));
 }
 
 
@@ -97,9 +108,12 @@ ANY         .|\n
 ID          [a-zA-Z\_][a-zA-Z0-9\_\'\-]*
 INT         [0-9]+
 FLOAT       (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH        [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
-HPATH       \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/?
-SPATH       \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\>
+PATH_CHAR   [a-zA-Z0-9\.\_\-\+]
+PATH        {PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
+PATH_SEG    {PATH_CHAR}*\/
+HPATH       \~(\/{PATH_CHAR}+)+\/?
+HPATH_START \~\/
+SPATH       \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
 URI         [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+
 
 
@@ -128,7 +142,7 @@ or          { return OR_KW; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
 
-{ID}        { yylval->id = strdup(yytext); return ID; }
+{ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
@@ -200,19 +214,77 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 
+{PATH_SEG}\$\{ |
+{HPATH_START}\$\{ {
+  PUSH_STATE(PATH_START);
+  yyless(0);
+  *yylloc = prev_yylloc;
+}
 
-{PATH}      { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path '%s' has a trailing slash", yytext);
-              yylval->path = strdup(yytext);
-              return PATH;
-            }
-{HPATH}     { if (yytext[yyleng-1] == '/')
-                  throw ParseError("path '%s' has a trailing slash", yytext);
-              yylval->path = strdup(yytext);
-              return HPATH;
-            }
-{SPATH}     { yylval->path = strdup(yytext); return SPATH; }
-{URI}       { yylval->uri = strdup(yytext); return URI; }
+<PATH_START>{PATH_SEG} {
+  POP_STATE();
+  PUSH_STATE(INPATH_SLASH);
+  yylval->path = {yytext, (size_t) yyleng};
+  return PATH;
+}
+
+<PATH_START>{HPATH_START} {
+  POP_STATE();
+  PUSH_STATE(INPATH_SLASH);
+  yylval->path = {yytext, (size_t) yyleng};
+  return HPATH;
+}
+
+{PATH} {
+  if (yytext[yyleng-1] == '/')
+    PUSH_STATE(INPATH_SLASH);
+  else
+    PUSH_STATE(INPATH);
+  yylval->path = {yytext, (size_t) yyleng};
+  return PATH;
+}
+{HPATH} {
+  if (yytext[yyleng-1] == '/')
+    PUSH_STATE(INPATH_SLASH);
+  else
+    PUSH_STATE(INPATH);
+  yylval->path = {yytext, (size_t) yyleng};
+  return HPATH;
+}
+
+<INPATH,INPATH_SLASH>\$\{ {
+  POP_STATE();
+  PUSH_STATE(INPATH);
+  PUSH_STATE(DEFAULT);
+  return DOLLAR_CURLY;
+}
+<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ {
+  POP_STATE();
+  if (yytext[yyleng-1] == '/')
+      PUSH_STATE(INPATH_SLASH);
+  else
+      PUSH_STATE(INPATH);
+  yylval->e = new ExprString(data->symbols.create(string(yytext)));
+  return STR;
+}
+<INPATH>{ANY} |
+<INPATH><<EOF>> {
+  /* if we encounter a non-path character we inform the parser that the path has
+     ended with a PATH_END token and re-parse this character in the default
+     context (it may be ')', ';', or something of that sort) */
+  POP_STATE();
+  yyless(0);
+  *yylloc = prev_yylloc;
+  return PATH_END;
+}
+
+<INPATH_SLASH>{ANY} |
+<INPATH_SLASH><<EOF>> {
+  throw ParseError("path has a trailing slash");
+}
+
+{SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
+{URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
 
 [ \t\r\n]+    /* eat up whitespace */
 \#[^\r\n]*    /* single-line comments */

src/libexpr/local.mk

@@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
 
 libexpr_LIBS = libutil libstore libfetchers
 
-libexpr_LDFLAGS = -lboost_context -pthread
+libexpr_LDFLAGS += -lboost_context -pthread
 ifdef HOST_LINUX
  libexpr_LDFLAGS += -ldl
 endif

src/libexpr/nixexpr.cc

@@ -124,7 +124,7 @@ void ExprList::show(std::ostream & str) const
 void ExprLambda::show(std::ostream & str) const
 {
     str << "(";
-    if (matchAttrs) {
+    if (hasFormals()) {
         str << "{ ";
         bool first = true;
         for (auto & i : formals->formals) {
@@ -143,6 +143,16 @@ void ExprLambda::show(std::ostream & str) const
     str << ": " << *body << ")";
 }
 
+void ExprCall::show(std::ostream & str) const
+{
+    str << '(' << *fun;
+    for (auto e : args) {
+        str <<  ' ';
+        str << *e;
+    }
+    str << ')';
+}
+
 void ExprLet::show(std::ostream & str) const
 {
     str << "(let ";
@@ -181,7 +191,7 @@ void ExprConcatStrings::show(std::ostream & str) const
     str << "(";
     for (auto & i : *es) {
         if (first) first = false; else str << " + ";
-        str << *i;
+        str << *i.second;
     }
     str << ")";
 }
@@ -263,13 +273,13 @@ void ExprVar::bindVars(const StaticEnv & env)
     /* Check whether the variable appears in the environment.  If so,
        set its level and displacement. */
     const StaticEnv * curEnv;
-    unsigned int level;
+    Level level;
     int withLevel = -1;
     for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) {
         if (curEnv->isWith) {
             if (withLevel == -1) withLevel = level;
         } else {
-            StaticEnv::Vars::const_iterator i = curEnv->vars.find(name);
+            auto i = curEnv->find(name);
             if (i != curEnv->vars.end()) {
                 fromWith = false;
                 this->level = level;
@@ -311,14 +321,16 @@ void ExprOpHasAttr::bindVars(const StaticEnv & env)
 void ExprAttrs::bindVars(const StaticEnv & env)
 {
     const StaticEnv * dynamicEnv = &env;
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0);
 
     if (recursive) {
         dynamicEnv = &newEnv;
 
-        unsigned int displ = 0;
+        Displacement displ = 0;
         for (auto & i : attrs)
-            newEnv.vars[i.first] = i.second.displ = displ++;
+            newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
+
+        // No need to sort newEnv since attrs is in sorted order.
 
         for (auto & i : attrs)
             i.second.e->bindVars(i.second.inherited ? env : newEnv);
@@ -342,15 +354,20 @@ void ExprList::bindVars(const StaticEnv & env)
 
 void ExprLambda::bindVars(const StaticEnv & env)
 {
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(
+        false, &env,
+        (hasFormals() ? formals->formals.size() : 0) +
+        (arg.empty() ? 0 : 1));
 
-    unsigned int displ = 0;
+    Displacement displ = 0;
 
-    if (!arg.empty()) newEnv.vars[arg] = displ++;
+    if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++);
 
-    if (matchAttrs) {
+    if (hasFormals()) {
         for (auto & i : formals->formals)
-            newEnv.vars[i.name] = displ++;
+            newEnv.vars.emplace_back(i.name, displ++);
+
+        newEnv.sort();
 
         for (auto & i : formals->formals)
             if (i.def) i.def->bindVars(newEnv);
@@ -359,13 +376,22 @@ void ExprLambda::bindVars(const StaticEnv & env)
     body->bindVars(newEnv);
 }
 
+void ExprCall::bindVars(const StaticEnv & env)
+{
+    fun->bindVars(env);
+    for (auto e : args)
+        e->bindVars(env);
+}
+
 void ExprLet::bindVars(const StaticEnv & env)
 {
-    StaticEnv newEnv(false, &env);
+    StaticEnv newEnv(false, &env, attrs->attrs.size());
 
-    unsigned int displ = 0;
+    Displacement displ = 0;
     for (auto & i : attrs->attrs)
-        newEnv.vars[i.first] = i.second.displ = displ++;
+        newEnv.vars.emplace_back(i.first, i.second.displ = displ++);
+
+    // No need to sort newEnv since attrs->attrs is in sorted order.
 
     for (auto & i : attrs->attrs)
         i.second.e->bindVars(i.second.inherited ? env : newEnv);
@@ -379,7 +405,7 @@ void ExprWith::bindVars(const StaticEnv & env)
        level so that `lookupVar' can look up variables in the previous
        `with' if this one doesn't contain the desired attribute. */
     const StaticEnv * curEnv;
-    unsigned int level;
+    Level level;
     prevWith = 0;
     for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++)
         if (curEnv->isWith) {
@@ -413,7 +439,7 @@ void ExprOpNot::bindVars(const StaticEnv & env)
 void ExprConcatStrings::bindVars(const StaticEnv & env)
 {
     for (auto & i : *es)
-        i->bindVars(env);
+        i.second->bindVars(env);
 }
 
 void ExprPos::bindVars(const StaticEnv & env)
@@ -447,10 +473,9 @@ string ExprLambda::showNamePos() const
 size_t SymbolTable::totalSize() const
 {
     size_t n = 0;
-    for (auto & i : symbols)
+    for (auto & i : store)
         n += i.size();
     return n;
 }
 
-
 }

src/libexpr/nixexpr.hh

@@ -4,8 +4,6 @@
 #include "symbol-table.hh"
 #include "error.hh"
 
-#include <map>
-
 
 namespace nix {
 
@@ -96,7 +94,7 @@ struct ExprInt : Expr
 {
     NixInt n;
     Value v;
-    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
+    ExprInt(NixInt n) : n(n) { v.mkInt(n); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -105,7 +103,7 @@ struct ExprFloat : Expr
 {
     NixFloat nf;
     Value v;
-    ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
+    ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -114,7 +112,7 @@ struct ExprString : Expr
 {
     Symbol s;
     Value v;
-    ExprString(const Symbol & s) : s(s) { mkString(v, s); };
+    ExprString(const Symbol & s) : s(s) { v.mkString(s); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -135,6 +133,9 @@ struct ExprPath : Expr
     Value * maybeThunk(EvalState & state, Env & env);
 };
 
+typedef uint32_t Level;
+typedef uint32_t Displacement;
+
 struct ExprVar : Expr
 {
     Pos pos;
@@ -150,8 +151,8 @@ struct ExprVar : Expr
        value is obtained by getting the attribute named `name' from
        the set stored in the environment that is `level' levels up
        from the current one.*/
-    unsigned int level;
-    unsigned int displ;
+    Level level;
+    Displacement displ;
 
     ExprVar(const Symbol & name) : name(name) { };
     ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { };
@@ -185,7 +186,7 @@ struct ExprAttrs : Expr
         bool inherited;
         Expr * e;
         Pos pos;
-        unsigned int displ; // displacement
+        Displacement displ; // displacement
         AttrDef(Expr * e, const Pos & pos, bool inherited=false)
             : inherited(inherited), e(e), pos(pos) { };
         AttrDef() { };
@@ -233,11 +234,10 @@ struct ExprLambda : Expr
     Pos pos;
     Symbol name;
     Symbol arg;
-    bool matchAttrs;
     Formals * formals;
     Expr * body;
-    ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
-        : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
+    ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body)
+        : pos(pos), arg(arg), formals(formals), body(body)
     {
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
             throw ParseError({
@@ -247,6 +247,18 @@ struct ExprLambda : Expr
     };
     void setName(Symbol & name);
     string showNamePos() const;
+    inline bool hasFormals() const { return formals != nullptr; }
+    COMMON_METHODS
+};
+
+struct ExprCall : Expr
+{
+    Expr * fun;
+    std::vector<Expr *> args;
+    Pos pos;
+    ExprCall(const Pos & pos, Expr * fun, std::vector<Expr *> && args)
+        : fun(fun), args(args), pos(pos)
+    { }
     COMMON_METHODS
 };
 
@@ -308,7 +320,6 @@ struct ExprOpNot : Expr
         void eval(EvalState & state, Env & env, Value & v); \
     };
 
-MakeBinOp(ExprApp, "")
 MakeBinOp(ExprOpEq, "==")
 MakeBinOp(ExprOpNEq, "!=")
 MakeBinOp(ExprOpAnd, "&&")
@@ -321,8 +332,8 @@ struct ExprConcatStrings : Expr
 {
     Pos pos;
     bool forceString;
-    vector<Expr *> * es;
-    ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es)
+    vector<std::pair<Pos, Expr *> > * es;
+    ExprConcatStrings(const Pos & pos, bool forceString, vector<std::pair<Pos, Expr *> > * es)
         : pos(pos), forceString(forceString), es(es) { };
     COMMON_METHODS
 };
@@ -342,9 +353,39 @@ struct StaticEnv
 {
     bool isWith;
     const StaticEnv * up;
-    typedef std::map<Symbol, unsigned int> Vars;
+
+    // Note: these must be in sorted order.
+    typedef std::vector<std::pair<Symbol, Displacement>> Vars;
     Vars vars;
-    StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { };
+
+    StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) {
+        vars.reserve(expectedSize);
+    };
+
+    void sort()
+    {
+        std::stable_sort(vars.begin(), vars.end(),
+            [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
+    }
+
+    void deduplicate()
+    {
+        auto it = vars.begin(), jt = it, end = vars.end();
+        while (jt != end) {
+            *it = *jt++;
+            while (jt != end && it->first == jt->first) *it = *jt++;
+            it++;
+        }
+        vars.erase(it, end);
+    }
+
+    Vars::const_iterator find(const Symbol & name) const
+    {
+        Vars::value_type key(name, 0);
+        auto i = std::lower_bound(vars.begin(), vars.end(), key);
+        if (i != vars.end() && i->first == name) return i;
+        return vars.end();
+    }
 };
 
 

src/libexpr/parser.y

@@ -33,11 +33,9 @@ namespace nix {
         Symbol file;
         FileOrigin origin;
         std::optional<ErrorInfo> error;
-        Symbol sLetBody;
         ParseData(EvalState & state)
             : state(state)
             , symbols(state.symbols)
-            , sLetBody(symbols.create("<let-body>"))
             { };
     };
 
@@ -126,14 +124,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
                     auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
                         dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs[ad.first] = ad.second;
+                    jAttrs->attrs.emplace(ad.first, ad.second);
                 }
             } else {
                 dupAttr(attrPath, pos, j->second.pos);
             }
         } else {
             // This attr path is not defined. Let's create it.
-            attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos);
+            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
             e->setName(i->symbol);
         }
     } else {
@@ -154,7 +152,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
 }
 
 
-static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Expr *> & es)
+static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<std::pair<Pos, Expr *> > & es)
 {
     if (es.empty()) return new ExprString(symbols.create(""));
 
@@ -164,7 +162,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
     size_t minIndent = 1000000;
     size_t curIndent = 0;
-    for (auto & i : es) {
+    for (auto & [i_pos, i] : es) {
         ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
         if (!e) {
             /* Anti-quotations end the current start-of-line whitespace. */
@@ -194,12 +192,12 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     }
 
     /* Strip spaces from each line. */
-    vector<Expr *> * es2 = new vector<Expr *>;
+    vector<std::pair<Pos, Expr *> > * es2 = new vector<std::pair<Pos, Expr *> >;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
-    for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
-        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
+    for (vector<std::pair<Pos, Expr *> >::iterator i = es.begin(); i != es.end(); ++i, --n) {
+        ExprIndStr * e = dynamic_cast<ExprIndStr *>(i->second);
         if (!e) {
             atStartOfLine = false;
             curDropped = 0;
@@ -236,11 +234,11 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
                 s2 = string(s2, 0, p + 1);
         }
 
-        es2->push_back(new ExprString(symbols.create(s2)));
+        es2->emplace_back(i->first, new ExprString(symbols.create(s2)));
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2);
+    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2);
 }
 
 
@@ -275,28 +273,35 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
   nix::Formal * formal;
   nix::NixInt n;
   nix::NixFloat nf;
-  const char * id; // !!! -> Symbol
-  char * path;
-  char * uri;
+  // using C a struct allows us to avoid having to define the special
+  // members that using string_view here would implicitly delete.
+  struct StringToken {
+    const char * p;
+    size_t l;
+    operator std::string_view() const { return {p, l}; }
+  };
+  StringToken id; // !!! -> Symbol
+  StringToken path;
+  StringToken uri;
   std::vector<nix::AttrName> * attrNames;
-  std::vector<nix::Expr *> * string_parts;
+  std::vector<std::pair<nix::Pos, nix::Expr *> > * string_parts;
 }
 
 %type <e> start expr expr_function expr_if expr_op
-%type <e> expr_app expr_select expr_simple
+%type <e> expr_select expr_simple expr_app
 %type <list> expr_list
 %type <attrs> binds
 %type <formals> formals
 %type <formal> formal
 %type <attrNames> attrs attrpath
 %type <string_parts> string_parts_interpolated ind_string_parts
-%type <e> string_parts string_attr
+%type <e> path_start string_parts string_attr
 %type <id> attr
 %token <id> ID ATTRPATH
 %token <e> STR IND_STR
 %token <n> INT
 %token <nf> FLOAT
-%token <path> PATH HPATH SPATH
+%token <path> PATH HPATH SPATH PATH_END
 %token <uri> URI
 %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW
 %token DOLLAR_CURLY /* == ${ */
@@ -324,13 +329,13 @@ expr: expr_function;
 
 expr_function
   : ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); }
   | '{' formals '}' '@' ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); }
   | ID '@' '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
@@ -353,31 +358,36 @@ expr_if
 
 expr_op
   : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); }
-  | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); }
+  | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); }
   | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); }
   | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); }
-  | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); }
-  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); }
-  | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); }
-  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); }
+  | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); }
+  | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); }
+  | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); }
+  | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); }
   | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); }
   | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); }
   | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); }
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); }
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
-  | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); }
-  | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); }
-  | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); }
+    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
+  | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
+  | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
+  | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
   | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); }
   | expr_app
   ;
 
 expr_app
-  : expr_app expr_select
-    { $$ = new ExprApp(CUR_POS, $1, $2); }
-  | expr_select { $$ = $1; }
+  : expr_app expr_select {
+      if (auto e2 = dynamic_cast<ExprCall *>($1)) {
+          e2->args.push_back($2);
+          $$ = $1;
+      } else
+          $$ = new ExprCall(CUR_POS, $1, {$2});
+  }
+  | expr_select
   ;
 
 expr_select
@@ -388,13 +398,14 @@ expr_select
   | /* Backwards compatibility: because Nixpkgs has a rarely used
        function named ‘or’, allow stuff like ‘map or [...]’. */
     expr_simple OR_KW
-    { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); }
+    { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); }
   | expr_simple { $$ = $1; }
   ;
 
 expr_simple
   : ID {
-      if (strcmp($1, "__curPos") == 0)
+      std::string_view s = "__curPos";
+      if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
           $$ = new ExprPos(CUR_POS);
       else
           $$ = new ExprVar(CUR_POS, data->symbols.create($1));
@@ -405,17 +416,20 @@ expr_simple
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = stripIndentation(CUR_POS, data->symbols, *$2);
   }
-  | PATH { $$ = new ExprPath(absPath($1, data->basePath)); }
-  | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); }
+  | path_start PATH_END { $$ = $1; }
+  | path_start string_parts_interpolated PATH_END {
+      $2->insert($2->begin(), {makeCurPos(@1, data), $1});
+      $$ = new ExprConcatStrings(CUR_POS, false, $2);
+  }
   | SPATH {
-      string path($1 + 1, strlen($1) - 2);
-      $$ = new ExprApp(CUR_POS,
-          new ExprApp(new ExprVar(data->symbols.create("__findFile")),
-              new ExprVar(data->symbols.create("__nixPath"))),
-          new ExprString(data->symbols.create(path)));
+      string path($1.p + 1, $1.l - 2);
+      $$ = new ExprCall(CUR_POS,
+          new ExprVar(data->symbols.create("__findFile")),
+          {new ExprVar(data->symbols.create("__nixPath")),
+           new ExprString(data->symbols.create(path))});
   }
   | URI {
-      static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
+      static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals);
       if (noURLLiterals)
           throw ParseError({
               .msg = hintfmt("URL literals are disabled"),
@@ -442,20 +456,34 @@ string_parts
   ;
 
 string_parts_interpolated
-  : string_parts_interpolated STR { $$ = $1; $1->push_back($2); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
-  | DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($2); }
+  : string_parts_interpolated STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
+  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | DOLLAR_CURLY expr '}' { $$ = new vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); }
   | STR DOLLAR_CURLY expr '}' {
-      $$ = new vector<Expr *>;
-      $$->push_back($1);
-      $$->push_back($3);
+      $$ = new vector<std::pair<Pos, Expr *> >;
+      $$->emplace_back(makeCurPos(@1, data), $1);
+      $$->emplace_back(makeCurPos(@2, data), $3);
     }
   ;
 
+path_start
+  : PATH {
+    Path path(absPath({$1.p, $1.l}, data->basePath));
+    /* add back in the trailing '/' to the first segment */
+    if ($1.p[$1.l-1] == '/' && $1.l > 1)
+      path += "/";
+    $$ = new ExprPath(path);
+  }
+  | HPATH {
+    Path path(getHome() + string($1.p + 1, $1.l - 1));
+    $$ = new ExprPath(path);
+  }
+  ;
+
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
-  | { $$ = new vector<Expr *>; }
+  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | { $$ = new vector<std::pair<Pos, Expr *> >; }
   ;
 
 binds
@@ -466,7 +494,7 @@ binds
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos);
           Pos pos = makeCurPos(@3, data);
-          $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true);
+          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
       }
     }
   | binds INHERIT '(' expr ')' attrs ';'
@@ -475,7 +503,7 @@ binds
       for (auto & i : *$6) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos);
-          $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data));
+          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)));
       }
     }
   | { $$ = new ExprAttrs(makeCurPos(@0, data)); }
@@ -523,7 +551,7 @@ attrpath
 
 attr
   : ID { $$ = $1; }
-  | OR_KW { $$ = "or"; }
+  | OR_KW { $$ = {"or", 2}; }
   ;
 
 string_attr
@@ -569,7 +597,7 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text, FileOrigin origin,
+Expr * EvalState::parse(char * text, size_t length, FileOrigin origin,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
@@ -589,7 +617,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
     data.basePath = basePath;
 
     yylex_init(&scanner);
-    yy_scan_string(text, scanner);
+    yy_scan_buffer(text, length, scanner);
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
@@ -635,26 +663,33 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
+    auto buffer = readFile(path);
+    // readFile should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), foFile, path, dirOf(path), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), foString, "", basePath, staticEnv);
+    s.append("\0\0", 2);
+    return parse(s.data(), s.size(), foString, "", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string s, const Path & basePath)
 {
-    return parseExprFromString(s, basePath, staticBaseEnv);
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
 }
 
 
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
+    auto buffer = drainFD(0);
+    // drainFD should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 
@@ -735,7 +770,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
             res = { true, path };
         else {
             logWarning({
-                .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second)
             });
             res = { false, "" };
         }

src/libexpr/primops.hh

@@ -15,7 +15,6 @@ struct RegisterPrimOp
         std::vector<std::string> args;
         size_t arity = 0;
         const char * doc;
-        std::optional<std::string> requiredFeature;
         PrimOpFun fun;
     };
 
@@ -28,8 +27,7 @@ struct RegisterPrimOp
     RegisterPrimOp(
         std::string name,
         size_t arity,
-        PrimOpFun fun,
-        std::optional<std::string> requiredFeature = {});
+        PrimOpFun fun);
 
     RegisterPrimOp(Info && info);
 };

src/libexpr/primops/context.cc

@@ -7,8 +7,7 @@ namespace nix {
 static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     PathSet context;
-    string s = state.coerceToString(pos, *args[0], context);
-    mkString(v, s, PathSet());
+    v.mkString(state.coerceToString(pos, *args[0], context));
 }
 
 static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext);
@@ -18,7 +17,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args,
 {
     PathSet context;
     state.forceString(*args[0], context, pos);
-    mkBool(v, !context.empty());
+    v.mkBool(!context.empty());
 }
 
 static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext);
@@ -39,7 +38,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po
     for (auto & p : context)
         context2.insert(p.at(0) == '=' ? string(p, 1) : p);
 
-    mkString(v, s, context2);
+    v.mkString(s, context2);
 }
 
 static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency);
@@ -103,28 +102,26 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
         }
     }
 
-    state.mkAttrs(v, contextInfos.size());
+    auto attrs = state.buildBindings(contextInfos.size());
 
     auto sPath = state.symbols.create("path");
     auto sAllOutputs = state.symbols.create("allOutputs");
     for (const auto & info : contextInfos) {
-        auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first));
-        state.mkAttrs(infoVal, 3);
+        auto infoAttrs = state.buildBindings(3);
         if (info.second.path)
-            mkBool(*state.allocAttr(infoVal, sPath), true);
+            infoAttrs.alloc(sPath).mkBool(true);
         if (info.second.allOutputs)
-            mkBool(*state.allocAttr(infoVal, sAllOutputs), true);
+            infoAttrs.alloc(sAllOutputs).mkBool(true);
         if (!info.second.outputs.empty()) {
-            auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs);
+            auto & outputsVal = infoAttrs.alloc(state.sOutputs);
             state.mkList(outputsVal, info.second.outputs.size());
-            size_t i = 0;
-            for (const auto & output : info.second.outputs) {
-                mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output);
-            }
+            for (const auto & [i, output] : enumerate(info.second.outputs))
+                (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
         }
-        infoVal.attrs->sort();
+        attrs.alloc(info.first).mkAttrs(infoAttrs);
     }
-    v.attrs->sort();
+
+    v.mkAttrs(attrs);
 }
 
 static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext);
@@ -181,14 +178,14 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                     .errPos = *i.pos
                 });
             }
-            for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
-                auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos);
+            for (auto elem : iter->value->listItems()) {
+                auto name = state.forceStringNoCtx(*elem, *iter->pos);
                 context.insert("!" + name + "!" + string(i.name));
             }
         }
     }
 
-    mkString(v, orig, context);
+    v.mkString(orig, context);
 }
 
 static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext);