Bump version

[backport] libfetchers/git: hardcode `--git-dir`

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,7 @@
+**Release Notes**
+Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed.
+
+
+**Testing**
+
+If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master).

.github/workflows/backport.yml

@@ -8,7 +8,7 @@ jobs:
     if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name))
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           # required to find all branches

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: "Test"
+name: "CI"
 
 on:
   pull_request:
@@ -14,10 +14,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - uses: cachix/install-nix-action@v16
+    - uses: cachix/install-nix-action@v17
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v10
       if: needs.check_cachix.outputs.secret == 'true'
@@ -25,7 +25,7 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)')
+    - run: nix --experimental-features 'nix-command flakes' flake check -L
 
   check_cachix:
     name: Cachix secret present for installer tests
@@ -46,11 +46,11 @@ jobs:
     outputs:
       installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 0
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v16
+    - uses: cachix/install-nix-action@v17
     - uses: cachix/cachix-action@v10
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -67,10 +67,42 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2.4.0
+    - uses: actions/checkout@v3
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v16
+    - uses: cachix/install-nix-action@v17
       with:
         install_url: '${{needs.installer.outputs.installerURL}}'
         install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
     - run: nix-instantiate -E 'builtins.currentTime' --eval
+
+  docker_push_image:
+    needs: [check_cachix, tests]
+    if: >-
+      github.event_name == 'push' &&
+      github.ref_name == 'master' &&
+      needs.check_cachix.outputs.secret == 'true'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v17
+    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - run: echo NIX_VERSION="$(nix-instantiate --eval -E '(import ./default.nix).defaultPackage.${builtins.currentSystem}.version' | tr -d \")" >> $GITHUB_ENV
+    - uses: cachix/cachix-action@v10
+      if: needs.check_cachix.outputs.secret == 'true'
+      with:
+        name: '${{ env.CACHIX_NAME }}'
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L
+    - run: docker load -i ./result/image.tar.gz
+    - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
+    - run: docker tag nix:$NIX_VERSION nixos/nix:master
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - run: docker push nixos/nix:$NIX_VERSION
+    - run: docker push nixos/nix:master

.github/workflows/hydra_status.yml

@@ -0,0 +1,16 @@
+name: Hydra status
+on:
+  schedule:
+    - cron: "12,42 * * * *"
+  workflow_dispatch:
+jobs:
+  check_hydra_status:
+    name: Check Hydra status
+    if: github.repository_owner == 'NixOS'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - run: bash scripts/check-hydra-status.sh
+

.gitignore

@@ -90,6 +90,7 @@ perl/Makefile.config
 
 /misc/systemd/nix-daemon.service
 /misc/systemd/nix-daemon.socket
+/misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 
 /src/resolve-system-dependencies/resolve-system-dependencies
@@ -120,3 +121,7 @@ GTAGS
 compile_commands.json
 
 nix-rust/target
+
+result
+
+.vscode/

.version

@@ -1 +1 @@
-2.5
+2.8.1

Makefile.config.in

@@ -16,6 +16,7 @@ LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
 LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 PACKAGE_NAME = @PACKAGE_NAME@

configure.ac

@@ -188,17 +188,24 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA
     [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])])
 ])
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Look for libbrotli{enc,dec}.
 PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"])
 
 # Look for libcpuid.
+have_libcpuid=
 if test "$machine_name" = "x86_64"; then
-  PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"])
-  have_libcpuid=1
-  AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])
+    AC_ARG_ENABLE([cpuid],
+                  AS_HELP_STRING([--disable-cpuid], [Do not determine microarchitecture levels with libcpuid (relevant to x86_64 only)]))
+    if test "x$enable_cpuid" != "xno"; then
+      PKG_CHECK_MODULES([LIBCPUID], [libcpuid],
+        [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"
+         have_libcpuid=1
+         AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])]
+      )
+    fi
 fi
 AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid])
 
@@ -255,13 +262,17 @@ fi
 PKG_CHECK_MODULES([GTEST], [gtest_main])
 
 
+# Look for nlohmann/json.
+PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
+
+
 # documentation generation switch
 AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]),
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
 # Look for lowdown library.
-PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.8.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
+PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])

default.nix

@@ -1,3 +1,3 @@
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") {
   src = ./.;
 }).defaultNix

doc/manual/generate-builtins.nix

@@ -6,9 +6,9 @@ builtins:
 concatStrings (map
   (name:
     let builtin = builtins.${name}; in
-    "<dt><code>${name} "
+    "<dt id=\"builtins-${name}\"><a href=\"#builtins-${name}\"><code>${name} "
     + concatStringsSep " " (map (s: "<var>${s}</var>") builtin.args)
-    + "</code></dt>"
+    + "</code></a></dt>"
     + "<dd>\n\n"
     + builtin.doc
     + "\n\n</dd>"

doc/manual/generate-options.nix

@@ -6,19 +6,22 @@ options:
 concatStrings (map
   (name:
     let option = options.${name}; in
-    "  - `${name}`  \n\n"
+    "  - [`${name}`](#conf-${name})"
+    + "<p id=\"conf-${name}\"></p>\n\n"
     + concatStrings (map (s: "    ${s}\n") (splitLines option.description)) + "\n\n"
-    + "    **Default:** " + (
-      if option.value == "" || option.value == []
-      then "*empty*"
-      else if isBool option.value
-      then (if option.value then "`true`" else "`false`")
-      else
-        # n.b. a StringMap value type is specified as a string, but
-        # this shows the value type.  The empty stringmap is "null" in
-        # JSON, but that converts to "{ }" here.
-        (if isAttrs option.value then "`\"\"`"
-         else "`" + toString option.value + "`")) + "\n\n"
+    + (if option.documentDefault
+       then "    **Default:** " + (
+       if option.value == "" || option.value == []
+       then "*empty*"
+       else if isBool option.value
+       then (if option.value then "`true`" else "`false`")
+       else
+         # n.b. a StringMap value type is specified as a string, but
+         # this shows the value type.  The empty stringmap is "null" in
+         # JSON, but that converts to "{ }" here.
+         (if isAttrs option.value then "`\"\"`"
+          else "`" + toString option.value + "`")) + "\n\n"
+       else "    **Default:** *machine-specific*\n")
     + (if option.aliases != []
        then "    **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n"
        else "")

doc/manual/local.mk

@@ -12,11 +12,13 @@ man-pages := $(foreach n, \
 clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8
 
 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox.
+# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine
 dummy-env = env -i \
 	HOME=/dummy \
 	NIX_CONF_DIR=/dummy \
 	NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \
-	NIX_STATE_DIR=/dummy
+	NIX_STATE_DIR=/dummy \
+	NIX_CONFIG='cores = 0'
 
 nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
 
@@ -70,6 +72,7 @@ $(d)/builtins.json: $(bindir)/nix
 	@mv $@.tmp $@
 
 # Generate the HTML manual.
+html: $(docdir)/manual/index.html
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.

doc/manual/src/SUMMARY.md.in

@@ -71,7 +71,10 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
+  - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md)
+  - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md)
+  - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md)
+  - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md)
   - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md)
   - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md)
   - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md)

doc/manual/src/advanced-topics/distributed-builds.md

@@ -53,8 +53,8 @@ example, the following command allows you to build a derivation for
 $ uname
 Linux
 
-$ nix build \
-  '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
+$ nix build --impure \
+  --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \
   --builders 'ssh://mac x86_64-darwin'
 [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac
 
@@ -110,7 +110,7 @@ default, set it to `-`.
 7.  A comma-separated list of *mandatory features*. A machine will only
     be used to build a derivation if all of the machine’s mandatory
     features appear in the derivation’s `requiredSystemFeatures`
-    attribute..
+    attribute.
 
 8.  The (base64-encoded) public host key of the remote machine. If omitted, SSH
     will use its regular known-hosts file. Specifically, the field is calculated

doc/manual/src/command-ref/nix-shell.md

@@ -101,7 +101,8 @@ The following common options are supported:
 
   - `NIX_BUILD_SHELL`\
     Shell used to start the interactive environment. Defaults to the
-    `bash` found in `PATH`.
+    `bash` found in `<nixpkgs>`, falling back to the `bash` found in
+    `PATH` if not found.
 
 # Examples
 

doc/manual/src/command-ref/nix-store.md

@@ -321,8 +321,8 @@ symlink.
     This query has one option:
 
       - `--include-outputs`
-        Also include the output path of store derivations, and their
-        closures.
+        Also include the existing output paths of store derivations,
+        and their closures.
 
     This query can be used to implement various kinds of deployment. A
     *source deployment* is obtained by distributing the closure of a

doc/manual/src/contributing/hacking.md

@@ -35,6 +35,25 @@ variables are set up so that those dependencies can be found:
 $ nix-shell
 ```
 
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop
+```
+
+To get a shell with a different compilation environment (e.g. stdenv,
+gccStdenv, clangStdenv, clang11Stdenv):
+
+```console
+$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages
+```
+
+or if you have a flake-enabled nix:
+
+```console
+$ nix develop .#clang11StdenvPackages
+```
+
 To build Nix itself in this shell:
 
 ```console

doc/manual/src/expressions/language-constructs.md

@@ -284,6 +284,10 @@ The points of interest are:
     function is called with the `localServer` argument set to `true` but
     the `db4` argument set to `null`, then the evaluation fails.
 
+    Note that `->` is the [logical
+    implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication)
+    Boolean operation.
+
 2.  This is a more subtle condition: if Subversion is built with Apache
     (`httpServer`) support, then the Expat library (an XML library) used
     by Subversion should be same as the one used by Apache. This is

doc/manual/src/glossary.md

@@ -47,7 +47,7 @@
     the store object at `P` contains the path `Q` somewhere. The
     *references* of a store path are the set of store paths to which it
     has a reference.
-    
+
     A derivation can reference other derivations and sources (but not
     output paths), whereas an output path only references other output
     paths.
@@ -66,7 +66,7 @@
     is necessary to deploy whole closures, since otherwise at runtime
     files could be missing. The command `nix-store -qR` prints out
     closures of store paths.
-    
+
     As an example, if the store object at path `P` contains a reference
     to path `Q`, then `Q` is in the closure of `P`. Further, if `Q`
     references `R` then `R` is also in the closure of `P`.
@@ -98,3 +98,7 @@
     store. It can contain regular files, directories and symbolic
     links.  NARs are generated and unpacked using `nix-store --dump`
     and `nix-store --restore`.
+  - `∅` \
+    The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.
+  - `ε` \
+    The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.

doc/manual/src/installation/installing-binary.md

@@ -84,7 +84,9 @@ The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist.
 The installer will first back up these files with a `.backup-before-nix`
 extension. The installer will also create `/etc/profile.d/nix.sh`.
 
-You can uninstall Nix with the following commands:
+## Uninstalling
+
+### Linux
 
 ```console
 sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
@@ -95,15 +97,95 @@ sudo systemctl stop nix-daemon.service
 sudo systemctl disable nix-daemon.socket
 sudo systemctl disable nix-daemon.service
 sudo systemctl daemon-reload
-
-# If you are on macOS, you will need to run:
-sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 ```
 
 There may also be references to Nix in `/etc/profile`, `/etc/bashrc`,
 and `/etc/zshrc` which you may remove.
 
+### macOS
+
+1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing
+   `nix-daemon.sh`, which should look like this:
+
+   ```bash
+   # Nix
+   if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then
+     . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh'
+   fi
+   # End Nix
+   ```
+
+   If these files haven't been altered since installing Nix you can simply put
+   the backups back in place:
+
+   ```console
+   sudo mv /etc/zshrc.backup-before-nix /etc/zshrc
+   sudo mv /etc/bashrc.backup-before-nix /etc/bashrc
+   ```
+
+   This will stop shells from sourcing the file and bringing everything you
+   installed using Nix in scope.
+
+2. Stop and remove the Nix daemon services:
+
+   ```console
+   sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+   sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
+   sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist
+   sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist
+   ```
+
+   This stops the Nix daemon and prevents it from being started next time you
+   boot the system.
+
+3. Remove the `nixbld` group and the `_nixbuildN` users:
+
+   ```console
+   sudo dscl . -delete /Groups/nixbld
+   for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done
+   ```
+
+   This will remove all the build users that no longer serve a purpose.
+
+4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store
+   volume on `/nix`, which looks like this,
+   `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic
+   mounting of the Nix Store volume.
+
+5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only
+   line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`.
+   This will prevent the creation of the empty `/nix` directory to provide a
+   mountpoint for the Nix Store volume.
+
+6. Remove the files Nix added to your system:
+
+   ```console
+   sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels
+   ```
+
+   This gets rid of any data Nix may have created except for the store which is
+   removed next.
+
+7. Remove the Nix Store volume:
+   
+   ```console
+   sudo diskutil apfs deleteVolume /nix
+   ```
+
+   This will remove the Nix Store volume and everything that was added to the
+   store.
+
+> **Note**
+> 
+> After you complete the steps here, you will still have an empty `/nix`
+> directory. This is an expected sign of a successful uninstall. The empty
+> `/nix` directory will disappear the next time you reboot.
+>
+> You do not have to reboot to finish uninstalling Nix. The uninstall is
+> complete. macOS (Catalina+) directly controls root directories and its
+> read-only root will prevent you from manually deleting the empty `/nix`
+> mountpoint.
+
 # macOS Installation <a name="sect-macos-installation-change-store-prefix"></a><a name="sect-macos-installation-encrypted-volume"></a><a name="sect-macos-installation-symlink"></a><a name="sect-macos-installation-recommended-notes"></a>
 <!-- Note: anchors above to catch permalinks to old explanations -->
 
@@ -119,6 +201,30 @@ this to run the installer, but it may help if you run into trouble:
 - update `/etc/synthetic.conf` to direct macOS to create a "synthetic"
   empty root directory to mount your volume
 - specify mount options for the volume in `/etc/fstab`
+  - `rw`: read-write
+  - `noauto`: prevent the system from auto-mounting the volume (so the
+    LaunchDaemon mentioned below can control mounting it, and to avoid
+    masking problems with that mounting service).
+  - `nobrowse`: prevent the Nix Store volume from showing up on your
+    desktop; also keeps Spotlight from spending resources to index
+    this volume
+  <!-- TODO:
+  - `suid`: honor setuid? surely not? ...
+  - `owners`: honor file ownership on the volume
+
+    For now I'll avoid pretending to understand suid/owners more
+    than I do. There've been some vague reports of file-ownership
+    and permission issues, particularly in cloud/VM/headless setups.
+    My pet theory is that this has something to do with these setups
+    not having a token that gets delegated to initial/admin accounts
+    on macOS. See scripts/create-darwin-volume.sh for a little more.
+
+    In any case, by Dec 4 2021, it _seems_ like some combination of
+    suid, owners, and calling diskutil enableOwnership have stopped
+    new reports from coming in. But I hesitate to celebrate because we
+    haven't really named and catalogued the behavior, understood what
+    we're fixing, and validated that all 3 components are essential.
+  -->
 - if you have FileVault enabled
     - generate an encryption password
     - put it in your system Keychain

doc/manual/src/installation/installing-docker.md

@@ -3,7 +3,7 @@
 To run the latest stable release of Nix with Docker run the following command:
 
 ```console
-$ docker -ti run nixos/nix
+$ docker run -ti nixos/nix
 Unable to find image 'nixos/nix:latest' locally
 latest: Pulling from nixos/nix
 5843afab3874: Pull complete
@@ -16,7 +16,7 @@ nix (Nix) 2.3.12
 35ca4ada6e96:/# exit
 ```
 
-# What is included in Nix' Docker image?
+# What is included in Nix's Docker image?
 
 The official Docker image is created using `pkgs.dockerTools.buildLayeredImage`
 (and not with `Dockerfile` as it is usual with Docker images). You can still
@@ -54,6 +54,6 @@ You can also build a Docker image from source yourself:
 
 ```console
 $ nix build ./\#hydraJobs.dockerImage.x86_64-linux
-$ docker load -i ./result
+$ docker load -i ./result/image.tar.gz
 $ docker run -ti nix:2.5pre20211105
 ```

doc/manual/src/installation/prerequisites-source.md

@@ -44,6 +44,11 @@
     obtained from the its repository
     <https://github.com/troglobit/editline>.
 
+  - The `libsodium` library for verifying cryptographic signatures
+    of contents fetched from binary caches.
+    It can be obtained from the official web site
+    <https://libsodium.org>.
+
   - Recent versions of Bison and Flex to build the parser. (This is
     because Nix needs GLR support in Bison and reentrancy support in
     Flex.) For Bison, you need version 2.6, which can be obtained from
@@ -58,3 +63,11 @@
     `--disable-seccomp-sandboxing` option to the `configure` script (Not
     recommended unless your system doesn't support `libseccomp`). To get
     the library, visit <https://github.com/seccomp/libseccomp>.
+
+  - On 64-bit x86 machines only, `libcpuid` library
+    is used to determine which microarchitecture levels are supported
+    (e.g., as whether to have `x86_64-v2-linux` among additional system types).
+    The library is available from its homepage
+    <http://libcpuid.sourceforge.net>.
+    This is an optional dependency and can be disabled
+    by providing a `--disable-cpuid` to the `configure` script.

doc/manual/src/installation/supported-platforms.md

@@ -4,4 +4,4 @@ Nix is currently supported on the following platforms:
 
   - Linux (i686, x86\_64, aarch64).
 
-  - macOS (x86\_64).
+  - macOS (x86\_64, aarch64).

doc/manual/src/package-management/basic-package-mgmt.md

@@ -40,7 +40,7 @@ $ nix-channel --update
 > 
 > On NixOS, you’re automatically subscribed to a NixOS channel
 > corresponding to your NixOS major release (e.g.
-> <http://nixos.org/channels/nixos-14.12>). A NixOS channel is identical
+> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical
 > to the Nixpkgs channel, except that it contains only Linux binaries
 > and is updated only if a set of regression tests succeed.
 

doc/manual/src/release-notes/rl-2.4.md

@@ -276,6 +276,9 @@ more than 2800 commits from 195 contributors since release 2.3.
 
 * Plugins can now register `nix` subcommands.
 
+* The `--indirect` flag to `nix-store --add-root` has become a no-op.
+  `--add-root` will always generate indirect GC roots from now on.
+
 ## Incompatible changes
 
 * The `nix` command is now marked as an experimental feature. This

doc/manual/src/release-notes/rl-2.5.md

@@ -0,0 +1,16 @@
+# Release 2.5 (2021-12-13)
+
+* The garbage collector no longer blocks new builds, so the message
+  `waiting for the big garbage collector lock...` is a thing of the
+  past.
+
+* Binary cache stores now have a setting `compression-level`.
+
+* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
+
+* Lists can now be compared lexicographically using the `<` operator.
+
+* New built-in function: `builtins.groupBy`, with the same functionality as
+  Nixpkgs' `lib.groupBy`, but faster.
+
+* `nix repl` now has a `:log` command.

doc/manual/src/release-notes/rl-2.6.md

@@ -0,0 +1,21 @@
+# Release 2.6 (2022-01-24)
+
+* The Nix CLI now searches for a `flake.nix` up until the root of the current
+  Git repository or a filesystem boundary rather than just in the current
+  directory.
+* The TOML parser used by `builtins.fromTOML` has been replaced by [a
+  more compliant one](https://github.com/ToruNiina/toml11).
+* Added `:st`/`:show-trace` commands to `nix repl`, which are used to
+  set or toggle display of error traces.
+* New builtin function `builtins.zipAttrsWith` with the same
+  functionality as `lib.zipAttrsWith` from Nixpkgs, but much more
+  efficient.
+* New command `nix store copy-log` to copy build logs from one store
+  to another.
+* The `commit-lockfile-summary` option can be set to a non-empty
+  string to override the commit summary used when commiting an updated
+  lockfile.  This may be used in conjunction with the `nixConfig`
+  attribute in `flake.nix` to better conform to repository
+  conventions.
+* `docker run -ti nixos/nix:master` will place you in the Docker
+  container with the latest version of Nix from the `master` branch.

doc/manual/src/release-notes/rl-2.7.md

@@ -0,0 +1,33 @@
+# Release 2.7 (2022-03-07)
+
+* Nix will now make some helpful suggestions when you mistype
+  something on the command line. For instance, if you type `nix build
+  nixpkgs#thunderbrd`, it will suggest `thunderbird`.
+
+* A number of "default" flake output attributes have been
+  renamed. These are:
+
+  * `defaultPackage.<system>` → `packages.<system>.default`
+  * `defaultApps.<system>` → `apps.<system>.default`
+  * `defaultTemplate` → `templates.default`
+  * `defaultBundler.<system>` → `bundlers.<system>.default`
+  * `overlay` → `overlays.default`
+  * `devShell.<system>` → `devShells.<system>.default`
+
+  The old flake output attributes still work, but `nix flake check`
+  will warn about them.
+
+* Breaking API change: `nix bundle` now supports bundlers of the form
+  `bundler.<system>.<name>= derivation: another-derivation;`. This
+  supports additional functionality to inspect evaluation information
+  during bundling. A new
+  [repository](https://github.com/NixOS/bundlers) has various bundlers
+  implemented.
+
+* `nix store ping` now reports the version of the remote Nix daemon.
+
+* `nix flake {init,new}` now display information about which files have been
+  created.
+
+* Templates can now define a `welcomeText` attribute, which is printed out by
+  `nix flake {init,new} --template <template>`.

doc/manual/src/release-notes/rl-2.8.md

@@ -0,0 +1,53 @@
+# Release 2.8 (2022-04-19)
+
+* New experimental command: `nix fmt`, which applies a formatter
+  defined by the `formatter.<system>` flake output to the Nix
+  expressions in a flake.
+
+* Various Nix commands can now read expressions from standard input
+  using `--file -`.
+
+* New experimental builtin function `builtins.fetchClosure` that
+  copies a closure from a binary cache at evaluation time and rewrites
+  it to content-addressed form (if it isn't already). Like
+  `builtins.storePath`, this allows importing pre-built store paths;
+  the difference is that it doesn't require the user to configure
+  binary caches and trusted public keys.
+
+  This function is only available if you enable the experimental
+  feature `fetch-closure`.
+
+* New experimental feature: *impure derivations*. These are
+  derivations that can produce a different result every time they're
+  built. Here is an example:
+
+  ```nix
+  stdenv.mkDerivation {
+    name = "impure";
+    __impure = true; # marks this derivation as impure
+    buildCommand = "date > $out";
+  }
+  ```
+
+  Running `nix build` twice on this expression will build the
+  derivation twice, producing two different content-addressed store
+  paths. Like fixed-output derivations, impure derivations have access
+  to the network. Only fixed-output derivations and impure derivations
+  can depend on an impure derivation.
+
+* `nix store make-content-addressable` has been renamed to `nix store
+  make-content-addressed`.
+
+* The `nixosModule` flake output attribute has been renamed consistent
+  with the `.default` renames in Nix 2.7.
+
+  * `nixosModule` → `nixosModules.default`
+
+  As before, the old output will continue to work, but `nix flake check` will
+  issue a warning about it.
+
+* `nix run` is now stricter in what it accepts: members of the `apps`
+  flake output are now required to be apps (as defined in [the
+  manual](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-run.html#apps)),
+  and members of `packages` or `legacyPackages` must be derivations
+  (not apps).

doc/manual/src/release-notes/rl-next.md

@@ -1,7 +1 @@
-# Release 2.5 (2021-XX-XX)
-
-* Binary cache stores now have a setting `compression-level`.
-
-* `nix develop` now has a flag `--unpack` to run `unpackPhase`.
-
-* Lists can now be compared lexicographically using the `<` operator.
+# Release X.Y (202?-??-??)

docker.nix

@@ -20,6 +20,8 @@ let
     man
     cacert.out
     findutils
+    iana-etc
+    git
   ];
 
   users = {
@@ -137,11 +139,8 @@ let
         name = "root-profile-env";
         paths = defaultPkgs;
       };
-      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
-        mkdir $out
-        cp -a ${rootEnv}/* $out/
-
-        cat > $out/manifest.nix <<EOF
+      manifest = pkgs.buildPackages.runCommand "manifest.nix" { } ''
+        cat > $out <<EOF
         [
         ${lib.concatStringsSep "\n" (builtins.map (drv: let
           outputs = drv.outputsToInstall or [ "out" ];
@@ -161,6 +160,11 @@ let
         ]
         EOF
       '';
+      profile = pkgs.buildPackages.runCommand "user-environment" { } ''
+        mkdir $out
+        cp -a ${rootEnv}/* $out/
+        ln -s ${manifest} $out/manifest.nix
+      '';
     in
     pkgs.runCommand "base-system"
       {
@@ -178,6 +182,9 @@ let
       set -x
       mkdir -p $out/etc
 
+      mkdir -p $out/etc/ssl/certs
+      ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs
+
       cat $passwdContentsPath > $out/etc/passwd
       echo "" >> $out/etc/passwd
 
@@ -194,6 +201,8 @@ let
 
       mkdir $out/tmp
 
+      mkdir -p $out/var/tmp
+
       mkdir -p $out/etc/nix
       cat $nixConfContentsPath > $out/etc/nix/nix.conf
 
@@ -227,6 +236,10 @@ pkgs.dockerTools.buildLayeredImageWithNixDb {
     rm -rf nix-support
     ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles
   '';
+  fakeRootCommands = ''
+    chmod 1777 tmp
+    chmod 1777 var/tmp
+  '';
 
   config = {
     Cmd = [ "/root/.nix-profile/bin/bash" ];

flake.lock

@@ -18,11 +18,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1632864508,
-        "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=",
+        "lastModified": 1645296114,
+        "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234",
+        "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
         "type": "github"
       },
       "original": {
@@ -31,10 +31,26 @@
         "type": "indirect"
       }
     },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "indirect"
+      }
+    },
     "root": {
       "inputs": {
         "lowdown-src": "lowdown-src",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-regression": "nixpkgs-regression"
       }
     }
   },

flake.nix

@@ -2,9 +2,10 @@
   description = "The purely functional package manager";
 
   inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small";
+  inputs.nixpkgs-regression.url = "nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
-  outputs = { self, nixpkgs, lowdown-src }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src }:
 
     let
 
@@ -14,7 +15,7 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];
@@ -22,15 +23,36 @@
 
       crossSystems = [ "armv6l-linux" "armv7l-linux" ];
 
+      stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ];
+
       forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+      forAllSystemsAndStdenvs = f: forAllSystems (system:
+        nixpkgs.lib.listToAttrs
+          (map
+            (n:
+            nixpkgs.lib.nameValuePair "${n}Packages" (
+              f system n
+            )) stdenvs
+          )
+      );
+
+      forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv);
 
       # Memoize nixpkgs for different platforms for efficiency.
-      nixpkgsFor = forAllSystems (system:
-        import nixpkgs {
-          inherit system;
-          overlays = [ self.overlay ];
-        }
-      );
+      nixpkgsFor =
+        let stdenvsPackages = forAllSystemsAndStdenvs
+          (system: stdenv:
+            import nixpkgs {
+              inherit system;
+              overlays = [
+                (overlayFor (p: p.${stdenv}))
+              ];
+            }
+          );
+        in
+        # Add the `stdenvPackages` at toplevel, both because these are the ones
+        # we want most of the time and for backwards compatibility
+        forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages);
 
       commonDeps = pkgs: with pkgs; rec {
         # Use "busybox-sandbox-shell" if present,
@@ -75,7 +97,7 @@
             buildPackages.mdbook
             buildPackages.autoconf-archive
             buildPackages.autoreconfHook
-            buildPackages.pkgconfig
+            buildPackages.pkg-config
 
             # Tests
             buildPackages.git
@@ -111,6 +133,7 @@
                 ./boehmgc-coroutine-sp-fallback.diff
               ];
             }))
+            nlohmann_json
           ];
 
         perlDeps =
@@ -119,8 +142,8 @@
           ];
       };
 
-    installScriptFor = systems:
-      with nixpkgsFor.x86_64-linux;
+      installScriptFor = systems:
+        with nixpkgsFor.x86_64-linux;
         runCommand "installer-script"
           { buildInputs = [ nix ];
           }
@@ -184,191 +207,209 @@
         installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES";
       };
 
-      binaryTarball = buildPackages: nix: pkgs: let
-        inherit (pkgs) cacert;
-        installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
-      in
+      binaryTarball = buildPackages: nix: pkgs:
+        let
+          inherit (pkgs) cacert;
+          installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; };
+        in
 
-      buildPackages.runCommand "nix-binary-tarball-${version}"
-        { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
-          meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
-        }
-        ''
-          cp ${installerClosureInfo}/registration $TMPDIR/reginfo
-          cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
-          substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
-            --subst-var-by nix ${nix} \
-            --subst-var-by cacert ${cacert}
+        buildPackages.runCommand "nix-binary-tarball-${version}"
+          { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+            meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}";
+          }
+          ''
+            cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+            cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
+            substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+              --subst-var-by nix ${nix} \
+              --subst-var-by cacert ${cacert}
 
-          substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
-            --subst-var-by nix ${nix} \
-            --subst-var-by cacert ${cacert}
-          substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
-            --subst-var-by nix ${nix} \
-            --subst-var-by cacert ${cacert}
-          substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
-            --subst-var-by nix ${nix} \
-            --subst-var-by cacert ${cacert}
+            substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+              --subst-var-by nix ${nix} \
+              --subst-var-by cacert ${cacert}
+            substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+              --subst-var-by nix ${nix} \
+              --subst-var-by cacert ${cacert}
+            substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+              --subst-var-by nix ${nix} \
+              --subst-var-by cacert ${cacert}
 
-          if type -p shellcheck; then
-            # SC1090: Don't worry about not being able to find
-            #         $nix/etc/profile.d/nix.sh
-            shellcheck --exclude SC1090 $TMPDIR/install
-            shellcheck $TMPDIR/create-darwin-volume.sh
-            shellcheck $TMPDIR/install-darwin-multi-user.sh
-            shellcheck $TMPDIR/install-systemd-multi-user.sh
+            if type -p shellcheck; then
+              # SC1090: Don't worry about not being able to find
+              #         $nix/etc/profile.d/nix.sh
+              shellcheck --exclude SC1090 $TMPDIR/install
+              shellcheck $TMPDIR/create-darwin-volume.sh
+              shellcheck $TMPDIR/install-darwin-multi-user.sh
+              shellcheck $TMPDIR/install-systemd-multi-user.sh
 
-            # SC1091: Don't panic about not being able to source
-            #         /etc/profile
-            # SC2002: Ignore "useless cat" "error", when loading
-            #         .reginfo, as the cat is a much cleaner
-            #         implementation, even though it is "useless"
-            # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
-            #         root's home directory
-            shellcheck --external-sources \
-              --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
-          fi
+              # SC1091: Don't panic about not being able to source
+              #         /etc/profile
+              # SC2002: Ignore "useless cat" "error", when loading
+              #         .reginfo, as the cat is a much cleaner
+              #         implementation, even though it is "useless"
+              # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+              #         root's home directory
+              shellcheck --external-sources \
+                --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+            fi
 
-          chmod +x $TMPDIR/install
-          chmod +x $TMPDIR/create-darwin-volume.sh
-          chmod +x $TMPDIR/install-darwin-multi-user.sh
-          chmod +x $TMPDIR/install-systemd-multi-user.sh
-          chmod +x $TMPDIR/install-multi-user
-          dir=nix-${version}-${pkgs.system}
-          fn=$out/$dir.tar.xz
-          mkdir -p $out/nix-support
-          echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-          tar cvfJ $fn \
-            --owner=0 --group=0 --mode=u+rw,uga+r \
-            --absolute-names \
-            --hard-dereference \
-            --transform "s,$TMPDIR/install,$dir/install," \
-            --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
-            --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
-            --transform "s,$NIX_STORE,$dir/store,S" \
-            $TMPDIR/install \
-            $TMPDIR/create-darwin-volume.sh \
-            $TMPDIR/install-darwin-multi-user.sh \
-            $TMPDIR/install-systemd-multi-user.sh \
-            $TMPDIR/install-multi-user \
-            $TMPDIR/reginfo \
-            $(cat ${installerClosureInfo}/store-paths)
-        '';
+            chmod +x $TMPDIR/install
+            chmod +x $TMPDIR/create-darwin-volume.sh
+            chmod +x $TMPDIR/install-darwin-multi-user.sh
+            chmod +x $TMPDIR/install-systemd-multi-user.sh
+            chmod +x $TMPDIR/install-multi-user
+            dir=nix-${version}-${pkgs.system}
+            fn=$out/$dir.tar.xz
+            mkdir -p $out/nix-support
+            echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+            tar cvfJ $fn \
+              --owner=0 --group=0 --mode=u+rw,uga+r \
+              --absolute-names \
+              --hard-dereference \
+              --transform "s,$TMPDIR/install,$dir/install," \
+              --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
+              --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+              --transform "s,$NIX_STORE,$dir/store,S" \
+              $TMPDIR/install \
+              $TMPDIR/create-darwin-volume.sh \
+              $TMPDIR/install-darwin-multi-user.sh \
+              $TMPDIR/install-systemd-multi-user.sh \
+              $TMPDIR/install-multi-user \
+              $TMPDIR/reginfo \
+              $(cat ${installerClosureInfo}/store-paths)
+          '';
+
+      overlayFor = getStdenv: final: prev:
+        let currentStdenv = getStdenv final; in
+        {
+          nixStable = prev.nix;
+
+          # Forward from the previous stage as we don’t want it to pick the lowdown override
+          nixUnstable = prev.nixUnstable;
+
+          nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation {
+            name = "nix-${version}";
+            inherit version;
+
+            src = self;
+
+            VERSION_SUFFIX = versionSuffix;
+
+            outputs = [ "out" "dev" "doc" ];
+
+            nativeBuildInputs = nativeBuildDeps;
+            buildInputs = buildDeps ++ awsDeps;
+
+            propagatedBuildInputs = propagatedDeps;
+
+            disallowedReferences = [ boost ];
+
+            preConfigure =
+              ''
+                # Copy libboost_context so we don't get all of Boost in our closure.
+                # https://github.com/NixOS/nixpkgs/issues/45462
+                mkdir -p $out/lib
+                cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+                rm -f $out/lib/*.a
+                ${lib.optionalString currentStdenv.isLinux ''
+                  chmod u+w $out/lib/*.so.*
+                  patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+                ''}
+                ${lib.optionalString currentStdenv.isDarwin ''
+                  for LIB in $out/lib/*.dylib; do
+                    chmod u+w $LIB
+                    install_name_tool -id $LIB $LIB
+                  done
+                  install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib
+                ''}
+              '';
+
+            configureFlags = configureFlags ++
+              [ "--sysconfdir=/etc" ];
+
+            enableParallelBuilding = true;
+
+            makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
+
+            doCheck = true;
+
+            installFlags = "sysconfdir=$(out)/etc";
+
+            postInstall = ''
+              mkdir -p $doc/nix-support
+              echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+              ${lib.optionalString currentStdenv.isDarwin ''
+              install_name_tool \
+                -change ${boost}/lib/libboost_context.dylib \
+                $out/lib/libboost_context.dylib \
+                $out/lib/libnixutil.dylib
+              ''}
+            '';
+
+            doInstallCheck = true;
+            installCheckFlags = "sysconfdir=$(out)/etc";
+
+            separateDebugInfo = true;
+
+            strictDeps = true;
+
+            passthru.perl-bindings = with final; currentStdenv.mkDerivation {
+              name = "nix-perl-${version}";
+
+              src = self;
+
+              nativeBuildInputs =
+                [ buildPackages.autoconf-archive
+                  buildPackages.autoreconfHook
+                  buildPackages.pkg-config
+                ];
+
+              buildInputs =
+                [ nix
+                  curl
+                  bzip2
+                  xz
+                  pkgs.perl
+                  boost
+                ]
+                ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium
+                ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security;
+
+              configureFlags = ''
+                --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
+                --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
+              '';
+
+              enableParallelBuilding = true;
+
+              postUnpack = "sourceRoot=$sourceRoot/perl";
+            };
+
+          };
+
+          lowdown-nix = with final; currentStdenv.mkDerivation rec {
+            name = "lowdown-0.9.0";
+
+            src = lowdown-src;
+
+            outputs = [ "out" "bin" "dev" ];
+
+            nativeBuildInputs = [ buildPackages.which ];
+
+            configurePhase = ''
+                ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
+                ./configure \
+                  PREFIX=${placeholder "dev"} \
+                  BINDIR=${placeholder "bin"}/bin
+            '';
+          };
+        };
 
     in {
 
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix.perl-bindings' packages.
-      overlay = final: prev: {
-
-        nixStable = prev.nix;
-
-        # Forward from the previous stage as we don’t want it to pick the lowdown override
-        nixUnstable = prev.nixUnstable;
-
-        nix = with final; with commonDeps pkgs; stdenv.mkDerivation {
-          name = "nix-${version}";
-          inherit version;
-
-          src = self;
-
-          VERSION_SUFFIX = versionSuffix;
-
-          outputs = [ "out" "dev" "doc" ];
-
-          nativeBuildInputs = nativeBuildDeps;
-          buildInputs = buildDeps ++ awsDeps;
-
-          propagatedBuildInputs = propagatedDeps;
-
-          preConfigure =
-            ''
-              # Copy libboost_context so we don't get all of Boost in our closure.
-              # https://github.com/NixOS/nixpkgs/issues/45462
-              mkdir -p $out/lib
-              cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
-              rm -f $out/lib/*.a
-              ${lib.optionalString stdenv.isLinux ''
-                chmod u+w $out/lib/*.so.*
-                patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
-              ''}
-            '';
-
-          configureFlags = configureFlags ++
-            [ "--sysconfdir=/etc" ];
-
-          enableParallelBuilding = true;
-
-          makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
-
-          doCheck = true;
-
-          installFlags = "sysconfdir=$(out)/etc";
-
-          postInstall = ''
-            mkdir -p $doc/nix-support
-            echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
-          '';
-
-          doInstallCheck = true;
-          installCheckFlags = "sysconfdir=$(out)/etc";
-
-          separateDebugInfo = true;
-
-          strictDeps = true;
-
-          passthru.perl-bindings = with final; stdenv.mkDerivation {
-            name = "nix-perl-${version}";
-
-            src = self;
-
-            nativeBuildInputs =
-              [ buildPackages.autoconf-archive
-                buildPackages.autoreconfHook
-                buildPackages.pkgconfig
-              ];
-
-            buildInputs =
-              [ nix
-                curl
-                bzip2
-                xz
-                pkgs.perl
-                boost
-              ]
-              ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
-              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
-
-            configureFlags = ''
-              --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
-              --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
-            '';
-
-            enableParallelBuilding = true;
-
-            postUnpack = "sourceRoot=$sourceRoot/perl";
-          };
-
-        };
-
-        lowdown-nix = with final; stdenv.mkDerivation rec {
-          name = "lowdown-0.9.0";
-
-          src = lowdown-src;
-
-          outputs = [ "out" "bin" "dev" ];
-
-          nativeBuildInputs = [ buildPackages.which ];
-
-          configurePhase = ''
-              ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""}
-              ./configure \
-                PREFIX=${placeholder "dev"} \
-                BINDIR=${placeholder "bin"}/bin
-            '';
-        };
-
-      };
+      overlay = overlayFor (p: p.stdenv);
 
       hydraJobs = {
 
@@ -406,19 +447,7 @@
         installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"];
 
         # docker image with Nix inside
-        dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system:
-          let
-            pkgs = nixpkgsFor.${system};
-            image = import ./docker.nix { inherit pkgs; tag = version; };
-          in pkgs.runCommand "docker-image-tarball-${version}"
-            { meta.description = "Docker image with Nix for ${system}";
-            }
-            ''
-              mkdir -p $out/nix-support
-              image=$out/image.tar.gz
-              ln -s ${image} $image
-              echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
-            '');
+        dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
 
         # Line coverage analysis.
         coverage =
@@ -472,6 +501,12 @@
           inherit (self) overlay;
         });
 
+        tests.sourcehutFlakes = (import ./tests/sourcehut-flakes.nix rec {
+          system = "x86_64-linux";
+          inherit nixpkgs;
+          inherit (self) overlay;
+        });
+
         tests.setuid = nixpkgs.lib.genAttrs
           ["i686-linux" "x86_64-linux"]
           (system:
@@ -480,29 +515,23 @@
               inherit (self) overlay;
             });
 
-        /*
-        # Check whether we can still evaluate all of Nixpkgs.
+        # Make sure that nix-env still produces the exact same result
+        # on a particular version of Nixpkgs.
         tests.evalNixpkgs =
-          import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
-            # FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
-            inherit nixpkgs;
-            pkgs = nixpkgsFor.x86_64-linux;
-            officialRelease = false;
-          };
-
-        # Check whether we can still evaluate NixOS.
-        tests.evalNixOS =
           with nixpkgsFor.x86_64-linux;
           runCommand "eval-nixos" { buildInputs = [ nix ]; }
             ''
-              export NIX_STATE_DIR=$TMPDIR
-
-              nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
-                --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
-
-              touch $out
+              type -p nix-env
+              # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
+              time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages
+              [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]]
+              mkdir $out
             '';
-        */
+
+        metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
+          pkgs = nixpkgsFor.x86_64-linux;
+          nixpkgs = nixpkgs-regression;
+        };
 
         installTests = forAllSystems (system:
           let pkgs = nixpkgsFor.${system}; in
@@ -524,9 +553,9 @@
         binaryTarball = self.hydraJobs.binaryTarball.${system};
         perlBindings = self.hydraJobs.perlBindings.${system};
         installTests = self.hydraJobs.installTests.${system};
-      } // (if system == "x86_64-linux" then {
+      } // (nixpkgs.lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
         dockerImage = self.hydraJobs.dockerImage.${system};
-      } else {}));
+      });
 
       packages = forAllSystems (system: {
         inherit (nixpkgsFor.${system}) nix;
@@ -571,6 +600,20 @@
 
           hardeningDisable = [ "pie" ];
         };
+        dockerImage =
+          let
+            pkgs = nixpkgsFor.${system};
+            image = import ./docker.nix { inherit pkgs; tag = version; };
+          in
+          pkgs.runCommand
+            "docker-image-tarball-${version}"
+            { meta.description = "Docker image with Nix for ${system}"; }
+            ''
+              mkdir -p $out/nix-support
+              image=$out/image.tar.gz
+              ln -s ${image} $image
+              echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
+            '';
       } // builtins.listToAttrs (map (crossSystem: {
         name = "nix-${crossSystem}";
         value = let
@@ -610,15 +653,21 @@
           doInstallCheck = true;
           installCheckFlags = "sysconfdir=$(out)/etc";
         };
-      }) crossSystems)));
+      }) crossSystems)) // (builtins.listToAttrs (map (stdenvName:
+        nixpkgsFor.${system}.lib.nameValuePair
+          "nix-${stdenvName}"
+          nixpkgsFor.${system}."${stdenvName}Packages".nix
+      ) stdenvs)));
 
       defaultPackage = forAllSystems (system: self.packages.${system}.nix);
 
-      devShell = forAllSystems (system:
+      devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages);
+
+      devShells = forAllSystemsAndStdenvs (system: stdenv:
         with nixpkgsFor.${system};
         with commonDeps pkgs;
 
-        stdenv.mkDerivation {
+        nixpkgsFor.${system}.${stdenv}.mkDerivation {
           name = "nix";
 
           outputs = [ "out" "dev" "doc" ];
@@ -637,6 +686,9 @@
               PATH=$prefix/bin:$PATH
               unset PYTHONPATH
               export MANPATH=$out/share/man:$MANPATH
+
+              # Make bash completion work.
+              XDG_DATA_DIRS+=:$out/share
             '';
         });
 

maintainers/upload-release.pl

@@ -55,6 +55,11 @@ my $releaseDir = "nix/$releaseName";
 my $tmpDir = "$TMPDIR/nix-release/$releaseName";
 File::Path::make_path($tmpDir);
 
+my $narCache = "$TMPDIR/nar-cache";
+File::Path::make_path($narCache);
+
+my $binaryCache = "https://cache.nixos.org/?local-nar-cache=$narCache";
+
 # S3 setup.
 my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
 my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
@@ -80,6 +85,7 @@ sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
 
     my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json'));
+    #print STDERR "$jobName: ", Dumper($buildInfo), "\n";
 
     my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n";
     $dstName //= basename($srcFile);
@@ -87,19 +93,27 @@ sub downloadFile {
 
     if (!-e $tmpFile) {
         print STDERR "downloading $srcFile to $tmpFile...\n";
-        system("NIX_REMOTE=https://cache.nixos.org/ nix store cat '$srcFile' > '$tmpFile'") == 0
+
+        my $fileInfo = decode_json(`NIX_REMOTE=$binaryCache nix store ls --json '$srcFile'`);
+
+        $srcFile = $fileInfo->{target} if $fileInfo->{type} eq 'symlink';
+
+        #print STDERR $srcFile, " ", Dumper($fileInfo), "\n";
+
+        system("NIX_REMOTE=$binaryCache nix store cat '$srcFile' > '$tmpFile'.tmp") == 0
             or die "unable to fetch $srcFile\n";
+        rename("$tmpFile.tmp", $tmpFile) or die;
     }
 
-    my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
+    my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash};
     my $sha256_actual = `nix hash file --base16 --type sha256 '$tmpFile'`;
     chomp $sha256_actual;
-    if ($sha256_expected ne $sha256_actual) {
+    if (defined($sha256_expected) && $sha256_expected ne $sha256_actual) {
         print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
         exit 1;
     }
 
-    write_file("$tmpFile.sha256", $sha256_expected);
+    write_file("$tmpFile.sha256", $sha256_actual);
 
     if (! -e "$tmpFile.asc") {
         system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
@@ -117,6 +131,60 @@ downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1");
 downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1");
 downloadFile("installerScript", "1");
 
+# Upload docker images to dockerhub.
+my $dockerManifest = "";
+my $dockerManifestLatest = "";
+
+for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
+    my $system = $platforms->[0];
+    my $dockerPlatform = $platforms->[1];
+    my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz";
+    downloadFile("dockerImage.$system", "1", $fn);
+
+    print STDERR "loading docker image for $dockerPlatform...\n";
+    system("docker load -i $tmpDir/$fn") == 0 or die;
+
+    my $tag = "nixos/nix:$version-$dockerPlatform";
+    my $latestTag = "nixos/nix:latest-$dockerPlatform";
+
+    print STDERR "tagging $version docker image for $dockerPlatform...\n";
+    system("docker tag nix:$version $tag") == 0 or die;
+
+    if ($isLatest) {
+        print STDERR "tagging latest docker image for $dockerPlatform...\n";
+        system("docker tag nix:$version $latestTag") == 0 or die;
+    }
+
+    print STDERR "pushing $version docker image for $dockerPlatform...\n";
+    system("docker push -q $tag") == 0 or die;
+
+    if ($isLatest) {
+        print STDERR "pushing latest docker image for $dockerPlatform...\n";
+        system("docker push -q $latestTag") == 0 or die;
+    }
+
+    $dockerManifest .= " --amend $tag";
+    $dockerManifestLatest .= " --amend $latestTag"
+}
+
+print STDERR "creating multi-platform docker manifest...\n";
+system("docker manifest rm nixos/nix:$version");
+system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
+if ($isLatest) {
+    print STDERR "creating latest multi-platform docker manifest...\n";
+    system("docker manifest rm nixos/nix:latest");
+    system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
+}
+
+print STDERR "pushing multi-platform docker manifest...\n";
+system("docker manifest push nixos/nix:$version") == 0 or die;
+
+if ($isLatest) {
+    print STDERR "pushing latest multi-platform docker manifest...\n";
+    system("docker manifest push nixos/nix:latest") == 0 or die;
+}
+
+# Upload release files to S3.
 for my $fn (glob "$tmpDir/*") {
     my $name = basename($fn);
     my $dstKey = "$releaseDir/" . $name;

misc/bash/completion.sh

@@ -7,13 +7,15 @@ function _complete_nix {
         local completion=${line%%	*}
         if [[ -z $have_type ]]; then
             have_type=1
-            if [[ $completion = filenames ]]; then
+            if [[ $completion == filenames ]]; then
                 compopt -o filenames
+            elif [[ $completion == attrs ]]; then
+                compopt -o nospace
             fi
         else
             COMPREPLY+=("$completion")
         fi
-    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
+    done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}" 2>/dev/null)
     __ltrim_colon_completions "$cur"
 }
 

misc/fish/completion.fish

@@ -19,7 +19,6 @@ end
 
 function _nix_accepts_files
   set -l response (_nix_complete)
-  # First line is either filenames or no-filenames.
   test $response[1] = 'filenames'
 end
 

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -25,5 +25,10 @@
     <string>/var/log/nix-daemon.log</string>
     <key>StandardOutPath</key>
     <string>/dev/null</string>
+    <key>SoftResourceLimits</key>
+    <dict>
+      <key>NumberOfFiles</key>
+      <integer>4096</integer>
+    </dict>
   </dict>
 </plist>

misc/systemd/local.mk

@@ -1,7 +1,8 @@
 ifdef HOST_LINUX
 
   $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
+  $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644)))
 
-  clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service
+  clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf
 
 endif

misc/systemd/nix-daemon.conf.in

@@ -0,0 +1 @@
+d @localstatedir@/nix/daemon-socket 0755 root root - -

misc/systemd/nix-daemon.service.in

@@ -1,7 +1,9 @@
 [Unit]
 Description=Nix Daemon
+Documentation=man:nix-daemon https://nixos.org/manual
 RequiresMountsFor=@storedir@
 RequiresMountsFor=@localstatedir@
+RequiresMountsFor=@localstatedir@/nix/db
 ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 
 [Service]

misc/zsh/completion.zsh

@@ -4,7 +4,7 @@ function _nix() {
   local ifs_bk="$IFS"
   local input=("${(Q)words[@]}")
   IFS=$'\n'
-  local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]"))
+  local res=($(NIX_GET_COMPLETIONS=$((CURRENT - 1)) "$input[@]" 2>/dev/null))
   IFS="$ifs_bk"
   local tpe="${${res[1]}%%>	*}"
   local -a suggestions

mk/run_test.sh

@@ -14,9 +14,27 @@ if [ -t 1 ]; then
     yellow=""
     normal=""
 fi
-(cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
-log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
-status=$?
+
+run_test () {
+    (cd tests && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+    log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+    status=$?
+}
+
+run_test "$1"
+
+# Hack: Retry the test if it fails with “unexpected EOF reading a line” as these
+# appear randomly without anyone knowing why.
+# See https://github.com/NixOS/nix/issues/3605 for more info
+if [[ $status -ne 0 && $status -ne 99 && \
+    "$(uname)" == "Darwin" && \
+    "$log" =~ "unexpected EOF reading a line" \
+]]; then
+    echo "$post_run_msg [${yellow}FAIL$normal] (possibly flaky, so will be retried)"
+    echo "$log" | sed 's/^/    /'
+    run_test "$1"
+fi
+
 if [ $status -eq 0 ]; then
   echo "$post_run_msg [${green}PASS$normal]"
 elif [ $status -eq 99 ]; then

nix-rust/Cargo.lock

@@ -1,399 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-[[package]]
-name = "assert_matches"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "autocfg"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "bit-set"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "bit-vec"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "bitflags"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "byteorder"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "c2-chacha"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "cloudabi"
-version = "0.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "fnv"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "fuchsia-cprng"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "getrandom"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "hex"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "libc"
-version = "0.2.66"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "nix-rust"
-version = "0.1.0"
-dependencies = [
- "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "proptest"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "quick-error"
-version = "1.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "rand"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "rand_core"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_hc"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_hc"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_isaac"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_jitter"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_os"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_pcg"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand_xorshift"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rdrand"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.1.56"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "regex-syntax"
-version = "0.6.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "remove_dir_all"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rusty-fork"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "tempfile"
-version = "3.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
- "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "wait-timeout"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "wasi"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[metadata]
-"checksum assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7deb0a829ca7bcfaf5da70b073a8d128619259a7be8216a355e23f00763059e5"
-"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
-"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
-"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
-"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
-"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
-"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
-"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
-"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
-"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
-"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
-"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
-"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77"
-"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
-"checksum num-traits 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c81ffc11c212fa327657cb19dd85eb7419e163b5b076bede2bdb5c974c07e4"
-"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
-"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
-"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
-"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
-"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412"
-"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
-"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
-"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
-"checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
-"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
-"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
-"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
-"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
-"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
-"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
-"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
-"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
-"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
-"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
-"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716"
-"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
-"checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
-"checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
-"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
-"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
-"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
-"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

nix-rust/Cargo.toml

@@ -1,23 +0,0 @@
-[package]
-name = "nix-rust"
-version = "0.1.0"
-authors = ["Eelco Dolstra <edolstra@gmail.com>"]
-edition = "2018"
-
-[lib]
-name = "nixrust"
-crate-type = ["cdylib"]
-
-[dependencies]
-libc = "0.2"
-#futures-preview = { version = "=0.3.0-alpha.19" }
-#hyper = "0.13.0-alpha.4"
-#http = "0.1"
-#tokio = { version = "0.2.0-alpha.6", default-features = false, features = ["rt-full"] }
-lazy_static = "1.4"
-#byteorder = "1.3"
-
-[dev-dependencies]
-hex = "0.3"
-assert_matches = "1.3"
-proptest = "0.9"

nix-rust/local.mk

@@ -1,48 +0,0 @@
-ifeq ($(OPTIMIZE), 1)
-  RUST_MODE = --release
-  RUST_DIR = release
-else
-  RUST_MODE =
-  RUST_DIR = debug
-endif
-
-libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT)
-libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT)
-libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust
-libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust
-
-ifdef HOST_LINUX
-libnixrust_LDFLAGS_USE += -ldl
-libnixrust_LDFLAGS_USE_INSTALLED += -ldl
-endif
-
-ifdef HOST_DARWIN
-libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup"
-else
-libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR))
-libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir)
-endif
-
-$(libnixrust_PATH): $(call rwildcard, $(d)/src, *.rs) $(d)/Cargo.toml
-	$(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \
-	$(libnixrust_BUILD_FLAGS) \
-	  cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \
-	&& touch target/$(RUST_DIR)/libnixrust.$(SO_EXT)
-
-$(libnixrust_INSTALL_PATH): $(libnixrust_PATH)
-	$(target-gen) cp $^ $@
-ifdef HOST_DARWIN
-	install_name_tool -id $@ $@
-endif
-
-clean: clean-rust
-
-clean-rust:
-	$(suppress) rm -rfv nix-rust/target
-
-ifndef HOST_DARWIN
-check: rust-tests
-
-rust-tests:
-	$(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi)
-endif

nix-rust/src/c.rs

@@ -1,77 +0,0 @@
-use super::{error, store::path, store::StorePath, util};
-
-#[no_mangle]
-pub unsafe extern "C" fn ffi_String_new(s: &str, out: *mut String) {
-    // FIXME: check whether 's' is valid UTF-8?
-    out.write(s.to_string())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn ffi_String_drop(self_: *mut String) {
-    std::ptr::drop_in_place(self_);
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_new(
-    path: &str,
-    store_dir: &str,
-) -> Result<StorePath, error::CppException> {
-    StorePath::new(std::path::Path::new(path), std::path::Path::new(store_dir))
-        .map_err(|err| err.into())
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_new2(
-    hash: &[u8; crate::store::path::STORE_PATH_HASH_BYTES],
-    name: &str,
-) -> Result<StorePath, error::CppException> {
-    StorePath::from_parts(*hash, name).map_err(|err| err.into())
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_fromBaseName(
-    base_name: &str,
-) -> Result<StorePath, error::CppException> {
-    StorePath::new_from_base_name(base_name).map_err(|err| err.into())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn ffi_StorePath_drop(self_: *mut StorePath) {
-    std::ptr::drop_in_place(self_);
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_to_string(self_: &StorePath) -> Vec<u8> {
-    let mut buf = vec![0; path::STORE_PATH_HASH_CHARS + 1 + self_.name.name().len()];
-    util::base32::encode_into(self_.hash.hash(), &mut buf[0..path::STORE_PATH_HASH_CHARS]);
-    buf[path::STORE_PATH_HASH_CHARS] = b'-';
-    buf[path::STORE_PATH_HASH_CHARS + 1..].clone_from_slice(self_.name.name().as_bytes());
-    buf
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_less_than(a: &StorePath, b: &StorePath) -> bool {
-    a < b
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_eq(a: &StorePath, b: &StorePath) -> bool {
-    a == b
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_clone(self_: &StorePath) -> StorePath {
-    self_.clone()
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_name(self_: &StorePath) -> &str {
-    self_.name.name()
-}
-
-#[no_mangle]
-pub extern "C" fn ffi_StorePath_hash_data(
-    self_: &StorePath,
-) -> &[u8; crate::store::path::STORE_PATH_HASH_BYTES] {
-    self_.hash.hash()
-}

nix-rust/src/error.rs

@@ -1,118 +0,0 @@
-use std::fmt;
-
-#[derive(Debug)]
-pub enum Error {
-    InvalidPath(crate::store::StorePath),
-    BadStorePath(std::path::PathBuf),
-    NotInStore(std::path::PathBuf),
-    BadNarInfo,
-    BadBase32,
-    StorePathNameEmpty,
-    StorePathNameTooLong,
-    BadStorePathName,
-    NarSizeFieldTooBig,
-    BadNarString,
-    BadNarPadding,
-    BadNarVersionMagic,
-    MissingNarOpenTag,
-    MissingNarCloseTag,
-    MissingNarField,
-    BadNarField(String),
-    BadExecutableField,
-    IOError(std::io::Error),
-    #[cfg(unused)]
-    HttpError(hyper::error::Error),
-    Misc(String),
-    #[cfg(not(test))]
-    Foreign(CppException),
-    BadTarFileMemberName(String),
-}
-
-impl From<std::io::Error> for Error {
-    fn from(err: std::io::Error) -> Self {
-        Error::IOError(err)
-    }
-}
-
-#[cfg(unused)]
-impl From<hyper::error::Error> for Error {
-    fn from(err: hyper::error::Error) -> Self {
-        Error::HttpError(err)
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Error::InvalidPath(_) => write!(f, "invalid path"),
-            Error::BadNarInfo => write!(f, ".narinfo file is corrupt"),
-            Error::BadStorePath(path) => write!(f, "path '{}' is not a store path", path.display()),
-            Error::NotInStore(path) => {
-                write!(f, "path '{}' is not in the Nix store", path.display())
-            }
-            Error::BadBase32 => write!(f, "invalid base32 string"),
-            Error::StorePathNameEmpty => write!(f, "store path name is empty"),
-            Error::StorePathNameTooLong => {
-                write!(f, "store path name is longer than 211 characters")
-            }
-            Error::BadStorePathName => write!(f, "store path name contains forbidden character"),
-            Error::NarSizeFieldTooBig => write!(f, "size field in NAR is too big"),
-            Error::BadNarString => write!(f, "NAR string is not valid UTF-8"),
-            Error::BadNarPadding => write!(f, "NAR padding is not zero"),
-            Error::BadNarVersionMagic => write!(f, "unsupported NAR version"),
-            Error::MissingNarOpenTag => write!(f, "NAR open tag is missing"),
-            Error::MissingNarCloseTag => write!(f, "NAR close tag is missing"),
-            Error::MissingNarField => write!(f, "expected NAR field is missing"),
-            Error::BadNarField(s) => write!(f, "unrecognized NAR field '{}'", s),
-            Error::BadExecutableField => write!(f, "bad 'executable' field in NAR"),
-            Error::IOError(err) => write!(f, "I/O error: {}", err),
-            #[cfg(unused)]
-            Error::HttpError(err) => write!(f, "HTTP error: {}", err),
-            #[cfg(not(test))]
-            Error::Foreign(_) => write!(f, "<C++ exception>"), // FIXME
-            Error::Misc(s) => write!(f, "{}", s),
-            Error::BadTarFileMemberName(s) => {
-                write!(f, "tar archive contains illegal file name '{}'", s)
-            }
-        }
-    }
-}
-
-#[cfg(not(test))]
-impl From<Error> for CppException {
-    fn from(err: Error) -> Self {
-        match err {
-            Error::Foreign(ex) => ex,
-            _ => CppException::new(&err.to_string()),
-        }
-    }
-}
-
-#[cfg(not(test))]
-#[repr(C)]
-#[derive(Debug)]
-pub struct CppException(*const libc::c_void); // == std::exception_ptr*
-
-#[cfg(not(test))]
-impl CppException {
-    fn new(s: &str) -> Self {
-        Self(unsafe { make_error(s) })
-    }
-}
-
-#[cfg(not(test))]
-impl Drop for CppException {
-    fn drop(&mut self) {
-        unsafe {
-            destroy_error(self.0);
-        }
-    }
-}
-
-#[cfg(not(test))]
-extern "C" {
-    #[allow(improper_ctypes)] // YOLO
-    fn make_error(s: &str) -> *const libc::c_void;
-
-    fn destroy_error(exc: *const libc::c_void);
-}

nix-rust/src/lib.rs

@@ -1,10 +0,0 @@
-#[allow(improper_ctypes_definitions)]
-#[cfg(not(test))]
-mod c;
-mod error;
-#[cfg(unused)]
-mod nar;
-mod store;
-mod util;
-
-pub use error::Error;

nix-rust/src/nar.rs

@@ -1,126 +0,0 @@
-use crate::Error;
-use byteorder::{LittleEndian, ReadBytesExt};
-use std::convert::TryFrom;
-use std::io::Read;
-
-pub fn parse<R: Read>(input: &mut R) -> Result<(), Error> {
-    if String::read(input)? != NAR_VERSION_MAGIC {
-        return Err(Error::BadNarVersionMagic);
-    }
-
-    parse_file(input)
-}
-
-const NAR_VERSION_MAGIC: &str = "nix-archive-1";
-
-fn parse_file<R: Read>(input: &mut R) -> Result<(), Error> {
-    if String::read(input)? != "(" {
-        return Err(Error::MissingNarOpenTag);
-    }
-
-    if String::read(input)? != "type" {
-        return Err(Error::MissingNarField);
-    }
-
-    match String::read(input)?.as_ref() {
-        "regular" => {
-            let mut _executable = false;
-            let mut tag = String::read(input)?;
-            if tag == "executable" {
-                _executable = true;
-                if String::read(input)? != "" {
-                    return Err(Error::BadExecutableField);
-                }
-                tag = String::read(input)?;
-            }
-            if tag != "contents" {
-                return Err(Error::MissingNarField);
-            }
-            let _contents = Vec::<u8>::read(input)?;
-            if String::read(input)? != ")" {
-                return Err(Error::MissingNarCloseTag);
-            }
-        }
-        "directory" => loop {
-            match String::read(input)?.as_ref() {
-                "entry" => {
-                    if String::read(input)? != "(" {
-                        return Err(Error::MissingNarOpenTag);
-                    }
-                    if String::read(input)? != "name" {
-                        return Err(Error::MissingNarField);
-                    }
-                    let _name = String::read(input)?;
-                    if String::read(input)? != "node" {
-                        return Err(Error::MissingNarField);
-                    }
-                    parse_file(input)?;
-                    let tag = String::read(input)?;
-                    if tag != ")" {
-                        return Err(Error::MissingNarCloseTag);
-                    }
-                }
-                ")" => break,
-                s => return Err(Error::BadNarField(s.into())),
-            }
-        },
-        "symlink" => {
-            if String::read(input)? != "target" {
-                return Err(Error::MissingNarField);
-            }
-            let _target = String::read(input)?;
-            if String::read(input)? != ")" {
-                return Err(Error::MissingNarCloseTag);
-            }
-        }
-        s => return Err(Error::BadNarField(s.into())),
-    }
-
-    Ok(())
-}
-
-trait Deserialize: Sized {
-    fn read<R: Read>(input: &mut R) -> Result<Self, Error>;
-}
-
-impl Deserialize for String {
-    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
-        let buf = Deserialize::read(input)?;
-        Ok(String::from_utf8(buf).map_err(|_| Error::BadNarString)?)
-    }
-}
-
-impl Deserialize for Vec<u8> {
-    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
-        let n: usize = Deserialize::read(input)?;
-        let mut buf = vec![0; n];
-        input.read_exact(&mut buf)?;
-        skip_padding(input, n)?;
-        Ok(buf)
-    }
-}
-
-fn skip_padding<R: Read>(input: &mut R, len: usize) -> Result<(), Error> {
-    if len % 8 != 0 {
-        let mut buf = [0; 8];
-        let buf = &mut buf[0..8 - (len % 8)];
-        input.read_exact(buf)?;
-        if !buf.iter().all(|b| *b == 0) {
-            return Err(Error::BadNarPadding);
-        }
-    }
-    Ok(())
-}
-
-impl Deserialize for u64 {
-    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
-        Ok(input.read_u64::<LittleEndian>()?)
-    }
-}
-
-impl Deserialize for usize {
-    fn read<R: Read>(input: &mut R) -> Result<Self, Error> {
-        let n: u64 = Deserialize::read(input)?;
-        Ok(usize::try_from(n).map_err(|_| Error::NarSizeFieldTooBig)?)
-    }
-}

nix-rust/src/store/binary_cache_store.rs

@@ -1,48 +0,0 @@
-use super::{PathInfo, Store, StorePath};
-use crate::Error;
-use hyper::client::Client;
-
-pub struct BinaryCacheStore {
-    base_uri: String,
-    client: Client<hyper::client::HttpConnector, hyper::Body>,
-}
-
-impl BinaryCacheStore {
-    pub fn new(base_uri: String) -> Self {
-        Self {
-            base_uri,
-            client: Client::new(),
-        }
-    }
-}
-
-impl Store for BinaryCacheStore {
-    fn query_path_info(
-        &self,
-        path: &StorePath,
-    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>> {
-        let uri = format!("{}/{}.narinfo", self.base_uri.clone(), path.hash);
-        let path = path.clone();
-        let client = self.client.clone();
-        let store_dir = self.store_dir().to_string();
-
-        Box::pin(async move {
-            let response = client.get(uri.parse::<hyper::Uri>().unwrap()).await?;
-
-            if response.status() == hyper::StatusCode::NOT_FOUND
-                || response.status() == hyper::StatusCode::FORBIDDEN
-            {
-                return Err(Error::InvalidPath(path));
-            }
-
-            let mut body = response.into_body();
-
-            let mut bytes = Vec::new();
-            while let Some(next) = body.next().await {
-                bytes.extend(next?);
-            }
-
-            PathInfo::parse_nar_info(std::str::from_utf8(&bytes).unwrap(), &store_dir)
-        })
-    }
-}

nix-rust/src/store/mod.rs

@@ -1,17 +0,0 @@
-pub mod path;
-
-#[cfg(unused)]
-mod binary_cache_store;
-#[cfg(unused)]
-mod path_info;
-#[cfg(unused)]
-mod store;
-
-pub use path::{StorePath, StorePathHash, StorePathName};
-
-#[cfg(unused)]
-pub use binary_cache_store::BinaryCacheStore;
-#[cfg(unused)]
-pub use path_info::PathInfo;
-#[cfg(unused)]
-pub use store::Store;

nix-rust/src/store/path.rs

@@ -1,224 +0,0 @@
-use crate::error::Error;
-use crate::util::base32;
-use std::fmt;
-use std::path::Path;
-
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub struct StorePath {
-    pub hash: StorePathHash,
-    pub name: StorePathName,
-}
-
-pub const STORE_PATH_HASH_BYTES: usize = 20;
-pub const STORE_PATH_HASH_CHARS: usize = 32;
-
-impl StorePath {
-    pub fn new(path: &Path, store_dir: &Path) -> Result<Self, Error> {
-        if path.parent() != Some(store_dir) {
-            return Err(Error::NotInStore(path.into()));
-        }
-        Self::new_from_base_name(
-            path.file_name()
-                .ok_or_else(|| Error::BadStorePath(path.into()))?
-                .to_str()
-                .ok_or_else(|| Error::BadStorePath(path.into()))?,
-        )
-    }
-
-    pub fn from_parts(hash: [u8; STORE_PATH_HASH_BYTES], name: &str) -> Result<Self, Error> {
-        Ok(StorePath {
-            hash: StorePathHash(hash),
-            name: StorePathName::new(name)?,
-        })
-    }
-
-    pub fn new_from_base_name(base_name: &str) -> Result<Self, Error> {
-        if base_name.len() < STORE_PATH_HASH_CHARS + 1
-            || base_name.as_bytes()[STORE_PATH_HASH_CHARS] != b'-'
-        {
-            return Err(Error::BadStorePath(base_name.into()));
-        }
-
-        Ok(StorePath {
-            hash: StorePathHash::new(&base_name[0..STORE_PATH_HASH_CHARS])?,
-            name: StorePathName::new(&base_name[STORE_PATH_HASH_CHARS + 1..])?,
-        })
-    }
-}
-
-impl fmt::Display for StorePath {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}-{}", self.hash, self.name)
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, Debug)]
-pub struct StorePathHash([u8; STORE_PATH_HASH_BYTES]);
-
-impl StorePathHash {
-    pub fn new(s: &str) -> Result<Self, Error> {
-        assert_eq!(s.len(), STORE_PATH_HASH_CHARS);
-        let v = base32::decode(s)?;
-        assert_eq!(v.len(), STORE_PATH_HASH_BYTES);
-        let mut bytes: [u8; 20] = Default::default();
-        bytes.copy_from_slice(&v[0..STORE_PATH_HASH_BYTES]);
-        Ok(Self(bytes))
-    }
-
-    pub fn hash(&self) -> &[u8; STORE_PATH_HASH_BYTES] {
-        &self.0
-    }
-}
-
-impl fmt::Display for StorePathHash {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut buf = vec![0; STORE_PATH_HASH_CHARS];
-        base32::encode_into(&self.0, &mut buf);
-        f.write_str(std::str::from_utf8(&buf).unwrap())
-    }
-}
-
-impl Ord for StorePathHash {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        // Historically we've sorted store paths by their base32
-        // serialization, but our base32 encodes bytes in reverse
-        // order. So compare them in reverse order as well.
-        self.0.iter().rev().cmp(other.0.iter().rev())
-    }
-}
-
-impl PartialOrd for StorePathHash {
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)]
-pub struct StorePathName(String);
-
-impl StorePathName {
-    pub fn new(s: &str) -> Result<Self, Error> {
-        if s.is_empty() {
-            return Err(Error::StorePathNameEmpty);
-        }
-
-        if s.len() > 211 {
-            return Err(Error::StorePathNameTooLong);
-        }
-
-        let is_good_path_name = s.chars().all(|c| {
-            c.is_ascii_alphabetic()
-                || c.is_ascii_digit()
-                || c == '+'
-                || c == '-'
-                || c == '.'
-                || c == '_'
-                || c == '?'
-                || c == '='
-        });
-        if s.starts_with('.') || !is_good_path_name {
-            return Err(Error::BadStorePathName);
-        }
-
-        Ok(Self(s.to_string()))
-    }
-
-    pub fn name(&self) -> &str {
-        &self.0
-    }
-}
-
-impl fmt::Display for StorePathName {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.write_str(&self.0)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use assert_matches::assert_matches;
-
-    #[test]
-    fn test_parse() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
-        let p = StorePath::new_from_base_name(&s).unwrap();
-        assert_eq!(p.name.0, "konsole-18.12.3");
-        assert_eq!(
-            p.hash.0,
-            [
-                0x9f, 0x76, 0x49, 0x20, 0xf6, 0x5d, 0xe9, 0x71, 0xc4, 0xca, 0x46, 0x21, 0xab, 0xff,
-                0x9b, 0x44, 0xef, 0x87, 0x0f, 0x3c
-            ]
-        );
-    }
-
-    #[test]
-    fn test_no_name() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-";
-        assert_matches!(
-            StorePath::new_from_base_name(&s),
-            Err(Error::StorePathNameEmpty)
-        );
-    }
-
-    #[test]
-    fn test_no_dash() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz";
-        assert_matches!(
-            StorePath::new_from_base_name(&s),
-            Err(Error::BadStorePath(_))
-        );
-    }
-
-    #[test]
-    fn test_short_hash() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxl-konsole-18.12.3";
-        assert_matches!(
-            StorePath::new_from_base_name(&s),
-            Err(Error::BadStorePath(_))
-        );
-    }
-
-    #[test]
-    fn test_invalid_hash() {
-        let s = "7h7qgvs4kgzsn8e6rb273saxyqh4jxlz-konsole-18.12.3";
-        assert_matches!(StorePath::new_from_base_name(&s), Err(Error::BadBase32));
-    }
-
-    #[test]
-    fn test_long_name() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
-        assert_matches!(StorePath::new_from_base_name(&s), Ok(_));
-    }
-
-    #[test]
-    fn test_too_long_name() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
-        assert_matches!(
-            StorePath::new_from_base_name(&s),
-            Err(Error::StorePathNameTooLong)
-        );
-    }
-
-    #[test]
-    fn test_bad_name() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-foo bar";
-        assert_matches!(
-            StorePath::new_from_base_name(&s),
-            Err(Error::BadStorePathName)
-        );
-
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-kónsole";
-        assert_matches!(
-            StorePath::new_from_base_name(&s),
-            Err(Error::BadStorePathName)
-        );
-    }
-
-    #[test]
-    fn test_roundtrip() {
-        let s = "7h7qgvs4kgzsn8a6rb273saxyqh4jxlz-konsole-18.12.3";
-        assert_eq!(StorePath::new_from_base_name(&s).unwrap().to_string(), s);
-    }
-}

nix-rust/src/store/path_info.rs

@@ -1,70 +0,0 @@
-use crate::store::StorePath;
-use crate::Error;
-use std::collections::BTreeSet;
-
-#[derive(Clone, Debug)]
-pub struct PathInfo {
-    pub path: StorePath,
-    pub references: BTreeSet<StorePath>,
-    pub nar_size: u64,
-    pub deriver: Option<StorePath>,
-
-    // Additional binary cache info.
-    pub url: Option<String>,
-    pub compression: Option<String>,
-    pub file_size: Option<u64>,
-}
-
-impl PathInfo {
-    pub fn parse_nar_info(nar_info: &str, store_dir: &str) -> Result<Self, Error> {
-        let mut path = None;
-        let mut references = BTreeSet::new();
-        let mut nar_size = None;
-        let mut deriver = None;
-        let mut url = None;
-        let mut compression = None;
-        let mut file_size = None;
-
-        for line in nar_info.lines() {
-            let colon = line.find(':').ok_or(Error::BadNarInfo)?;
-
-            let (name, value) = line.split_at(colon);
-
-            if !value.starts_with(": ") {
-                return Err(Error::BadNarInfo);
-            }
-
-            let value = &value[2..];
-
-            if name == "StorePath" {
-                path = Some(StorePath::new(std::path::Path::new(value), store_dir)?);
-            } else if name == "NarSize" {
-                nar_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
-            } else if name == "References" {
-                if !value.is_empty() {
-                    for r in value.split(' ') {
-                        references.insert(StorePath::new_from_base_name(r)?);
-                    }
-                }
-            } else if name == "Deriver" {
-                deriver = Some(StorePath::new_from_base_name(value)?);
-            } else if name == "URL" {
-                url = Some(value.into());
-            } else if name == "Compression" {
-                compression = Some(value.into());
-            } else if name == "FileSize" {
-                file_size = Some(u64::from_str_radix(value, 10).map_err(|_| Error::BadNarInfo)?);
-            }
-        }
-
-        Ok(PathInfo {
-            path: path.ok_or(Error::BadNarInfo)?,
-            references,
-            nar_size: nar_size.ok_or(Error::BadNarInfo)?,
-            deriver,
-            url: Some(url.ok_or(Error::BadNarInfo)?),
-            compression,
-            file_size,
-        })
-    }
-}

nix-rust/src/store/store.rs

@@ -1,53 +0,0 @@
-use super::{PathInfo, StorePath};
-use crate::Error;
-use std::collections::{BTreeMap, BTreeSet};
-use std::path::Path;
-
-pub trait Store: Send + Sync {
-    fn store_dir(&self) -> &str {
-        "/nix/store"
-    }
-
-    fn query_path_info(
-        &self,
-        store_path: &StorePath,
-    ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<PathInfo, Error>> + Send>>;
-}
-
-impl dyn Store {
-    pub fn parse_store_path(&self, path: &Path) -> Result<StorePath, Error> {
-        StorePath::new(path, self.store_dir())
-    }
-
-    pub async fn compute_path_closure(
-        &self,
-        roots: BTreeSet<StorePath>,
-    ) -> Result<BTreeMap<StorePath, PathInfo>, Error> {
-        let mut done = BTreeSet::new();
-        let mut result = BTreeMap::new();
-        let mut pending = vec![];
-
-        for root in roots {
-            pending.push(self.query_path_info(&root));
-            done.insert(root);
-        }
-
-        while !pending.is_empty() {
-            let (info, _, remaining) = futures::future::select_all(pending).await;
-            pending = remaining;
-
-            let info = info?;
-
-            for path in &info.references {
-                if !done.contains(path) {
-                    pending.push(self.query_path_info(&path));
-                    done.insert(path.clone());
-                }
-            }
-
-            result.insert(info.path.clone(), info);
-        }
-
-        Ok(result)
-    }
-}

nix-rust/src/util/base32.rs

@@ -1,160 +0,0 @@
-use crate::error::Error;
-use lazy_static::lazy_static;
-
-pub fn encoded_len(input_len: usize) -> usize {
-    if input_len == 0 {
-        0
-    } else {
-        (input_len * 8 - 1) / 5 + 1
-    }
-}
-
-pub fn decoded_len(input_len: usize) -> usize {
-    input_len * 5 / 8
-}
-
-static BASE32_CHARS: &[u8; 32] = &b"0123456789abcdfghijklmnpqrsvwxyz";
-
-lazy_static! {
-    static ref BASE32_CHARS_REVERSE: Box<[u8; 256]> = {
-        let mut xs = [0xffu8; 256];
-        for (n, c) in BASE32_CHARS.iter().enumerate() {
-            xs[*c as usize] = n as u8;
-        }
-        Box::new(xs)
-    };
-}
-
-pub fn encode(input: &[u8]) -> String {
-    let mut buf = vec![0; encoded_len(input.len())];
-    encode_into(input, &mut buf);
-    std::str::from_utf8(&buf).unwrap().to_string()
-}
-
-pub fn encode_into(input: &[u8], output: &mut [u8]) {
-    let len = encoded_len(input.len());
-    assert_eq!(len, output.len());
-
-    let mut nr_bits_left: usize = 0;
-    let mut bits_left: u16 = 0;
-    let mut pos = len;
-
-    for b in input {
-        bits_left |= (*b as u16) << nr_bits_left;
-        nr_bits_left += 8;
-        while nr_bits_left > 5 {
-            output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
-            pos -= 1;
-            bits_left >>= 5;
-            nr_bits_left -= 5;
-        }
-    }
-
-    if nr_bits_left > 0 {
-        output[pos - 1] = BASE32_CHARS[(bits_left & 0x1f) as usize];
-        pos -= 1;
-    }
-
-    assert_eq!(pos, 0);
-}
-
-pub fn decode(input: &str) -> Result<Vec<u8>, crate::Error> {
-    let mut res = Vec::with_capacity(decoded_len(input.len()));
-
-    let mut nr_bits_left: usize = 0;
-    let mut bits_left: u16 = 0;
-
-    for c in input.chars().rev() {
-        let b = BASE32_CHARS_REVERSE[c as usize];
-        if b == 0xff {
-            return Err(Error::BadBase32);
-        }
-        bits_left |= (b as u16) << nr_bits_left;
-        nr_bits_left += 5;
-        if nr_bits_left >= 8 {
-            res.push((bits_left & 0xff) as u8);
-            bits_left >>= 8;
-            nr_bits_left -= 8;
-        }
-    }
-
-    if nr_bits_left > 0 && bits_left != 0 {
-        return Err(Error::BadBase32);
-    }
-
-    Ok(res)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use assert_matches::assert_matches;
-    use hex;
-    use proptest::proptest;
-
-    #[test]
-    fn test_encode() {
-        assert_eq!(encode(&[]), "");
-
-        assert_eq!(
-            encode(&hex::decode("0839703786356bca59b0f4a32987eb2e6de43ae8").unwrap()),
-            "x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88"
-        );
-
-        assert_eq!(
-            encode(
-                &hex::decode("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
-                    .unwrap()
-            ),
-            "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
-        );
-
-        assert_eq!(
-            encode(
-                &hex::decode("ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f")
-                    .unwrap()
-            ),
-            "2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx"
-        );
-    }
-
-    #[test]
-    fn test_decode() {
-        assert_eq!(hex::encode(decode("").unwrap()), "");
-
-        assert_eq!(
-            hex::encode(decode("x0xf8v9fxf3jk8zln1cwlsrmhqvp0f88").unwrap()),
-            "0839703786356bca59b0f4a32987eb2e6de43ae8"
-        );
-
-        assert_eq!(
-            hex::encode(decode("1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s").unwrap()),
-            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
-        );
-
-        assert_eq!(
-            hex::encode(decode("2gs8k559z4rlahfx0y688s49m2vvszylcikrfinm30ly9rak69236nkam5ydvly1ai7xac99vxfc4ii84hawjbk876blyk1jfhkbbyx").unwrap()),
-            "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
-        );
-
-        assert_matches!(
-            decode("xoxf8v9fxf3jk8zln1cwlsrmhqvp0f88"),
-            Err(Error::BadBase32)
-        );
-        assert_matches!(
-            decode("2b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"),
-            Err(Error::BadBase32)
-        );
-        assert_matches!(decode("2"), Err(Error::BadBase32));
-        assert_matches!(decode("2gs"), Err(Error::BadBase32));
-        assert_matches!(decode("2gs8"), Err(Error::BadBase32));
-    }
-
-    proptest! {
-
-        #[test]
-        fn roundtrip(s: Vec<u8>) {
-            assert_eq!(s, decode(&encode(&s)).unwrap());
-        }
-    }
-}

nix-rust/src/util/mod.rs

@@ -1 +0,0 @@
-pub mod base32;

perl/configure.ac

@@ -41,7 +41,7 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};')
 AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname])
 AC_MSG_RESULT($perllibdir)
 
-# Look for libsodium, an optional dependency.
+# Look for libsodium.
 PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"])
 
 # Check for the required Perl dependencies (DBI and DBD::SQLite).

perl/lib/Nix/Store.xs

@@ -240,7 +240,7 @@ SV * convertHash(char * algo, char * s, int toBase32)
     PPCODE:
         try {
             auto h = Hash::parseAny(s, parseHashType(algo));
-            string s = h.to_string(toBase32 ? Base32 : Base16, false);
+            auto s = h.to_string(toBase32 ? Base32 : Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

scripts/check-hydra-status.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+# set -x
+
+
+# mapfile BUILDS_FOR_LATEST_EVAL < <(
+# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+#   jq -r '.evals[0].builds[] | @sh')
+BUILDS_FOR_LATEST_EVAL=$(
+curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \
+  jq -r '.evals[0].builds[]')
+
+someBuildFailed=0
+
+for buildId in $BUILDS_FOR_LATEST_EVAL; do
+  buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId")
+
+  finished=$(echo "$buildInfo" | jq -r '.finished')
+
+  if [[ $finished = 0 ]]; then
+    continue
+  fi
+
+  buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus')
+
+  if [[ $buildStatus != 0 ]]; then
+    someBuildFailed=1
+    echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra: $buildInfo"
+  fi
+done
+
+exit "$someBuildFailed"

scripts/create-darwin-volume.sh

@@ -246,7 +246,8 @@ get_volume_pass() {
 verify_volume_pass() {
     local volume_special="$1" # (i.e., disk1s7)
     local volume_uuid="$2"
-    /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid"
+    _sudo "to confirm the password actually unlocks the volume" \
+        /usr/sbin/diskutil apfs unlockVolume "$volume_special" -verify -stdinpassphrase -user "$volume_uuid"
 }
 
 volume_pass_works() {
@@ -440,7 +441,22 @@ add_nix_vol_fstab_line() {
     # shellcheck disable=SC1003,SC2026
     local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}"
     shift
-    EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <<EOF
+
+    # wrap `ex` to work around a problem with vim plugins breaking exit codes;
+    # (see https://github.com/NixOS/nix/issues/5468)
+    # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split
+    # the EDITOR env.
+    #
+    # TODO: at some point we should switch to `--clean`, but it wasn't added
+    # until https://github.com/vim/vim/releases/tag/v8.0.1554 while the macOS
+    # minver 10.12.6 seems to have released with vim 7.4
+    cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF
+#!/bin/sh
+/usr/bin/ex --noplugin "\$@"
+EOF
+    chmod 755 "$SCRATCH/ex_cleanroom_wrapper"
+
+    EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" <<EOF
 :a
 UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners
 .
@@ -631,7 +647,7 @@ EOF
         # technically /etc/synthetic.d/nix is supported in Big Sur+
         # but handling both takes even more code...
         _sudo "to add Nix to /etc/synthetic.conf" \
-            /usr/bin/ex /etc/synthetic.conf <<EOF
+            /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF
 :a
 ${NIX_ROOT:1}
 .
@@ -670,22 +686,27 @@ encrypt_volume() {
     local volume_uuid="$1"
     local volume_label="$2"
     local password
+
+    task "Encrypt the Nix volume" >&2
+
     # Note: mount/unmount are late additions to support the right order
     # of operations for creating the volume and then baking its uuid into
     # other artifacts; not as well-trod wrt to potential errors, race
     # conditions, etc.
 
-    /usr/sbin/diskutil mount "$volume_label"
+    _sudo "to mount your Nix volume for encrypting" \
+        /usr/sbin/diskutil mount "$volume_label"
 
     password="$(/usr/bin/xxd -l 32 -p -c 256 /dev/random)"
     _sudo "to add your Nix volume's password to Keychain" \
         /usr/bin/security -i <<EOF
 add-generic-password -a "$volume_label" -s "$volume_uuid" -l "$volume_label encryption password" -D "Encrypted volume password" -j "Added automatically by the Nix installer for use by $NIX_VOLUME_MOUNTD_DEST" -w "$password" -T /System/Library/CoreServices/APFSUserAgent -T /System/Library/CoreServices/CSUserAgent -T /usr/bin/security "/Library/Keychains/System.keychain"
 EOF
-    builtin printf "%s" "$password" | _sudo "to encrypt your Nix volume" \
+    builtin printf "%s" "$password" | _sudo "to actually encrypt your Nix volume" \
         /usr/sbin/diskutil apfs encryptVolume "$volume_label" -user disk -stdinpassphrase
 
-    /usr/sbin/diskutil unmount force "$volume_label"
+    _sudo "to unmount the encrypted volume" \
+        /usr/sbin/diskutil unmount force "$volume_label"
 }
 
 create_volume() {
@@ -794,7 +815,7 @@ setup_volume_daemon() {
     local volume_uuid="$2"
     if ! test_voldaemon; then
         task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2
-        _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <<EOF
+        _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF
 :a
 $(generate_mount_daemon "$cmd_type" "$volume_uuid")
 .

scripts/install-darwin-multi-user.sh

@@ -218,7 +218,7 @@ EOF
         setup_darwin_volume
     fi
 
-    if [ "$(diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
-        failure "This script needs a /nix volume with global permissions! This may require running sudo diskutil enableOwnership /nix."
+    if [ "$(/usr/sbin/diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then
+        failure "This script needs a /nix volume with global permissions! This may require running sudo /usr/sbin/diskutil enableOwnership /nix."
     fi
 }

scripts/install-multi-user.sh

@@ -23,10 +23,10 @@ readonly RED='\033[31m'
 # installer allows overriding build user count to speed up installation
 # as creating each user takes non-trivial amount of time on macos
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
-readonly NIX_BUILD_GROUP_ID="30000"
+readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 # darwin installer needs to override these
-NIX_FIRST_BUILD_UID="30001"
+NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
 NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
@@ -377,6 +377,11 @@ cure_artifacts() {
 }
 
 validate_starting_assumptions() {
+    task "Checking for artifacts of previous installs"
+    cat <<EOF
+Before I try to install, I'll check for signs Nix already is or has
+been installed on this system.
+EOF
     if type nix-env 2> /dev/null >&2; then
         warning <<EOF
 Nix already appears to be installed. This installer may run into issues.
@@ -386,6 +391,11 @@ $(uninstall_directions)
 EOF
     fi
 
+    # TODO: I think it would be good for this step to accumulate more
+    #       knowledge of older obsolete artifacts, if there are any.
+    #       We could issue a "reminder" here that the user might want
+    #       to clean them up?
+
     for profile_target in "${PROFILE_TARGETS[@]}"; do
         # TODO: I think it would be good to accumulate a list of all
         #       of the copies so that people don't hit this 2 or 3x in
@@ -413,6 +423,18 @@ EOF
         fi
     done
 
+    if [ "$(uname -s)" = "Linux" ] && [ ! -e /run/systemd/system ]; then
+        warning <<EOF
+We did not detect systemd on your system. With a multi-user install
+without systemd you will have to manually configure your init system to
+launch the Nix daemon after installation.
+EOF
+        if ! ui_confirm "Do you want to proceed with a multi-user installation?"; then
+            failure <<EOF
+You have aborted the installation.
+EOF
+        fi
+    fi
 }
 
 setup_report() {
@@ -566,21 +588,40 @@ create_directories() {
         # since this bit is cross-platform:
         # - first try with `command -vp` to try and find
         #   chown in the usual places
+        #   * to work around some sort of deficiency in
+        #     `command -p` in macOS bash 3.2, we also add
+        #     PATH="$(getconf PATH 2>/dev/null)". As long as
+        #     getconf is found, this should set a sane PATH
+        #     which `command -p` in bash 3.2 appears to use.
+        #     A bash with a properly-working `command -p`
+        #     should ignore this hard-set PATH in favor of
+        #     whatever it obtains internally. See
+        #     github.com/NixOS/nix/issues/5768
         # - fall back on `command -v` which would find
         #   any chown on path
         # if we don't find one, the command is already
         # hiding behind || true, and the general state
         # should be one the user can repair once they
         # figure out where chown is...
-        local get_chr_own="$(command -vp chown)"
+        local get_chr_own="$(PATH="$(getconf PATH 2>/dev/null)" command -vp chown)"
         if [[ -z "$get_chr_own" ]]; then
             get_chr_own="$(command -v chown)"
         fi
-        _sudo "to take root ownership of existing Nix store files" \
-              "$get_chr_own" -R "root:$NIX_BUILD_GROUP_NAME" "$NIX_ROOT" || true
+
+        if [[ -z "$get_chr_own" ]]; then
+            reminder <<EOF
+I wanted to take root ownership of existing Nix store files,
+but I couldn't locate 'chown'. (You may need to fix your PATH.)
+To manually change file ownership, you can run:
+    sudo chown -R 'root:$NIX_BUILD_GROUP_NAME' '$NIX_ROOT'
+EOF
+        else
+            _sudo "to take root ownership of existing Nix store files" \
+                  "$get_chr_own" -R "root:$NIX_BUILD_GROUP_NAME" "$NIX_ROOT" || true
+        fi
     fi
     _sudo "to make the basic directory structure of Nix (part 1)" \
-          install -dv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user
+          install -dv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool,/daemon-socket} /nix/var/nix/{gcroots,profiles}/per-user
 
     _sudo "to make the basic directory structure of Nix (part 2)" \
           install -dv -g "$NIX_BUILD_GROUP_NAME" -m 1775 /nix/store
@@ -710,7 +751,7 @@ install_from_extracted_nix() {
         cd "$EXTRACTED_NIX_PATH"
 
         _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \
-              cp -RLp ./store/* "$NIX_ROOT/store/"
+              cp -RPp ./store/* "$NIX_ROOT/store/"
 
         _sudo "to make the new store non-writable at $NIX_ROOT/store" \
               chmod -R ugo-w "$NIX_ROOT/store/"

scripts/install-nix-from-closure.sh

@@ -38,7 +38,7 @@ fi
 
 # Determine if we could use the multi-user installer or not
 if [ "$(uname -s)" = "Linux" ]; then
-    echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
+    echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2
 fi
 
 case "$(uname -s)" in
@@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do
                 echo "              providing multi-user support and better isolation for local builds."
                 echo "              Both for security and reproducibility, this method is recommended if"
                 echo "              supported on your platform."
-                echo "              See https://nixos.org/nix/manual/#sect-multi-user-installation"
+                echo "              See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation"
                 echo ""
                 echo " --no-daemon: Simple, single-user installation that does not require root and is"
                 echo "              trivial to uninstall."
@@ -144,7 +144,7 @@ if ! [ -e "$dest" ]; then
 fi
 
 if ! [ -w "$dest" ]; then
-    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
+    echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2
     exit 1
 fi
 

scripts/install-systemd-multi-user.sh

@@ -9,6 +9,8 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service
 readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket
 readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket
 
+readonly TMPFILES_SRC=/lib/tmpfiles.d/nix-daemon.conf
+readonly TMPFILES_DEST=/etc/tmpfiles.d/nix-daemon.conf
 
 # Path for the systemd override unit file to contain the proxy settings
 readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf
@@ -83,6 +85,13 @@ EOF
 poly_configure_nix_daemon_service() {
     if [ -e /run/systemd/system ]; then
         task "Setting up the nix-daemon systemd service"
+
+        _sudo "to create the nix-daemon tmpfiles config" \
+              ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST
+
+        _sudo "to run systemd-tmpfiles once to pick that path up" \
+             systemd-tmpfiles --create --prefix=/nix/var/nix
+
         _sudo "to set up the nix-daemon service" \
               systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
 

scripts/install.in

@@ -82,7 +82,7 @@ if [ "$(uname -s)" != "Darwin" ]; then
 fi
 
 if command -v curl > /dev/null 2>&1; then
-    fetch() { curl -L "$1" -o "$2"; }
+    fetch() { curl --fail -L "$1" -o "$2"; }
 elif command -v wget > /dev/null 2>&1; then
     fetch() { wget "$1" -O "$2"; }
 else

scripts/nix-profile.sh.in

@@ -24,6 +24,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then
         export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt"
     fi
 
+    # Only use MANPATH if it is already set. In general `man` will just simply
+    # pick up `.nix-profile/share/man` because is it close to `.nix-profile/bin`
+    # which is in the $PATH. For more info, run `manpath -d`.
     if [ -n "${MANPATH-}" ]; then
         export MANPATH="$NIX_LINK/share/man:$MANPATH"
     fi

shell.nix

@@ -1,3 +1,3 @@
-(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") {
   src = ./.;
 }).shellNix

src/build-remote/build-remote.cc

@@ -14,6 +14,7 @@
 #include "pathlocks.hh"
 #include "globals.hh"
 #include "serialise.hh"
+#include "build-result.hh"
 #include "store-api.hh"
 #include "derivations.hh"
 #include "local-store.hh"
@@ -32,7 +33,7 @@ std::string escapeUri(std::string uri)
     return uri;
 }
 
-static string currentLoad;
+static std::string currentLoad;
 
 static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
 {
@@ -97,7 +98,7 @@ static int main_build_remote(int argc, char * * argv)
         }
 
         std::optional<StorePath> drvPath;
-        string storeUri;
+        std::string storeUri;
 
         while (true) {
 
@@ -183,7 +184,7 @@ static int main_build_remote(int argc, char * * argv)
                     else
                     {
                         // build the hint template.
-                        string errorText =
+                        std::string errorText =
                             "Failed to find a machine for remote build!\n"
                             "derivation: %s\nrequired (system, features): (%s, %s)";
                         errorText += "\n%s available machines:";
@@ -193,7 +194,7 @@ static int main_build_remote(int argc, char * * argv)
                             errorText += "\n(%s, %s, %s, %s)";
 
                         // add the template values.
-                        string drvstr;
+                        std::string drvstr;
                         if (drvPath.has_value())
                             drvstr = drvPath->to_string();
                         else
@@ -208,7 +209,7 @@ static int main_build_remote(int argc, char * * argv)
 
                         for (auto & m : machines)
                             error
-                                % concatStringsSep<vector<string>>(", ", m.systemTypes)
+                                % concatStringsSep<std::vector<std::string>>(", ", m.systemTypes)
                                 % m.maxJobs
                                 % concatStringsSep<StringSet>(", ", m.supportedFeatures)
                                 % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
@@ -299,7 +300,7 @@ connected:
 
         std::set<Realisation> missingRealisations;
         StorePathSet missingPaths;
-        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
             for (auto & outputName : wantedOutputs) {
                 auto thisOutputHash = outputHashes.at(outputName);
                 auto thisOutputId = DrvOutput{ thisOutputHash, outputName };

src/cpptoml/LICENSE

@@ -1,18 +0,0 @@
-Copyright (c) 2014 Chase Geigle
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

src/libcmd/command.cc

@@ -54,6 +54,36 @@ void StoreCommand::run()
     run(getStore());
 }
 
+CopyCommand::CopyCommand()
+{
+    addFlag({
+        .longName = "from",
+        .description = "URL of the source Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&srcUri},
+    });
+
+    addFlag({
+        .longName = "to",
+        .description = "URL of the destination Nix store.",
+        .labels = {"store-uri"},
+        .handler = {&dstUri},
+    });
+}
+
+ref<Store> CopyCommand::createStore()
+{
+    return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
+}
+
+ref<Store> CopyCommand::getDstStore()
+{
+    if (srcUri.empty() && dstUri.empty())
+        throw UsageError("you must pass '--from' and/or '--to'");
+
+    return dstUri.empty() ? openStore() : openStore(dstUri);
+}
+
 EvalCommand::EvalCommand()
 {
 }
@@ -74,7 +104,15 @@ ref<Store> EvalCommand::getEvalStore()
 ref<EvalState> EvalCommand::getEvalState()
 {
     if (!evalState)
-        evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
+        evalState =
+            #if HAVE_BOEHMGC
+            std::allocate_shared<EvalState>(traceable_allocator<EvalState>(),
+                searchPath, getEvalStore(), getStore())
+            #else
+            std::make_shared<EvalState>(
+                searchPath, getEvalStore(), getStore())
+            #endif
+            ;
     return ref<EvalState>(evalState);
 }
 
@@ -115,7 +153,7 @@ void BuiltPathsCommand::run(ref<Store> store)
         for (auto & p : store->queryAllValidPaths())
             paths.push_back(BuiltPath::Opaque{p});
     } else {
-        paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
+        paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations
@@ -166,7 +204,8 @@ Strings editorFor(const Pos & pos)
     if (pos.line > 0 && (
         editor.find("emacs") != std::string::npos ||
         editor.find("nano") != std::string::npos ||
-        editor.find("vim") != std::string::npos))
+        editor.find("vim") != std::string::npos ||
+        editor.find("kak") != std::string::npos))
         args.push_back(fmt("+%d", pos.line));
     args.push_back(pos.file);
     return args;

src/libcmd/command.hh

@@ -5,7 +5,6 @@
 #include "common-eval-args.hh"
 #include "path.hh"
 #include "flake/lockfile.hh"
-#include "store-api.hh"
 
 #include <optional>
 
@@ -43,6 +42,19 @@ private:
     std::shared_ptr<Store> _store;
 };
 
+/* A command that copies something between `--from` and `--to`
+   stores. */
+struct CopyCommand : virtual StoreCommand
+{
+    std::string srcUri, dstUri;
+
+    CopyCommand();
+
+    ref<Store> createStore() override;
+
+    ref<Store> getDstStore();
+};
+
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
     EvalCommand();
@@ -69,23 +81,16 @@ struct MixFlakeOptions : virtual Args, EvalCommand
     { return {}; }
 };
 
-/* How to handle derivations in commands that operate on store paths. */
-enum class OperateOn {
-    /* Operate on the output path. */
-    Output,
-    /* Operate on the .drv path. */
-    Derivation
-};
-
 struct SourceExprCommand : virtual Args, MixFlakeOptions
 {
     std::optional<Path> file;
     std::optional<std::string> expr;
+    bool readOnlyMode = false;
 
     // FIXME: move this; not all commands (e.g. 'nix run') use it.
     OperateOn operateOn = OperateOn::Output;
 
-    SourceExprCommand();
+    SourceExprCommand(bool supportReadOnlyMode = false);
 
     std::vector<std::shared_ptr<Installable>> parseInstallables(
         ref<Store> store, std::vector<std::string> ss);
@@ -100,19 +105,6 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
     void completeInstallable(std::string_view prefix);
 };
 
-enum class Realise {
-    /* Build the derivation. Postcondition: the
-       derivation outputs exist. */
-    Outputs,
-    /* Don't build the derivation. Postcondition: the store derivation
-       exists. */
-    Derivation,
-    /* Evaluate in dry-run mode. Postcondition: nothing. */
-    // FIXME: currently unused, but could be revived if we can
-    // evaluate derivations in-memory.
-    Nothing
-};
-
 /* A command that operates on a list of "installables", which can be
    store paths, attribute paths, Nix expressions, etc. */
 struct InstallablesCommand : virtual Args, SourceExprCommand
@@ -137,13 +129,13 @@ struct InstallableCommand : virtual Args, SourceExprCommand
 {
     std::shared_ptr<Installable> installable;
 
-    InstallableCommand();
+    InstallableCommand(bool supportReadOnlyMode = false);
 
     void prepare() override;
 
     std::optional<FlakeRef> getFlakeRefForCompletion() override
     {
-        return parseFlakeRef(_installable, absPath("."));
+        return parseFlakeRefWithFragment(_installable, absPath(".")).first;
     }
 
 private:
@@ -225,38 +217,6 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-BuiltPaths build(
-    ref<Store> evalStore,
-    ref<Store> store, Realise mode,
-    const std::vector<std::shared_ptr<Installable>> & installables,
-    BuildMode bMode = bmNormal);
-
-std::set<StorePath> toStorePaths(
-    ref<Store> evalStore,
-    ref<Store> store,
-    Realise mode,
-    OperateOn operateOn,
-    const std::vector<std::shared_ptr<Installable>> & installables);
-
-StorePath toStorePath(
-    ref<Store> evalStore,
-    ref<Store> store,
-    Realise mode,
-    OperateOn operateOn,
-    std::shared_ptr<Installable> installable);
-
-std::set<StorePath> toDerivations(
-    ref<Store> store,
-    const std::vector<std::shared_ptr<Installable>> & installables,
-    bool useDeriver = false);
-
-BuiltPaths toBuiltPaths(
-    ref<Store> evalStore,
-    ref<Store> store,
-    Realise mode,
-    OperateOn operateOn,
-    const std::vector<std::shared_ptr<Installable>> & installables);
-
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */
 Strings editorFor(const Pos & pos);

src/libcmd/common-eval-args.cc

@@ -7,6 +7,7 @@
 #include "registry.hh"
 #include "flake/flakeref.hh"
 #include "store-api.hh"
+#include "command.hh"
 
 namespace nix {
 
@@ -59,6 +60,9 @@ MixEvalArgs::MixEvalArgs()
             fetchers::Attrs extraAttrs;
             if (to.subdir != "") extraAttrs["dir"] = to.subdir;
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
+        }},
+        .completer = {[&](size_t, std::string_view prefix) {
+            completeFlakeRef(openStore(), prefix);
         }}
     });
 
@@ -73,30 +77,29 @@ MixEvalArgs::MixEvalArgs()
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 {
-    Bindings * res = state.allocBindings(autoArgs.size());
+    auto res = state.buildBindings(autoArgs.size());
     for (auto & i : autoArgs) {
-        Value * v = state.allocValue();
+        auto v = state.allocValue();
         if (i.second[0] == 'E')
-            state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath(".")));
+            state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath(".")));
         else
-            mkString(*v, string(i.second, 1));
-        res->push_back(Attr(state.symbols.create(i.first), v));
+            v->mkString(((std::string_view) i.second).substr(1));
+        res.insert(state.symbols.create(i.first), v);
     }
-    res->sort();
-    return res;
+    return res.finish();
 }
 
-Path lookupFileArg(EvalState & state, string s)
+Path lookupFileArg(EvalState & state, std::string_view s)
 {
     if (isUri(s)) {
         return state.store->toRealPath(
             fetchers::downloadTarball(
                 state.store, resolveUri(s), "source", false).first.storePath);
     } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
-        Path p = s.substr(1, s.size() - 2);
+        Path p(s.substr(1, s.size() - 2));
         return state.findFile(p);
     } else
-        return absPath(s);
+        return absPath(std::string(s));
 }
 
 }

src/libcmd/common-eval-args.hh

@@ -22,6 +22,6 @@ private:
     std::map<std::string, std::string> autoArgs;
 };
 
-Path lookupFileArg(EvalState & state, string s);
+Path lookupFileArg(EvalState & state, std::string_view s);
 
 }

src/libcmd/installables.cc

@@ -1,4 +1,6 @@
+#include "globals.hh"
 #include "installables.hh"
+#include "util.hh"
 #include "command.hh"
 #include "attr-path.hh"
 #include "common-eval-args.hh"
@@ -12,6 +14,7 @@
 #include "eval-cache.hh"
 #include "url.hh"
 #include "registry.hh"
+#include "build-result.hh"
 
 #include <regex>
 #include <queue>
@@ -97,7 +100,15 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
-                parseFlakeRef(flakeRef, absPath(".")));
+                parseFlakeRef(flakeRef, absPath("."), true));
+        }},
+        .completer = {[&](size_t n, std::string_view prefix) {
+            if (n == 0) {
+                if (auto flakeRef = getFlakeRefForCompletion())
+                    completeFlakeInputPath(getEvalState(), *flakeRef, prefix);
+            } else if (n == 1) {
+                completeFlakeRef(getEvalState()->store, prefix);
+            }
         }}
     });
 
@@ -128,12 +139,14 @@ MixFlakeOptions::MixFlakeOptions()
     });
 }
 
-SourceExprCommand::SourceExprCommand()
+SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode)
 {
     addFlag({
         .longName = "file",
         .shortName = 'f',
-        .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.",
+        .description =
+            "Interpret installables as attribute paths relative to the Nix expression stored in *file*. "
+            "If *file* is the character -, then a Nix expression will be read from standard input.",
         .category = installablesCategory,
         .labels = {"file"},
         .handler = {&file},
@@ -154,11 +167,25 @@ SourceExprCommand::SourceExprCommand()
         .category = installablesCategory,
         .handler = {&operateOn, OperateOn::Derivation},
     });
+
+    if (supportReadOnlyMode) {
+        addFlag({
+            .longName = "read-only",
+            .description =
+                "Do not instantiate each evaluated derivation. "
+                "This improves performance, but can cause errors when accessing "
+                "store paths of derivations during evaluation.",
+            .handler = {&readOnlyMode, true},
+        });
+    }
 }
 
 Strings SourceExprCommand::getDefaultFlakeAttrPaths()
 {
-    return {"defaultPackage." + settings.thisSystem.get()};
+    return {
+        "packages." + settings.thisSystem.get() + ".default",
+        "defaultPackage." + settings.thisSystem.get()
+    };
 }
 
 Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
@@ -176,6 +203,8 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
     if (file) {
+        completionType = ctAttrs;
+
         evalSettings.pureEval = false;
         auto state = getEvalState();
         Expr *e = state->parseExprFromFile(
@@ -191,15 +220,16 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
         auto sep = prefix_.rfind('.');
         std::string searchWord;
         if (sep != std::string::npos) {
-            searchWord = prefix_.substr(sep, std::string::npos);
+            searchWord = prefix_.substr(sep + 1, std::string::npos);
             prefix_ = prefix_.substr(0, sep);
         } else {
             searchWord = prefix_;
             prefix_ = "";
         }
 
-        Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first);
-        state->forceValue(v1);
+        auto [v, pos] = findAlongAttrPath(*state, prefix_, *autoArgs, root);
+        Value &v1(*v);
+        state->forceValue(v1, pos);
         Value v2;
         state->autoCallFunction(*autoArgs, v1, v2);
 
@@ -207,7 +237,10 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
             for (auto & i : *v2.attrs) {
                 std::string name = i.name;
                 if (name.find(searchWord) == 0) {
-                    completions->add(i.name);
+                    if (prefix_ == "")
+                        completions->add(name);
+                    else
+                        completions->add(prefix_ + "." + name);
                 }
             }
         }
@@ -232,11 +265,14 @@ void completeFlakeRefWithFragment(
        prefix. */
     try {
         auto hash = prefix.find('#');
-        if (hash != std::string::npos) {
+        if (hash == std::string::npos) {
+            completeFlakeRef(evalState->store, prefix);
+        } else {
+            completionType = ctAttrs;
+
             auto fragment = prefix.substr(hash + 1);
             auto flakeRefS = std::string(prefix.substr(0, hash));
-            // FIXME: do tilde expansion.
-            auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
+            auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
                 std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
@@ -262,9 +298,9 @@ void completeFlakeRefWithFragment(
                 auto attr = root->findAlongAttrPath(attrPath);
                 if (!attr) continue;
 
-                for (auto & attr2 : attr->getAttrs()) {
+                for (auto & attr2 : (*attr)->getAttrs()) {
                     if (hasPrefix(attr2, lastAttr)) {
-                        auto attrPath2 = attr->getAttrPath(attr2);
+                        auto attrPath2 = (*attr)->getAttrPath(attr2);
                         /* Strip the attrpath prefix. */
                         attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
                         completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2));
@@ -285,12 +321,13 @@ void completeFlakeRefWithFragment(
     } catch (Error & e) {
         warn(e.msg());
     }
-
-    completeFlakeRef(evalState->store, prefix);
 }
 
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
+    if (!settings.isExperimentalFeatureEnabled(Xp::Flakes))
+        return;
+
     if (prefix == "")
         completions->add(".");
 
@@ -320,16 +357,16 @@ DerivedPath Installable::toDerivedPath()
     return std::move(buildables[0]);
 }
 
-std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+std::vector<ref<eval_cache::AttrCursor>>
 Installable::getCursors(EvalState & state)
 {
     auto evalCache =
         std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
             [&]() { return toValue(state).first; });
-    return {{evalCache->getRoot(), ""}};
+    return {evalCache->getRoot()};
 }
 
-std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+ref<eval_cache::AttrCursor>
 Installable::getCursor(EvalState & state)
 {
     auto cursors = getCursors(state);
@@ -338,6 +375,18 @@ Installable::getCursor(EvalState & state)
     return cursors[0];
 }
 
+static StorePath getDeriver(
+    ref<Store> store,
+    const Installable & i,
+    const StorePath & drvPath)
+{
+    auto derivers = store->queryValidDerivers(drvPath);
+    if (derivers.empty())
+        throw Error("'%s' does not have a known deriver", i.what());
+    // FIXME: use all derivers?
+    return *derivers.begin();
+}
+
 struct InstallableStorePath : Installable
 {
     ref<Store> store;
@@ -346,7 +395,7 @@ struct InstallableStorePath : Installable
     InstallableStorePath(ref<Store> store, StorePath && storePath)
         : store(store), storePath(std::move(storePath)) { }
 
-    std::string what() override { return store->printStorePath(storePath); }
+    std::string what() const override { return store->printStorePath(storePath); }
 
     DerivedPaths toDerivedPaths() override
     {
@@ -367,6 +416,15 @@ struct InstallableStorePath : Installable
         }
     }
 
+    StorePathSet toDrvPaths(ref<Store> store) override
+    {
+        if (storePath.isDerivation()) {
+            return {storePath};
+        } else {
+            return {getDeriver(store, *this, storePath)};
+        }
+    }
+
     std::optional<StorePath> getStorePath() override
     {
         return storePath;
@@ -395,6 +453,14 @@ DerivedPaths InstallableValue::toDerivedPaths()
     return res;
 }
 
+StorePathSet InstallableValue::toDrvPaths(ref<Store> store)
+{
+    StorePathSet res;
+    for (auto & drv : toDerivations())
+        res.insert(drv.drvPath);
+    return res;
+}
+
 struct InstallableAttrPath : InstallableValue
 {
     SourceExprCommand & cmd;
@@ -405,12 +471,12 @@ struct InstallableAttrPath : InstallableValue
         : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
     { }
 
-    std::string what() override { return attrPath; }
+    std::string what() const override { return attrPath; }
 
     std::pair<Value *, Pos> toValue(EvalState & state) override
     {
         auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
-        state.forceValue(*vRes);
+        state.forceValue(*vRes, pos);
         return {vRes, pos};
     }
 
@@ -428,11 +494,10 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
 
     std::vector<DerivationInfo> res;
     for (auto & drvInfo : drvInfos) {
-        res.push_back({
-            state->store->parseStorePath(drvInfo.queryDrvPath()),
-            state->store->maybeParseStorePath(drvInfo.queryOutPath()),
-            drvInfo.queryOutputName()
-        });
+        auto drvPath = drvInfo.queryDrvPath();
+        if (!drvPath)
+            throw Error("'%s' is not a derivation", what());
+        res.push_back({ *drvPath, drvInfo.queryOutputName() });
     }
 
     return res;
@@ -460,7 +525,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
     auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
     assert(aOutputs);
 
-    state.forceValue(*aOutputs->value);
+    state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); });
 
     return aOutputs->value;
 }
@@ -485,7 +550,7 @@ ref<eval_cache::EvalCache> openEvalCache(
             auto vFlake = state.allocValue();
             flake::callFlake(state, *lockedFlake, *vFlake);
 
-            state.forceAttrs(*vFlake);
+            state.forceAttrs(*vFlake, noPos);
 
             auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
             assert(aOutputs);
@@ -508,13 +573,14 @@ InstallableFlake::InstallableFlake(
     SourceExprCommand * cmd,
     ref<EvalState> state,
     FlakeRef && flakeRef,
-    Strings && attrPaths,
-    Strings && prefixes,
+    std::string_view fragment,
+    Strings attrPaths,
+    Strings prefixes,
     const flake::LockFlags & lockFlags)
     : InstallableValue(state),
       flakeRef(flakeRef),
-      attrPaths(attrPaths),
-      prefixes(prefixes),
+      attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
+      prefixes(fragment == "" ? Strings{} : prefixes),
       lockFlags(lockFlags)
 {
     if (cmd && cmd->getAutoArgs(*state)->size())
@@ -523,35 +589,21 @@ InstallableFlake::InstallableFlake(
 
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
-    auto lockedFlake = getLockedFlake();
+    auto attr = getCursor(*state);
 
-    auto cache = openEvalCache(*state, lockedFlake);
-    auto root = cache->getRoot();
+    auto attrPath = attr->getAttrPathStr();
 
-    for (auto & attrPath : getActualAttrPaths()) {
-        auto attr = root->findAlongAttrPath(
-            parseAttrPath(*state, attrPath),
-            true
-        );
+    if (!attr->isDerivation())
+        throw Error("flake output attribute '%s' is not a derivation", attrPath);
 
-        if (!attr) continue;
+    auto drvPath = attr->forceDerivation();
 
-        if (!attr->isDerivation())
-            throw Error("flake output attribute '%s' is not a derivation", attrPath);
+    auto drvInfo = DerivationInfo {
+        std::move(drvPath),
+        attr->getAttr(state->sOutputName)->getString()
+    };
 
-        auto drvPath = attr->forceDerivation();
-
-        auto drvInfo = DerivationInfo{
-            std::move(drvPath),
-            state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
-            attr->getAttr(state->sOutputName)->getString()
-        };
-
-        return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
-    }
-
-    throw Error("flake '%s' does not provide attribute %s",
-        flakeRef, showAttrPaths(getActualAttrPaths()));
+    return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};
 }
 
 std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
@@ -563,26 +615,10 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
 
 std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 {
-    auto lockedFlake = getLockedFlake();
-
-    auto vOutputs = getFlakeOutputs(state, *lockedFlake);
-
-    auto emptyArgs = state.allocBindings(0);
-
-    for (auto & attrPath : getActualAttrPaths()) {
-        try {
-            auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
-            state.forceValue(*v);
-            return {v, pos};
-        } catch (AttrPathNotFound & e) {
-        }
-    }
-
-    throw Error("flake '%s' does not provide attribute %s",
-        flakeRef, showAttrPaths(getActualAttrPaths()));
+    return {&getCursor(state)->forceValue(), noPos};
 }
 
-std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+std::vector<ref<eval_cache::AttrCursor>>
 InstallableFlake::getCursors(EvalState & state)
 {
     auto evalCache = openEvalCache(state,
@@ -590,21 +626,55 @@ InstallableFlake::getCursors(EvalState & state)
 
     auto root = evalCache->getRoot();
 
-    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res;
+    std::vector<ref<eval_cache::AttrCursor>> res;
 
     for (auto & attrPath : getActualAttrPaths()) {
         auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
-        if (attr) res.push_back({attr, attrPath});
+        if (attr) res.push_back(ref(*attr));
     }
 
     return res;
 }
 
+ref<eval_cache::AttrCursor> InstallableFlake::getCursor(EvalState & state)
+{
+    auto lockedFlake = getLockedFlake();
+
+    auto cache = openEvalCache(state, lockedFlake);
+    auto root = cache->getRoot();
+
+    Suggestions suggestions;
+
+    auto attrPaths = getActualAttrPaths();
+
+    for (auto & attrPath : attrPaths) {
+        debug("trying flake output attribute '%s'", attrPath);
+
+        auto attrOrSuggestions = root->findAlongAttrPath(
+            parseAttrPath(state, attrPath),
+            true
+        );
+
+        if (!attrOrSuggestions) {
+            suggestions += attrOrSuggestions.getSuggestions();
+            continue;
+        }
+
+        return *attrOrSuggestions;
+    }
+
+    throw Error(
+        suggestions,
+        "flake '%s' does not provide attribute %s",
+        flakeRef,
+        showAttrPaths(attrPaths));
+}
+
 std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
 {
-    flake::LockFlags lockFlagsApplyConfig = lockFlags;
-    lockFlagsApplyConfig.applyNixConfig = true;
     if (!_lockedFlake) {
+        flake::LockFlags lockFlagsApplyConfig = lockFlags;
+        lockFlagsApplyConfig.applyNixConfig = true;
         _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
@@ -629,6 +699,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
 {
     std::vector<std::shared_ptr<Installable>> result;
 
+    if (readOnlyMode) {
+        settings.readOnlyMode = true;
+    }
+
     if (file || expr) {
         if (file && expr)
             throw UsageError("'--file' and '--expr' are exclusive");
@@ -639,7 +713,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         auto state = getEvalState();
         auto vFile = state->allocValue();
 
-        if (file)
+        if (file == "-") {
+            auto e = state->parseStdin();
+            state->eval(e, *vFile);
+        } else if (file)
             state->evalFile(lookupFileArg(*state, *file), *vFile);
         else {
             auto e = state->parseExprFromString(*expr, absPath("."));
@@ -671,7 +748,8 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
                         this,
                         getEvalState(),
                         std::move(flakeRef),
-                        fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
+                        fragment,
+                        getDefaultFlakeAttrPaths(),
                         getDefaultFlakeAttrPathPrefixes(),
                         lockFlags));
                 continue;
@@ -694,56 +772,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
+BuiltPaths Installable::build(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode)
 {
     BuiltPaths res;
-    for (const auto & b : hopefullyBuiltPaths)
-        std::visit(
-            overloaded{
-                [&](const DerivedPath::Opaque & bo) {
-                    res.push_back(BuiltPath::Opaque{bo.path});
-                },
-                [&](const DerivedPath::Built & bfd) {
-                    OutputPathMap outputs;
-                    auto drv = evalStore->readDerivation(bfd.drvPath);
-                    auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
-                    auto drvOutputs = drv.outputsAndOptPaths(*store);
-                    for (auto & output : bfd.outputs) {
-                        if (!outputHashes.count(output))
-                            throw Error(
-                                "the derivation '%s' doesn't have an output named '%s'",
-                                store->printStorePath(bfd.drvPath), output);
-                        if (settings.isExperimentalFeatureEnabled(
-                                Xp::CaDerivations)) {
-                            auto outputId =
-                                DrvOutput{outputHashes.at(output), output};
-                            auto realisation =
-                                store->queryRealisation(outputId);
-                            if (!realisation)
-                                throw Error(
-                                    "cannot operate on an output of unbuilt "
-                                    "content-addressed derivation '%s'",
-                                    outputId.to_string());
-                            outputs.insert_or_assign(
-                                output, realisation->outPath);
-                        } else {
-                            // If ca-derivations isn't enabled, assume that
-                            // the output path is statically known.
-                            assert(drvOutputs.count(output));
-                            assert(drvOutputs.at(output).second);
-                            outputs.insert_or_assign(
-                                output, *drvOutputs.at(output).second);
-                        }
-                    }
-                    res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
-                },
-            },
-            b.raw());
-
+    for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
+        res.push_back(builtPath);
     return res;
 }
 
-BuiltPaths build(
+std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -754,21 +796,96 @@ BuiltPaths build(
         settings.readOnlyMode = true;
 
     std::vector<DerivedPath> pathsToBuild;
+    std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap;
 
     for (auto & i : installables) {
-        auto b = i->toDerivedPaths();
-        pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
+        for (auto b : i->toDerivedPaths()) {
+            pathsToBuild.push_back(b);
+            backmap[b].push_back(i);
+        }
     }
 
-    if (mode == Realise::Nothing || mode == Realise::Derivation)
-        printMissing(store, pathsToBuild, lvlError);
-    else if (mode == Realise::Outputs)
-        store->buildPaths(pathsToBuild, bMode, evalStore);
+    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
 
-    return getBuiltPaths(evalStore, store, pathsToBuild);
+    switch (mode) {
+
+    case Realise::Nothing:
+    case Realise::Derivation:
+        printMissing(store, pathsToBuild, lvlError);
+
+        for (auto & path : pathsToBuild) {
+            for (auto & installable : backmap[path]) {
+                std::visit(overloaded {
+                    [&](const DerivedPath::Built & bfd) {
+                        OutputPathMap outputs;
+                        auto drv = evalStore->readDerivation(bfd.drvPath);
+                        auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
+                        auto drvOutputs = drv.outputsAndOptPaths(*store);
+                        for (auto & output : bfd.outputs) {
+                            if (!outputHashes.count(output))
+                                throw Error(
+                                    "the derivation '%s' doesn't have an output named '%s'",
+                                    store->printStorePath(bfd.drvPath), output);
+                            if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+                                DrvOutput outputId { outputHashes.at(output), output };
+                                auto realisation = store->queryRealisation(outputId);
+                                if (!realisation)
+                                    throw Error(
+                                        "cannot operate on an output of unbuilt "
+                                        "content-addressed derivation '%s'",
+                                        outputId.to_string());
+                                outputs.insert_or_assign(output, realisation->outPath);
+                            } else {
+                                // If ca-derivations isn't enabled, assume that
+                                // the output path is statically known.
+                                assert(drvOutputs.count(output));
+                                assert(drvOutputs.at(output).second);
+                                outputs.insert_or_assign(
+                                    output, *drvOutputs.at(output).second);
+                            }
+                        }
+                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                    },
+                    [&](const DerivedPath::Opaque & bo) {
+                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                    },
+                }, path.raw());
+            }
+        }
+
+        break;
+
+    case Realise::Outputs: {
+        for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
+            if (!buildResult.success())
+                buildResult.rethrow();
+
+            for (auto & installable : backmap[buildResult.path]) {
+                std::visit(overloaded {
+                    [&](const DerivedPath::Built & bfd) {
+                        std::map<std::string, StorePath> outputs;
+                        for (auto & path : buildResult.builtOutputs)
+                            outputs.emplace(path.first.outputName, path.second.outPath);
+                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                    },
+                    [&](const DerivedPath::Opaque & bo) {
+                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                    },
+                }, buildResult.path.raw());
+            }
+        }
+
+        break;
+    }
+
+    default:
+        assert(false);
+    }
+
+    return res;
 }
 
-BuiltPaths toBuiltPaths(
+BuiltPaths Installable::toBuiltPaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -776,19 +893,19 @@ BuiltPaths toBuiltPaths(
     const std::vector<std::shared_ptr<Installable>> & installables)
 {
     if (operateOn == OperateOn::Output)
-        return build(evalStore, store, mode, installables);
+        return Installable::build(evalStore, store, mode, installables);
     else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
         BuiltPaths res;
-        for (auto & drvPath : toDerivations(store, installables, true))
+        for (auto & drvPath : Installable::toDerivations(store, installables, true))
             res.push_back(BuiltPath::Opaque{drvPath});
         return res;
     }
 }
 
-StorePathSet toStorePaths(
+StorePathSet Installable::toStorePaths(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -802,7 +919,7 @@ StorePathSet toStorePaths(
     return outPaths;
 }
 
-StorePath toStorePath(
+StorePath Installable::toStorePath(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode, OperateOn operateOn,
@@ -816,7 +933,7 @@ StorePath toStorePath(
     return *paths.begin();
 }
 
-StorePathSet toDerivations(
+StorePathSet Installable::toDerivations(
     ref<Store> store,
     const std::vector<std::shared_ptr<Installable>> & installables,
     bool useDeriver)
@@ -829,11 +946,7 @@ StorePathSet toDerivations(
                 [&](const DerivedPath::Opaque & bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
-                    auto derivers = store->queryValidDerivers(bo.path);
-                    if (derivers.empty())
-                        throw Error("'%s' does not have a known deriver", i->what());
-                    // FIXME: use all derivers?
-                    drvPaths.insert(*derivers.begin());
+                    drvPaths.insert(getDeriver(store, *i, bo.path));
                 },
                 [&](const DerivedPath::Built & bfd) {
                     drvPaths.insert(bfd.drvPath);
@@ -857,7 +970,7 @@ InstallablesCommand::InstallablesCommand()
 void InstallablesCommand::prepare()
 {
     if (_installables.empty() && useDefaultInstallables())
-        // FIXME: commands like "nix install" should not have a
+        // FIXME: commands like "nix profile install" should not have a
         // default, probably.
         _installables.push_back(".");
     installables = parseInstallables(getStore(), _installables);
@@ -867,13 +980,14 @@ std::optional<FlakeRef> InstallablesCommand::getFlakeRefForCompletion()
 {
     if (_installables.empty()) {
         if (useDefaultInstallables())
-            return parseFlakeRef(".", absPath("."));
+            return parseFlakeRefWithFragment(".", absPath(".")).first;
         return {};
     }
-    return parseFlakeRef(_installables.front(), absPath("."));
+    return parseFlakeRefWithFragment(_installables.front(), absPath(".")).first;
 }
 
-InstallableCommand::InstallableCommand()
+InstallableCommand::InstallableCommand(bool supportReadOnlyMode)
+    : SourceExprCommand(supportReadOnlyMode)
 {
     expectArgs({
         .label = "installable",

src/libcmd/installables.hh

@@ -5,6 +5,7 @@
 #include "path-with-outputs.hh"
 #include "derived-path.hh"
 #include "eval.hh"
+#include "store-api.hh"
 #include "flake/flake.hh"
 
 #include <optional>
@@ -29,14 +30,40 @@ struct UnresolvedApp
     App resolve(ref<Store> evalStore, ref<Store> store);
 };
 
+enum class Realise {
+    /* Build the derivation. Postcondition: the
+       derivation outputs exist. */
+    Outputs,
+    /* Don't build the derivation. Postcondition: the store derivation
+       exists. */
+    Derivation,
+    /* Evaluate in dry-run mode. Postcondition: nothing. */
+    // FIXME: currently unused, but could be revived if we can
+    // evaluate derivations in-memory.
+    Nothing
+};
+
+/* How to handle derivations in commands that operate on store paths. */
+enum class OperateOn {
+    /* Operate on the output path. */
+    Output,
+    /* Operate on the .drv path. */
+    Derivation
+};
+
 struct Installable
 {
     virtual ~Installable() { }
 
-    virtual std::string what() = 0;
+    virtual std::string what() const = 0;
 
     virtual DerivedPaths toDerivedPaths() = 0;
 
+    virtual StorePathSet toDrvPaths(ref<Store> store)
+    {
+        throw Error("'%s' cannot be converted to a derivation path", what());
+    }
+
     DerivedPath toDerivedPath();
 
     UnresolvedApp toApp(EvalState & state);
@@ -53,16 +80,56 @@ struct Installable
         return {};
     }
 
-    virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    virtual std::vector<ref<eval_cache::AttrCursor>>
     getCursors(EvalState & state);
 
-    std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+    virtual ref<eval_cache::AttrCursor>
     getCursor(EvalState & state);
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
         return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
     }
+
+    static BuiltPaths build(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        const std::vector<std::shared_ptr<Installable>> & installables,
+        BuildMode bMode = bmNormal);
+
+    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        const std::vector<std::shared_ptr<Installable>> & installables,
+        BuildMode bMode = bmNormal);
+
+    static std::set<StorePath> toStorePaths(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const std::vector<std::shared_ptr<Installable>> & installables);
+
+    static StorePath toStorePath(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        std::shared_ptr<Installable> installable);
+
+    static std::set<StorePath> toDerivations(
+        ref<Store> store,
+        const std::vector<std::shared_ptr<Installable>> & installables,
+        bool useDeriver = false);
+
+    static BuiltPaths toBuiltPaths(
+        ref<Store> evalStore,
+        ref<Store> store,
+        Realise mode,
+        OperateOn operateOn,
+        const std::vector<std::shared_ptr<Installable>> & installables);
 };
 
 struct InstallableValue : Installable
@@ -74,13 +141,14 @@ struct InstallableValue : Installable
     struct DerivationInfo
     {
         StorePath drvPath;
-        std::optional<StorePath> outPath;
         std::string outputName;
     };
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;
 
     DerivedPaths toDerivedPaths() override;
+
+    StorePathSet toDrvPaths(ref<Store> store) override;
 };
 
 struct InstallableFlake : InstallableValue
@@ -95,11 +163,12 @@ struct InstallableFlake : InstallableValue
         SourceExprCommand * cmd,
         ref<EvalState> state,
         FlakeRef && flakeRef,
-        Strings && attrPaths,
-        Strings && prefixes,
+        std::string_view fragment,
+        Strings attrPaths,
+        Strings prefixes,
         const flake::LockFlags & lockFlags);
 
-    std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
+    std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
 
     std::vector<std::string> getActualAttrPaths();
 
@@ -111,9 +180,15 @@ struct InstallableFlake : InstallableValue
 
     std::pair<Value *, Pos> toValue(EvalState & state) override;
 
-    std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+    /* Get a cursor to every attrpath in getActualAttrPaths() that
+       exists. */
+    std::vector<ref<eval_cache::AttrCursor>>
     getCursors(EvalState & state) override;
 
+    /* Get a cursor to the first attrpath in getActualAttrPaths() that
+       exists, or throw an exception with suggestions if none exists. */
+    ref<eval_cache::AttrCursor> getCursor(EvalState & state) override;
+
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
     FlakeRef nixpkgsFlakeRef() const override;

src/libcmd/local.mk

@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS += -llowdown -pthread
+libcmd_LDFLAGS += $(LOWDOWN_LIBS) -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 

src/libexpr/attr-path.cc

@@ -9,7 +9,7 @@ namespace nix {
 static Strings parseAttrPath(std::string_view s)
 {
     Strings res;
-    string cur;
+    std::string cur;
     auto i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
@@ -41,7 +41,7 @@ std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
 }
 
 
-std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
+std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const std::string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {
     Strings tokens = parseAttrPath(attrPath);
@@ -58,7 +58,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
         Value * vNew = state.allocValue();
         state.autoCallFunction(autoArgs, *v, *vNew);
         v = vNew;
-        state.forceValue(*v);
+        state.forceValue(*v, noPos);
 
         /* It should evaluate to either a set or an expression,
            according to what is specified in the attrPath. */
@@ -74,8 +74,14 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
                 throw Error("empty attribute name in selection path '%1%'", attrPath);
 
             Bindings::iterator a = v->attrs->find(state.symbols.create(attr));
-            if (a == v->attrs->end())
-                throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath);
+            if (a == v->attrs->end()) {
+                std::set<std::string> attrNames;
+                for (auto & attr : *v->attrs)
+                    attrNames.insert(attr.name);
+
+                auto suggestions = Suggestions::bestMatches(attrNames, attr);
+                throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath);
+            }
             v = &*a->value;
             pos = *a->pos;
         }
@@ -121,7 +127,7 @@ Pos findPackageFilename(EvalState & state, Value & v, std::string what)
     std::string filename(pos, 0, colon);
     unsigned int lineno;
     try {
-        lineno = std::stoi(std::string(pos, colon + 1));
+        lineno = std::stoi(std::string(pos, colon + 1, std::string::npos));
     } catch (std::invalid_argument & e) {
         throw ParseError("cannot parse line number '%s'", pos);
     }

src/libexpr/attr-path.hh

@@ -10,8 +10,11 @@ namespace nix {
 MakeError(AttrPathNotFound, Error);
 MakeError(NoPositionInfo, Error);
 
-std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
-    Bindings & autoArgs, Value & vIn);
+std::pair<Value *, Pos> findAlongAttrPath(
+    EvalState & state,
+    const std::string & attrPath,
+    Bindings & autoArgs,
+    Value & vIn);
 
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findPackageFilename(EvalState & state, Value & v, std::string what);

src/libexpr/attr-set.cc

@@ -7,26 +7,19 @@
 namespace nix {
 
 
+
 /* Allocate a new array of attributes for an attribute set with a specific
    capacity. The space is implicitly reserved after the Bindings
    structure. */
 Bindings * EvalState::allocBindings(size_t capacity)
 {
+    if (capacity == 0)
+        return &emptyBindings;
     if (capacity > std::numeric_limits<Bindings::size_t>::max())
         throw Error("attribute set of size %d is too big", capacity);
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
-}
-
-
-void EvalState::mkAttrs(Value & v, size_t capacity)
-{
-    if (capacity == 0) {
-        v = vEmptySet;
-        return;
-    }
-    v.mkAttrs(allocBindings(capacity));
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
 }
 
 
@@ -41,15 +34,36 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name)
 }
 
 
-Value * EvalState::allocAttr(Value & vAttrs, const std::string & name)
+Value * EvalState::allocAttr(Value & vAttrs, std::string_view name)
 {
     return allocAttr(vAttrs, symbols.create(name));
 }
 
 
+Value & BindingsBuilder::alloc(const Symbol & name, ptr<Pos> pos)
+{
+    auto value = state.allocValue();
+    bindings->push_back(Attr(name, value, pos));
+    return *value;
+}
+
+
+Value & BindingsBuilder::alloc(std::string_view name, ptr<Pos> pos)
+{
+    return alloc(state.symbols.create(name), pos);
+}
+
+
 void Bindings::sort()
 {
-    std::sort(begin(), end());
+    if (size_) std::sort(begin(), end());
+}
+
+
+Value & Value::mkAttrs(BindingsBuilder & bindings)
+{
+    mkAttrs(bindings.finish());
+    return *this;
 }
 
 

src/libexpr/attr-set.hh

@@ -105,7 +105,7 @@ public:
         for (size_t n = 0; n < size_; n++)
             res.emplace_back(&attrs[n]);
         std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) {
-            return (const string &) a->name < (const string &) b->name;
+            return (const std::string &) a->name < (const std::string &) b->name;
         });
         return res;
     }
@@ -113,5 +113,52 @@ public:
     friend class EvalState;
 };
 
+/* A wrapper around Bindings that ensures that its always in sorted
+   order at the end. The only way to consume a BindingsBuilder is to
+   call finish(), which sorts the bindings. */
+class BindingsBuilder
+{
+    Bindings * bindings;
+
+public:
+    // needed by std::back_inserter
+    using value_type = Attr;
+
+    EvalState & state;
+
+    BindingsBuilder(EvalState & state, Bindings * bindings)
+        : bindings(bindings), state(state)
+    { }
+
+    void insert(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos))
+    {
+        insert(Attr(name, value, pos));
+    }
+
+    void insert(const Attr & attr)
+    {
+        push_back(attr);
+    }
+
+    void push_back(const Attr & attr)
+    {
+        bindings->push_back(attr);
+    }
+
+    Value & alloc(const Symbol & name, ptr<Pos> pos = ptr(&noPos));
+
+    Value & alloc(std::string_view name, ptr<Pos> pos = ptr(&noPos));
+
+    Bindings * finish()
+    {
+        bindings->sort();
+        return bindings;
+    }
+
+    Bindings * alreadySorted()
+    {
+        return bindings;
+    }
+};
 
 }

src/libexpr/eval-cache.cc

@@ -21,6 +21,8 @@ struct AttrDb
 {
     std::atomic_bool failed{false};
 
+    const Store & cfg;
+
     struct State
     {
         SQLite db;
@@ -33,8 +35,9 @@ struct AttrDb
 
     std::unique_ptr<Sync<State>> _state;
 
-    AttrDb(const Hash & fingerprint)
-        : _state(std::make_unique<Sync<State>>())
+    AttrDb(const Store & cfg, const Hash & fingerprint)
+        : cfg(cfg)
+        , _state(std::make_unique<Sync<State>>())
     {
         auto state(_state->lock());
 
@@ -254,10 +257,10 @@ struct AttrDb
                 return {{rowId, attrs}};
             }
             case AttrType::String: {
-                std::vector<std::pair<Path, std::string>> context;
+                NixStringContext context;
                 if (!queryAttribute.isNull(3))
                     for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(s));
+                        context.push_back(decodeContext(cfg, s));
                 return {{rowId, string_t{queryAttribute.getStr(2), context}}};
             }
             case AttrType::Bool:
@@ -274,10 +277,10 @@ struct AttrDb
     }
 };
 
-static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+static std::shared_ptr<AttrDb> makeAttrDb(const Store & cfg, const Hash & fingerprint)
 {
     try {
-        return std::make_shared<AttrDb>(fingerprint);
+        return std::make_shared<AttrDb>(cfg, fingerprint);
     } catch (SQLiteError &) {
         ignoreException();
         return nullptr;
@@ -288,7 +291,7 @@ EvalCache::EvalCache(
     std::optional<std::reference_wrapper<const Hash>> useCache,
     EvalState & state,
     RootLoader rootLoader)
-    : db(useCache ? makeAttrDb(*useCache) : nullptr)
+    : db(useCache ? makeAttrDb(*state.store, *useCache) : nullptr)
     , state(state)
     , rootLoader(rootLoader)
 {
@@ -303,9 +306,9 @@ Value * EvalCache::getRootValue()
     return *value;
 }
 
-std::shared_ptr<AttrCursor> EvalCache::getRoot()
+ref<AttrCursor> EvalCache::getRoot()
 {
-    return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
+    return make_ref<AttrCursor>(ref(shared_from_this()), std::nullopt);
 }
 
 AttrCursor::AttrCursor(
@@ -336,7 +339,7 @@ Value & AttrCursor::getValue()
     if (!_value) {
         if (parent) {
             auto & vParent = parent->first->getValue();
-            root->state.forceAttrs(vParent);
+            root->state.forceAttrs(vParent, noPos);
             auto attr = vParent.attrs->get(parent->second);
             if (!attr)
                 throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
@@ -381,7 +384,7 @@ Value & AttrCursor::forceValue()
     auto & v = getValue();
 
     try {
-        root->state.forceValue(v);
+        root->state.forceValue(v, noPos);
     } catch (EvalError &) {
         debug("setting '%s' to failed", getAttrPathStr());
         if (root->db)
@@ -406,6 +409,16 @@ Value & AttrCursor::forceValue()
     return v;
 }
 
+Suggestions AttrCursor::getSuggestionsForAttr(Symbol name)
+{
+    auto attrNames = getAttrs();
+    std::set<std::string> strAttrNames;
+    for (auto & name : attrNames)
+        strAttrNames.insert(std::string(name));
+
+    return Suggestions::bestMatches(strAttrNames, name);
+}
+
 std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
 {
     if (root->db) {
@@ -446,6 +459,11 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
         return nullptr;
         //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
 
+    for (auto & attr : *v.attrs) {
+        if (root->db)
+            root->db->setPlaceholder({cachedValue->first, attr.name});
+    }
+
     auto attr = v.attrs->get(name);
 
     if (!attr) {
@@ -464,7 +482,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
         cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
     }
 
-    return std::make_shared<AttrCursor>(
+    return make_ref<AttrCursor>(
         root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
 }
 
@@ -473,27 +491,31 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
     return maybeGetAttr(root->state.symbols.create(name));
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
+ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors)
 {
     auto p = maybeGetAttr(name, forceErrors);
     if (!p)
         throw Error("attribute '%s' does not exist", getAttrPathStr(name));
-    return p;
+    return ref(p);
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+ref<AttrCursor> AttrCursor::getAttr(std::string_view name)
 {
     return getAttr(root->state.symbols.create(name));
 }
 
-std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
+OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force)
 {
     auto res = shared_from_this();
     for (auto & attr : attrPath) {
-        res = res->maybeGetAttr(attr, force);
-        if (!res) return {};
+        auto child = res->maybeGetAttr(attr, force);
+        if (!child) {
+            auto suggestions = res->getSuggestionsForAttr(attr);
+            return OrSuggestions<ref<AttrCursor>>::failed(suggestions);
+        }
+        res = child;
     }
-    return res;
+    return ref(res);
 }
 
 std::string AttrCursor::getString()
@@ -527,7 +549,7 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+                    if (!root->state.store->isValidPath(c.first)) {
                         valid = false;
                         break;
                     }
@@ -544,7 +566,7 @@ string_t AttrCursor::getStringWithContext()
     auto & v = forceValue();
 
     if (v.type() == nString)
-        return {v.string.s, v.getContext()};
+        return {v.string.s, v.getContext(*root->state.store)};
     else if (v.type() == nPath)
         return {v.path, {}};
     else
@@ -596,7 +618,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
     for (auto & attr : *getValue().attrs)
         attrs.push_back(attr.name);
     std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
-        return (const string &) a < (const string &) b;
+        return (const std::string &) a < (const std::string &) b;
     });
 
     if (root->db)

src/libexpr/eval-cache.hh

@@ -33,7 +33,7 @@ public:
         EvalState & state,
         RootLoader rootLoader);
 
-    std::shared_ptr<AttrCursor> getRoot();
+    ref<AttrCursor> getRoot();
 };
 
 enum AttrType {
@@ -52,7 +52,7 @@ struct misc_t {};
 struct failed_t {};
 typedef uint64_t AttrId;
 typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+typedef std::pair<std::string, NixStringContext> string_t;
 
 typedef std::variant<
     std::vector<Symbol>,
@@ -94,15 +94,19 @@ public:
 
     std::string getAttrPathStr(Symbol name) const;
 
+    Suggestions getSuggestionsForAttr(Symbol name);
+
     std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false);
 
     std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
 
-    std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
+    ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false);
 
-    std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+    ref<AttrCursor> getAttr(std::string_view name);
 
-    std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
+    /* Get an attribute along a chain of attrsets. Note that this does
+       not auto-call functors or functions. */
+    OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false);
 
     std::string getString();
 

src/libexpr/eval-inline.hh

@@ -15,12 +15,6 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
     });
 }
 
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
-{
-    throw TypeError(s, showType(v));
-}
-
-
 LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
 {
     throw TypeError({
@@ -30,7 +24,89 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 }
 
 
+/* Note: Various places expect the allocated memory to be zeroed. */
+[[gnu::always_inline]]
+inline void * allocBytes(size_t n)
+{
+    void * p;
+#if HAVE_BOEHMGC
+    p = GC_MALLOC(n);
+#else
+    p = calloc(n, 1);
+#endif
+    if (!p) throw std::bad_alloc();
+    return p;
+}
+
+
+[[gnu::always_inline]]
+Value * EvalState::allocValue()
+{
+#if HAVE_BOEHMGC
+    /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
+       GC_malloc_many returns a linked list of objects of the given size, where the first word
+       of each object is also the pointer to the next object in the list. This also means that we
+       have to explicitly clear the first word of every object we take. */
+    if (!*valueAllocCache) {
+        *valueAllocCache = GC_malloc_many(sizeof(Value));
+        if (!*valueAllocCache) throw std::bad_alloc();
+    }
+
+    /* GC_NEXT is a convenience macro for accessing the first word of an object.
+       Take the first list item, advance the list to the next item, and clear the next pointer. */
+    void * p = *valueAllocCache;
+    *valueAllocCache = GC_NEXT(p);
+    GC_NEXT(p) = nullptr;
+#else
+    void * p = allocBytes(sizeof(Value));
+#endif
+
+    nrValues++;
+    return (Value *) p;
+}
+
+
+[[gnu::always_inline]]
+Env & EvalState::allocEnv(size_t size)
+{
+    nrEnvs++;
+    nrValuesInEnvs += size;
+
+    Env * env;
+
+#if HAVE_BOEHMGC
+    if (size == 1) {
+        /* see allocValue for explanations. */
+        if (!*env1AllocCache) {
+            *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *));
+            if (!*env1AllocCache) throw std::bad_alloc();
+        }
+
+        void * p = *env1AllocCache;
+        *env1AllocCache = GC_NEXT(p);
+        GC_NEXT(p) = nullptr;
+        env = (Env *) p;
+    } else
+#endif
+        env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
+
+    env->type = Env::Plain;
+
+    /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
+
+    return *env;
+}
+
+
+[[gnu::always_inline]]
 void EvalState::forceValue(Value & v, const Pos & pos)
+{
+    forceValue(v, [&]() { return pos; });
+}
+
+
+template<typename Callable>
+void EvalState::forceValue(Value & v, Callable getPos)
 {
     if (v.isThunk()) {
         Env * env = v.thunk.env;
@@ -47,34 +123,28 @@ void EvalState::forceValue(Value & v, const Pos & pos)
     else if (v.isApp())
         callFunction(*v.app.left, *v.app.right, v, noPos);
     else if (v.isBlackhole())
-        throwEvalError(pos, "infinite recursion encountered");
-}
-
-
-inline void EvalState::forceAttrs(Value & v)
-{
-    forceValue(v);
-    if (v.type() != nAttrs)
-        throwTypeError("value is %1% while a set was expected", v);
+        throwEvalError(getPos(), "infinite recursion encountered");
 }
 
 
+[[gnu::always_inline]]
 inline void EvalState::forceAttrs(Value & v, const Pos & pos)
 {
-    forceValue(v, pos);
-    if (v.type() != nAttrs)
-        throwTypeError(pos, "value is %1% while a set was expected", v);
+    forceAttrs(v, [&]() { return pos; });
 }
 
 
-inline void EvalState::forceList(Value & v)
+template <typename Callable>
+[[gnu::always_inline]]
+inline void EvalState::forceAttrs(Value & v, Callable getPos)
 {
-    forceValue(v);
-    if (!v.isList())
-        throwTypeError("value is %1% while a list was expected", v);
+    forceValue(v, getPos);
+    if (v.type() != nAttrs)
+        throwTypeError(getPos(), "value is %1% while a set was expected", v);
 }
 
 
+[[gnu::always_inline]]
 inline void EvalState::forceList(Value & v, const Pos & pos)
 {
     forceValue(v, pos);
@@ -82,18 +152,5 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
         throwTypeError(pos, "value is %1% while a list was expected", v);
 }
 
-/* Note: Various places expect the allocated memory to be zeroed. */
-inline void * allocBytes(size_t n)
-{
-    void * p;
-#if HAVE_BOEHMGC
-    p = GC_MALLOC(n);
-#else
-    p = calloc(n, 1);
-#endif
-    if (!p) throw std::bad_alloc();
-    return p;
-}
-
 
 }

src/libexpr/eval.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "attr-set.hh"
+#include "types.hh"
 #include "value.hh"
 #include "nixexpr.hh"
 #include "symbol-table.hh"
@@ -44,8 +45,6 @@ struct Env
 };
 
 
-Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet());
-
 void copyContext(const Value & v, PathSet & context);
 
 
@@ -79,10 +78,11 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sContentAddressed,
+        sContentAddressed, sImpure,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
-        sDescription, sSelf, sEpsilon;
+        sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
+        sPrefix;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -93,7 +93,7 @@ public:
        mode. */
     std::optional<PathSet> allowedPaths;
 
-    Value vEmptySet;
+    Bindings emptyBindings;
 
     /* Store used to materialise .drv files. */
     const ref<Store> store;
@@ -133,6 +133,14 @@ private:
     /* Cache used by prim_match(). */
     std::shared_ptr<RegexCache> regexCache;
 
+#if HAVE_BOEHMGC
+    /* Allocation cache for GC'd Value objects. */
+    std::shared_ptr<void *> valueAllocCache;
+
+    /* Allocation cache for size-1 Env objects. */
+    std::shared_ptr<void *> env1AllocCache;
+#endif
+
 public:
 
     EvalState(
@@ -141,13 +149,7 @@ public:
         std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
-    void requireExperimentalFeatureOnEvaluation(
-        const ExperimentalFeature &,
-        const std::string_view fName,
-        const Pos & pos
-    );
-
-    void addToSearchPath(const string & s);
+    void addToSearchPath(const std::string & s);
 
     SearchPath getSearchPath() { return searchPath; }
 
@@ -158,6 +160,9 @@ public:
        the real store path if `store` is a chroot store. */
     void allowPath(const StorePath & storePath);
 
+    /* Allow access to a store path and return it as a string. */
+    void allowAndSetStorePathString(const StorePath & storePath, Value & v);
+
     /* Check whether access to a path is allowed and throw an error if
        not. Otherwise return the canonicalised path. */
     Path checkSourcePath(const Path & path);
@@ -178,8 +183,8 @@ public:
     Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv);
 
     /* Parse a Nix expression from the specified string. */
-    Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv);
-    Expr * parseExprFromString(std::string_view s, const Path & basePath);
+    Expr * parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv);
+    Expr * parseExprFromString(std::string s, const Path & basePath);
 
     Expr * parseStdin();
 
@@ -199,8 +204,8 @@ public:
     void resetFileCache();
 
     /* Look up a file in the search path. */
-    Path findFile(const string & path);
-    Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos);
+    Path findFile(const std::string_view path);
+    Path findFile(SearchPath & searchPath, const std::string_view path, const Pos & pos = noPos);
 
     /* If the specified search path element is a URI, download it. */
     std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
@@ -219,7 +224,10 @@ public:
        of the evaluation of the thunk.  If `v' is a delayed function
        application, call the function and overwrite `v' with the
        result.  Otherwise, this is a no-op. */
-    inline void forceValue(Value & v, const Pos & pos = noPos);
+    inline void forceValue(Value & v, const Pos & pos);
+
+    template <typename Callable>
+    inline void forceValue(Value & v, Callable getPos);
 
     /* Force a value, then recursively force list elements and
        attributes. */
@@ -229,37 +237,43 @@ public:
     NixInt forceInt(Value & v, const Pos & pos);
     NixFloat forceFloat(Value & v, const Pos & pos);
     bool forceBool(Value & v, const Pos & pos);
-    inline void forceAttrs(Value & v);
-    inline void forceAttrs(Value & v, const Pos & pos);
-    inline void forceList(Value & v);
+
+    void forceAttrs(Value & v, const Pos & pos);
+
+    template <typename Callable>
+    inline void forceAttrs(Value & v, Callable getPos);
+
     inline void forceList(Value & v, const Pos & pos);
     void forceFunction(Value & v, const Pos & pos); // either lambda or primop
-    string forceString(Value & v, const Pos & pos = noPos);
-    string forceString(Value & v, PathSet & context, const Pos & pos = noPos);
-    string forceStringNoCtx(Value & v, const Pos & pos = noPos);
+    std::string_view forceString(Value & v, const Pos & pos = noPos);
+    std::string_view forceString(Value & v, PathSet & context, const Pos & pos = noPos);
+    std::string_view forceStringNoCtx(Value & v, const Pos & pos = noPos);
 
     /* Return true iff the value `v' denotes a derivation (i.e. a
        set with attribute `type = "derivation"'). */
     bool isDerivation(Value & v);
 
-    std::optional<string> tryAttrsToString(const Pos & pos, Value & v,
+    std::optional<std::string> tryAttrsToString(const Pos & pos, Value & v,
         PathSet & context, bool coerceMore = false, bool copyToStore = true);
 
     /* String coercion.  Converts strings, paths and derivations to a
        string.  If `coerceMore' is set, also converts nulls, integers,
        booleans and lists to a string.  If `copyToStore' is set,
        referenced paths are copied to the Nix store as a side effect. */
-    string coerceToString(const Pos & pos, Value & v, PathSet & context,
+    BackedStringView coerceToString(const Pos & pos, Value & v, PathSet & context,
         bool coerceMore = false, bool copyToStore = true,
         bool canonicalizePath = true);
 
-    string copyPathToStore(PathSet & context, const Path & path);
+    std::string copyPathToStore(PathSet & context, const Path & path);
 
     /* Path coercion.  Converts strings, paths and derivations to a
        path.  The result is guaranteed to be a canonicalised, absolute
        path.  Nothing is copied to the store. */
     Path coerceToPath(const Pos & pos, Value & v, PathSet & context);
 
+    /* Like coerceToPath, but the result must be a store path. */
+    StorePath coerceToStorePath(const Pos & pos, Value & v, PathSet & context);
+
 public:
 
     /* The base environment, containing the builtin functions and
@@ -275,18 +289,18 @@ private:
 
     void createBaseEnv();
 
-    Value * addConstant(const string & name, Value & v);
+    Value * addConstant(const std::string & name, Value & v);
 
-    void addConstant(const string & name, Value * v);
+    void addConstant(const std::string & name, Value * v);
 
-    Value * addPrimOp(const string & name,
+    Value * addPrimOp(const std::string & name,
         size_t arity, PrimOpFun primOp);
 
     Value * addPrimOp(PrimOp && primOp);
 
 public:
 
-    Value & getBuiltin(const string & name);
+    Value & getBuiltin(const std::string & name);
 
     struct Doc
     {
@@ -307,8 +321,8 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, FileOrigin origin, const Path & path,
-        const Path & basePath, StaticEnv & staticEnv);
+    Expr * parse(char * text, size_t length, FileOrigin origin, const PathView path,
+        const PathView basePath, StaticEnv & staticEnv);
 
 public:
 
@@ -332,16 +346,20 @@ public:
     void autoCallFunction(Bindings & args, Value & fun, Value & res);
 
     /* Allocation primitives. */
-    Value * allocValue();
-    Env & allocEnv(size_t size);
+    inline Value * allocValue();
+    inline Env & allocEnv(size_t size);
 
     Value * allocAttr(Value & vAttrs, const Symbol & name);
-    Value * allocAttr(Value & vAttrs, const std::string & name);
+    Value * allocAttr(Value & vAttrs, std::string_view name);
 
     Bindings * allocBindings(size_t capacity);
 
+    BindingsBuilder buildBindings(size_t capacity)
+    {
+        return BindingsBuilder(*this, allocBindings(capacity));
+    }
+
     void mkList(Value & v, size_t length);
-    void mkAttrs(Value & v, size_t capacity);
     void mkThunk_(Value & v, Expr * expr);
     void mkPos(Value & v, ptr<Pos> pos);
 
@@ -350,7 +368,10 @@ public:
     /* Print statistics. */
     void printStats();
 
-    void realiseContext(const PathSet & context);
+    /* Realise the given context, and return a mapping from the placeholders
+     * used to construct the associated value to their final store path
+     */
+    [[nodiscard]] StringMap realiseContext(const PathSet & context);
 
 private:
 
@@ -391,16 +412,19 @@ private:
     friend struct ExprSelect;
     friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v);
     friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v);
+    friend void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v);
+
+    friend struct Value;
 };
 
 
 /* Return a string representing the type of the value `v'. */
-string showType(ValueType type);
-string showType(const Value & v);
+std::string_view showType(ValueType type);
+std::string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,
    name>. */
-std::pair<string, string> decodeContext(std::string_view s);
+NixStringContextElem decodeContext(const Store & store, std::string_view s);
 
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
@@ -484,3 +508,5 @@ extern EvalSettings evalSettings;
 static const std::string corepkgsPrefix{"/__corepkgs__/"};
 
 }
+
+#include "eval-inline.hh"

src/libexpr/flake/config.cc

@@ -1,5 +1,6 @@
 #include "flake.hh"
 #include "globals.hh"
+#include "fetch-settings.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -38,11 +39,11 @@ void ConfigFile::apply()
 
         // FIXME: Move into libutil/config.cc.
         std::string valueS;
-        if (auto s = std::get_if<std::string>(&value))
+        if (auto* s = std::get_if<std::string>(&value))
             valueS = *s;
-        else if (auto n = std::get_if<int64_t>(&value))
-            valueS = fmt("%d", n);
-        else if (auto b = std::get_if<Explicit<bool>>(&value))
+        else if (auto* n = std::get_if<int64_t>(&value))
+            valueS = fmt("%d", *n);
+        else if (auto* b = std::get_if<Explicit<bool>>(&value))
             valueS = b->t ? "true" : "false";
         else if (auto ss = std::get_if<std::vector<std::string>>(&value))
             valueS = concatStringsSep(" ", *ss); // FIXME: evil
@@ -53,7 +54,7 @@ void ConfigFile::apply()
             auto trustedList = readTrustedList();
 
             bool trusted = false;
-            if (nix::settings.acceptFlakeConfig){
+            if (nix::fetchSettings.acceptFlakeConfig){
                 trusted = true;
             } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
                 trusted = *saved;

src/libexpr/flake/flake.cc

@@ -6,6 +6,7 @@
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "finally.hh"
+#include "fetch-settings.hh"
 
 namespace nix {
 
@@ -89,11 +90,11 @@ static void expectType(EvalState & state, ValueType type,
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
     EvalState & state, Value * value, const Pos & pos,
-    const std::optional<Path> & baseDir);
+    const std::optional<Path> & baseDir, InputPath lockRootPath);
 
 static FlakeInput parseFlakeInput(EvalState & state,
     const std::string & inputName, Value * value, const Pos & pos,
-    const std::optional<Path> & baseDir)
+    const std::optional<Path> & baseDir, InputPath lockRootPath)
 {
     expectType(state, nAttrs, *value, pos);
 
@@ -117,10 +118,12 @@ static FlakeInput parseFlakeInput(EvalState & state,
                 expectType(state, nBool, *attr.value, *attr.pos);
                 input.isFlake = attr.value->boolean;
             } else if (attr.name == sInputs) {
-                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir);
+                input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir, lockRootPath);
             } else if (attr.name == sFollows) {
                 expectType(state, nString, *attr.value, *attr.pos);
-                input.follows = parseInputPath(attr.value->string.s);
+                auto follows(parseInputPath(attr.value->string.s));
+                follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end());
+                input.follows = follows;
             } else {
                 switch (attr.value->type()) {
                     case nString:
@@ -155,7 +158,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
         if (!attrs.empty())
             throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
         if (url)
-            input.ref = parseFlakeRef(*url, baseDir, true);
+            input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake);
     }
 
     if (!input.follows && !input.ref)
@@ -166,7 +169,7 @@ static FlakeInput parseFlakeInput(EvalState & state,
 
 static std::map<FlakeId, FlakeInput> parseFlakeInputs(
     EvalState & state, Value * value, const Pos & pos,
-    const std::optional<Path> & baseDir)
+    const std::optional<Path> & baseDir, InputPath lockRootPath)
 {
     std::map<FlakeId, FlakeInput> inputs;
 
@@ -178,7 +181,8 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
                 inputAttr.name,
                 inputAttr.value,
                 *inputAttr.pos,
-                baseDir));
+                baseDir,
+                lockRootPath));
     }
 
     return inputs;
@@ -188,14 +192,15 @@ static Flake getFlake(
     EvalState & state,
     const FlakeRef & originalRef,
     bool allowLookup,
-    FlakeCache & flakeCache)
+    FlakeCache & flakeCache,
+    InputPath lockRootPath)
 {
     auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
         state, originalRef, allowLookup, flakeCache);
 
     // Guard against symlink attacks.
-    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir);
-    auto flakeFile = canonPath(flakeDir + "/flake.nix");
+    auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true);
+    auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
     if (!isInDir(flakeFile, sourceInfo.actualPath))
         throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
             lockedRef, state.store->printStorePath(sourceInfo.storePath));
@@ -223,7 +228,7 @@ static Flake getFlake(
     auto sInputs = state.symbols.create("inputs");
 
     if (auto inputs = vInfo.attrs->get(sInputs))
-        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir);
+        flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir, lockRootPath);
 
     auto sOutputs = state.symbols.create("outputs");
 
@@ -250,18 +255,24 @@ static Flake getFlake(
         for (auto & setting : *nixConfig->value->attrs) {
             forceTrivialValue(state, *setting.value, *setting.pos);
             if (setting.value->type() == nString)
-                flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
+                flake.config.settings.insert({setting.name, std::string(state.forceStringNoCtx(*setting.value, *setting.pos))});
+            else if (setting.value->type() == nPath) {
+                PathSet emptyContext = {};
+                flake.config.settings.emplace(
+                    setting.name,
+                    state.coerceToString(*setting.pos, *setting.value, emptyContext, false, true, true) .toOwned());
+            }
             else if (setting.value->type() == nInt)
                 flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
             else if (setting.value->type() == nBool)
-                flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
+                flake.config.settings.insert({setting.name, Explicit<bool> { state.forceBool(*setting.value, *setting.pos) }});
             else if (setting.value->type() == nList) {
                 std::vector<std::string> ss;
                 for (auto elem : setting.value->listItems()) {
                     if (elem->type() != nString)
                         throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
                             setting.name, showType(*setting.value));
-                    ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));
+                    ss.emplace_back(state.forceStringNoCtx(*elem, *setting.pos));
                 }
                 flake.config.settings.insert({setting.name, ss});
             }
@@ -283,6 +294,11 @@ static Flake getFlake(
     return flake;
 }
 
+Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache)
+{
+    return getFlake(state, originalRef, allowLookup, flakeCache, {});
+}
+
 Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
 {
     FlakeCache flakeCache;
@@ -300,7 +316,7 @@ LockedFlake lockFlake(
 
     FlakeCache flakeCache;
 
-    auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries);
+    auto useRegistries = lockFlags.useRegistries.value_or(fetchSettings.useRegistries);
 
     auto flake = getFlake(state, topRef, useRegistries, flakeCache);
 
@@ -328,23 +344,14 @@ LockedFlake lockFlake(
 
         std::vector<FlakeRef> parents;
 
-        struct LockParent {
-            /* The path to this parent. */
-            InputPath path;
-
-            /* Whether we are currently inside a top-level lockfile
-               (inputs absolute) or subordinate lockfile (inputs
-               relative). */
-            bool absolute;
-        };
-
         std::function<void(
             const FlakeInputs & flakeInputs,
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
             std::shared_ptr<const Node> oldNode,
-            const LockParent & parent,
-            const Path & parentPath)>
+            const InputPath & lockRootPath,
+            const Path & parentPath,
+            bool trustLock)>
             computeLocks;
 
         computeLocks = [&](
@@ -352,8 +359,9 @@ LockedFlake lockFlake(
             std::shared_ptr<Node> node,
             const InputPath & inputPathPrefix,
             std::shared_ptr<const Node> oldNode,
-            const LockParent & parent,
-            const Path & parentPath)
+            const InputPath & lockRootPath,
+            const Path & parentPath,
+            bool trustLock)
         {
             debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
 
@@ -396,17 +404,7 @@ LockedFlake lockFlake(
                     if (input.follows) {
                         InputPath target;
 
-                        if (parent.absolute && !hasOverride) {
-                            target = *input.follows;
-                        } else {
-                            if (hasOverride) {
-                                target = inputPathPrefix;
-                                target.pop_back();
-                            } else
-                                target = parent.path;
-
-                            for (auto & i : *input.follows) target.push_back(i);
-                        }
+                        target.insert(target.end(), input.follows->begin(), input.follows->end());
 
                         debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
                         node->inputs.insert_or_assign(id, target);
@@ -464,34 +462,47 @@ LockedFlake lockFlake(
                                         .isFlake = (*lockedNode)->isFlake,
                                     });
                                 } else if (auto follows = std::get_if<1>(&i.second)) {
-                                    auto o = input.overrides.find(i.first);
-                                    // If the override disappeared, we have to refetch the flake,
-                                    // since some of the inputs may not be present in the lockfile.
-                                    if (o == input.overrides.end()) {
-                                        mustRefetch = true;
-                                        // There's no point populating the rest of the fake inputs,
-                                        // since we'll refetch the flake anyways.
-                                        break;
+                                    if (! trustLock) {
+                                        // It is possible that the flake has changed,
+                                        // so we must confirm all the follows that are in the lockfile are also in the flake.
+                                        auto overridePath(inputPath);
+                                        overridePath.push_back(i.first);
+                                        auto o = overrides.find(overridePath);
+                                        // If the override disappeared, we have to refetch the flake,
+                                        // since some of the inputs may not be present in the lockfile.
+                                        if (o == overrides.end()) {
+                                            mustRefetch = true;
+                                            // There's no point populating the rest of the fake inputs,
+                                            // since we'll refetch the flake anyways.
+                                            break;
+                                        }
                                     }
+                                    auto absoluteFollows(lockRootPath);
+                                    absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end());
                                     fakeInputs.emplace(i.first, FlakeInput {
-                                        .follows = *follows,
+                                        .follows = absoluteFollows,
                                     });
                                 }
                             }
                         }
 
+                        auto localPath(parentPath);
+                        // If this input is a path, recurse it down.
+                        // This allows us to resolve path inputs relative to the current flake.
+                        if ((*input.ref).input.getType() == "path")
+                            localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
                         computeLocks(
                             mustRefetch
-                            ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs
+                            ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs
                             : fakeInputs,
-                            childNode, inputPath, oldLock, parent, parentPath);
+                            childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch);
 
                     } else {
                         /* We need to create a new lock file entry. So fetch
                            this input. */
                         debug("creating new input '%s'", inputPathS);
 
-                        if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
+                        if (!lockFlags.allowMutable && !input.ref->input.isLocked())
                             throw Error("cannot update flake input '%s' in pure mode", inputPathS);
 
                         if (input.isFlake) {
@@ -503,7 +514,7 @@ LockedFlake lockFlake(
                             if (localRef.input.getType() == "path")
                                 localPath = absPath(*input.ref->input.getSourcePath(), parentPath);
 
-                            auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache);
+                            auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath);
 
                             /* Note: in case of an --override-input, we use
                                the *original* ref (input2.ref) for the
@@ -524,13 +535,6 @@ LockedFlake lockFlake(
                             parents.push_back(*input.ref);
                             Finally cleanup([&]() { parents.pop_back(); });
 
-                            // Follows paths from existing inputs in the top-level lockfile are absolute,
-                            // whereas paths in subordinate lockfiles are relative to those lockfiles.
-                            LockParent newParent {
-                                .path = inputPath,
-                                .absolute = oldLock ? true : false
-                            };
-
                             /* Recursively process the inputs of this
                                flake. Also, unless we already have this flake
                                in the top-level lock file, use this flake's
@@ -541,7 +545,7 @@ LockedFlake lockFlake(
                                 ? std::dynamic_pointer_cast<const Node>(oldLock)
                                 : LockFile::read(
                                     inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root,
-                                newParent, localPath);
+                                oldLock ? lockRootPath : inputPath, localPath, false);
                         }
 
                         else {
@@ -559,17 +563,12 @@ LockedFlake lockFlake(
             }
         };
 
-        LockParent parent {
-            .path = {},
-            .absolute = true
-        };
-
         // Bring in the current ref for relative path resolution if we have it
-        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir);
+        auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true);
 
         computeLocks(
             flake.inputs, newLockFile.root, {},
-            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath);
+            lockFlags.recreateLockFile ? nullptr : oldLockFile.root, {}, parentPath, false);
 
         for (auto & i : lockFlags.inputOverrides)
             if (!overridesUsed.count(i.first))
@@ -593,7 +592,7 @@ LockedFlake lockFlake(
             if (lockFlags.writeLockFile) {
                 if (auto sourcePath = topRef.input.getSourcePath()) {
                     if (!newLockFile.isImmutable()) {
-                        if (settings.warnDirty)
+                        if (fetchSettings.warnDirty)
                             warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
                     } else {
                         if (!lockFlags.updateLockFile)
@@ -616,12 +615,24 @@ LockedFlake lockFlake(
 
                         newLockFile.write(path);
 
+                        std::optional<std::string> commitMessage = std::nullopt;
+                        if (lockFlags.commitLockFile) {
+                            std::string cm;
+
+                            cm = fetchSettings.commitLockFileSummary.get();
+
+                            if (cm == "") {
+                                cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add");
+                            }
+
+                            cm += "\n\nFlake lock file updates:\n\n";
+                            cm += filterANSIEscapes(diff, true);
+                            commitMessage = cm;
+                        }
+
                         topRef.input.markChangedFile(
                             (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
-                            lockFlags.commitLockFile
-                            ? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s",
-                                    relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true)))
-                            : std::nullopt);
+                            commitMessage);
 
                         /* Rewriting the lockfile changed the top-level
                            repo, so we should re-read it. FIXME: we could
@@ -640,7 +651,7 @@ LockedFlake lockFlake(
                            now. Corner case: we could have reverted from a
                            dirty to a clean tree! */
                         if (flake.lockedRef.input == prevLockedRef.input
-                            && !flake.lockedRef.input.isImmutable())
+                            && !flake.lockedRef.input.isLocked())
                             throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
                     }
                 } else
@@ -669,7 +680,7 @@ void callFlake(EvalState & state,
     auto vTmp1 = state.allocValue();
     auto vTmp2 = state.allocValue();
 
-    mkString(*vLocks, lockedFlake.lockFile.to_string());
+    vLocks->mkString(lockedFlake.lockFile.to_string());
 
     emitTreeAttrs(
         state,
@@ -679,7 +690,7 @@ void callFlake(EvalState & state,
         false,
         lockedFlake.flake.forceDirty);
 
-    mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+    vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
 
     if (!state.vCallFlake) {
         state.vCallFlake = allocRootValue(state.allocValue());
@@ -695,24 +706,45 @@ void callFlake(EvalState & state,
 
 static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
-    state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
-
-    auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
+    std::string flakeRefS(state.forceStringNoCtx(*args[0], pos));
     auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
-    if (evalSettings.pureEval && !flakeRef.input.isImmutable())
-        throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
+    if (evalSettings.pureEval && !flakeRef.input.isLocked())
+        throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
 
     callFlake(state,
         lockFlake(state, flakeRef,
             LockFlags {
                 .updateLockFile = false,
-                .useRegistries = !evalSettings.pureEval && settings.useRegistries,
+                .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries,
                 .allowMutable  = !evalSettings.pureEval,
             }),
         v);
 }
 
-static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
+static RegisterPrimOp r2({
+    .name =  "__getFlake",
+    .args = {"args"},
+    .doc = R"(
+      Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
+
+      ```nix
+      (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
+      ```
+
+      Unless impure evaluation is allowed (`--impure`), the flake reference
+      must be "locked", e.g. contain a Git revision or content hash. An
+      example of an unlocked usage is:
+
+      ```nix
+      (builtins.getFlake "github:edolstra/dwarffs").rev
+      ```
+
+      This function is only available if you enable the experimental feature
+      `flakes`.
+    )",
+    .fun = prim_getFlake,
+    .experimentalFeature = Xp::Flakes,
+});
 
 }
 

src/libexpr/flake/flakeref.cc

@@ -48,9 +48,12 @@ FlakeRef FlakeRef::resolve(ref<Store> store) const
 }
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
-    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
     if (fragment != "")
         throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
     return flakeRef;
@@ -67,7 +70,10 @@ std::optional<FlakeRef> maybeParseFlakeRef(
 }
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
 {
     using namespace fetchers;
 
@@ -92,7 +98,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
     if (std::regex_match(url, match, flakeRegex)) {
         auto parsedURL = ParsedURL{
             .url = url,
-            .base = "flake:" + std::string(match[1]),
+            .base = "flake:" + match.str(1),
             .scheme = "flake",
             .authority = "",
             .path = match[1],
@@ -100,58 +106,83 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 
         return std::make_pair(
             FlakeRef(Input::fromURL(parsedURL), ""),
-            percentDecode(std::string(match[6])));
+            percentDecode(match.str(6)));
     }
 
     else if (std::regex_match(url, match, pathUrlRegex)) {
         std::string path = match[1];
-        std::string fragment = percentDecode(std::string(match[3]));
+        std::string fragment = percentDecode(match.str(3));
 
         if (baseDir) {
             /* Check if 'url' is a path (either absolute or relative
                to 'baseDir'). If so, search upward to the root of the
                repo (i.e. the directory containing .git). */
 
-            path = absPath(path, baseDir, true);
+            path = absPath(path, baseDir);
 
-            if (!S_ISDIR(lstat(path).st_mode))
-                throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+            if (isFlake) {
 
-            if (!allowMissing && !pathExists(path + "/flake.nix"))
-                throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+                if (!allowMissing && !pathExists(path + "/flake.nix")){
+                    notice("path '%s' does not contain a 'flake.nix', searching up",path);
 
-            auto flakeRoot = path;
-            std::string subdir;
-
-            while (flakeRoot != "/") {
-                if (pathExists(flakeRoot + "/.git")) {
-                    auto base = std::string("git+file://") + flakeRoot;
-
-                    auto parsedURL = ParsedURL{
-                        .url = base, // FIXME
-                        .base = base,
-                        .scheme = "git+file",
-                        .authority = "",
-                        .path = flakeRoot,
-                        .query = decodeQuery(match[2]),
-                    };
-
-                    if (subdir != "") {
-                        if (parsedURL.query.count("dir"))
-                            throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
-                        parsedURL.query.insert_or_assign("dir", subdir);
+                    // Save device to detect filesystem boundary
+                    dev_t device = lstat(path).st_dev;
+                    bool found = false;
+                    while (path != "/") {
+                        if (pathExists(path + "/flake.nix")) {
+                            found = true;
+                            break;
+                        } else if (pathExists(path + "/.git"))
+                            throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
+                        else {
+                            if (lstat(path).st_dev != device)
+                                throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
+                        }
+                        path = dirOf(path);
                     }
-
-                    if (pathExists(flakeRoot + "/.git/shallow"))
-                        parsedURL.query.insert_or_assign("shallow", "1");
-
-                    return std::make_pair(
-                        FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
-                        fragment);
+                    if (!found)
+                        throw BadURL("could not find a flake.nix file");
                 }
 
-                subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
-                flakeRoot = dirOf(flakeRoot);
+                if (!S_ISDIR(lstat(path).st_mode))
+                    throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+
+                if (!allowMissing && !pathExists(path + "/flake.nix"))
+                    throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+
+                auto flakeRoot = path;
+                std::string subdir;
+
+                while (flakeRoot != "/") {
+                    if (pathExists(flakeRoot + "/.git")) {
+                        auto base = std::string("git+file://") + flakeRoot;
+
+                        auto parsedURL = ParsedURL{
+                            .url = base, // FIXME
+                            .base = base,
+                            .scheme = "git+file",
+                            .authority = "",
+                            .path = flakeRoot,
+                            .query = decodeQuery(match[2]),
+                        };
+
+                        if (subdir != "") {
+                            if (parsedURL.query.count("dir"))
+                                throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                            parsedURL.query.insert_or_assign("dir", subdir);
+                        }
+
+                        if (pathExists(flakeRoot + "/.git/shallow"))
+                            parsedURL.query.insert_or_assign("shallow", "1");
+
+                        return std::make_pair(
+                            FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+                            fragment);
+                    }
+
+                    subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+                    flakeRoot = dirOf(flakeRoot);
+                }
             }
 
         } else {

src/libexpr/flake/flakeref.hh

@@ -62,13 +62,19 @@ struct FlakeRef
 std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
 
 FlakeRef parseFlakeRef(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<FlakeRef> maybeParseFlake(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
-    const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
 
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});

src/libexpr/flake/lockfile.cc

@@ -35,7 +35,7 @@ LockedNode::LockedNode(const nlohmann::json & json)
     , originalRef(getFlakeRef(json, "original", nullptr))
     , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
 {
-    if (!lockedRef.input.isImmutable())
+    if (!lockedRef.input.isLocked())
         throw Error("lockfile contains mutable lock '%s'",
             fetchers::attrsToJSON(lockedRef.input.toAttrs()));
 }
@@ -220,7 +220,7 @@ bool LockFile::isImmutable() const
     for (auto & i : nodes) {
         if (i == root) continue;
         auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
-        if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
+        if (lockedNode && !lockedNode->lockedRef.input.isLocked()) return false;
     }
 
     return true;

src/libexpr/get-drvs.cc

@@ -1,6 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
+#include "derivations.hh"
 #include "store-api.hh"
 #include "path-with-outputs.hh"
 
@@ -11,8 +12,8 @@
 namespace nix {
 
 
-DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
-    : state(&state), attrs(attrs), attrPath(attrPath)
+DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs)
+    : state(&state), attrs(attrs), attrPath(std::move(attrPath))
 {
 }
 
@@ -22,7 +23,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
 {
     auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
 
-    this->drvPath = store->printStorePath(drvPath);
+    this->drvPath = drvPath;
 
     auto drv = store->derivationFromPath(drvPath);
 
@@ -41,13 +42,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
         throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
     auto & [outputName, output] = *i;
 
-    auto optStorePath = output.path(*store, drv.name, outputName);
-    if (optStorePath)
-        outPath = store->printStorePath(*optStorePath);
+    outPath = {output.path(*store, drv.name, outputName)};
 }
 
 
-string DrvInfo::queryName() const
+std::string DrvInfo::queryName() const
 {
     if (name == "" && attrs) {
         auto i = attrs->find(state->sName);
@@ -58,7 +57,7 @@ string DrvInfo::queryName() const
 }
 
 
-string DrvInfo::querySystem() const
+std::string DrvInfo::querySystem() const
 {
     if (system == "" && attrs) {
         auto i = attrs->find(state->sSystem);
@@ -68,24 +67,35 @@ string DrvInfo::querySystem() const
 }
 
 
-string DrvInfo::queryDrvPath() const
+std::optional<StorePath> DrvInfo::queryDrvPath() const
 {
-    if (drvPath == "" && attrs) {
+    if (!drvPath && attrs) {
         Bindings::iterator i = attrs->find(state->sDrvPath);
         PathSet context;
-        drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : "";
+        if (i == attrs->end())
+            drvPath = {std::nullopt};
+        else
+            drvPath = {state->coerceToStorePath(*i->pos, *i->value, context)};
     }
-    return drvPath;
+    return drvPath.value_or(std::nullopt);
 }
 
 
-string DrvInfo::queryOutPath() const
+StorePath DrvInfo::requireDrvPath() const
+{
+    if (auto drvPath = queryDrvPath())
+        return *drvPath;
+    throw Error("derivation does not contain a 'drvPath' attribute");
+}
+
+
+StorePath DrvInfo::queryOutPath() const
 {
     if (!outPath && attrs) {
         Bindings::iterator i = attrs->find(state->sOutPath);
         PathSet context;
         if (i != attrs->end())
-            outPath = state->coerceToPath(*i->pos, *i->value, context);
+            outPath = state->coerceToStorePath(*i->pos, *i->value, context);
     }
     if (!outPath)
         throw UnimplementedError("CA derivations are not yet supported");
@@ -93,7 +103,7 @@ string DrvInfo::queryOutPath() const
 }
 
 
-DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
+DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
 {
     if (outputs.empty()) {
         /* Get the ‘outputs’ list. */
@@ -103,20 +113,24 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 
             /* For each output... */
             for (auto elem : i->value->listItems()) {
-                /* Evaluate the corresponding set. */
-                string name = state->forceStringNoCtx(*elem, *i->pos);
-                Bindings::iterator out = attrs->find(state->symbols.create(name));
-                if (out == attrs->end()) continue; // FIXME: throw error?
-                state->forceAttrs(*out->value);
+                std::string output(state->forceStringNoCtx(*elem, *i->pos));
 
-                /* And evaluate its ‘outPath’ attribute. */
-                Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
-                if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
-                PathSet context;
-                outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context);
+                if (withPaths) {
+                    /* Evaluate the corresponding set. */
+                    Bindings::iterator out = attrs->find(state->symbols.create(output));
+                    if (out == attrs->end()) continue; // FIXME: throw error?
+                    state->forceAttrs(*out->value, *i->pos);
+
+                    /* And evaluate its ‘outPath’ attribute. */
+                    Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
+                    if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
+                    PathSet context;
+                    outputs.emplace(output, state->coerceToStorePath(*outPath->pos, *outPath->value, context));
+                } else
+                    outputs.emplace(output, std::nullopt);
             }
         } else
-            outputs["out"] = queryOutPath();
+            outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt);
     }
     if (!onlyOutputsToInstall || !attrs)
         return outputs;
@@ -138,7 +152,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
 }
 
 
-string DrvInfo::queryOutputName() const
+std::string DrvInfo::queryOutputName() const
 {
     if (outputName == "" && attrs) {
         Bindings::iterator i = attrs->find(state->sOutputName);
@@ -172,7 +186,7 @@ StringSet DrvInfo::queryMetaNames()
 
 bool DrvInfo::checkMeta(Value & v)
 {
-    state->forceValue(v);
+    state->forceValue(v, [&]() { return v.determinePos(noPos); });
     if (v.type() == nList) {
         for (auto elem : v.listItems())
             if (!checkMeta(*elem)) return false;
@@ -190,7 +204,7 @@ bool DrvInfo::checkMeta(Value & v)
 }
 
 
-Value * DrvInfo::queryMeta(const string & name)
+Value * DrvInfo::queryMeta(const std::string & name)
 {
     if (!getMeta()) return 0;
     Bindings::iterator a = meta->find(state->symbols.create(name));
@@ -199,7 +213,7 @@ Value * DrvInfo::queryMeta(const string & name)
 }
 
 
-string DrvInfo::queryMetaString(const string & name)
+std::string DrvInfo::queryMetaString(const std::string & name)
 {
     Value * v = queryMeta(name);
     if (!v || v->type() != nString) return "";
@@ -207,7 +221,7 @@ string DrvInfo::queryMetaString(const string & name)
 }
 
 
-NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
+NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -221,7 +235,7 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
     return def;
 }
 
-NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
+NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -236,7 +250,7 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
 }
 
 
-bool DrvInfo::queryMetaBool(const string & name, bool def)
+bool DrvInfo::queryMetaBool(const std::string & name, bool def)
 {
     Value * v = queryMeta(name);
     if (!v) return def;
@@ -251,23 +265,22 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
 }
 
 
-void DrvInfo::setMeta(const string & name, Value * v)
+void DrvInfo::setMeta(const std::string & name, Value * v)
 {
     getMeta();
-    Bindings * old = meta;
-    meta = state->allocBindings(1 + (old ? old->size() : 0));
+    auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0));
     Symbol sym = state->symbols.create(name);
-    if (old)
-        for (auto i : *old)
+    if (meta)
+        for (auto i : *meta)
             if (i.name != sym)
-                meta->push_back(i);
-    if (v) meta->push_back(Attr(sym, v));
-    meta->sort();
+                attrs.insert(i);
+    if (v) attrs.insert(sym, v);
+    meta = attrs.finish();
 }
 
 
 /* Cache for already considered attrsets. */
-typedef set<Bindings *> Done;
+typedef std::set<Bindings *> Done;
 
 
 /* Evaluate value `v'.  If it evaluates to a set of type `derivation',
@@ -275,11 +288,11 @@ typedef set<Bindings *> Done;
    The result boolean indicates whether it makes sense
    for the caller to recursively search for derivations in `v'. */
 static bool getDerivation(EvalState & state, Value & v,
-    const string & attrPath, DrvInfos & drvs, Done & done,
+    const std::string & attrPath, DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
     try {
-        state.forceValue(v);
+        state.forceValue(v, [&]() { return v.determinePos(noPos); });
         if (!state.isDerivation(v)) return true;
 
         /* Remove spurious duplicates (e.g., a set like `rec { x =
@@ -312,7 +325,7 @@ std::optional<DrvInfo> getDerivation(EvalState & state, Value & v,
 }
 
 
-static string addToPath(const string & s1, const string & s2)
+static std::string addToPath(const std::string & s1, const std::string & s2)
 {
     return s1.empty() ? s2 : s1 + "." + s2;
 }
@@ -322,7 +335,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*");
 
 
 static void getDerivations(EvalState & state, Value & vIn,
-    const string & pathPrefix, Bindings & autoArgs,
+    const std::string & pathPrefix, Bindings & autoArgs,
     DrvInfos & drvs, Done & done,
     bool ignoreAssertionFailures)
 {
@@ -347,7 +360,7 @@ static void getDerivations(EvalState & state, Value & vIn,
             debug("evaluating attribute '%1%'", i->name);
             if (!std::regex_match(std::string(i->name), attrRegex))
                 continue;
-            string pathPrefix2 = addToPath(pathPrefix, i->name);
+            std::string pathPrefix2 = addToPath(pathPrefix, i->name);
             if (combineChannels)
                 getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
             else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) {
@@ -365,7 +378,7 @@ static void getDerivations(EvalState & state, Value & vIn,
 
     else if (v.type() == nList) {
         for (auto [n, elem] : enumerate(v.listItems())) {
-            string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
+            std::string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n));
             if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures))
                 getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
         }
@@ -375,7 +388,7 @@ static void getDerivations(EvalState & state, Value & vIn,
 }
 
 
-void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
     Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures)
 {
     Done done;

src/libexpr/get-drvs.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "eval.hh"
+#include "path.hh"
 
 #include <string>
 #include <map>
@@ -12,16 +13,16 @@ namespace nix {
 struct DrvInfo
 {
 public:
-    typedef std::map<string, Path> Outputs;
+    typedef std::map<std::string, std::optional<StorePath>> Outputs;
 
 private:
     EvalState * state;
 
-    mutable string name;
-    mutable string system;
-    mutable string drvPath;
-    mutable std::optional<string> outPath;
-    mutable string outputName;
+    mutable std::string name;
+    mutable std::string system;
+    mutable std::optional<std::optional<StorePath>> drvPath;
+    mutable std::optional<StorePath> outPath;
+    mutable std::string outputName;
     Outputs outputs;
 
     bool failed = false; // set if we get an AssertionError
@@ -33,36 +34,38 @@ private:
     bool checkMeta(Value & v);
 
 public:
-    string attrPath; /* path towards the derivation */
+    std::string attrPath; /* path towards the derivation */
 
     DrvInfo(EvalState & state) : state(&state) { };
-    DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs);
+    DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs);
     DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs);
 
-    string queryName() const;
-    string querySystem() const;
-    string queryDrvPath() const;
-    string queryOutPath() const;
-    string queryOutputName() const;
-    /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
-    Outputs queryOutputs(bool onlyOutputsToInstall = false);
+    std::string queryName() const;
+    std::string querySystem() const;
+    std::optional<StorePath> queryDrvPath() const;
+    StorePath requireDrvPath() const;
+    StorePath queryOutPath() const;
+    std::string queryOutputName() const;
+    /** Return the unordered map of output names to (optional) output paths.
+     * The "outputs to install" are determined by `meta.outputsToInstall`. */
+    Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false);
 
     StringSet queryMetaNames();
-    Value * queryMeta(const string & name);
-    string queryMetaString(const string & name);
-    NixInt queryMetaInt(const string & name, NixInt def);
-    NixFloat queryMetaFloat(const string & name, NixFloat def);
-    bool queryMetaBool(const string & name, bool def);
-    void setMeta(const string & name, Value * v);
+    Value * queryMeta(const std::string & name);
+    std::string queryMetaString(const std::string & name);
+    NixInt queryMetaInt(const std::string & name, NixInt def);
+    NixFloat queryMetaFloat(const std::string & name, NixFloat def);
+    bool queryMetaBool(const std::string & name, bool def);
+    void setMeta(const std::string & name, Value * v);
 
     /*
     MetaInfo queryMetaInfo(EvalState & state) const;
     MetaValue queryMetaInfo(EvalState & state, const string & name) const;
     */
 
-    void setName(const string & s) { name = s; }
-    void setDrvPath(const string & s) { drvPath = s; }
-    void setOutPath(const string & s) { outPath = s; }
+    void setName(const std::string & s) { name = s; }
+    void setDrvPath(StorePath path) { drvPath = {{std::move(path)}}; }
+    void setOutPath(StorePath path) { outPath = {{std::move(path)}}; }
 
     void setFailed() { failed = true; };
     bool hasFailed() { return failed; };
@@ -70,9 +73,9 @@ public:
 
 
 #if HAVE_BOEHMGC
-typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
+typedef std::list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos;
 #else
-typedef list<DrvInfo> DrvInfos;
+typedef std::list<DrvInfo> DrvInfos;
 #endif
 
 
@@ -81,7 +84,7 @@ typedef list<DrvInfo> DrvInfos;
 std::optional<DrvInfo> getDerivation(EvalState & state,
     Value & v, bool ignoreAssertionFailures);
 
-void getDerivations(EvalState & state, Value & v, const string & pathPrefix,
+void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix,
     Bindings & autoArgs, DrvInfos & drvs,
     bool ignoreAssertionFailures);
 

src/libexpr/json-to-value.cc

@@ -37,10 +37,10 @@ class JSONSax : nlohmann::json_sax<json> {
         ValueMap attrs;
         std::unique_ptr<JSONState> resolve(EvalState & state) override
         {
-            Value & v = parent->value(state);
-            state.mkAttrs(v, attrs.size());
+            auto attrs2 = state.buildBindings(attrs.size());
             for (auto & i : attrs)
-                v.attrs->push_back(Attr(i.first, i.second));
+                attrs2.insert(i.first, i.second);
+            parent->value(state).mkAttrs(attrs2.alreadySorted());
             return std::move(parent);
         }
         void add() override { v = nullptr; }
@@ -76,45 +76,51 @@ class JSONSax : nlohmann::json_sax<json> {
     EvalState & state;
     std::unique_ptr<JSONState> rs;
 
-    template<typename T, typename... Args> inline bool handle_value(T f, Args... args)
-    {
-        f(rs->value(state), args...);
-        rs->add();
-        return true;
-    }
-
 public:
     JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {};
 
     bool null()
     {
-        return handle_value(mkNull);
+        rs->value(state).mkNull();
+        rs->add();
+        return true;
     }
 
     bool boolean(bool val)
     {
-        return handle_value(mkBool, val);
+        rs->value(state).mkBool(val);
+        rs->add();
+        return true;
     }
 
     bool number_integer(number_integer_t val)
     {
-        return handle_value(mkInt, val);
+        rs->value(state).mkInt(val);
+        rs->add();
+        return true;
     }
 
     bool number_unsigned(number_unsigned_t val)
     {
-        return handle_value(mkInt, val);
+        rs->value(state).mkInt(val);
+        rs->add();
+        return true;
     }
 
     bool number_float(number_float_t val, const string_t & s)
     {
-        return handle_value(mkFloat, val);
+        rs->value(state).mkFloat(val);
+        rs->add();
+        return true;
     }
 
     bool string(string_t & val)
     {
-        return handle_value<void(Value&, const char*)>(mkString, val.c_str());
+        rs->value(state).mkString(val);
+        rs->add();
+        return true;
     }
+
 #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8
     bool binary(binary_t&)
     {
@@ -157,7 +163,7 @@ public:
     }
 };
 
-void parseJSON(EvalState & state, const string & s_, Value & v)
+void parseJSON(EvalState & state, const std::string_view & s_, Value & v)
 {
     JSONSax parser(state, v);
     bool res = json::sax_parse(s_, &parser);

src/libexpr/json-to-value.hh

@@ -8,6 +8,6 @@ namespace nix {
 
 MakeError(JSONParseError, EvalError);
 
-void parseJSON(EvalState & state, const string & s, Value & v);
+void parseJSON(EvalState & state, const std::string_view & s, Value & v);
 
 }

src/libexpr/lexer.l

@@ -28,6 +28,13 @@ using namespace nix;
 
 namespace nix {
 
+static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
+{
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
+}
+
+#define CUR_POS makeCurPos(*yylloc, data)
+
 // backup to recover from yyless(0)
 YYLTYPE prev_yylloc;
 
@@ -37,7 +44,6 @@ static void initLoc(YYLTYPE * loc)
     loc->first_column = loc->last_column = 1;
 }
 
-
 static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 {
     prev_yylloc = *loc;
@@ -64,29 +70,32 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
 }
 
 
-// FIXME: optimize
-static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length)
+// we make use of the fact that the parser receives a private copy of the input
+// string and can munge around in it.
+static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
 {
-    string t;
-    t.reserve(length);
+    char * result = s;
+    char * t = s;
     char c;
+    // the input string is terminated with *two* NULs, so we can safely take
+    // *one* character after the one being checked against.
     while ((c = *s++)) {
         if (c == '\\') {
-            assert(*s);
             c = *s++;
-            if (c == 'n') t += '\n';
-            else if (c == 'r') t += '\r';
-            else if (c == 't') t += '\t';
-            else t += c;
+            if (c == 'n') *t = '\n';
+            else if (c == 'r') *t = '\r';
+            else if (c == 't') *t = '\t';
+            else *t = c;
         }
         else if (c == '\r') {
             /* Normalise CR and CR/LF into LF. */
-            t += '\n';
+            *t = '\n';
             if (*s == '\n') s++; /* cr/lf */
         }
-        else t += c;
+        else *t = c;
+        t++;
     }
-    return new ExprString(symbols.create(t));
+    return {result, size_t(t - result)};
 }
 
 
@@ -139,19 +148,25 @@ or          { return OR_KW; }
 \/\/        { return UPDATE; }
 \+\+        { return CONCAT; }
 
-{ID}        { yylval->id = strdup(yytext); return ID; }
+{ID}        { yylval->id = {yytext, (size_t) yyleng}; return ID; }
 {INT}       { errno = 0;
               try {
                   yylval->n = boost::lexical_cast<int64_t>(yytext);
               } catch (const boost::bad_lexical_cast &) {
-                  throw ParseError("invalid integer '%1%'", yytext);
+                  throw ParseError({
+                      .msg = hintfmt("invalid integer '%1%'", yytext),
+                      .errPos = CUR_POS,
+                  });
               }
               return INT;
             }
 {FLOAT}     { errno = 0;
               yylval->nf = strtod(yytext, 0);
               if (errno != 0)
-                  throw ParseError("invalid float '%1%'", yytext);
+                  throw ParseError({
+                      .msg = hintfmt("invalid float '%1%'", yytext),
+                      .errPos = CUR_POS,
+                  });
               return FLOAT;
             }
 
@@ -173,7 +188,7 @@ or          { return OR_KW; }
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
-                yylval->e = unescapeStr(data->symbols, yytext, yyleng);
+                yylval->str = unescapeStr(data->symbols, yytext, yyleng);
                 return STR;
               }
 <STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -188,26 +203,26 @@ or          { return OR_KW; }
 
 \'\'(\ *\n)?     { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
 <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
-                   yylval->e = new ExprIndStr(yytext);
+                   yylval->str = {yytext, (size_t) yyleng, true};
                    return IND_STR;
                  }
 <IND_STRING>\'\'\$ |
 <IND_STRING>\$   {
-                   yylval->e = new ExprIndStr("$");
+                   yylval->str = {"$", 1};
                    return IND_STR;
                  }
 <IND_STRING>\'\'\' {
-                   yylval->e = new ExprIndStr("''");
+                   yylval->str = {"''", 2};
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\{ANY} {
-                   yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
+                   yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
 <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
 <IND_STRING>\'   {
-                   yylval->e = new ExprIndStr("'");
+                   yylval->str = {"'", 1};
                    return IND_STR;
                  }
 
@@ -221,14 +236,14 @@ or          { return OR_KW; }
 <PATH_START>{PATH_SEG} {
   POP_STATE();
   PUSH_STATE(INPATH_SLASH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return PATH;
 }
 
 <PATH_START>{HPATH_START} {
   POP_STATE();
   PUSH_STATE(INPATH_SLASH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return HPATH;
 }
 
@@ -237,7 +252,7 @@ or          { return OR_KW; }
     PUSH_STATE(INPATH_SLASH);
   else
     PUSH_STATE(INPATH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return PATH;
 }
 {HPATH} {
@@ -245,7 +260,7 @@ or          { return OR_KW; }
     PUSH_STATE(INPATH_SLASH);
   else
     PUSH_STATE(INPATH);
-  yylval->path = strdup(yytext);
+  yylval->path = {yytext, (size_t) yyleng};
   return HPATH;
 }
 
@@ -261,7 +276,7 @@ or          { return OR_KW; }
       PUSH_STATE(INPATH_SLASH);
   else
       PUSH_STATE(INPATH);
-  yylval->e = new ExprString(data->symbols.create(string(yytext)));
+  yylval->str = {yytext, (size_t) yyleng};
   return STR;
 }
 <INPATH>{ANY} |
@@ -277,11 +292,14 @@ or          { return OR_KW; }
 
 <INPATH_SLASH>{ANY} |
 <INPATH_SLASH><<EOF>> {
-  throw ParseError("path has a trailing slash");
+  throw ParseError({
+      .msg = hintfmt("path has a trailing slash"),
+      .errPos = CUR_POS,
+  });
 }
 
-{SPATH}     { yylval->path = strdup(yytext); return SPATH; }
-{URI}       { yylval->uri = strdup(yytext); return URI; }
+{SPATH}     { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
+{URI}       { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
 
 [ \t\r\n]+    /* eat up whitespace */
 \#[^\r\n]*    /* single-line comments */

src/libexpr/nixexpr.cc

@@ -16,10 +16,10 @@ std::ostream & operator << (std::ostream & str, const Expr & e)
     return str;
 }
 
-static void showString(std::ostream & str, const string & s)
+static void showString(std::ostream & str, std::string_view s)
 {
     str << '"';
-    for (auto c : (string) s)
+    for (auto c : s)
         if (c == '"' || c == '\\' || c == '$') str << "\\" << c;
         else if (c == '\n') str << "\\n";
         else if (c == '\r') str << "\\r";
@@ -28,7 +28,7 @@ static void showString(std::ostream & str, const string & s)
     str << '"';
 }
 
-static void showId(std::ostream & str, const string & s)
+static void showId(std::ostream & str, std::string_view s)
 {
     if (s.empty())
         str << "\"\"";
@@ -103,11 +103,18 @@ void ExprAttrs::show(std::ostream & str) const
 {
     if (recursive) str << "rec ";
     str << "{ ";
-    for (auto & i : attrs)
-        if (i.second.inherited)
-            str << "inherit " << i.first << " " << "; ";
+    typedef const decltype(attrs)::value_type * Attr;
+    std::vector<Attr> sorted;
+    for (auto & i : attrs) sorted.push_back(&i);
+        std::sort(sorted.begin(), sorted.end(), [](Attr a, Attr b) {
+            return (const std::string &) a->first < (const std::string &) b->first;
+        });
+    for (auto & i : sorted) {
+        if (i->second.inherited)
+            str << "inherit " << i->first << " " << "; ";
         else
-            str << i.first << " = " << *i.second.e << "; ";
+            str << i->first << " = " << *i->second.e << "; ";
+    }
     for (auto & i : dynamicAttrs)
         str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; ";
     str << "}";
@@ -191,7 +198,7 @@ void ExprConcatStrings::show(std::ostream & str) const
     str << "(";
     for (auto & i : *es) {
         if (first) first = false; else str << " + ";
-        str << *i;
+        str << *i.second;
     }
     str << ")";
 }
@@ -211,7 +218,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
         auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
         switch (pos.origin) {
             case foFile:
-                f % (string) pos.file;
+                f % (const std::string &) pos.file;
                 break;
             case foStdin:
             case foString:
@@ -227,7 +234,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
 }
 
 
-string showAttrPath(const AttrPath & attrPath)
+std::string showAttrPath(const AttrPath & attrPath)
 {
     std::ostringstream out;
     bool first = true;
@@ -439,7 +446,7 @@ void ExprOpNot::bindVars(const StaticEnv & env)
 void ExprConcatStrings::bindVars(const StaticEnv & env)
 {
     for (auto & i : *es)
-        i->bindVars(env);
+        i.second->bindVars(env);
 }
 
 void ExprPos::bindVars(const StaticEnv & env)
@@ -461,9 +468,9 @@ void ExprLambda::setName(Symbol & name)
 }
 
 
-string ExprLambda::showNamePos() const
+std::string ExprLambda::showNamePos() const
 {
-    return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str();
+    return fmt("%1% at %2%", name.set() ? "'" + (std::string) name + "'" : "anonymous function", pos);
 }
 
 
@@ -473,7 +480,7 @@ string ExprLambda::showNamePos() const
 size_t SymbolTable::totalSize() const
 {
     size_t n = 0;
-    for (auto & i : symbols)
+    for (auto & i : store)
         n += i.size();
     return n;
 }

src/libexpr/nixexpr.hh

@@ -23,21 +23,23 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
-    FileOrigin origin;
     Symbol file;
-    unsigned int line, column;
-    Pos() : origin(foString), line(0), column(0) { };
-    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
-        : origin(origin), file(file), line(line), column(column) { };
+    uint32_t line;
+    FileOrigin origin:2;
+    uint32_t column:30;
+    Pos() : line(0), origin(foString), column(0) { };
+    Pos(FileOrigin origin, const Symbol & file, uint32_t line, uint32_t column)
+        : file(file), line(line), origin(origin), column(column) { };
     operator bool() const
     {
         return line != 0;
     }
+
     bool operator < (const Pos & p2) const
     {
         if (!line) return p2.line;
         if (!p2.line) return false;
-        int d = ((string) file).compare((string) p2.file);
+        int d = ((const std::string &) file).compare((const std::string &) p2.file);
         if (d < 0) return true;
         if (d > 0) return false;
         if (line < p2.line) return true;
@@ -68,7 +70,7 @@ struct AttrName
 
 typedef std::vector<AttrName> AttrPath;
 
-string showAttrPath(const AttrPath & attrPath);
+std::string showAttrPath(const AttrPath & attrPath);
 
 
 /* Abstract syntax of Nix expressions. */
@@ -94,7 +96,7 @@ struct ExprInt : Expr
 {
     NixInt n;
     Value v;
-    ExprInt(NixInt n) : n(n) { mkInt(v, n); };
+    ExprInt(NixInt n) : n(n) { v.mkInt(n); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -103,32 +105,25 @@ struct ExprFloat : Expr
 {
     NixFloat nf;
     Value v;
-    ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); };
+    ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
 
 struct ExprString : Expr
 {
-    Symbol s;
+    std::string s;
     Value v;
-    ExprString(const Symbol & s) : s(s) { mkString(v, s); };
+    ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
 
-/* Temporary class used during parsing of indented strings. */
-struct ExprIndStr : Expr
-{
-    string s;
-    ExprIndStr(const string & s) : s(s) { };
-};
-
 struct ExprPath : Expr
 {
-    string s;
+    std::string s;
     Value v;
-    ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); };
+    ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); };
     COMMON_METHODS
     Value * maybeThunk(EvalState & state, Env & env);
 };
@@ -223,10 +218,25 @@ struct Formal
 
 struct Formals
 {
-    typedef std::list<Formal> Formals_;
+    typedef std::vector<Formal> Formals_;
     Formals_ formals;
-    std::set<Symbol> argNames; // used during parsing
     bool ellipsis;
+
+    bool has(Symbol arg) const {
+        auto it = std::lower_bound(formals.begin(), formals.end(), arg,
+            [] (const Formal & f, const Symbol & sym) { return f.name < sym; });
+        return it != formals.end() && it->name == arg;
+    }
+
+    std::vector<Formal> lexicographicOrder() const
+    {
+        std::vector<Formal> result(formals.begin(), formals.end());
+        std::sort(result.begin(), result.end(),
+            [] (const Formal & a, const Formal & b) {
+                return std::string_view(a.name) < std::string_view(b.name);
+            });
+        return result;
+    }
 };
 
 struct ExprLambda : Expr
@@ -239,14 +249,9 @@ struct ExprLambda : Expr
     ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body)
         : pos(pos), arg(arg), formals(formals), body(body)
     {
-        if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
-            throw ParseError({
-                .msg = hintfmt("duplicate formal function argument '%1%'", arg),
-                .errPos = pos
-            });
     };
     void setName(Symbol & name);
-    string showNamePos() const;
+    std::string showNamePos() const;
     inline bool hasFormals() const { return formals != nullptr; }
     COMMON_METHODS
 };
@@ -332,8 +337,8 @@ struct ExprConcatStrings : Expr
 {
     Pos pos;
     bool forceString;
-    vector<Expr *> * es;
-    ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es)
+    std::vector<std::pair<Pos, Expr *> > * es;
+    ExprConcatStrings(const Pos & pos, bool forceString, std::vector<std::pair<Pos, Expr *> > * es)
         : pos(pos), forceString(forceString), es(es) { };
     COMMON_METHODS
 };
@@ -364,10 +369,21 @@ struct StaticEnv
 
     void sort()
     {
-        std::sort(vars.begin(), vars.end(),
+        std::stable_sort(vars.begin(), vars.end(),
             [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; });
     }
 
+    void deduplicate()
+    {
+        auto it = vars.begin(), jt = it, end = vars.end();
+        while (jt != end) {
+            *it = *jt++;
+            while (jt != end && it->first == jt->first) *it = *jt++;
+            it++;
+        }
+        vars.erase(it, end);
+    }
+
     Vars::const_iterator find(const Symbol & name) const
     {
         Vars::value_type key(name, 0);

src/libexpr/parser.y

@@ -16,6 +16,8 @@
 #ifndef BISON_HEADER
 #define BISON_HEADER
 
+#include <variant>
+
 #include "util.hh"
 
 #include "nixexpr.hh"
@@ -39,8 +41,22 @@ namespace nix {
             { };
     };
 
+    struct ParserFormals {
+        std::vector<Formal> formals;
+        bool ellipsis = false;
+    };
+
 }
 
+// using C a struct allows us to avoid having to define the special
+// members that using string_view here would implicitly delete.
+struct StringToken {
+  const char * p;
+  size_t l;
+  bool hasIndentation;
+  operator std::string_view() const { return {p, l}; }
+};
+
 #define YY_DECL int yylex \
     (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data)
 
@@ -140,21 +156,46 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath,
 }
 
 
-static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
+static Formals * toFormals(ParseData & data, ParserFormals * formals,
+    Pos pos = noPos, Symbol arg = {})
 {
-    if (!formals->argNames.insert(formal.name).second)
+    std::sort(formals->formals.begin(), formals->formals.end(),
+        [] (const auto & a, const auto & b) {
+            return std::tie(a.name, a.pos) < std::tie(b.name, b.pos);
+        });
+
+    std::optional<std::pair<Symbol, Pos>> duplicate;
+    for (size_t i = 0; i + 1 < formals->formals.size(); i++) {
+        if (formals->formals[i].name != formals->formals[i + 1].name)
+            continue;
+        std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos};
+        duplicate = std::min(thisDup, duplicate.value_or(thisDup));
+    }
+    if (duplicate)
         throw ParseError({
-            .msg = hintfmt("duplicate formal function argument '%1%'",
-                formal.name),
+            .msg = hintfmt("duplicate formal function argument '%1%'", duplicate->first),
+            .errPos = duplicate->second
+        });
+
+    Formals result;
+    result.ellipsis = formals->ellipsis;
+    result.formals = std::move(formals->formals);
+
+    if (arg.set() && result.has(arg))
+        throw ParseError({
+            .msg = hintfmt("duplicate formal function argument '%1%'", arg),
             .errPos = pos
         });
-    formals->formals.push_front(formal);
+
+    delete formals;
+    return new Formals(std::move(result));
 }
 
 
-static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Expr *> & es)
+static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols,
+    std::vector<std::pair<Pos, std::variant<Expr *, StringToken> > > & es)
 {
-    if (es.empty()) return new ExprString(symbols.create(""));
+    if (es.empty()) return new ExprString("");
 
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
@@ -162,21 +203,21 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     bool atStartOfLine = true; /* = seen only whitespace in the current line */
     size_t minIndent = 1000000;
     size_t curIndent = 0;
-    for (auto & i : es) {
-        ExprIndStr * e = dynamic_cast<ExprIndStr *>(i);
-        if (!e) {
-            /* Anti-quotations end the current start-of-line whitespace. */
+    for (auto & [i_pos, i] : es) {
+        auto * str = std::get_if<StringToken>(&i);
+        if (!str || !str->hasIndentation) {
+            /* Anti-quotations and escaped characters end the current start-of-line whitespace. */
             if (atStartOfLine) {
                 atStartOfLine = false;
                 if (curIndent < minIndent) minIndent = curIndent;
             }
             continue;
         }
-        for (size_t j = 0; j < e->s.size(); ++j) {
+        for (size_t j = 0; j < str->l; ++j) {
             if (atStartOfLine) {
-                if (e->s[j] == ' ')
+                if (str->p[j] == ' ')
                     curIndent++;
-                else if (e->s[j] == '\n') {
+                else if (str->p[j] == '\n') {
                     /* Empty line, doesn't influence minimum
                        indentation. */
                     curIndent = 0;
@@ -184,7 +225,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
                     atStartOfLine = false;
                     if (curIndent < minIndent) minIndent = curIndent;
                 }
-            } else if (e->s[j] == '\n') {
+            } else if (str->p[j] == '\n') {
                 atStartOfLine = true;
                 curIndent = 0;
             }
@@ -192,53 +233,54 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
     }
 
     /* Strip spaces from each line. */
-    vector<Expr *> * es2 = new vector<Expr *>;
+    std::vector<std::pair<Pos, Expr *> > * es2 = new std::vector<std::pair<Pos, Expr *> >;
     atStartOfLine = true;
     size_t curDropped = 0;
     size_t n = es.size();
-    for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) {
-        ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i);
-        if (!e) {
-            atStartOfLine = false;
-            curDropped = 0;
-            es2->push_back(*i);
-            continue;
-        }
-
-        string s2;
-        for (size_t j = 0; j < e->s.size(); ++j) {
+    auto i = es.begin();
+    const auto trimExpr = [&] (Expr * e) {
+        atStartOfLine = false;
+        curDropped = 0;
+        es2->emplace_back(i->first, e);
+    };
+    const auto trimString = [&] (const StringToken & t) {
+        std::string s2;
+        for (size_t j = 0; j < t.l; ++j) {
             if (atStartOfLine) {
-                if (e->s[j] == ' ') {
+                if (t.p[j] == ' ') {
                     if (curDropped++ >= minIndent)
-                        s2 += e->s[j];
+                        s2 += t.p[j];
                 }
-                else if (e->s[j] == '\n') {
+                else if (t.p[j] == '\n') {
                     curDropped = 0;
-                    s2 += e->s[j];
+                    s2 += t.p[j];
                 } else {
                     atStartOfLine = false;
                     curDropped = 0;
-                    s2 += e->s[j];
+                    s2 += t.p[j];
                 }
             } else {
-                s2 += e->s[j];
-                if (e->s[j] == '\n') atStartOfLine = true;
+                s2 += t.p[j];
+                if (t.p[j] == '\n') atStartOfLine = true;
             }
         }
 
         /* Remove the last line if it is empty and consists only of
            spaces. */
         if (n == 1) {
-            string::size_type p = s2.find_last_of('\n');
-            if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos)
-                s2 = string(s2, 0, p + 1);
+            std::string::size_type p = s2.find_last_of('\n');
+            if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos)
+                s2 = std::string(s2, 0, p + 1);
         }
 
-        es2->push_back(new ExprString(symbols.create(s2)));
+        es2->emplace_back(i->first, new ExprString(s2));
+    };
+    for (; i != es.end(); ++i, --n) {
+        std::visit(overloaded { trimExpr, trimString }, i->second);
     }
 
     /* If this is a single string, then don't do a concatenation. */
-    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2);
+    return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2);
 }
 
 
@@ -269,15 +311,17 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
   nix::Expr * e;
   nix::ExprList * list;
   nix::ExprAttrs * attrs;
-  nix::Formals * formals;
+  nix::ParserFormals * formals;
   nix::Formal * formal;
   nix::NixInt n;
   nix::NixFloat nf;
-  const char * id; // !!! -> Symbol
-  char * path;
-  char * uri;
+  StringToken id; // !!! -> Symbol
+  StringToken path;
+  StringToken uri;
+  StringToken str;
   std::vector<nix::AttrName> * attrNames;
-  std::vector<nix::Expr *> * string_parts;
+  std::vector<std::pair<nix::Pos, nix::Expr *> > * string_parts;
+  std::vector<std::pair<nix::Pos, std::variant<nix::Expr *, StringToken> > > * ind_string_parts;
 }
 
 %type <e> start expr expr_function expr_if expr_op
@@ -287,11 +331,12 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 %type <formals> formals
 %type <formal> formal
 %type <attrNames> attrs attrpath
-%type <string_parts> string_parts_interpolated ind_string_parts
+%type <string_parts> string_parts_interpolated
+%type <ind_string_parts> ind_string_parts
 %type <e> path_start string_parts string_attr
 %type <id> attr
 %token <id> ID ATTRPATH
-%token <e> STR IND_STR
+%token <str> STR IND_STR
 %token <n> INT
 %token <nf> FLOAT
 %token <path> PATH HPATH SPATH PATH_END
@@ -324,11 +369,17 @@ expr_function
   : ID ':' expr_function
     { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
   | '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); }
+    { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), toFormals(*data, $2), $5); }
   | '{' formals '}' '@' ID ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); }
+    {
+      Symbol arg = data->symbols.create($5);
+      $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $2, CUR_POS, arg), $7);
+    }
   | ID '@' '{' formals '}' ':' expr_function
-    { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); }
+    {
+      Symbol arg = data->symbols.create($1);
+      $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $4, CUR_POS, arg), $7);
+    }
   | ASSERT expr ';' expr_function
     { $$ = new ExprAssert(CUR_POS, $2, $4); }
   | WITH expr ';' expr_function
@@ -364,7 +415,7 @@ expr_op
   | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); }
   | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); }
   | expr_op '+' expr_op
-    { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); }
+    { $$ = new ExprConcatStrings(CUR_POS, false, new std::vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); }
   | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); }
   | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); }
   | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); }
@@ -397,7 +448,8 @@ expr_select
 
 expr_simple
   : ID {
-      if (strcmp($1, "__curPos") == 0)
+      std::string_view s = "__curPos";
+      if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0)
           $$ = new ExprPos(CUR_POS);
       else
           $$ = new ExprVar(CUR_POS, data->symbols.create($1));
@@ -410,15 +462,15 @@ expr_simple
   }
   | path_start PATH_END { $$ = $1; }
   | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), $1);
+      $2->insert($2->begin(), {makeCurPos(@1, data), $1});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
-      string path($1 + 1, strlen($1) - 2);
+      std::string path($1.p + 1, $1.l - 2);
       $$ = new ExprCall(CUR_POS,
           new ExprVar(data->symbols.create("__findFile")),
           {new ExprVar(data->symbols.create("__nixPath")),
-           new ExprString(data->symbols.create(path))});
+           new ExprString(path)});
   }
   | URI {
       static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals);
@@ -427,7 +479,7 @@ expr_simple
               .msg = hintfmt("URL literals are disabled"),
               .errPos = CUR_POS
           });
-      $$ = new ExprString(data->symbols.create($1));
+      $$ = new ExprString(std::string($1));
   }
   | '(' expr ')' { $$ = $2; }
   /* Let expressions `let {..., body = ...}' are just desugared
@@ -442,40 +494,41 @@ expr_simple
   ;
 
 string_parts
-  : STR
+  : STR { $$ = new ExprString(std::string($1)); }
   | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); }
-  | { $$ = new ExprString(data->symbols.create("")); }
+  | { $$ = new ExprString(""); }
   ;
 
 string_parts_interpolated
-  : string_parts_interpolated STR { $$ = $1; $1->push_back($2); }
-  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
-  | DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($2); }
+  : string_parts_interpolated STR
+  { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); }
+  | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); }
   | STR DOLLAR_CURLY expr '}' {
-      $$ = new vector<Expr *>;
-      $$->push_back($1);
-      $$->push_back($3);
+      $$ = new std::vector<std::pair<Pos, Expr *> >;
+      $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1)));
+      $$->emplace_back(makeCurPos(@2, data), $3);
     }
   ;
 
 path_start
   : PATH {
-    Path path(absPath($1, data->basePath));
+    Path path(absPath({$1.p, $1.l}, data->basePath));
     /* add back in the trailing '/' to the first segment */
-    if ($1[strlen($1)-1] == '/' && strlen($1) > 1)
+    if ($1.p[$1.l-1] == '/' && $1.l > 1)
       path += "/";
     $$ = new ExprPath(path);
   }
   | HPATH {
-    Path path(getHome() + string($1 + 1));
+    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
     $$ = new ExprPath(path);
   }
   ;
 
 ind_string_parts
-  : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); }
-  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); }
-  | { $$ = new vector<Expr *>; }
+  : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); }
+  | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); }
+  | { $$ = new std::vector<std::pair<Pos, std::variant<Expr *, StringToken> > >; }
   ;
 
 binds
@@ -507,7 +560,7 @@ attrs
     { $$ = $1;
       ExprString * str = dynamic_cast<ExprString *>($2);
       if (str) {
-          $$->push_back(AttrName(str->s));
+          $$->push_back(AttrName(data->symbols.create(str->s)));
           delete str;
       } else
           throw ParseError({
@@ -524,17 +577,17 @@ attrpath
     { $$ = $1;
       ExprString * str = dynamic_cast<ExprString *>($3);
       if (str) {
-          $$->push_back(AttrName(str->s));
+          $$->push_back(AttrName(data->symbols.create(str->s)));
           delete str;
       } else
           $$->push_back(AttrName($3));
     }
-  | attr { $$ = new vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); }
+  | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); }
   | string_attr
-    { $$ = new vector<AttrName>;
+    { $$ = new std::vector<AttrName>;
       ExprString *str = dynamic_cast<ExprString *>($1);
       if (str) {
-          $$->push_back(AttrName(str->s));
+          $$->push_back(AttrName(data->symbols.create(str->s)));
           delete str;
       } else
           $$->push_back(AttrName($1));
@@ -543,7 +596,7 @@ attrpath
 
 attr
   : ID { $$ = $1; }
-  | OR_KW { $$ = "or"; }
+  | OR_KW { $$ = {"or", 2}; }
   ;
 
 string_attr
@@ -558,13 +611,13 @@ expr_list
 
 formals
   : formal ',' formals
-    { $$ = $3; addFormal(CUR_POS, $$, *$1); }
+    { $$ = $3; $$->formals.push_back(*$1); }
   | formal
-    { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; }
+    { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; }
   |
-    { $$ = new Formals; $$->ellipsis = false; }
+    { $$ = new ParserFormals; $$->ellipsis = false; }
   | ELLIPSIS
-    { $$ = new Formals; $$->ellipsis = true; }
+    { $$ = new ParserFormals; $$->ellipsis = true; }
   ;
 
 formal
@@ -589,8 +642,8 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text, FileOrigin origin,
-    const Path & path, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parse(char * text, size_t length, FileOrigin origin,
+    const PathView path, const PathView basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
@@ -609,7 +662,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
     data.basePath = basePath;
 
     yylex_init(&scanner);
-    yy_scan_string(text, scanner);
+    yy_scan_buffer(text, length, scanner);
     int res = yyparse(scanner, &data);
     yylex_destroy(scanner);
 
@@ -655,63 +708,70 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
+    auto buffer = readFile(path);
+    // readFile should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), foFile, path, dirOf(path), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
+Expr * EvalState::parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), foString, "", basePath, staticEnv);
+    s.append("\0\0", 2);
+    return parse(s.data(), s.size(), foString, "", basePath, staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
+Expr * EvalState::parseExprFromString(std::string s, const Path & basePath)
 {
-    return parseExprFromString(s, basePath, staticBaseEnv);
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
 }
 
 
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
+    auto buffer = drainFD(0);
+    // drainFD should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 
-void EvalState::addToSearchPath(const string & s)
+void EvalState::addToSearchPath(const std::string & s)
 {
     size_t pos = s.find('=');
-    string prefix;
+    std::string prefix;
     Path path;
-    if (pos == string::npos) {
+    if (pos == std::string::npos) {
         path = s;
     } else {
-        prefix = string(s, 0, pos);
-        path = string(s, pos + 1);
+        prefix = std::string(s, 0, pos);
+        path = std::string(s, pos + 1);
     }
 
     searchPath.emplace_back(prefix, path);
 }
 
 
-Path EvalState::findFile(const string & path)
+Path EvalState::findFile(const std::string_view path)
 {
     return findFile(searchPath, path);
 }
 
 
-Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos)
+Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, const Pos & pos)
 {
     for (auto & i : searchPath) {
         std::string suffix;
         if (i.first.empty())
-            suffix = "/" + path;
+            suffix = concatStrings("/", path);
         else {
             auto s = i.first.size();
             if (path.compare(0, s, i.first) != 0 ||
                 (path.size() > s && path[s] != '/'))
                 continue;
-            suffix = path.size() == s ? "" : "/" + string(path, s);
+            suffix = path.size() == s ? "" : concatStrings("/", path.substr(s));
         }
         auto r = resolveSearchPathElem(i);
         if (!r.first) continue;
@@ -720,7 +780,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
     }
 
     if (hasPrefix(path, "nix/"))
-        return corepkgsPrefix + path.substr(4);
+        return concatStrings(corepkgsPrefix, path.substr(4));
 
     throw ThrownError({
         .msg = hintfmt(evalSettings.pureEval