Merge pull request #12319 from NixOS/mergify/bp/2.25-maintenance/pr-12315

tests: Wait for network *online* targets and/or multi-user targets (more) (backport #12315)

.github/workflows/ci.yml

@@ -7,14 +7,28 @@ on:
 permissions: read-all
 
 jobs:
+  eval:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v30
+    - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json
 
   tests:
-    needs: [check_secrets]
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        include:
-    runs-on: ${{ matrix.os }}
+          - scenario: on ubuntu
+            runs-on: ubuntu-24.04
+            os: linux
+          - scenario: on macos
+            runs-on: macos-14
+            os: darwin
+    name: tests ${{ matrix.scenario }}
+    runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
     steps:
     - uses: actions/checkout@v4
@@ -23,103 +37,54 @@ jobs:
     - uses: cachix/install-nix-action@v30
       with:
         # The sandbox would otherwise be disabled by default on Darwin
-        extra_nix_config: "sandbox = true"
+        extra_nix_config: |
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+          sandbox = true
-    - uses: cachix/cachix-action@v15
+          max-jobs = 1
-      if: needs.check_secrets.outputs.cachix == 'true'
+    - uses: DeterminateSystems/magic-nix-cache-action@main
+    # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user:
+    # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces
+    - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+      if: matrix.os == 'linux'
+    - run: scripts/build-checks
+    - run: scripts/prepare-installer-for-github-actions
+    - name: Upload installer tarball
+      uses: actions/upload-artifact@v4
       with:
-        name: '${{ env.CACHIX_NAME }}'
+        name: installer-${{matrix.os}}
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+        path: out/*
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    - if: matrix.os == 'ubuntu-latest'
-      run: |
-        free -h
-        swapon --show
-        swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}')
-        echo "Found swap: $swap"
-        sudo swapoff $swap
-        # resize it (fallocate)
-        sudo fallocate -l 10G $swap
-        sudo mkswap $swap
-        sudo swapon $swap
-        free -h
-        (
-          while sleep 60; do
-            free -h
-          done
-        ) &
-    - run: nix --experimental-features 'nix-command flakes' flake check -L
-    - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json
-
-  # Steps to test CI automation in your own fork.
-  # Cachix:
-  # 1. Sign-up for https://www.cachix.org/
-  # 2. Create a cache for $githubuser-nix-install-tests
-  # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN
-  # Dockerhub:
-  # 1. Sign-up for https://hub.docker.com/
-  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
-  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
-  check_secrets:
-    permissions:
-      contents: none
-    name: Check Cachix and Docker secrets present for installer tests
-    runs-on: ubuntu-latest
-    outputs:
-      cachix: ${{ steps.secret.outputs.cachix }}
-      docker: ${{ steps.secret.outputs.docker }}
-    steps:
-      - name: Check for secrets
-        id: secret
-        env:
-          _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
-          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}"
-          echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
-
-  installer:
-    needs: [tests, check_secrets]
-    if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
-    runs-on: ubuntu-latest
-    outputs:
-      installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v30
-      with:
-        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
-    - uses: cachix/cachix-action@v15
-      with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-        cachixArgs: '-v'
-    - id: prepare-installer
-      run: scripts/prepare-installer-for-github-actions
 
   installer_test:
-    needs: [installer, check_secrets]
+    needs: [tests]
-    if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        include:
-    runs-on: ${{ matrix.os }}
+          - scenario: on ubuntu
+            runs-on: ubuntu-24.04
+            os: linux
+          - scenario: on macos
+            runs-on: macos-14
+            os: darwin
+    name: installer test ${{ matrix.scenario }}
+    runs-on: ${{ matrix.runs-on }}
     steps:
     - uses: actions/checkout@v4
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - name: Download installer tarball
+      uses: actions/download-artifact@v4
+      with:
+        name: installer-${{matrix.os}}
+        path: out
+    - name: Serving installer
+      id: serving_installer
+      run: ./scripts/serve-installer-for-github-actions
     - uses: cachix/install-nix-action@v30
       with:
-        install_url: '${{needs.installer.outputs.installerURL}}'
+        install_url: 'http://localhost:8126/install'
-        install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
+        install_options: "--tarball-url-prefix http://localhost:8126/"
     - run: sudo apt install fish zsh
-      if: matrix.os == 'ubuntu-latest'
+      if: matrix.os == 'linux'
     - run: brew install fish
-      if: matrix.os == 'macos-latest'
+      if: matrix.os == 'darwin'
     - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval"
     - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval"
     - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval"
@@ -127,32 +92,50 @@ jobs:
     - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
     - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
 
+  # Steps to test CI automation in your own fork.
+  # 1. Sign-up for https://hub.docker.com/
+  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
+  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
+  check_secrets:
+    permissions:
+      contents: none
+    name: Check Docker secrets present for installer tests
+    runs-on: ubuntu-24.04
+    outputs:
+      docker: ${{ steps.secret.outputs.docker }}
+    steps:
+      - name: Check for secrets
+        id: secret
+        env:
+          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
+        run: |
+          echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
+
   docker_push_image:
-    needs: [check_secrets, tests]
+    needs: [tests, vm_tests, check_secrets]
     permissions:
       contents: read
       packages: write
     if: >-
+      needs.check_secrets.outputs.docker == 'true' &&
       github.event_name == 'push' &&
-      github.ref_name == 'master' &&
+      github.ref_name == 'master'
-      needs.check_secrets.outputs.cachix == 'true' &&
+    runs-on: ubuntu-24.04
-      needs.check_secrets.outputs.docker == 'true'
-    runs-on: ubuntu-latest
     steps:
+    - name: Check for secrets
+      id: secret
+      env:
+        _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
+      run: |
+        echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - uses: DeterminateSystems/magic-nix-cache-action@main
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v15
-      if: needs.check_secrets.outputs.cachix == 'true'
-      with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L
     - run: docker load -i ./result/image.tar.gz
     - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
@@ -189,7 +172,7 @@ jobs:
         docker push $IMAGE_ID:master
 
   vm_tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
@@ -198,7 +181,7 @@ jobs:
 
   flake_regressions:
     needs: vm_tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout nix
         uses: actions/checkout@v4

.github/workflows/labels.yml

@@ -15,7 +15,7 @@ permissions:
 
 jobs:
   labels:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: github.repository_owner == 'NixOS'
     steps:
     - uses: actions/labeler@v5

.mergify.yml

@@ -5,8 +5,10 @@ queue_rules:
       - check-success=installer
       - check-success=installer_test (macos-latest)
       - check-success=installer_test (ubuntu-latest)
-      - check-success=tests (macos-latest)
+      - check-success=tests on macos
-      - check-success=tests (ubuntu-latest)
+      - check-success=tests on ubuntu
+      - check-success=installer test on macos
+      - check-success=installer test on ubuntu
       - check-success=vm_tests
     merge_method: rebase
     batch_size: 5

.version

@@ -1 +1 @@
-2.25.0
+2.25.5

doc/manual/source/command-ref/nix-collect-garbage.md

@@ -62,6 +62,15 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto
   This is the equivalent of invoking [`nix-env --delete-generations <period>`](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile.
   See the documentation of that command for additional information about the *period* argument.
 
+  - <span id="opt-max-freed">[`--max-freed`](#opt-max-freed)</span> *bytes*
+
+<!-- duplication from https://github.com/NixOS/nix/blob/442a2623e48357ff72c77bb11cf2cf06d94d2f90/doc/manual/source/command-ref/nix-store/gc.md?plain=1#L39-L44 -->
+
+  Keep deleting paths until at least *bytes* bytes have been deleted,
+  then stop. The argument *bytes* can be followed by the
+  multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB
+  or TiB units.
+  
 {{#include ./opt-common.md}}
 
 {{#include ./env-common.md}}

doc/manual/source/command-ref/nix-store/add-fixed.md

@@ -21,6 +21,9 @@ This operation has the following options:
   Use recursive instead of flat hashing mode, used when adding
   directories to the store.
 
+  *paths* that refer to symlinks are not dereferenced, but added to the store
+  as symlinks with the same target.
+
 {{#include ./opt-common.md}}
 
 {{#include ../opt-common.md}}

doc/manual/source/command-ref/nix-store/add.md

@@ -11,6 +11,9 @@
 The operation `--add` adds the specified paths to the Nix store. It
 prints the resulting paths in the Nix store on standard output.
 
+*paths* that refer to symlinks are not dereferenced, but added to the store
+as symlinks with the same target.
+
 {{#include ./opt-common.md}}
 
 {{#include ../opt-common.md}}

doc/manual/source/development/testing.md

@@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to
   - `armv7l-linux`
   - `x86_64-darwin`
 
-- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
+- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command.
 
 ### One-time setup
 

docker.nix

@@ -241,14 +241,14 @@ let
       mkdir -p $out/nix/var/nix/profiles/per-user/root
 
       ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
-      ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+      ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
       ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
 
       ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
-      ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
+      ln -s /nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
 
       mkdir -p $out/root/.nix-defexpr
-      ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
+      ln -s /nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
       echo "${channelURL} ${channelName}" > $out/root/.nix-channels
 
       mkdir -p $out/bin $out/usr/bin

flake.nix

@@ -24,7 +24,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux32BitSystems = [ "i686-linux" ];
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
@@ -168,7 +168,7 @@
       };
 
       checks = forAllSystems (system: {
-        binaryTarball = self.hydraJobs.binaryTarball.${system};
+        installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
         rl-next =
@@ -220,7 +220,9 @@
           # for which we don't apply the full build matrix such as cross or static.
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
-          default = self.packages.${system}.nix-ng;
+          default = self.packages.${system}.nix;
+          binaryTarball = self.hydraJobs.binaryTarball.${system};
+          installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
           nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual;
           nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
           nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;

packaging/everything.nix

@@ -95,6 +95,8 @@
     nix-functional-tests
   ];
   passthru = prevAttrs.passthru // {
+    inherit (nix-cli) version;
+
     /**
       These are the libraries that are part of the Nix project. They are used
       by the Nix CLI and other tools.

packaging/hydra.nix

@@ -128,15 +128,10 @@ in
     self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
     self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
   ];
-  installerScriptForGHA = installScriptFor [
+
-    # Native
+  installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix {
-    self.hydraJobs.binaryTarball."x86_64-linux"
+    tarballs = [ self.hydraJobs.binaryTarball.${system} ];
-    self.hydraJobs.binaryTarball."aarch64-darwin"
+  });
-    # Cross
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
-  ];
 
   # docker image with Nix inside
   dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);

scripts/binary-tarball.nix

@@ -65,7 +65,7 @@ runCommand "nix-binary-tarball-${version}" env ''
   fn=$out/$dir.tar.xz
   mkdir -p $out/nix-support
   echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-  tar cvfJ $fn \
+  tar cfJ $fn \
     --owner=0 --group=0 --mode=u+rw,uga+r \
     --mtime='1970-01-01' \
     --absolute-names \

scripts/build-checks

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+system=$(nix eval --raw --impure --expr builtins.currentSystem)
+nix eval --json ".#checks.$system" --apply builtins.attrNames | \
+  jq -r '.[]' | \
+  xargs -P0 -I '{}' sh -c "nix build -L .#checks.$system.{} || { echo 'FAILED: \033[0;31mnix build -L .#checks.$system.{}\\033[0m'; kill 0; }"

scripts/create-darwin-volume.sh

@@ -463,7 +463,7 @@ EOF
 
     EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" <<EOF
 :a
-UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners
+UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,nosuid,noatime,owners
 .
 :x
 EOF

scripts/install-multi-user.sh

@@ -557,7 +557,7 @@ create_build_user_for_core() {
         if [ "$actual_uid" != "$uid" ]; then
             failure <<EOF
 It seems the build user $username already exists, but with the UID
-with the UID '$actual_uid'. This script can't really handle that right
+'$actual_uid'. This script can't really handle that right
 now, so I'm going to give up.
 
 If you already created the users and you know they start from

scripts/prepare-installer-for-github-actions

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
-script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
+nix build -L ".#installerScriptForGHA" ".#binaryTarball"
-installerHash=$(echo "$script" | cut -b12-43 -)
 
-installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
+mkdir -p out
-
+cp ./result/install "out/install"
-echo "::set-output name=installerURL::$installerURL"
+name="$(basename "$(realpath ./result-1)")"
+# everything before the first dash
+cp -r ./result-1 "out/${name%%-*}"

scripts/serve-installer-for-github-actions

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+if [[ ! -d out ]]; then
+  echo "run prepare-installer-for-github-actions first"
+  exit 1
+fi
+cd out
+PORT=${PORT:-8126}
+nohup python -m http.server "$PORT" >/dev/null 2>&1 &
+pid=$!
+
+while ! curl -s "http://localhost:$PORT"; do
+  sleep 1
+  if ! kill -0 $pid; then
+    echo "Failed to start http server"
+    exit 1
+  fi
+done
+
+echo 'To install nix, run the following command:'
+echo "sh <(curl http://localhost:$PORT/install) --tarball-url-prefix http://localhost:$PORT"

src/libcmd/installable-derived-path.cc

@@ -32,16 +32,6 @@ InstallableDerivedPath InstallableDerivedPath::parse(
         // store path.
         [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
             auto storePath = store->followLinksToStorePath(prefix);
-            // Remove this prior to stabilizing the new CLI.
-            if (storePath.isDerivation()) {
-                auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(storePath),
-                    .outputs = OutputsSpec::All { },
-                };
-                warn(
-                    "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'",
-                    oldDerivedPath.to_string(*store));
-            };
             return DerivedPath::Opaque {
                 .path = std::move(storePath),
             };

src/libcmd/markdown.cc

@@ -16,13 +16,25 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown)
 {
     int windowWidth = getWindowSize().second;
 
-    struct lowdown_opts opts
+#if HAVE_LOWDOWN_1_4
-    {
+    struct lowdown_opts_term opts_term {
-        .type = LOWDOWN_TERM,
-        .maxdepth = 20,
         .cols = (size_t) std::max(windowWidth - 5, 60),
         .hmargin = 0,
         .vmargin = 0,
+    };
+#endif
+    struct lowdown_opts opts
+    {
+        .type = LOWDOWN_TERM,
+#if HAVE_LOWDOWN_1_4
+        .term = opts_term,
+#endif
+        .maxdepth = 20,
+#if !HAVE_LOWDOWN_1_4
+        .cols = (size_t) std::max(windowWidth - 5, 60),
+        .hmargin = 0,
+        .vmargin = 0,
+#endif
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
         .oflags = LOWDOWN_TERM_NOLINK,
     };

src/libcmd/meson.build

@@ -38,6 +38,8 @@ deps_public += nlohmann_json
 lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown'))
 deps_private += lowdown
 configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
+# The API changed slightly around terminal initialization.
+configdata.set('HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int())
 
 readline_flavor = get_option('readline-flavor')
 if readline_flavor == 'editline'

src/libexpr-tests/trivial.cc

@@ -177,6 +177,57 @@ namespace nix {
             )
     );
 
+// The following macros ultimately define 48 tests (16 variations on three
+// templates). Each template tests an expression that can be written in 2^4
+// different ways, by making four choices about whether to write a particular
+// attribute path segment as `x.y = ...;` (collapsed) or `x = { y = ...; };`
+// (expanded).
+//
+// The nestedAttrsetMergeXXXX tests check that the expression
+// `{ a.b.c = 1; a.b.d = 2; }` has the same value regardless of how it is
+// expanded. (That exact expression is exercised in test
+// nestedAttrsetMerge0000, because it is fully collapsed. The test
+// nestedAttrsetMerge1001 would instead examine
+// `{ a = { b.c = 1; }; a.b = { d = 2; }; }`.)
+//
+// The nestedAttrsetMergeDupXXXX tests check that the expression
+// `{ a.b.c = 1; a.b.c = 2; }` throws a duplicate attribute error, again
+// regardless of how it is expanded.
+//
+// The nestedAttrsetMergeLetXXXX tests check that the expression
+// `let a.b.c = 1; a.b.d = 2; in a` has the same value regardless of how it is
+// expanded.
+#define X_EXPAND_IF0(k, v) k "." v
+#define X_EXPAND_IF1(k, v) k " = { " v " };"
+#define X4(w, x, y, z) \
+    TEST_F(TrivialExpressionTest, nestedAttrsetMerge##w##x##y##z) { \
+        auto v = eval("{ a.b = { c = 1; d = 2; }; } == { " \
+                X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
+                X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " }"); \
+        ASSERT_THAT(v, IsTrue()); \
+    }; \
+    TEST_F(TrivialExpressionTest, nestedAttrsetMergeDup##w##x##y##z) { \
+        ASSERT_THROW(eval("{ " \
+                X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
+                X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "c = 2;")) " }"), Error); \
+    }; \
+    TEST_F(TrivialExpressionTest, nestedAttrsetMergeLet##w##x##y##z) { \
+        auto v = eval("{ b = { c = 1; d = 2; }; } == (let " \
+                X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \
+                X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " in a)"); \
+        ASSERT_THAT(v, IsTrue()); \
+    };
+#define X3(...) X4(__VA_ARGS__, 0) X4(__VA_ARGS__, 1)
+#define X2(...) X3(__VA_ARGS__, 0) X3(__VA_ARGS__, 1)
+#define X1(...) X2(__VA_ARGS__, 0) X2(__VA_ARGS__, 1)
+    X1(0) X1(1)
+#undef X_EXPAND_IF0
+#undef X_EXPAND_IF1
+#undef X1
+#undef X2
+#undef X3
+#undef X4
+
     TEST_F(TrivialExpressionTest, functor) {
         auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5");
         ASSERT_THAT(v, IsIntEq(15));

src/libexpr/eval.cc

@@ -396,7 +396,7 @@ void EvalState::checkURI(const std::string & uri)
 
     /* If the URI is a path, then check it against allowedPaths as
        well. */
-    if (hasPrefix(uri, "/")) {
+    if (isAbsolute(uri)) {
         if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
             rootFS2->checkAccess(CanonPath(uri));
         return;

src/libexpr/parser-state.hh

@@ -88,6 +88,7 @@ struct ParserState
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
     void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
+    void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def);
     Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
     Expr * stripIndentation(const PosIdx pos,
         std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
@@ -120,64 +121,29 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const
     // Checking attrPath validity.
     // ===========================
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
+        ExprAttrs * nested;
         if (i->symbol) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
-                if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
+                nested = dynamic_cast<ExprAttrs *>(j->second.e);
-                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
+                if (!nested) {
-                    if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
+                    attrPath.erase(i + 1, attrPath.end());
-                    attrs = attrs2;
-                } else
                     dupAttr(attrPath, pos, j->second.pos);
+                }
             } else {
-                ExprAttrs * nested = new ExprAttrs;
+                nested = new ExprAttrs;
                 attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
-                attrs = nested;
             }
         } else {
-            ExprAttrs *nested = new ExprAttrs;
+            nested = new ExprAttrs;
             attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
-            attrs = nested;
         }
+        attrs = nested;
     }
     // Expr insertion.
     // ==========================
     if (i->symbol) {
-        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
+        addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
-        if (j != attrs->attrs.end()) {
-            // This attr path is already defined. However, if both
-            // e and the expr pointed by the attr path are two attribute sets,
-            // we want to merge them.
-            // Otherwise, throw an error.
-            auto ae = dynamic_cast<ExprAttrs *>(e);
-            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
-            if (jAttrs && ae) {
-                if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                    jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-                for (auto & ad : ae->attrs) {
-                    auto j2 = jAttrs->attrs.find(ad.first);
-                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
-                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs.emplace(ad.first, ad.second);
-                    if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
-                        auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
-                        auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
-                        from.displ += jAttrs->inheritFromExprs->size();
-                    }
-                }
-                jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
-                if (ae->inheritFromExprs) {
-                    jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
-                        ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
-                }
-            } else {
-                dupAttr(attrPath, pos, j->second.pos);
-            }
-        } else {
-            // This attr path is not defined. Let's create it.
-            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
-            e->setName(i->symbol);
-        }
     } else {
         attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
     }
@@ -189,6 +155,60 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const
     }
 }
 
+/**
+ * Precondition: attrPath is used for error messages and should already contain
+ * symbol as its last element.
+ */
+inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
+{
+    ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
+    if (j != attrs->attrs.end()) {
+        // This attr path is already defined. However, if both
+        // e and the expr pointed by the attr path are two attribute sets,
+        // we want to merge them.
+        // Otherwise, throw an error.
+        auto ae = dynamic_cast<ExprAttrs *>(def.e);
+        auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+
+        // N.B. In a world in which we are less bound by our past mistakes, we
+        // would also test that jAttrs and ae are not recursive. The effect of
+        // not doing so is that any `rec` marker on ae is discarded, and any
+        // `rec` marker on jAttrs will apply to the attributes in ae.
+        // See https://github.com/NixOS/nix/issues/9020.
+        if (jAttrs && ae) {
+            if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
+                jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+            for (auto & ad : ae->attrs) {
+                if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
+                    auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
+                    auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
+                    from.displ += jAttrs->inheritFromExprs->size();
+                }
+                attrPath.emplace_back(AttrName(ad.first));
+                addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
+                attrPath.pop_back();
+            }
+            ae->attrs.clear();
+            jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(),
+                std::make_move_iterator(ae->dynamicAttrs.begin()),
+                std::make_move_iterator(ae->dynamicAttrs.end()));
+            ae->dynamicAttrs.clear();
+            if (ae->inheritFromExprs) {
+                jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
+                    std::make_move_iterator(ae->inheritFromExprs->begin()),
+                    std::make_move_iterator(ae->inheritFromExprs->end()));
+                ae->inheritFromExprs = nullptr;
+            }
+        } else {
+            dupAttr(attrPath, def.pos, j->second.pos);
+        }
+    } else {
+        // This attr path is not defined. Let's create it.
+        attrs->attrs.emplace(symbol, def);
+        def.e->setName(symbol);
+    }
+}
+
 inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)
 {
     std::sort(formals->formals.begin(), formals->formals.end(),

src/libexpr/primops.cc

@@ -4061,7 +4061,7 @@ static RegisterPrimOp primop_toString({
 });
 
 /* `substring start len str' returns the substring of `str' starting
-   at character position `min(start, stringLength str)' inclusive and
+   at byte position `min(start, stringLength str)' inclusive and
    ending at `min(start + len, stringLength str)'.  `start' must be
    non-negative. */
 static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -4100,7 +4100,7 @@ static RegisterPrimOp primop_substring({
     .name = "__substring",
     .args = {"start", "len", "s"},
     .doc = R"(
-      Return the substring of *s* from character position *start*
+      Return the substring of *s* from byte position *start*
       (zero-based) up to but not including *start + len*. If *start* is
       greater than the length of the string, an empty string is returned.
       If *start + len* lies beyond the end of the string or *len* is `-1`,

src/libexpr/value-to-json.cc

@@ -108,7 +108,11 @@ json printValueAsJSON(EvalState & state, bool strict,
 void printValueAsJSON(EvalState & state, bool strict,
     Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore)
 {
-    str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
+    try {
+        str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
+    } catch (nlohmann::json::exception & e) {
+        throw JSONSerializationError("JSON serialization error: %s", e.what());
+    }
 }
 
 json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,

src/libexpr/value-to-json.hh

@@ -16,4 +16,7 @@ nlohmann::json printValueAsJSON(EvalState & state, bool strict,
 void printValueAsJSON(EvalState & state, bool strict,
     Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true);
 
+
+MakeError(JSONSerializationError, Error);
+
 }

src/libfetchers/cache.cc

@@ -36,7 +36,7 @@ struct CacheImpl : Cache
     {
         auto state(_state.lock());
 
-        auto dbPath = getCacheDir() + "/fetcher-cache-v2.sqlite";
+        auto dbPath = getCacheDir() + "/fetcher-cache-v3.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);

src/libfetchers/fetchers.cc

@@ -66,7 +66,7 @@ Input Input::fromURL(
         }
     }
 
-    throw Error("input '%s' is unsupported", url.url);
+    throw Error("input '%s' is unsupported", url);
 }
 
 Input Input::fromAttrs(const Settings & settings, Attrs && attrs)

src/libfetchers/git.cc

@@ -425,7 +425,16 @@ struct GitInputScheme : InputScheme
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
         repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
-        repoInfo.url = repoInfo.isLocal ? url.path : url.base;
+        //
+        // FIXME: here we turn a possibly relative path into an absolute path.
+        // This allows relative git flake inputs to be resolved against the
+        // **current working directory** (as in POSIX), which tends to work out
+        // ok in the context of flakes, but is the wrong behavior,
+        // as it should resolve against the flake.nix base directory instead.
+        //
+        // See: https://discourse.nixos.org/t/57783 and #9708
+        //
+        repoInfo.url = repoInfo.isLocal ? std::filesystem::absolute(url.path).string() : url.to_string();
 
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.

src/libfetchers/github.cc

@@ -50,7 +50,7 @@ struct GitArchiveInputScheme : InputScheme
             else if (std::regex_match(path[2], refRegex))
                 ref = path[2];
             else
-                throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+                throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url, path[2]);
         } else if (size > 3) {
             std::string rs;
             for (auto i = std::next(path.begin(), 2); i != path.end(); i++) {
@@ -63,34 +63,34 @@ struct GitArchiveInputScheme : InputScheme
             if (std::regex_match(rs, refRegex)) {
                 ref = rs;
             } else {
-                throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs);
+                throw BadURL("in URL '%s', '%s' is not a branch/tag name", url, rs);
             }
         } else if (size < 2)
-            throw BadURL("URL '%s' is invalid", url.url);
+            throw BadURL("URL '%s' is invalid", url);
 
         for (auto &[name, value] : url.query) {
             if (name == "rev") {
                 if (rev)
-                    throw BadURL("URL '%s' contains multiple commit hashes", url.url);
+                    throw BadURL("URL '%s' contains multiple commit hashes", url);
                 rev = Hash::parseAny(value, HashAlgorithm::SHA1);
             }
             else if (name == "ref") {
                 if (!std::regex_match(value, refRegex))
-                    throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
+                    throw BadURL("URL '%s' contains an invalid branch/tag name", url);
                 if (ref)
-                    throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
+                    throw BadURL("URL '%s' contains multiple branch/tag names", url);
                 ref = value;
             }
             else if (name == "host") {
                 if (!std::regex_match(value, hostRegex))
-                    throw BadURL("URL '%s' contains an invalid instance host", url.url);
+                    throw BadURL("URL '%s' contains an invalid instance host", url);
                 host_url = value;
             }
             // FIXME: barf on unsupported attributes
         }
 
         if (ref && rev)
-            throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
+            throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev());
 
         Input input{settings};
         input.attrs.insert_or_assign("type", std::string { schemeName() });

src/libfetchers/indirect.cc

@@ -26,16 +26,16 @@ struct IndirectInputScheme : InputScheme
             else if (std::regex_match(path[1], refRegex))
                 ref = path[1];
             else
-                throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url, path[1]);
         } else if (path.size() == 3) {
             if (!std::regex_match(path[1], refRegex))
-                throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
+                throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url, path[1]);
             ref = path[1];
             if (!std::regex_match(path[2], revRegex))
-                throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
+                throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]);
             rev = Hash::parseAny(path[2], HashAlgorithm::SHA1);
         } else
-            throw BadURL("GitHub URL '%s' is invalid", url.url);
+            throw BadURL("GitHub URL '%s' is invalid", url);
 
         std::string id = path[0];
         if (!std::regex_match(id, flakeRegex))

src/libfetchers/mercurial.cc

@@ -161,7 +161,7 @@ struct MercurialInputScheme : InputScheme
     {
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isLocal = url.scheme == "file";
-        return {isLocal, isLocal ? url.path : url.base};
+        return {isLocal, isLocal ? url.path : url.to_string()};
     }
 
     StorePath fetchToStore(ref<Store> store, Input & input) const

src/libfetchers/path.cc

@@ -14,7 +14,7 @@ struct PathInputScheme : InputScheme
         if (url.scheme != "path") return {};
 
         if (url.authority && *url.authority != "")
-            throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
+            throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority);
 
         Input input{settings};
         input.attrs.insert_or_assign("type", "path");
@@ -27,10 +27,10 @@ struct PathInputScheme : InputScheme
                 if (auto n = string2Int<uint64_t>(value))
                     input.attrs.insert_or_assign(name, *n);
                 else
-                    throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+                    throw Error("path URL '%s' has invalid parameter '%s'", url, name);
             }
             else
-                throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
+                throw Error("path URL '%s' has unsupported parameter '%s'", url, name);
 
         return input;
     }
@@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme
     std::optional<std::string> isRelative(const Input & input) const
     {
         auto path = getStrAttr(input.attrs, "path");
-        if (hasPrefix(path, "/"))
+        if (isAbsolute(path))
             return std::nullopt;
         else
             return path;

src/libfetchers/registry.cc

@@ -156,7 +156,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
             return std::make_shared<Registry>(settings, Registry::Global); // empty registry
         }
 
-        if (!hasPrefix(path, "/")) {
+        if (!isAbsolute(path)) {
             auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");

src/libflake-tests/flakeref.cc

@@ -7,18 +7,60 @@ namespace nix {
 
 /* ----------- tests for flake/flakeref.hh --------------------------------------------------*/
 
-    /* ----------------------------------------------------------------------------
+    TEST(parseFlakeRef, path) {
-     * to_string
+        experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes);
-     * --------------------------------------------------------------------------*/
+
+        fetchers::Settings fetchSettings;
+
+        {
+            auto s = "/foo/bar";
+            auto flakeref = parseFlakeRef(fetchSettings, s);
+            ASSERT_EQ(flakeref.to_string(), "path:/foo/bar");
+        }
+
+        {
+            auto s = "/foo/bar?revCount=123&rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+            auto flakeref = parseFlakeRef(fetchSettings, s);
+            ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&revCount=123");
+        }
+
+        {
+            auto s = "/foo/bar?xyzzy=123";
+            EXPECT_THROW(
+                parseFlakeRef(fetchSettings, s),
+                Error);
+        }
+
+        {
+            auto s = "/foo/bar#bla";
+            EXPECT_THROW(
+                parseFlakeRef(fetchSettings, s),
+                Error);
+        }
+
+        {
+            auto s = "/foo/bar#bla";
+            auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s);
+            ASSERT_EQ(flakeref.to_string(), "path:/foo/bar");
+            ASSERT_EQ(fragment, "bla");
+        }
+
+        {
+            auto s = "/foo/bar?revCount=123#bla";
+            auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s);
+            ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?revCount=123");
+            ASSERT_EQ(fragment, "bla");
+        }
+    }
 
     TEST(to_string, doesntReencodeUrl) {
         fetchers::Settings fetchSettings;
         auto s = "http://localhost:8181/test/+3d.tar.gz";
         auto flakeref = parseFlakeRef(fetchSettings, s);
-        auto parsed = flakeref.to_string();
+        auto unparsed = flakeref.to_string();
         auto expected = "http://localhost:8181/test/%2B3d.tar.gz";
 
-        ASSERT_EQ(parsed, expected);
+        ASSERT_EQ(unparsed, expected);
     }
 
 }

src/libflake/flake/flakeref.cc

@@ -67,6 +67,20 @@ std::optional<FlakeRef> maybeParseFlakeRef(
     }
 }
 
+static std::pair<FlakeRef, std::string> fromParsedURL(
+    const fetchers::Settings & fetchSettings,
+    ParsedURL && parsedURL,
+    bool isFlake)
+{
+    auto dir = getOr(parsedURL.query, "dir", "");
+    parsedURL.query.erase("dir");
+
+    std::string fragment;
+    std::swap(fragment, parsedURL.fragment);
+
+    return {FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), dir), fragment};
+}
+
 std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
     const fetchers::Settings & fetchSettings,
     const std::string & url,
@@ -74,28 +88,21 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
     bool allowMissing,
     bool isFlake)
 {
-    std::string path = url;
+    static std::regex pathFlakeRegex(
-    std::string fragment = "";
+        R"(([^?#]*)(\?([^#]*))?(#(.*))?)",
-    std::map<std::string, std::string> query;
+        std::regex::ECMAScript);
-    auto pathEnd = url.find_first_of("#?");
+
-    auto fragmentStart = pathEnd;
+    std::smatch match;
-    if (pathEnd != std::string::npos && url[pathEnd] == '?') {
+    auto succeeds = std::regex_match(url, match, pathFlakeRegex);
-        fragmentStart = url.find("#");
+    assert(succeeds);
-    }
+    auto path = match[1].str();
-    if (pathEnd != std::string::npos) {
+    auto query = decodeQuery(match[3]);
-        path = url.substr(0, pathEnd);
+    auto fragment = percentDecode(match[5].str());
-    }
-    if (fragmentStart != std::string::npos) {
-        fragment = percentDecode(url.substr(fragmentStart+1));
-    }
-    if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') {
-        query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1));
-    }
 
     if (baseDir) {
         /* Check if 'url' is a path (either absolute or relative
-            to 'baseDir'). If so, search upward to the root of the
+           to 'baseDir'). If so, search upward to the root of the
-            repo (i.e. the directory containing .git). */
+           repo (i.e. the directory containing .git). */
 
         path = absPath(path, baseDir);
 
@@ -144,15 +151,12 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
 
             while (flakeRoot != "/") {
                 if (pathExists(flakeRoot + "/.git")) {
-                    auto base = std::string("git+file://") + flakeRoot;
-
                     auto parsedURL = ParsedURL{
-                        .url = base, // FIXME
-                        .base = base,
                         .scheme = "git+file",
                         .authority = "",
                         .path = flakeRoot,
                         .query = query,
+                        .fragment = fragment,
                     };
 
                     if (subdir != "") {
@@ -164,9 +168,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
                     if (pathExists(flakeRoot + "/.git/shallow"))
                         parsedURL.query.insert_or_assign("shallow", "1");
 
-                    return std::make_pair(
+                    return fromParsedURL(fetchSettings, std::move(parsedURL), isFlake);
-                        FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")),
-                        fragment);
                 }
 
                 subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
@@ -175,21 +177,24 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
         }
 
     } else {
-        if (!hasPrefix(path, "/"))
+        if (!isAbsolute(path))
             throw BadURL("flake reference '%s' is not an absolute path", url);
         path = canonPath(path + "/" + getOr(query, "dir", ""));
     }
 
-    fetchers::Attrs attrs;
+    return fromParsedURL(fetchSettings, {
-    attrs.insert_or_assign("type", "path");
+        .scheme = "path",
-    attrs.insert_or_assign("path", path);
+        .authority = "",
+        .path = path,
+        .query = query,
+        .fragment = fragment
+    }, isFlake);
+}
 
-    return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment);
+/**
-};
+ * Check if `url` is a flake ID. This is an abbreviated syntax for
-
+ * `flake:<flake-id>?ref=<ref>&rev=<rev>`.
-
+ */
-/* Check if 'url' is a flake ID. This is an abbreviated syntax for
-    'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
 static std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
     const fetchers::Settings & fetchSettings,
     const std::string & url,
@@ -205,8 +210,6 @@ static std::optional<std::pair<FlakeRef, std::string>> parseFlakeIdRef(
 
     if (std::regex_match(url, match, flakeRegex)) {
         auto parsedURL = ParsedURL{
-            .url = url,
-            .base = "flake:" + match.str(1),
             .scheme = "flake",
             .authority = "",
             .path = match[1],
@@ -227,22 +230,16 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
     bool isFlake
 )
 {
-    ParsedURL parsedURL;
     try {
-        parsedURL = parseURL(url);
+        auto parsed = parseURL(url);
+        if (baseDir
+            && parsed.scheme == "path"
+            && !isAbsolute(parsed.path))
+            parsed.path = absPath(parsed.path, *baseDir);
+        return fromParsedURL(fetchSettings, std::move(parsed), isFlake);
     } catch (BadURL &) {
         return std::nullopt;
     }
-
-    std::string fragment;
-    std::swap(fragment, parsedURL.fragment);
-
-    auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake);
-    input.parent = baseDir;
-
-    return std::make_pair(
-        FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")),
-        fragment);
 }
 
 std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(

src/libmain/common-args.cc

@@ -17,7 +17,9 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
         .shortName = 'v',
         .description = "Increase the logging verbosity level.",
         .category = loggingCategory,
-        .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
+        .handler = {[]() {
+            verbosity = (Verbosity) std::min<std::underlying_type_t<Verbosity>>(verbosity + 1, lvlVomit);
+        }},
     });
 
     addFlag({

src/libstore/build/derivation-goal.cc

@@ -1158,7 +1158,7 @@ HookReply DerivationGoal::tryBuildHook()
                     throw;
                 }
             }();
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
+            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, "the build hook", true))
                 ;
             else if (s.substr(0, 2) == "# ") {
                 reply = s.substr(2);
@@ -1343,9 +1343,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
     if (hook && fd == hook->fromHook.readSide.get()) {
         for (auto c : data)
             if (c == '\n') {
-                auto json = parseJSONMessage(currentHookLine);
+                auto json = parseJSONMessage(currentHookLine, "the derivation builder");
                 if (json) {
-                    auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
+                    auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true);
                     // ensure that logs from a builder using `ssh-ng://` as protocol
                     // are also available to `nix log`.
                     if (s && !isWrittenToLog && logSink) {
@@ -1387,7 +1387,7 @@ void DerivationGoal::handleEOF(Descriptor fd)
 
 void DerivationGoal::flushLine()
 {
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
+    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false))
         ;
 
     else {

src/libstore/filetransfer.cc

@@ -767,7 +767,7 @@ struct curlFileTransfer : public FileTransfer
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;
                 if (!s3Res.data)
-                    throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri);
+                    throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri);
                 res.data = std::move(*s3Res.data);
                 res.urls.push_back(request.uri);
                 callback(std::move(res));

src/libstore/gc.cc

@@ -4,6 +4,7 @@
 #include "finally.hh"
 #include "unix-domain-socket.hh"
 #include "signals.hh"
+#include "posix-fs-canonicalise.hh"
 
 #if !defined(__linux__)
 // For shelling out to lsof
@@ -763,13 +764,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 }
             }
         }
-
         for (auto & path : topoSortPaths(visited)) {
             if (!dead.insert(path).second) continue;
             if (shouldDelete) {
-                invalidatePathChecked(path);
+                try {
-                deleteFromStore(path.to_string());
+                    invalidatePathChecked(path);
-                referrersCache.erase(path);
+                    deleteFromStore(path.to_string());
+                    referrersCache.erase(path);
+                } catch (PathInUse &e) {
+                    // If we end up here, it's likely a new occurence
+                    // of https://github.com/NixOS/nix/issues/11923
+                    printError("BUG: %s", e.what());
+                }
             }
         }
     };

src/libstore/remote-store.cc

@@ -534,14 +534,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 void RemoteStore::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
     auto source = sinkToSource([&](Sink & sink) {
         sink << pathsToCopy.size();
-        for (auto & [pathInfo, pathSource] : pathsToCopy) {
+        // Reverse, so we can release memory at the original start
+        std::reverse(pathsToCopy.begin(), pathsToCopy.end());
+        while (!pathsToCopy.empty()) {
+            auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {
                      .to = sink,
@@ -549,6 +552,7 @@ void RemoteStore::addMultipleToStore(
                  },
                  pathInfo);
             pathSource->drainInto(sink);
+            pathsToCopy.pop_back();
         }
     });
 

src/libstore/remote-store.hh

@@ -102,7 +102,7 @@ public:
         CheckSigsFlag checkSigs) override;
 
     void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;

src/libstore/store-api.cc

@@ -223,7 +223,7 @@ StorePath Store::addToStore(
 }
 
 void Store::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
@@ -246,9 +246,7 @@ void Store::addMultipleToStore(
         act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
     };
 
-    ThreadPool pool;
+    processGraph<StorePath>(
-
-    processGraph<StorePath>(pool,
         storePathsToAdd,
 
         [&](const StorePath & path) {
@@ -1028,12 +1026,10 @@ std::map<StorePath, StorePath> copyPaths(
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
 
-    ThreadPool pool;
-
     try {
         // Copy the realisation closure
         processGraph<Realisation>(
-            pool, Realisation::closure(srcStore, toplevelRealisations),
+            Realisation::closure(srcStore, toplevelRealisations),
             [&](const Realisation & current) -> std::set<Realisation> {
                 std::set<Realisation> children;
                 for (const auto & [drvOutput, _] : current.dependentRealisations) {
@@ -1142,7 +1138,7 @@ std::map<StorePath, StorePath> copyPaths(
         pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
     }
 
-    dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
+    dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);
 
     return pathsMap;
 }

src/libstore/store-api.hh

@@ -425,7 +425,7 @@ public:
         CheckSigsFlag checkSigs = CheckSigs);
 
     virtual void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs);

src/libutil-tests/file-system.cc

@@ -261,4 +261,18 @@ TEST(pathExists, bogusPathDoesNotExist)
 {
     ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes"));
 }
+
+/* ----------------------------------------------------------------------------
+ * makeParentCanonical
+ * --------------------------------------------------------------------------*/
+
+TEST(makeParentCanonical, noParent)
+{
+    ASSERT_EQ(makeParentCanonical("file"), absPath(std::filesystem::path("file")));
+}
+
+TEST(makeParentCanonical, root)
+{
+    ASSERT_EQ(makeParentCanonical("/"), "/");
+}
 }

src/libutil-tests/terminal.cc

@@ -57,4 +57,9 @@ TEST(filterANSIEscapes, utf8)
     ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b");
 }
 
+TEST(filterANSIEscapes, osc8)
+{
+    ASSERT_EQ(filterANSIEscapes("\e]8;;http://example.com\e\\This is a link\e]8;;\e\\"), "This is a link");
+}
+
 } // namespace nix

src/libutil-tests/url.cc

@@ -20,24 +20,11 @@ namespace nix {
     }
 
 
-    std::ostream& operator<<(std::ostream& os, const ParsedURL& p) {
-        return os << "\n"
-                  << "url:       " << p.url << "\n"
-                  << "base:      " << p.base << "\n"
-                  << "scheme:    " << p.scheme << "\n"
-                  << "authority: " << p.authority.value() << "\n"
-                  << "path:      " << p.path << "\n"
-                  << "query:     " << print_map(p.query) << "\n"
-                  << "fragment:  " << p.fragment << "\n";
-    }
-
     TEST(parseURL, parsesSimpleHttpUrl) {
         auto s = "http://www.example.org/file.tar.gz";
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "http://www.example.org/file.tar.gz",
-            .base = "http://www.example.org/file.tar.gz",
             .scheme = "http",
             .authority = "www.example.org",
             .path = "/file.tar.gz",
@@ -53,8 +40,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "https://www.example.org/file.tar.gz",
-            .base = "https://www.example.org/file.tar.gz",
             .scheme = "https",
             .authority = "www.example.org",
             .path = "/file.tar.gz",
@@ -70,8 +55,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "https://www.example.org/file.tar.gz",
-            .base = "https://www.example.org/file.tar.gz",
             .scheme = "https",
             .authority = "www.example.org",
             .path = "/file.tar.gz",
@@ -87,8 +70,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "http://www.example.org/file.tar.gz",
-            .base = "http://www.example.org/file.tar.gz",
             .scheme = "http",
             .authority = "www.example.org",
             .path = "/file.tar.gz",
@@ -104,8 +85,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "file+https://www.example.org/video.mp4",
-            .base = "https://www.example.org/video.mp4",
             .scheme = "file+https",
             .authority = "www.example.org",
             .path = "/video.mp4",
@@ -126,8 +105,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "http://127.0.0.1:8080/file.tar.gz",
-            .base = "https://127.0.0.1:8080/file.tar.gz",
             .scheme = "http",
             .authority = "127.0.0.1:8080",
             .path = "/file.tar.gz",
@@ -143,8 +120,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
-            .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
             .scheme = "http",
             .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
             .path = "",
@@ -161,8 +136,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
-            .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
             .scheme = "http",
             .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
             .path = "",
@@ -185,8 +158,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "http://user:pass@www.example.org/file.tar.gz",
-            .base = "http://user:pass@www.example.org/file.tar.gz",
             .scheme = "http",
             .authority = "user:pass@www.example.org:8080",
             .path = "/file.tar.gz",
@@ -203,8 +174,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "",
-            .base = "",
             .scheme = "file",
             .authority = "",
             .path = "/none/of//your/business",
@@ -228,8 +197,6 @@ namespace nix {
         auto parsed = parseURL(s);
 
         ParsedURL expected {
-            .url = "ftp://ftp.nixos.org/downloads/nixos.iso",
-            .base = "ftp://ftp.nixos.org/downloads/nixos.iso",
             .scheme = "ftp",
             .authority = "ftp.nixos.org",
             .path = "/downloads/nixos.iso",

src/libutil/config.hh

@@ -260,6 +260,7 @@ public:
     operator const T &() const { return value; }
     operator T &() { return value; }
     const T & get() const { return value; }
+    T & get() { return value; }
     template<typename U>
     bool operator ==(const U & v2) const { return value == v2; }
     template<typename U>

src/libutil/executable-path.cc

@@ -73,7 +73,7 @@ ExecutablePath::findName(const OsString & exe, std::function<bool(const fs::path
     for (auto & dir : directories) {
         auto candidate = dir / exe;
         if (isExecutable(candidate))
-            return std::filesystem::canonical(candidate);
+            return candidate.lexically_normal();
     }
 
     return std::nullopt;

src/libutil/file-system.cc

@@ -31,12 +31,7 @@ namespace nix {
 
 namespace fs { using namespace std::filesystem; }
 
-/**
+bool isAbsolute(PathView path)
- * Treat the string as possibly an absolute path, by inspecting the
- * start of it. Return whether it was probably intended to be
- * absolute.
- */
-static bool isAbsolute(PathView path)
 {
     return fs::path { path }.is_absolute();
 }
@@ -331,7 +326,7 @@ void syncParent(const Path & path)
 
 void recursiveSync(const Path & path)
 {
-    /* If it's a file, just fsync and return. */
+    /* If it's a file or symlink, just fsync and return. */
     auto st = lstat(path);
     if (S_ISREG(st.st_mode)) {
         AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY, 0));
@@ -339,7 +334,8 @@ void recursiveSync(const Path & path)
             throw SysError("opening file '%1%'", path);
         fd.fsync();
         return;
-    }
+    } else if (S_ISLNK(st.st_mode))
+        return;
 
     /* Otherwise, perform a depth-first traversal of the directory and
        fsync all the files. */
@@ -602,7 +598,11 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
 
 void createSymlink(const Path & target, const Path & link)
 {
-    fs::create_symlink(target, link);
+    try {
+        fs::create_symlink(target, link);
+    } catch (fs::filesystem_error & e) {
+        throw SysError("creating symlink '%1%' -> '%2%'", link, target);
+    }
 }
 
 void replaceSymlink(const fs::path & target, const fs::path & link)
@@ -615,10 +615,16 @@ void replaceSymlink(const fs::path & target, const fs::path & link)
             fs::create_symlink(target, tmp);
         } catch (fs::filesystem_error & e) {
             if (e.code() == std::errc::file_exists) continue;
-            throw;
+            throw SysError("creating symlink '%1%' -> '%2%'", tmp, target);
+        }
+
+        try {
+            fs::rename(tmp, link);
+        } catch (fs::filesystem_error & e) {
+            if (e.code() == std::errc::file_exists) continue;
+            throw SysError("renaming '%1%' to '%2%'", tmp, link);
         }
 
-        fs::rename(tmp, link);
 
         break;
     }
@@ -755,4 +761,19 @@ bool isExecutableFileAmbient(const fs::path & exe) {
         ) == 0;
 }
 
+std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath)
+{
+    std::filesystem::path path(absPath(rawPath));;
+    try {
+        auto parent = path.parent_path();
+        if (parent == path) {
+            // `path` is a root directory => trivially canonical
+            return parent;
+        }
+        return std::filesystem::canonical(parent) / path.filename();
+    } catch (fs::filesystem_error & e) {
+        throw SysError("canonicalising parent path of '%1%'", path);
+    }
 }
+
+} // namespace nix

src/libutil/file-system.hh

@@ -42,6 +42,11 @@ namespace nix {
 struct Sink;
 struct Source;
 
+/**
+ * Return whether the path denotes an absolute path.
+ */
+bool isAbsolute(PathView path);
+
 /**
  * @return An absolutized path, resolving paths relative to the
  * specified directory, or the current directory otherwise.  The path
@@ -143,6 +148,23 @@ inline bool symlink_exists(const std::filesystem::path & path) {
 
 } // namespace fs
 
+/**
+ * Canonicalize a path except for the last component.
+ *
+ * This is useful for getting the canonical location of a symlink.
+ *
+ * Consider the case where `foo/l` is a symlink. `canonical("foo/l")` will
+ * resolve the symlink `l` to its target.
+ * `makeParentCanonical("foo/l")` will not resolve the symlink `l` to its target,
+ * but does ensure that the returned parent part of the path, `foo` is resolved
+ * to `canonical("foo")`, and can therefore be retrieved without traversing any
+ * symlinks.
+ *
+ * If a relative path is passed, it will be made absolute, so that the parent
+ * can always be canonicalized.
+ */
+std::filesystem::path makeParentCanonical(const std::filesystem::path & path);
+
 /**
  * A version of pathExists that returns false on a permission error.
  * Useful for inferring default paths across directories that might not
@@ -250,8 +272,6 @@ void setWriteTime(const std::filesystem::path & path, const struct stat & st);
 /**
  * Create a symlink.
  *
- * In the process of being deprecated for
- * `std::filesystem::create_symlink`.
  */
 void createSymlink(const Path & target, const Path & link);
 

src/libutil/hash.cc

@@ -134,7 +134,8 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const
 
 Hash Hash::dummy(HashAlgorithm::SHA256);
 
-Hash Hash::parseSRI(std::string_view original) {
+Hash Hash::parseSRI(std::string_view original)
+{
     auto rest = original;
 
     // Parse the has type before the separater, if there was one.

src/libutil/logging.cc

@@ -280,61 +280,72 @@ static Logger::Fields getFields(nlohmann::json & json)
     return fields;
 }
 
-std::optional<nlohmann::json> parseJSONMessage(const std::string & msg)
+std::optional<nlohmann::json> parseJSONMessage(const std::string & msg, std::string_view source)
 {
     if (!hasPrefix(msg, "@nix ")) return std::nullopt;
     try {
         return nlohmann::json::parse(std::string(msg, 5));
     } catch (std::exception & e) {
-        printError("bad JSON log message from builder: %s", e.what());
+        printError("bad JSON log message from %s: %s",
+            Uncolored(source),
+            e.what());
     }
     return std::nullopt;
 }
 
 bool handleJSONLogMessage(nlohmann::json & json,
     const Activity & act, std::map<ActivityId, Activity> & activities,
-    bool trusted)
+    std::string_view source, bool trusted)
 {
-    std::string action = json["action"];
+    try {
+        std::string action = json["action"];
 
-    if (action == "start") {
+        if (action == "start") {
-        auto type = (ActivityType) json["type"];
+            auto type = (ActivityType) json["type"];
-        if (trusted || type == actFileTransfer)
+            if (trusted || type == actFileTransfer)
-            activities.emplace(std::piecewise_construct,
+                activities.emplace(std::piecewise_construct,
-                std::forward_as_tuple(json["id"]),
+                    std::forward_as_tuple(json["id"]),
-                std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
+                    std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
-                    json["text"], getFields(json["fields"]), act.id));
+                        json["text"], getFields(json["fields"]), act.id));
+        }
+
+        else if (action == "stop")
+            activities.erase((ActivityId) json["id"]);
+
+        else if (action == "result") {
+            auto i = activities.find((ActivityId) json["id"]);
+            if (i != activities.end())
+                i->second.result((ResultType) json["type"], getFields(json["fields"]));
+        }
+
+        else if (action == "setPhase") {
+            std::string phase = json["phase"];
+            act.result(resSetPhase, phase);
+        }
+
+        else if (action == "msg") {
+            std::string msg = json["msg"];
+            logger->log((Verbosity) json["level"], msg);
+        }
+
+        return true;
+    } catch (const nlohmann::json::exception &e) {
+        warn(
+            "Unable to handle a JSON message from %s: %s",
+            Uncolored(source),
+            e.what()
+        );
+        return false;
     }
-
-    else if (action == "stop")
-        activities.erase((ActivityId) json["id"]);
-
-    else if (action == "result") {
-        auto i = activities.find((ActivityId) json["id"]);
-        if (i != activities.end())
-            i->second.result((ResultType) json["type"], getFields(json["fields"]));
-    }
-
-    else if (action == "setPhase") {
-        std::string phase = json["phase"];
-        act.result(resSetPhase, phase);
-    }
-
-    else if (action == "msg") {
-        std::string msg = json["msg"];
-        logger->log((Verbosity) json["level"], msg);
-    }
-
-    return true;
 }
 
 bool handleJSONLogMessage(const std::string & msg,
-    const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted)
+    const Activity & act, std::map<ActivityId, Activity> & activities, std::string_view source, bool trusted)
 {
-    auto json = parseJSONMessage(msg);
+    auto json = parseJSONMessage(msg, source);
     if (!json) return false;
 
-    return handleJSONLogMessage(*json, act, activities, trusted);
+    return handleJSONLogMessage(*json, act, activities, source, trusted);
 }
 
 Activity::~Activity()

src/libutil/logging.hh

@@ -185,14 +185,25 @@ Logger * makeSimpleLogger(bool printBuildLogs = true);
 
 Logger * makeJSONLogger(Logger & prevLogger);
 
-std::optional<nlohmann::json> parseJSONMessage(const std::string & msg);
+/**
+ * @param source A noun phrase describing the source of the message, e.g. "the builder".
+ */
+std::optional<nlohmann::json> parseJSONMessage(const std::string & msg, std::string_view source);
 
+/**
+ * @param source A noun phrase describing the source of the message, e.g. "the builder".
+ */
 bool handleJSONLogMessage(nlohmann::json & json,
     const Activity & act, std::map<ActivityId, Activity> & activities,
+    std::string_view source,
     bool trusted);
 
+/**
+ * @param source A noun phrase describing the source of the message, e.g. "the builder".
+ */
 bool handleJSONLogMessage(const std::string & msg,
     const Activity & act, std::map<ActivityId, Activity> & activities,
+    std::string_view source,
     bool trusted);
 
 /**

src/libutil/posix-source-accessor.hh

@@ -43,13 +43,25 @@ struct PosixSourceAccessor : virtual SourceAccessor
     std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
 
     /**
-     * Create a `PosixSourceAccessor` and `CanonPath` corresponding to
+     * Create a `PosixSourceAccessor` and `SourcePath` corresponding to
      * some native path.
      *
      * The `PosixSourceAccessor` is rooted as far up the tree as
      * possible, (e.g. on Windows it could scoped to a drive like
      * `C:\`). This allows more `..` parent accessing to work.
      *
+     * @note When `path` is trusted user input, canonicalize it using
+     * `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc,
+     * as appropriate for the use case. At least weak canonicalization is
+     * required for the `SourcePath` to do anything useful at the location it
+     * points to.
+     *
+     * @note A canonicalizing behavior is not built in `createAtRoot` so that
+     * callers do not accidentally introduce symlink-related security vulnerabilities.
+     * Furthermore, `createAtRoot` does not know whether the file pointed to by
+     * `path` should be resolved if it is itself a symlink. In other words,
+     * `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers.
+     *
      * See
      * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path)
      * and

src/libutil/source-accessor.cc

@@ -94,7 +94,7 @@ CanonPath SourceAccessor::resolveSymlinks(
                         throw Error("infinite symlink recursion in path '%s'", showPath(path));
                     auto target = readLink(res);
                     res.pop();
-                    if (hasPrefix(target, "/"))
+                    if (isAbsolute(target))
                         res = CanonPath::root;
                     todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
                 }

src/libutil/terminal.cc

@@ -45,6 +45,13 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w
                 while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
                 // eat final byte
                 if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
+            } else if (i != s.end() && *i == ']') {
+                // OSC
+                e += *i++;
+                // eat ESC
+                while (i != s.end() && *i != '\e') e += *i++;
+                // eat backslash
+                if (i != s.end() && *i == '\\') e += last = *i++;
             } else {
                 if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
             }

src/libutil/thread-pool.hh

@@ -83,7 +83,6 @@ private:
  */
 template<typename T>
 void processGraph(
-    ThreadPool & pool,
     const std::set<T> & nodes,
     std::function<std::set<T>(const T &)> getEdges,
     std::function<void(const T &)> processNode)
@@ -97,6 +96,10 @@ void processGraph(
 
     std::function<void(const T &)> worker;
 
+    /* Create pool last to ensure threads are stopped before other destructors
+     * run */
+    ThreadPool pool;
+
     worker = [&](const T & node) {
 
         {
@@ -147,8 +150,16 @@ void processGraph(
         }
     };
 
-    for (auto & node : nodes)
+    for (auto & node : nodes) {
-        pool.enqueue(std::bind(worker, std::ref(node)));
+        try {
+            pool.enqueue(std::bind(worker, std::ref(node)));
+        } catch (ThreadPoolShutDown &) {
+            /* Stop if the thread pool is shutting down. It means a
+               previous work item threw an exception, so process()
+               below will rethrow it. */
+            break;
+        }
+    }
 
     pool.process();
 

src/libutil/url.cc

@@ -22,7 +22,6 @@ ParsedURL parseURL(const std::string & url)
     std::smatch match;
 
     if (std::regex_match(url, match, uriRegex)) {
-        auto & base = match[1];
         std::string scheme = match[2];
         auto authority = match[3].matched
             ? std::optional<std::string>(match[3]) : std::nullopt;
@@ -40,8 +39,6 @@ ParsedURL parseURL(const std::string & url)
             path = "/";
 
         return ParsedURL{
-            .url = url,
-            .base = base,
             .scheme = scheme,
             .authority = authority,
             .path = percentDecode(path),
@@ -136,6 +133,12 @@ std::string ParsedURL::to_string() const
         + (fragment.empty() ? "" : "#" + percentEncode(fragment));
 }
 
+std::ostream & operator << (std::ostream & os, const ParsedURL & url)
+{
+    os << url.to_string();
+    return os;
+}
+
 bool ParsedURL::operator ==(const ParsedURL & other) const noexcept
 {
     return

src/libutil/url.hh

@@ -7,9 +7,6 @@ namespace nix {
 
 struct ParsedURL
 {
-    std::string url;
-    /// URL without query/fragment
-    std::string base;
     std::string scheme;
     std::optional<std::string> authority;
     std::string path;
@@ -26,6 +23,8 @@ struct ParsedURL
     ParsedURL canonicalise();
 };
 
+std::ostream & operator << (std::ostream & os, const ParsedURL & url);
+
 MakeError(BadURL, Error);
 
 std::string percentDecode(std::string_view in);

src/nix-build/nix-build.cc

@@ -340,13 +340,15 @@ static void main_nix_build(int argc, char * * argv)
         exprs = {state->parseStdin()};
     else
         for (auto i : remainingArgs) {
-            auto baseDir = inShebang && !packages ? absPath(dirOf(script)) : i;
+            if (fromArgs) {
-
+                auto shebangBaseDir = absPath(dirOf(script));
-            if (fromArgs)
                 exprs.push_back(state->parseExprFromString(
                     std::move(i),
-                    (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) ? lookupFileArg(*state, baseDir) : state->rootPath(".")
+                    (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript)
+                        ? lookupFileArg(*state, shebangBaseDir)
+                        : state->rootPath(".")
                 ));
+            }
             else {
                 auto absolute = i;
                 try {

src/nix-store/nix-store.cc

@@ -183,9 +183,9 @@ static void opAdd(Strings opFlags, Strings opArgs)
     if (!opFlags.empty()) throw UsageError("unknown flag");
 
     for (auto & i : opArgs) {
-        auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i);
+        auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
         cout << fmt("%s\n", store->printStorePath(store->addToStore(
-            std::string(baseNameOf(i)), {accessor, canonPath})));
+            std::string(baseNameOf(i)), sourcePath)));
     }
 }
 
@@ -207,10 +207,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     opArgs.pop_front();
 
     for (auto & i : opArgs) {
-        auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i);
+        auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
         std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(
             baseNameOf(i),
-            {accessor, canonPath},
+            sourcePath,
             method,
             hashAlgo).path));
     }

src/nix/add-to-store.cc

@@ -37,13 +37,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     {
         if (!namePart) namePart = baseNameOf(path);
 
-        auto [accessor, path2] = PosixSourceAccessor::createAtRoot(path);
+        auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
 
         auto storePath = dryRun
             ? store->computeStorePath(
-                *namePart, {accessor, path2}, caMethod, hashAlgo, {}).first
+                *namePart, sourcePath, caMethod, hashAlgo, {}).first
             : store->addToStoreSlow(
-                *namePart, {accessor, path2}, caMethod, hashAlgo, {}).path;
+                *namePart, sourcePath, caMethod, hashAlgo, {}).path;
 
         logger->cout("%s", store->printStorePath(storePath));
     }

src/nix/flake.cc

@@ -941,7 +941,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
                         }
                         continue;
                     } else
-                          fs::create_symlink(target, to2);
+                        createSymlink(target, to2);
                 }
                 else
                     throw Error("file '%s' has unsupported type", from2);

src/nix/flake.md

@@ -648,7 +648,7 @@ following fields:
 * `inputs`: The dependencies of this node, as a mapping from input
   names (e.g. `nixpkgs`) to node labels (e.g. `n2`).
 
-* `original`: The original input specification from `flake.lock`, as a
+* `original`: The original input specification from `flake.nix`, as a
   set of `builtins.fetchTree` arguments.
 
 * `locked`: The locked input specification, as a set of

src/nix/hash.cc

@@ -87,18 +87,35 @@ struct CmdHashBase : Command
                     return std::make_unique<HashSink>(hashAlgo);
             };
 
-            auto path2 = PosixSourceAccessor::createAtRoot(path);
+            auto makeSourcePath = [&]() -> SourcePath {
+                return PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
+            };
+
             Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++
             switch (mode) {
             case FileIngestionMethod::Flat:
+            {
+                // While usually we could use the some code as for NixArchive,
+                // the Flat method needs to support FIFOs, such as those
+                // produced by bash process substitution, e.g.:
+                //     nix hash --mode flat <(echo hi)
+                // Also symlinks semantics are unambiguous in the flat case,
+                // so we don't need to go low-level, or reject symlink `path`s.
+                auto hashSink = makeSink();
+                readFile(path, *hashSink);
+                h = hashSink->finish().first;
+                break;
+            }
             case FileIngestionMethod::NixArchive:
             {
+                auto sourcePath = makeSourcePath();
                 auto hashSink = makeSink();
-                dumpPath(path2, *hashSink, (FileSerialisationMethod) mode);
+                dumpPath(sourcePath, *hashSink, (FileSerialisationMethod) mode);
                 h = hashSink->finish().first;
                 break;
             }
             case FileIngestionMethod::Git: {
+                auto sourcePath = makeSourcePath();
                 std::function<git::DumpHook> hook;
                 hook = [&](const SourcePath & path) -> git::TreeEntry {
                     auto hashSink = makeSink();
@@ -109,7 +126,7 @@ struct CmdHashBase : Command
                         .hash = hash,
                     };
                 };
-                h = hook(path2).hash;
+                h = hook(sourcePath).hash;
                 break;
             }
             }
@@ -163,8 +180,11 @@ struct CmdToBase : Command
     HashFormat hashFormat;
     std::optional<HashAlgorithm> hashAlgo;
     std::vector<std::string> args;
+    bool legacyCli;
 
-    CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat)
+    CmdToBase(HashFormat hashFormat, bool legacyCli = false)
+        : hashFormat(hashFormat)
+        , legacyCli(legacyCli)
     {
         addFlag(flag::hashAlgoOpt("type", &hashAlgo));
         expectArgs("strings", &args);
@@ -181,8 +201,9 @@ struct CmdToBase : Command
 
     void run() override
     {
-        warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`.");
+        if (!legacyCli)
-        for (auto s : args)
+            warn("The old format conversion subcommands of `nix hash` were deprecated in favor of `nix hash convert`.");
+        for (const auto & s : args)
             logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI));
     }
 };
@@ -222,11 +243,18 @@ struct CmdHashConvert : Command
     Category category() override { return catUtility; }
 
     void run() override {
-        for (const auto& s: hashStrings) {
+        for (const auto & s : hashStrings) {
-            Hash h = Hash::parseAny(s, algo);
+            Hash h =
-            if (from && h.to_string(*from, from == HashFormat::SRI) != s) {
+                from == HashFormat::SRI
+                ? Hash::parseSRI(s)
+                : Hash::parseAny(s, algo);
+            if (from
+                && from != HashFormat::SRI
+                && h.to_string(*from, false) !=
+                    (from == HashFormat::Base16 ? toLower(s) : s))
+            {
                 auto from_as_string = printHashFormat(*from);
-                throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string);
+                throw BadHash("input hash '%s' does not have the expected format for '--from %s'", s, from_as_string);
             }
             logger->cout(h.to_string(to, to == HashFormat::SRI));
         }
@@ -321,7 +349,7 @@ static int compatNixHash(int argc, char * * argv)
     }
 
     else {
-        CmdToBase cmd(hashFormat);
+        CmdToBase cmd(hashFormat, true);
         cmd.args = ss;
         if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo;
         cmd.run();

src/nix/path-info.cc

@@ -139,12 +139,12 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
 
     Category category() override { return catSecondary; }
 
-    void printSize(uint64_t value)
+    void printSize(std::ostream & str, uint64_t value)
     {
         if (humanReadable)
-            std::cout << fmt("\t%s", renderSize(value, true));
+            str << fmt("\t%s", renderSize(value, true));
         else
-            std::cout << fmt("\t%11d", value);
+            str << fmt("\t%11d", value);
     }
 
     void run(ref<Store> store, StorePaths && storePaths) override
@@ -154,11 +154,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
             pathLen = std::max(pathLen, store->printStorePath(storePath).size());
 
         if (json) {
-            std::cout << pathInfoToJSON(
+            logger->cout(pathInfoToJSON(
                 *store,
                 // FIXME: preserve order?
                 StorePathSet(storePaths.begin(), storePaths.end()),
-                showClosureSize).dump();
+                showClosureSize).dump());
         }
 
         else {
@@ -167,30 +167,32 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                 auto info = store->queryPathInfo(storePath);
                 auto storePathS = store->printStorePath(info->path);
 
-                std::cout << storePathS;
+                std::ostringstream str;
+
+                str << storePathS;
 
                 if (showSize || showClosureSize || showSigs)
-                    std::cout << std::string(std::max(0, (int) pathLen - (int) storePathS.size()), ' ');
+                    str << std::string(std::max(0, (int) pathLen - (int) storePathS.size()), ' ');
 
                 if (showSize)
-                    printSize(info->narSize);
+                    printSize(str, info->narSize);
 
                 if (showClosureSize) {
                     StorePathSet closure;
                     store->computeFSClosure(storePath, closure, false, false);
-                    printSize(getStoreObjectsTotalSize(*store, closure));
+                    printSize(str, getStoreObjectsTotalSize(*store, closure));
                 }
 
                 if (showSigs) {
-                    std::cout << '\t';
+                    str << '\t';
                     Strings ss;
                     if (info->ultimate) ss.push_back("ultimate");
                     if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca));
                     for (auto & sig : info->sigs) ss.push_back(sig);
-                    std::cout << concatStringsSep(" ", ss);
+                    str << concatStringsSep(" ", ss);
                 }
 
-                std::cout << std::endl;
+                logger->cout(str.str());
             }
 
         }

src/nix/upgrade-nix.cc

@@ -15,7 +15,7 @@ using namespace nix;
 
 struct CmdUpgradeNix : MixDryRun, StoreCommand
 {
-    Path profileDir;
+    std::filesystem::path profileDir;
 
     CmdUpgradeNix()
     {
@@ -64,7 +64,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         if (profileDir == "")
             profileDir = getProfileDir(store);
 
-        printInfo("upgrading Nix in profile '%s'", profileDir);
+        printInfo("upgrading Nix in profile %s", profileDir);
 
         auto storePath = getLatestNix(store);
 
@@ -93,7 +93,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
         {
             Activity act(*logger, lvlInfo, actUnknown,
-                fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir));
+                fmt("installing '%s' into profile %s...", store->printStorePath(storePath), profileDir));
+
+            // FIXME: don't call an external process.
             runProgram(getNixBin("nix-env").string(), false,
                 {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"});
         }
@@ -102,31 +104,33 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
     }
 
     /* Return the profile in which Nix is installed. */
-    Path getProfileDir(ref<Store> store)
+    std::filesystem::path getProfileDir(ref<Store> store)
     {
         auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env"));
         if (!whereOpt)
             throw Error("couldn't figure out how Nix is installed, so I can't upgrade it");
-        auto & where = *whereOpt;
+        const auto & where = whereOpt->parent_path();
 
-        printInfo("found Nix in '%s'", where);
+        printInfo("found Nix in %s", where);
 
         if (hasPrefix(where.string(), "/run/current-system"))
             throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'");
 
-        Path profileDir = where.parent_path().string();
+        auto profileDir = where.parent_path();
 
         // Resolve profile to /nix/var/nix/profiles/<name> link.
-        while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir))
+        while (canonPath(profileDir.string()).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir))
             profileDir = readLink(profileDir);
 
-        printInfo("found profile '%s'", profileDir);
+        printInfo("found profile %s", profileDir);
 
-        Path userEnv = canonPath(profileDir, true);
+        Path userEnv = canonPath(profileDir.string(), true);
 
-        if (where.filename() != "bin" ||
+        if (std::filesystem::exists(profileDir / "manifest.json"))
-            !hasSuffix(userEnv, "user-environment"))
+            throw Error("directory %s is managed by 'nix profile' and currently cannot be upgraded by 'nix upgrade-nix'", profileDir);
-            throw Error("directory '%s' does not appear to be part of a Nix profile", where);
+
+        if (!std::filesystem::exists(profileDir / "manifest.nix"))
+            throw Error("directory %s does not appear to be part of a Nix profile", profileDir);
 
         if (!store->isValidPath(store->parseStorePath(userEnv)))
             throw Error("directory '%s' is not in the Nix store", userEnv);

tests/functional/add.sh

@@ -29,6 +29,47 @@ echo "$hash2"
 
 test "$hash1" = "sha256:$hash2"
 
+# The contents can be accessed through a symlink, and this symlink has no effect on the hash
+# https://github.com/NixOS/nix/issues/11941
+test_issue_11941() {
+    local expected actual
+    mkdir -p "$TEST_ROOT/foo/bar" && ln -s "$TEST_ROOT/foo" "$TEST_ROOT/foo-link"
+
+    # legacy
+    expected=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo/bar")
+    actual=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo-link/bar")
+    [[ "$expected" == "$actual" ]]
+    actual=$(nix-store --add "$TEST_ROOT/foo-link/bar")
+    [[ "$expected" == "$actual" ]]
+
+    # nix store add
+    actual=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/foo/bar")
+    [[ "$expected" == "$actual" ]]
+
+    # cleanup
+    rm -r "$TEST_ROOT/foo" "$TEST_ROOT/foo-link"
+}
+test_issue_11941
+
+# A symlink is added to the store as a symlink, not as a copy of the target
+test_add_symlink() {
+    ln -s /bin "$TEST_ROOT/my-bin"
+
+    # legacy
+    path=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/my-bin")
+    [[ "$(readlink "$path")" == /bin ]]
+    path=$(nix-store --add "$TEST_ROOT/my-bin")
+    [[ "$(readlink "$path")" == /bin ]]
+
+    # nix store add
+    path=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/my-bin")
+    [[ "$(readlink "$path")" == /bin ]]
+
+    # cleanup
+    rm "$TEST_ROOT/my-bin"
+}
+test_add_symlink
+
 #### New style commands
 
 clearStoreIfPossible

tests/functional/build-remote-trustless-should-fail-0.sh

@@ -8,6 +8,7 @@ TODO_NixOS
 restartDaemon
 
 requireSandboxSupport
+requiresUnprivilegedUserNamespaces
 [[ $busybox =~ busybox ]] || skipTest "no busybox"
 
 unset NIX_STORE_DIR

tests/functional/build-remote-trustless.sh

@@ -5,6 +5,7 @@
 # shellcheck disable=SC2154
 
 requireSandboxSupport
+requiresUnprivilegedUserNamespaces
 [[ "$busybox" =~ busybox ]] || skipTest "no busybox"
 
 unset NIX_STORE_DIR

tests/functional/build-remote.sh

@@ -3,6 +3,7 @@
 : "${file?must be defined by caller (remote building test case using this)}"
 
 requireSandboxSupport
+requiresUnprivilegedUserNamespaces
 [[ "${busybox-}" =~ busybox ]] || skipTest "no busybox"
 
 # Avoid store dir being inside sandbox build-dir
@@ -27,6 +28,7 @@ builders=(
 chmod -R +w "$TEST_ROOT/machine"* || true
 rm -rf "$TEST_ROOT/machine"* || true
 
+
 # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a
 # child process. This allows us to test LegacySSHStore::buildDerivation().
 # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation().

tests/functional/chroot-store.sh

@@ -43,6 +43,7 @@ EOF
     removeBuildDirRef "$flakeDir"/*.nix
 
     TODO_NixOS
+    requiresUnprivilegedUserNamespaces
 
     outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir")
 

tests/functional/common/functions.sh

@@ -354,4 +354,15 @@ removeBuildDirRef() {
 
 trap onError ERR
 
+requiresUnprivilegedUserNamespaces() {
+  if [[ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]] && [[ $(< /proc/sys/kernel/apparmor_restrict_unprivileged_userns) -eq 1 ]]; then
+    skipTest "Unprivileged user namespaces are disabled. Run 'sudo sysctl -w /proc/sys/kernel/apparmor_restrict_unprivileged_userns=0' to allow, and run these tests."
+  fi
+}
+
+execUnshare () {
+  requiresUnprivilegedUserNamespaces
+  exec unshare --mount --map-root-user "$SHELL" "$@"
+}
+
 fi # COMMON_FUNCTIONS_SH_SOURCED

tests/functional/dependencies.nix

@@ -1,7 +1,7 @@
 { hashInvalidator ? "" }:
 with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix";
 
-let {
+let
 
   input0 = mkDerivation {
     name = "dependencies-input-0";
@@ -33,16 +33,15 @@ let {
     outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d";
   };
 
-  body = mkDerivation {
+in
-    name = "dependencies-top";
+mkDerivation {
-    builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
+  name = "dependencies-top";
-    input1 = input1 + "/.";
+  builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
-    input2 = "${input2}/.";
+  input1 = input1 + "/.";
-    input1_drv = input1;
+  input2 = "${input2}/.";
-    input2_drv = input2;
+  input1_drv = input1;
-    input0_drv = input0;
+  input2_drv = input2;
-    fod_input_drv = fod_input;
+  input0_drv = input0;
-    meta.description = "Random test package";
+  fod_input_drv = fod_input;
-  };
+  meta.description = "Random test package";
-
 }

tests/functional/flakes/develop.sh

@@ -125,7 +125,7 @@ expectStderr 1 nix develop --unset-env-var FOO --set-env-var FOO 'BAR' --no-writ
   grepQuiet "error: Cannot set environment variable 'FOO' that is unset with '--unset-env-var'"
 
 # Check that multiple `--ignore-env`'s are okay.
-expectStderr 0 nix develop --ignore-env --set-env-var FOO 'BAR' --ignore-env .#hello
+expectStderr 0 nix develop --ignore-env --set-env-var FOO 'BAR' --ignore-env .#hello < /dev/null
 
 # Determine the bashInteractive executable.
 nix build --no-write-lock-file './nixpkgs#bashInteractive' --out-link ./bash-interactive

tests/functional/flakes/flake-in-submodule.sh

@@ -63,3 +63,34 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule
 echo '"foo"' > "$rootRepo"/submodule/sub.nix
 [[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]]
 [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]]
+
+# Test that `nix flake metadata` parses `submodule` correctly.
+cat > "$rootRepo"/flake.nix <<EOF
+{
+    outputs = { self }: {
+    };
+}
+EOF
+git -C "$rootRepo" add flake.nix
+git -C "$rootRepo" commit -m "Add flake.nix"
+
+storePath=$(nix flake metadata --json "$rootRepo?submodules=1" | jq -r .path)
+[[ -e "$storePath/submodule" ]]
+
+# The root repo may use the submodule repo as an input
+# through the relative path. This may change in the future;
+# see: https://discourse.nixos.org/t/57783 and #9708.
+cat > "$rootRepo"/flake.nix <<EOF
+{
+    inputs.subRepo.url = "git+file:./submodule";
+    outputs = { ... }: { };
+}
+EOF
+git -C "$rootRepo" add flake.nix
+git -C "$rootRepo" commit -m "Add subRepo input"
+(
+  cd "$rootRepo"
+  # The submodule must be locked to the relative path,
+  # _not_ the absolute path:
+  [[ $(nix flake metadata --json | jq -r .locks.nodes.subRepo.locked.url) = "file:./submodule" ]]
+)

tests/functional/flakes/flakes.sh

@@ -222,6 +222,8 @@ nix build -o "$TEST_ROOT/result" flake1
 
 nix build -o "$TEST_ROOT/result" "$flake1Dir"
 nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir"
+(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" ".")
+(cd "$flake1Dir" && nix build -o "$TEST_ROOT/result" "path:.")
 
 # Test explicit packages.default.
 nix build -o "$TEST_ROOT/result" "$flake1Dir#default"
@@ -231,6 +233,15 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default"
 nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default"
 nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default"
 
+# Check that relative paths are allowed for git flakes.
+# This may change in the future once git submodule support is refined.
+# See: https://discourse.nixos.org/t/57783 and #9708.
+(
+  # This `cd` should not be required and is indicative of aforementioned bug.
+  cd "$flake1Dir/.."
+  nix build -o "$TEST_ROOT/result" "git+file:./$(basename "$flake1Dir")"
+)
+
 # Check that store symlinks inside a flake are not interpreted as flakes.
 nix build -o "$flake1Dir/result" "git+file://$flake1Dir"
 nix path-info "$flake1Dir/result"

tests/functional/hash-convert.sh

@@ -93,15 +93,19 @@ try3() {
     # Asserting input format fails.
     #
 
-    fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?")
+    expectStderr 1 nix hash convert --hash-algo "$1" --from sri "$2" | grepQuiet "is not SRI"
-    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
+    expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$2" | grepQuiet "input hash"
-    fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?")
+    expectStderr 1 nix hash convert --hash-algo "$1" --from base16 "$3" | grepQuiet "input hash"
-    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
+    expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash"
-    fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?")
-    [[ "$fail" == *"error: input hash"*"exit: 1" ]]
 
+    # Base-16 hashes can be in uppercase.
+    nix hash convert --hash-algo "$1" --from base16 "$(echo $2 | tr [a-z] [A-Z])"
 }
 
 try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8="
 try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="
 try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ=="
+
+# Test SRI hashes that lack trailing '=' characters. These are incorrect but we need to support them for backward compatibility.
+[[ $(nix hash convert --from sri "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]]
+[[ $(nix hash convert --from sri "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ") = sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ== ]]

tests/functional/hash-path.sh

@@ -92,3 +92,32 @@ try2 md5 "20f3ffe011d4cfa7d72bfabef7882836"
 rm "$TEST_ROOT/hash-path/hello"
 ln -s x "$TEST_ROOT/hash-path/hello"
 try2 md5 "f78b733a68f5edbdf9413899339eaa4a"
+
+# Flat mode supports process substitution
+h=$(nix hash path --mode flat --type sha256 --base32 <(printf "SMASH THE STATE"))
+[[ 0d9n3r2i4m1zgy0wpqbsyabsfzgs952066bfp8gwvcg4mkr4r5g8 == "$h" ]]
+
+# Flat mode supports process substitution (hash file)
+h=$(nix hash file --type sha256 --base32 <(printf "SMASH THE STATE"))
+[[ 0d9n3r2i4m1zgy0wpqbsyabsfzgs952066bfp8gwvcg4mkr4r5g8 == "$h" ]]
+
+# Symlinks in the ancestry are ok and don't affect the result
+mkdir -p "$TEST_ROOT/simple" "$TEST_ROOT/try/to/mess/with/it"
+echo hi > "$TEST_ROOT/simple/hi"
+ln -s "$TEST_ROOT/simple" "$TEST_ROOT/try/to/mess/with/it/simple-link"
+h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/simple/hi")
+[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]]
+h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/try/to/mess/with/it/simple-link/hi")
+[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]]
+
+# nix hash --mode nar does not canonicalize a symlink argument.
+#   Otherwise it can't generate a NAR whose root is a symlink.
+#   If you want to follow the symlink, pass $(realpath -s ...) instead.
+ln -s /non-existent-48cujwe8ndf4as0bne "$TEST_ROOT/symlink-to-nowhere"
+h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-nowhere")
+[[ 1bl5ry3x1fcbwgr5c2x50bn572iixh4j1p6ax5isxly2ddgn8pbp == "$h" ]]  # manually verified hash
+if [[ -e /bin ]]; then
+    ln -s /bin "$TEST_ROOT/symlink-to-bin"
+    h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-bin")
+    [[ 0z2mdmkd43l0ijdxfbj1y8vzli15yh9b09n3a3rrygmjshbyypsw == "$h" ]] # manually verified hash
+fi

tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp

@@ -0,0 +1,5 @@
+error: undefined variable 'd'
+       at /pwd/lang/eval-fail-attrset-merge-drops-later-rec.nix:1:26:
+            1| { a.b = 1; a = rec { c = d + 2; d = 3; }; }.c
+             |                          ^
+            2|

tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix

@@ -0,0 +1 @@
+{ a.b = 1; a = rec { c = d + 2; d = 3; }; }.c

tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp

@@ -0,0 +1,8 @@
+error:
+       … while calling the 'toJSON' builtin
+         at /pwd/lang/eval-fail-toJSON-non-utf-8.nix:1:1:
+            1| builtins.toJSON "_invalid UTF-8: ÿ_"
+             | ^
+            2|
+
+       error: JSON serialization error: [json.exception.type_error.316] invalid UTF-8 byte at index 16: 0xFF

tests/functional/lang/eval-fail-toJSON-non-utf-8.nix

@@ -0,0 +1 @@
+builtins.toJSON "_invalid UTF-8: ÿ_"

tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp

@@ -0,0 +1 @@
+6

tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix

@@ -0,0 +1,3 @@
+# This is for backwards compatibility, not because we like it.
+# See https://github.com/NixOS/nix/issues/9020.
+{ a = rec { b = c + 1; d = 2; }; a.c = d + 3; }.a.b

tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp

@@ -1,6 +1,6 @@
-error: attribute 'z' already defined at «stdin»:3:16
+error: attribute 'x.z' already defined at «stdin»:2:3
-       at «stdin»:2:3:
+       at «stdin»:3:16:
-            1| {
             2|   x.z = 3;
-             |   ^
             3|   x = { y = 3; z = 3; };
+             |                ^
+            4| }

tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp

@@ -1,6 +1,6 @@
-error: attribute 'y' already defined at «stdin»:3:9
+error: attribute 'x.y.y' already defined at «stdin»:2:3
-       at «stdin»:2:3:
+       at «stdin»:3:9:
-            1| {
             2|   x.y.y = 3;
-             |   ^
             3|   x = { y.y= 3; z = 3; };
+             |         ^
+            4| }

tests/functional/linux-sandbox.sh

@@ -9,6 +9,7 @@ TODO_NixOS
 clearStore
 
 requireSandboxSupport
+requiresUnprivilegedUserNamespaces
 
 # Note: we need to bind-mount $SHELL into the chroot. Currently we
 # only support the case where $SHELL is in the Nix store, because

tests/functional/local-overlay-store/bad-uris.sh

@@ -19,7 +19,7 @@ TODO_NixOS
 
 for i in "${storesBad[@]}"; do
     echo $i
-    unshare --mount --map-root-user bash <<EOF
+    execUnshare <<EOF
         source common.sh
         setupStoreDirs
         mountOverlayfs

tests/functional/local-overlay-store/common.sh

@@ -94,10 +94,6 @@ initLowerStore () {
   pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath)
 }
 
-execUnshare () {
-  exec unshare --mount --map-root-user "$SHELL" "$@"
-}
-
 addTextToStore() {
   storeDir=$1; shift
   filename=$1; shift

tests/functional/logging.sh

@@ -28,3 +28,8 @@ outp="$(nix-build -E \
 test -d "$outp"
 
 nix log "$outp"
+
+if isDaemonNewer "2.26"; then
+    # Build works despite ill-formed structured build log entries.
+    expectStderr 0 nix build -f ./logging/unusual-logging.nix --no-link | grepQuiet 'warning: Unable to handle a JSON message from the derivation builder:'
+fi

tests/functional/logging/unusual-logging.nix

@@ -0,0 +1,16 @@
+let
+  inherit (import ../config.nix) mkDerivation;
+in
+mkDerivation {
+  name = "unusual-logging";
+  buildCommand = ''
+    {
+      echo "@nix 1"
+      echo "@nix {}"
+      echo '@nix {"action": null}'
+      echo '@nix {"action": 123}'
+      echo '@nix ]['
+    } >&$NIX_LOG_FD
+    touch $out
+  '';
+}

tests/functional/nested-sandboxing.sh

@@ -7,6 +7,7 @@ source common.sh
 TODO_NixOS
 
 requireSandboxSupport
+requiresUnprivilegedUserNamespaces
 
 start="$TEST_ROOT/start"
 mkdir -p "$start"

tests/functional/nested-sandboxing/command.sh

@@ -18,6 +18,7 @@ goodStoreUrl () {
 # whether this test is being run in a derivation as part of the nix build or
 # being manually run by a developer outside a derivation
 runNixBuild () {
+
     local storeFun=$1
     local altitude=$2
     nix-build \

tests/functional/nix-shell.sh

@@ -167,6 +167,35 @@ EOF
 chmod a+x $TEST_ROOT/marco/polo/default.nix
 (cd $TEST_ROOT/marco && ./polo/default.nix | grepQuiet "Polo")
 
+# https://github.com/NixOS/nix/issues/11892
+mkdir $TEST_ROOT/issue-11892
+cat >$TEST_ROOT/issue-11892/shebangscript <<EOF
+#!$(type -P env) nix-shell
+#! nix-shell -I nixpkgs=$shellDotNix
+#! nix-shell -p 'callPackage (import ./my_package.nix) {}'
+#! nix-shell -i bash
+set -euxo pipefail
+my_package
+EOF
+cat >$TEST_ROOT/issue-11892/my_package.nix <<EOF
+{ stdenv, shell, ... }:
+stdenv.mkDerivation {
+  name = "my_package";
+  buildCommand = ''
+    mkdir -p \$out/bin
+    ( echo "#!\${shell}"
+      echo "echo 'ok' 'baz11892'"
+    ) > \$out/bin/my_package
+    cat \$out/bin/my_package
+    chmod a+x \$out/bin/my_package
+  '';
+}
+EOF
+chmod a+x $TEST_ROOT/issue-11892/shebangscript
+$TEST_ROOT/issue-11892/shebangscript \
+  | tee /dev/stderr \
+  | grepQuiet "ok baz11892"
+
 
 #####################
 # Flake equivalents #

tests/functional/shell.nix

@@ -37,7 +37,7 @@ let pkgs = rec {
       mkdir -p $out
       ln -s ${setupSh} $out/setup
     '';
-  };
+  } // { inherit mkDerivation; };
 
   shellDrv = mkDerivation {
     name = "shellDrv";
@@ -94,5 +94,9 @@ let pkgs = rec {
     chmod a+rx $out/bin/ruby
   '';
 
+  inherit (cfg) shell;
+
+  callPackage = f: args: f (pkgs // args);
+
   inherit pkgs;
 }; in pkgs

tests/functional/shell.sh

@@ -52,6 +52,7 @@ if isDaemonNewer "2.20.0pre20231220"; then
 fi
 
 requireSandboxSupport
+requiresUnprivilegedUserNamespaces
 
 chmod -R u+w "$TEST_ROOT/store0" || true
 rm -rf "$TEST_ROOT/store0"

tests/functional/supplementary-groups.sh

@@ -9,7 +9,7 @@ needLocalStore "The test uses --store always so we would just be bypassing the d
 
 TODO_NixOS
 
-unshare --mount --map-root-user -- bash -e -x <<EOF
+execUnshare <<EOF
   source common.sh
 
   # Avoid store dir being inside sandbox build-dir