Merge pull request #12287 from bryango/2.24-maintenance

Backport git+file:./ fixes to 2.24 (#12107 + #12277)

.github/workflows/backport.yml

@@ -1,32 +0,0 @@
-name: Backport
-on:
-  pull_request_target:
-    types: [closed, labeled]
-permissions:
-  contents: read
-jobs:
-  backport:
-    name: Backport Pull Request
-    permissions:
-      # for zeebe-io/backport-action
-      contents: write
-      pull-requests: write
-    if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name))
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          # required to find all branches
-          fetch-depth: 0
-      - name: Create backport PRs
-        # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v3.0.2
-        with:
-          # Config README: https://github.com/zeebe-io/backport-action#backport-action
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_workspace: ${{ github.workspace }}
-          pull_description: |-
-            Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}.
-          # should be kept in sync with `uses`
-          version: v0.0.5

.github/workflows/ci.yml

@@ -7,14 +7,28 @@ on:
 permissions: read-all
 
 jobs:
+  eval:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v30
+    - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json
 
   tests:
-    needs: [check_secrets]
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
+        include:
+          - scenario: on ubuntu
+            runs-on: ubuntu-24.04
+            os: linux
+          - scenario: on macos
+            runs-on: macos-14
+            os: darwin
+    name: tests ${{ matrix.scenario }}
+    runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
     steps:
     - uses: actions/checkout@v4
@@ -23,102 +37,54 @@ jobs:
     - uses: cachix/install-nix-action@V27
       with:
         # The sandbox would otherwise be disabled by default on Darwin
-        extra_nix_config: "sandbox = true"
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v15
-      if: needs.check_secrets.outputs.cachix == 'true'
+        extra_nix_config: |
+          sandbox = true
+          max-jobs = 1
+    - uses: DeterminateSystems/magic-nix-cache-action@main
+    # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user:
+    # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces
+    - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+      if: matrix.os == 'linux'
+    - run: scripts/build-checks
+    - run: scripts/prepare-installer-for-github-actions
+    - name: Upload installer tarball
+      uses: actions/upload-artifact@v4
       with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-    - if: matrix.os == 'ubuntu-latest'
-      run: |
-        free -h
-        swapon --show
-        swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}')
-        echo "Found swap: $swap"
-        sudo swapoff $swap
-        # resize it (fallocate)
-        sudo fallocate -l 10G $swap
-        sudo mkswap $swap
-        sudo swapon $swap
-        free -h
-        (
-          while sleep 60; do
-            free -h
-          done
-        ) &
-    - run: nix --experimental-features 'nix-command flakes' flake check -L
-
-  # Steps to test CI automation in your own fork.
-  # Cachix:
-  # 1. Sign-up for https://www.cachix.org/
-  # 2. Create a cache for $githubuser-nix-install-tests
-  # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN
-  # Dockerhub:
-  # 1. Sign-up for https://hub.docker.com/
-  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
-  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
-  check_secrets:
-    permissions:
-      contents: none
-    name: Check Cachix and Docker secrets present for installer tests
-    runs-on: ubuntu-latest
-    outputs:
-      cachix: ${{ steps.secret.outputs.cachix }}
-      docker: ${{ steps.secret.outputs.docker }}
-    steps:
-      - name: Check for secrets
-        id: secret
-        env:
-          _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
-          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}"
-          echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
-
-  installer:
-    needs: [tests, check_secrets]
-    if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
-    runs-on: ubuntu-latest
-    outputs:
-      installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@V27
-      with:
-        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
-    - uses: cachix/cachix-action@v15
-      with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-        cachixArgs: '-v'
-    - id: prepare-installer
-      run: scripts/prepare-installer-for-github-actions
+        name: installer-${{matrix.os}}
+        path: out/*
 
   installer_test:
-    needs: [installer, check_secrets]
-    if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
+    needs: [tests]
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
+        include:
+          - scenario: on ubuntu
+            runs-on: ubuntu-24.04
+            os: linux
+          - scenario: on macos
+            runs-on: macos-14
+            os: darwin
+    name: installer test ${{ matrix.scenario }}
+    runs-on: ${{ matrix.runs-on }}
     steps:
     - uses: actions/checkout@v4
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@V27
+    - name: Download installer tarball
+      uses: actions/download-artifact@v4
       with:
-        install_url: '${{needs.installer.outputs.installerURL}}'
-        install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
+        name: installer-${{matrix.os}}
+        path: out
+    - name: Serving installer
+      id: serving_installer
+      run: ./scripts/serve-installer-for-github-actions
+    - uses: cachix/install-nix-action@v30
+      with:
+        install_url: 'http://localhost:8126/install'
+        install_options: "--tarball-url-prefix http://localhost:8126/"
     - run: sudo apt install fish zsh
-      if: matrix.os == 'ubuntu-latest'
+      if: matrix.os == 'linux'
     - run: brew install fish
-      if: matrix.os == 'macos-latest'
+      if: matrix.os == 'darwin'
     - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval"
     - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval"
     - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval"
@@ -126,32 +92,50 @@ jobs:
     - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
     - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
 
+  # Steps to test CI automation in your own fork.
+  # 1. Sign-up for https://hub.docker.com/
+  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
+  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
+  check_secrets:
+    permissions:
+      contents: none
+    name: Check Docker secrets present for installer tests
+    runs-on: ubuntu-24.04
+    outputs:
+      docker: ${{ steps.secret.outputs.docker }}
+    steps:
+      - name: Check for secrets
+        id: secret
+        env:
+          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
+        run: |
+          echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
+
   docker_push_image:
-    needs: [check_secrets, tests]
+    needs: [tests, vm_tests, check_secrets]
     permissions:
       contents: read
       packages: write
     if: >-
+      needs.check_secrets.outputs.docker == 'true' &&
       github.event_name == 'push' &&
-      github.ref_name == 'master' &&
-      needs.check_secrets.outputs.cachix == 'true' &&
-      needs.check_secrets.outputs.docker == 'true'
-    runs-on: ubuntu-latest
+      github.ref_name == 'master'
+    runs-on: ubuntu-24.04
     steps:
+    - name: Check for secrets
+      id: secret
+      env:
+        _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
+      run: |
+        echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@V27
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v15
-      if: needs.check_secrets.outputs.cachix == 'true'
-      with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - uses: DeterminateSystems/magic-nix-cache-action@main
+    - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
     - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L
     - run: docker load -i ./result/image.tar.gz
     - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
@@ -188,7 +172,7 @@ jobs:
         docker push $IMAGE_ID:master
 
   vm_tests:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
@@ -211,7 +195,7 @@ jobs:
 
   flake_regressions:
     needs: vm_tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout nix
         uses: actions/checkout@v4

.github/workflows/labels.yml

@@ -15,7 +15,7 @@ permissions:
 
 jobs:
   labels:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: github.repository_owner == 'NixOS'
     steps:
     - uses: actions/labeler@v5

.mergify.yml

@@ -0,0 +1,94 @@
+queue_rules:
+  - name: default
+    # all required tests need to go here
+    merge_conditions:
+      - check-success=installer
+      - check-success=installer_test (macos-latest)
+      - check-success=installer_test (ubuntu-latest)
+      - check-success=tests on macos
+      - check-success=tests on ubuntu
+      - check-success=installer test on macos
+      - check-success=installer test on ubuntu
+      - check-success=vm_tests
+    merge_method: rebase
+    batch_size: 5
+
+pull_request_rules:
+  - name: merge using the merge queue
+    conditions:
+      - base=master
+      - label~=merge-queue|dependencies
+    actions:
+      queue: {}
+
+# The rules below will first create backport pull requests and put those in a merge queue.
+
+  - name: backport patches to 2.18
+    conditions:
+      - label=backport 2.18-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.18-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.19
+    conditions:
+      - label=backport 2.19-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.19-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.20
+    conditions:
+      - label=backport 2.20-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.20-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.21
+    conditions:
+      - label=backport 2.21-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.21-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.22
+    conditions:
+      - label=backport 2.22-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.22-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.23
+    conditions:
+      - label=backport 2.23-maintenance
+    actions:
+      backport:
+        branches:
+          - 2.23-maintenance
+        labels:
+          - merge-queue
+
+  - name: backport patches to 2.24
+    conditions:
+      - label=backport 2.24-maintenance
+    actions:
+      backport:
+        branches:
+          - "2.24-maintenance"
+        labels:
+          - merge-queue

.version

@@ -1 +1 @@
-2.24.0
+2.24.12

configure.ac

@@ -62,12 +62,16 @@ AC_CHECK_TOOL([AR], [ar])
 AC_SYS_LARGEFILE
 
 
-# Solaris-specific stuff.
+# OS-specific stuff.
 case "$host_os" in
   solaris*)
     # Solaris requires -lsocket -lnsl for network functions
     LDFLAGS="-lsocket -lnsl $LDFLAGS"
     ;;
+  darwin*)
+    # Need to link to libsandbox.
+    LDFLAGS="-lsandbox $LDFLAGS"
+    ;;
 esac
 
 
@@ -89,9 +93,10 @@ AC_LANG_POP(C++)
 AC_CHECK_FUNCS([statvfs pipe2])
 
 
-# Check for lutimes, optionally used for changing the mtime of
-# symlinks.
-AC_CHECK_FUNCS([lutimes])
+# Check for lutimes and utimensat, optionally used for changing the
+# mtime of symlinks.
+AC_CHECK_DECLS([AT_SYMLINK_NOFOLLOW], [], [], [[#include <fcntl.h>]])
+AC_CHECK_FUNCS([lutimes utimensat])
 
 
 # Check whether the store optimiser can optimise symlinks.
@@ -170,6 +175,10 @@ AS_IF(
   [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"],
   [NEED_PROG(jq, jq)])
 
+AS_IF(
+  [test "$ENABLE_DOC_GEN" == "yes"],
+  [NEED_PROG(man, man)])
+
 AS_IF([test "$ENABLE_BUILD" == "yes"],[
 
 # Look for boost, a required dependency.

doc/manual/source/language/string-literals.md

@@ -0,0 +1,205 @@
+# String literals
+
+A *string literal* represents a [string](types.md#type-string) value.
+
+> **Syntax**
+>
+> *expression* → *string*
+>
+> *string* → `"` ( *string_char*\* [*interpolation_element*][string interpolation] )* *string_char*\* `"`
+>
+> *string* → `''` ( *indented_string_char*\* [*interpolation_element*][string interpolation] )* *indented_string_char*\* `''`
+>
+> *string* → *uri*
+>
+> *string_char* ~ `[^"$\\]|\$(?!\{)|\\.`
+>
+> *indented_string_char* ~ `[^$']|\$\$|\$(?!\{)|''[$']|''\\.|'(?!')`
+>
+> *uri* ~ `[A-Za-z][+\-.0-9A-Za-z]*:[!$%&'*+,\-./0-9:=?@A-Z_a-z~]+`
+
+Strings can be written in three ways.
+
+The most common way is to enclose the string between double quotes, e.g., `"foo bar"`.
+Strings can span multiple lines.
+The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation].
+
+[string interpolation]: ./string-interpolation.md
+
+The following must be escaped to represent them within a string, by prefixing with a backslash (`\`):
+
+- Double quote (`"`)
+
+> **Example**
+>
+> ```nix
+> "\""
+> ```
+>
+>     "\""
+
+- Backslash (`\`)
+
+> **Example**
+>
+> ```nix
+> "\\"
+> ```
+>
+>     "\\"
+
+- Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly"
+
+> **Example**
+>
+> ```nix
+> "\${"
+> ```
+>
+>     "\${"
+
+The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively.
+
+A "double-dollar-curly" (`$${`) can be written literally.
+
+> **Example**
+>
+> ```nix
+> "$${"
+> ```
+>
+>     "$\${"
+
+String values are output on the terminal with Nix-specific escaping.
+Strings written to files will contain the characters encoded by the escaping.
+
+The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so:
+
+```nix
+''
+This is the first line.
+This is the second line.
+  This is the third line.
+''
+```
+
+This kind of string literal intelligently strips indentation from
+the start of each line. To be precise, it strips from each line a
+number of spaces equal to the minimal indentation of the string as a
+whole (disregarding the indentation of empty lines). For instance,
+the first and second line are indented two spaces, while the third
+line is indented four spaces. Thus, two spaces are stripped from
+each line, so the resulting string is
+
+```nix
+"This is the first line.\nThis is the second line.\n  This is the third line.\n"
+```
+
+> **Note**
+>
+> Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line.
+
+> **Warning**
+>
+> Prefixed tab characters are not stripped.
+>
+> > **Example**
+> >
+> > The following indented string is prefixed with tabs:
+> >
+> > <pre><code class="nohighlight">''
+> > 	all:
+> > 		@echo hello
+> > ''
+> > </code></pre>
+> >
+> >     "\tall:\n\t\t@echo hello\n"
+
+Indented strings support [string interpolation].
+
+The following must be escaped to represent them in an indented string:
+
+- `$` is escaped by prefixing it with two single quotes (`''`)
+
+> **Example**
+>
+> ```nix
+> ''
+>   ''$
+> ''
+> ```
+>
+>     "$\n"
+
+- `''` is escaped by prefixing it with one single quote (`'`)
+
+> **Example**
+>
+> ```nix
+> ''
+>   '''
+> ''
+> ```
+>
+>     "''\n"
+
+These special characters are escaped as follows:
+- Linefeed (`\n`): `''\n`
+- Carriage return (`\r`): `''\r`
+- Tab (`\t`): `''\t`
+
+`''\` escapes any other character.
+
+A "dollar-curly" (`${`) can be written as follows: 
+> **Example**
+>
+> ```nix
+> ''
+>   echo ''${PATH}
+> ''
+> ```
+>
+>     "echo ${PATH}\n"
+
+> **Note**
+>
+> This differs from the syntax for escaping a dollar-curly within double quotes (`"\${"`). Be aware of which one is needed at a given moment.
+
+A "double-dollar-curly" (`$${`) can be written literally.
+
+> **Example**
+>
+> ```nix
+> ''
+>   $${
+> ''
+> ```
+>
+>     "$\${\n"
+
+Indented strings are primarily useful in that they allow multi-line
+string literals to follow the indentation of the enclosing Nix
+expression, and that less escaping is typically necessary for
+strings representing languages such as shell scripts and
+configuration files because `''` is much less common than `"`.
+Example:
+
+```nix
+stdenv.mkDerivation {
+...
+postInstall =
+  ''
+    mkdir $out/bin $out/etc
+    cp foo $out/bin
+    echo "Hello World" > $out/etc/foo.conf
+    ${if enableBar then "cp bar $out/bin" else ""}
+  '';
+...
+}
+```
+
+Finally, as a convenience, *URIs* as defined in appendix B of
+[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as
+is*, without quotes. For instance, the string
+`"http://example.org/foo.tar.bz2"` can also be written as
+`http://example.org/foo.tar.bz2`.

doc/manual/src/command-ref/nix-collect-garbage.md

@@ -62,6 +62,15 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto
   This is the equivalent of invoking [`nix-env --delete-generations <period>`](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile.
   See the documentation of that command for additional information about the *period* argument.
 
+  - <span id="opt-max-freed">[`--max-freed`](#opt-max-freed)</span> *bytes*
+
+<!-- duplication from https://github.com/NixOS/nix/blob/442a2623e48357ff72c77bb11cf2cf06d94d2f90/doc/manual/source/command-ref/nix-store/gc.md?plain=1#L39-L44 -->
+
+  Keep deleting paths until at least *bytes* bytes have been deleted,
+  then stop. The argument *bytes* can be followed by the
+  multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB
+  or TiB units.
+  
 {{#include ./opt-common.md}}
 
 {{#include ./env-common.md}}

doc/manual/src/command-ref/nix-env.md

@@ -62,7 +62,7 @@ These pages can be viewed offline:
 
 Several operations, such as [`nix-env --query`](./nix-env/query.md) and [`nix-env --install`](./nix-env/install.md), take a list of *arguments* that specify the packages on which to operate.
 
-Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-names):
+Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-name):
 
 - `name`: Everything up to but not including the first dash (`-`) that is *not* followed by a letter.
 - `version`: The rest, excluding the separating dash.

doc/manual/src/command-ref/nix-store/add-fixed.md

@@ -21,6 +21,9 @@ This operation has the following options:
   Use recursive instead of flat hashing mode, used when adding
   directories to the store.
 
+  *paths* that refer to symlinks are not dereferenced, but added to the store
+  as symlinks with the same target.
+
 {{#include ./opt-common.md}}
 
 {{#include ../opt-common.md}}

doc/manual/src/command-ref/nix-store/add.md

@@ -11,6 +11,9 @@
 The operation `--add` adds the specified paths to the Nix store. It
 prints the resulting paths in the Nix store on standard output.
 
+*paths* that refer to symlinks are not dereferenced, but added to the store
+as symlinks with the same target.
+
 {{#include ./opt-common.md}}
 
 {{#include ../opt-common.md}}

doc/manual/src/command-ref/nix-store/query.md

@@ -104,7 +104,7 @@ symlink.
 
   Prints a set of derivation files (`.drv`) which are supposed produce
   said paths when realized. Might print nothing, for example for source paths
-  or paths subsituted from a binary cache.
+  or paths substituted from a binary cache.
 
 - `--graph`
 
@@ -241,4 +241,3 @@ $ nix-store --query --roots $(which svn)
 /nix/var/nix/profiles/default-82-link
 /home/eelco/.local/state/nix/profiles/profile-97-link
 ```
-

doc/manual/src/development/json-guideline.md

@@ -90,7 +90,7 @@ This representation is extensible and preserves the ordering:
 
 ## Self-describing values
 
-As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility.
+As described in the previous section, it's crucial that schemas can be extended with new fields without breaking compatibility.
 However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema.
 Instead, always include the field, and use `null` to indicate the "nothing" case.
 

doc/manual/src/development/testing.md

@@ -308,7 +308,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to
   - `armv7l-linux`
   - `x86_64-darwin`
 
-- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command.
+- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command.
 
 ### One-time setup
 

doc/manual/src/installation/index.md

@@ -14,6 +14,14 @@ This option requires either:
 * Linux running systemd, with SELinux disabled
 * MacOS
 
+> **Updating to macOS 15 Sequoia**
+>
+> If you recently updated to macOS 15 Sequoia and are getting
+> ```console
+> error: the user '_nixbld1' in the group 'nixbld' does not exist
+> ```
+> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling.
+
 ```console
 $ bash <(curl -L https://nixos.org/nix/install) --daemon
 ```

doc/manual/src/installation/installing-binary.md

@@ -1,5 +1,13 @@
 # Installing a Binary Distribution
 
+> **Updating to macOS 15 Sequoia**
+>
+> If you recently updated to macOS 15 Sequoia and are getting
+> ```console
+> error: the user '_nixbld1' in the group 'nixbld' does not exist
+> ```
+> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling.
+
 To install the latest version Nix, run the following command:
 
 ```console

doc/manual/src/installation/prerequisites-source.md

@@ -39,8 +39,6 @@
     `pkgconfig` and the Boehm garbage collector, and pass the flag
     `--enable-gc` to `configure`.
 
-    For `bdw-gc` <= 8.2.4 Nix needs a [small patch](https://github.com/NixOS/nix/blob/ac4d2e7b857acdfeac35ac8a592bdecee2d29838/boehmgc-traceable_allocator-public.diff) to be applied.
-
   - The `boost` library of version 1.66.0 or higher. It can be obtained
     from the official web site <https://www.boost.org/>.
 

doc/manual/src/installation/uninstall.md

@@ -19,7 +19,7 @@ If you are on Linux with systemd:
 Remove files created by Nix:
 
 ```console
-sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile
+sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile ~root/.cache/nix
 ```
 
 Remove build users and their group:
@@ -43,6 +43,14 @@ which you may remove.
 
 ### macOS
 
+> **Updating to macOS 15 Sequoia**
+>
+> If you recently updated to macOS 15 Sequoia and are getting
+> ```console
+> error: the user '_nixbld1' in the group 'nixbld' does not exist
+> ```
+> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling.
+
 1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer:
 
    ```console
@@ -133,7 +141,9 @@ which you may remove.
    diskutil list
    ```
 
-   If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier.
+   If you _do_ find a "Nix Store" volume, delete it by running `diskutil apfs deleteVolume` with the store volume's `diskXsY` identifier.
+
+   If you get an error that the volume is in use by the kernel, reboot and immediately delete the volume before starting any other process.
 
 > **Note**
 >

doc/manual/src/language/derivations.md

@@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
 
 ### Optional
 
-- [`args`]{#attr-args} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string))
+- [`args`]{#attr-args} ([List](@docroot@/language/types.md#type-list) of [String](@docroot@/language/types.md#type-string))
 
   Default: `[ ]`
 
@@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
   > };
   > ```
 
-- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#list) of [String](@docroot@/language/types.md#type-string))
+- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#type-list) of [String](@docroot@/language/types.md#type-string))
 
   Default: `[ "out" ]`
 

doc/manual/src/language/operators.md

@@ -3,7 +3,7 @@
 | Name                                   | Syntax                                     | Associativity | Precedence |
 |----------------------------------------|--------------------------------------------|---------------|------------|
 | [Attribute selection]                  | *attrset* `.` *attrpath* \[ `or` *expr* \] | none          | 1          |
-| Function application                   | *func* *expr*                              | left          | 2          |
+| [Function application]                 | *func* *expr*                              | left          | 2          |
 | [Arithmetic negation][arithmetic]      | `-` *number*                               | none          | 3          |
 | [Has attribute]                        | *attrset* `?` *attrpath*                   | none          | 4          |
 | List concatenation                     | *list* `++` *list*                         | right         | 5          |
@@ -32,8 +32,8 @@
 [string]: ./types.md#type-string
 [path]: ./types.md#type-path
 [number]: ./types.md#type-float
-[list]: ./types.md#list
-[attribute set]: ./types.md#attribute-set
+[list]: ./types.md#type-list
+[attribute set]: ./types.md#type-attrs
 
 <!-- TODO(@rhendric, #10970): ^ rationalize number -> int/float -->
 
@@ -48,6 +48,22 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe
 
 [Attribute selection]: #attribute-selection
 
+## Function application
+
+> **Syntax**
+>
+> *func* *expr*
+
+Apply the callable value *func* to the argument *expr*. Note the absence of any visible operator symbol.
+A callable value is either:
+- a [user-defined function][function]
+- a [built-in][builtins] function
+- an attribute set with a [`__functor` attribute](./syntax.md#attr-__functor)
+
+> **Warning**
+>
+> [List][list] items are also separated by whitespace, which means that function calls in list items must be enclosed by parentheses.
+
 ## Has attribute
 
 > **Syntax**
@@ -59,7 +75,7 @@ The result is a [Boolean] value.
 
 See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr)
 
-[Boolean]: ./types.md#type-boolean
+[Boolean]: ./types.md#type-bool
 
 [Has attribute]: #has-attribute
 
@@ -211,3 +227,5 @@ Equivalent to `!`*b1* `||` *b2*.
 > ```
 
 [Pipe operator]: #pipe-operators
+[builtins]: ./builtins.md
+[Function application]: #function-application

doc/manual/src/language/syntax.md

@@ -379,7 +379,7 @@ a string), that attribute is simply not added to the set:
 
 This will evaluate to `{}` if `foo` evaluates to `false`.
 
-A set that has a `__functor` attribute whose value is callable (i.e. is
+A set that has a [`__functor`]{#attr-__functor} attribute whose value is callable (i.e. is
 itself a function or a set with a `__functor` attribute whose value is
 callable) can be applied as if it were a function, with the set itself
 passed in first , e.g.,

doc/manual/src/protocols/store-path.md

@@ -82,7 +82,7 @@ where
 
   - if `type` = `"source:" ...`:
 
-    the the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object.
+    the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object.
 
   - if `type` = `"output:" id`:
 

doc/manual/src/release-notes/rl-2.19.md

@@ -75,3 +75,7 @@
   (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html).
 
 - A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated.
+
+- A new option [`always-allow-substitutes`](@docroot@/command-ref/conf-file.md#conf-always-allow-substitutes) has been added.
+
+  When set to `true`, Nix will always try to substitute a derivation, even if it has the [`allowSubstitutes`]{#adv-attr-allowSubstitutes} attribute set to `false`.

doc/manual/src/release-notes/rl-2.23.md

@@ -85,7 +85,7 @@
 - Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995)
 
   The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value.
-  For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely.
+  For example, `"ca": null` is used to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely.
   This makes records of this sort more self-describing, and easier to consume programmatically.
 
   We will follow this design principle going forward;
@@ -96,7 +96,7 @@
   Nix can now warn when evaluation of a Nix expression causes a large
   path to be copied to the Nix store. The threshold for this warning can
   be configured using [the `warn-large-path-threshold`
-  setting](@docroot@/command-ref/conf-file.md#warn-large-path-threshold),
+  setting](@docroot@/command-ref/conf-file.md#conf-warn-large-path-threshold),
   e.g. `--warn-large-path-threshold 100M` will warn about paths larger
   than 100 MiB.
 

doc/manual/src/release-notes/rl-2.24.md

@@ -274,6 +274,21 @@
   be configured using the `warn-large-path-threshold` setting,
   e.g. `--warn-large-path-threshold 100M`.
 
+- Wrap filesystem exceptions more correctly [#11378](https://github.com/NixOS/nix/pull/11378)
+
+  With the switch to `std::filesystem` in different places, Nix started to throw `std::filesystem::filesystem_error` in many places instead of its own exceptions.
+
+  This led to no longer generating error traces, for example when listing a non-existing directory.
+
+  This version catches these types of exception correctly and wraps them into Nix's own exeception type.
+
+  Author: [**@Mic92**](https://github.com/Mic92)
+
+- `<nix/fetchurl.nix>` uses TLS verification [#11585](https://github.com/NixOS/nix/pull/11585)
+
+  Previously `<nix/fetchurl.nix>` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `<nix/fetchurl.nix>` will now fail if you're fetching from a HTTPS server that does not have a valid certificate.
+
+  `<nix/fetchurl.nix>` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue.
 
 # Contributors
 

docker.nix

@@ -241,14 +241,14 @@ let
       mkdir -p $out/nix/var/nix/profiles/per-user/root
 
       ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
-      ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+      ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
       ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
 
       ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
-      ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
+      ln -s /nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
 
       mkdir -p $out/root/.nix-defexpr
-      ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
+      ln -s /nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
       echo "${channelURL} ${channelName}" > $out/root/.nix-channels
 
       mkdir -p $out/bin $out/usr/bin

flake.lock

@@ -80,11 +80,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1721548954,
-        "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=",
+        "lastModified": 1723688146,
+        "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a",
+        "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0",
         "type": "github"
       },
       "original": {

flake.nix

@@ -24,7 +24,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -174,7 +174,7 @@
       };
 
       checks = forAllSystems (system: {
-        binaryTarball = self.hydraJobs.binaryTarball.${system};
+        installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
         rl-next =
@@ -221,6 +221,8 @@
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
           default = self.packages.${system}.nix;
+          binaryTarball = self.hydraJobs.binaryTarball.${system};
+          installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
           nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs;
           nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
         }

maintainers/flake-module.nix

@@ -10,6 +10,27 @@
     # https://flake.parts/options/pre-commit-hooks-nix.html#options
     pre-commit.settings = {
       hooks = {
+        # Conflicts are usually found by other checks, but not those in docs,
+        # and potentially other places.
+        check-merge-conflicts.enable = true;
+        # built-in check-merge-conflicts seems ineffective against those produced by mergify backports
+        check-merge-conflicts-2 = {
+          enable = true;
+          entry = "${pkgs.writeScript "check-merge-conflicts" ''
+            #!${pkgs.runtimeShell}
+            conflicts=false
+            for file in "$@"; do
+              if grep --with-filename --line-number -E '^>>>>>>> ' -- "$file"; then
+                conflicts=true
+              fi
+            done
+            if $conflicts; then
+              echo "ERROR: found merge/patch conflicts in files"
+              exit 1
+            fi
+            touch $out
+          ''}";
+        };
         clang-format = {
           enable = true;
           excludes = [

maintainers/upload-release.pl

@@ -42,7 +42,7 @@ my $flakeUrl = $evalInfo->{flake};
 my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl;
 my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die;
 
-my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json'));
+my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json'));
 #print Dumper($buildInfo);
 
 my $releaseName = $buildInfo->{nixname};
@@ -91,7 +91,7 @@ sub getStorePath {
 sub copyManual {
     my $manual;
     eval {
-        $manual = getStorePath("build.x86_64-linux", "doc");
+        $manual = getStorePath("build.nix.x86_64-linux", "doc");
     };
     if ($@) {
         warn "$@";
@@ -112,7 +112,7 @@ sub copyManual {
         system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0
             or die "unable to unpack $manualNar\n";
         rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die;
-        system("rm -rf '$tmpDir/manual.tmp'") == 0 or die;
+        File::Path::remove_tree("$tmpDir/manual.tmp", {safe => 1});
     }
 
     system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
@@ -240,12 +240,12 @@ if ($haveDocker) {
 # Upload nix-fallback-paths.nix.
 write_file("$tmpDir/fallback-paths.nix",
     "{\n" .
-    "  x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" .
-    "  i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" .
-    "  aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" .
-    "  riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
-    "  x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" .
-    "  aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" .
+    "  x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" .
+    "  i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" .
+    "  aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" .
+    "  riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
+    "  x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" .
+    "  aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" .
     "}\n");
 
 # Upload release files to S3.
@@ -281,3 +281,6 @@ system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
 system("git push --tags") == 0 or die;
 system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
+
+File::Path::remove_tree($narCache, {safe => 1});
+File::Path::remove_tree($tmpDir, {safe => 1});

misc/bash/completion.sh

@@ -12,9 +12,16 @@ function _complete_nix {
             elif [[ $completion == attrs ]]; then
                 compopt -o nospace
             fi
-        else
-            COMPREPLY+=("$completion")
+            continue
         fi
+
+        if [[ "${cur}" =~ "=" ]]; then
+            # drop everything up to the first =. if a = is included, bash assumes this to be
+            # an arg=value argument and the completion gets mangled (see #11208)
+            completion="${completion#*=}"
+        fi
+
+        COMPREPLY+=("${completion}")
     done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}" 2>/dev/null)
     __ltrim_colon_completions "$cur"
 }

mk/libraries.mk

@@ -86,7 +86,9 @@ define build-library
     else
       ifndef HOST_DARWIN
         ifndef HOST_WINDOWS
-          $(1)_LDFLAGS += -Wl,-z,defs
+	  ifndef HOST_OPENBSD
+            $(1)_LDFLAGS += -Wl,-z,defs
+          endif
         endif
       endif
     endif

mk/platform.mk

@@ -21,6 +21,10 @@ ifdef HOST_OS
     HOST_NETBSD = 1
     HOST_UNIX = 1
   endif
+  ifeq ($(patsubst openbsd%,,$(HOST_KERNEL)),)
+    HOST_OPENBSD = 1
+    HOST_UNIX = 1
+  endif
   ifeq ($(HOST_KERNEL), linux)
     HOST_LINUX = 1
     HOST_UNIX = 1

package.nix

@@ -23,6 +23,7 @@
 , libseccomp
 , libsodium
 , man
+, darwin
 , lowdown
 , mdbook
 , mdbook-linkcheck
@@ -59,7 +60,7 @@
 # Run the functional tests as part of the build.
 , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled
 
-# Check test coverage of Nix. Probably want to use with with at least
+# Check test coverage of Nix. Probably want to use with at least
 # one of `doCHeck` or `doInstallCheck` enabled.
 , withCoverageChecks ? false
 
@@ -75,7 +76,9 @@
 #
 # Temporarily disabled on Windows because the `GC_throw_bad_alloc`
 # symbol is missing during linking.
-, enableGC ? !stdenv.hostPlatform.isWindows
+#
+# Disabled on OpenBSD because of missing `_data_start` symbol while linking
+, enableGC ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isOpenBSD
 
 # Whether to enable Markdown rendering in the Nix binary.
 , enableMarkdown ? !stdenv.hostPlatform.isWindows
@@ -210,9 +213,10 @@ in {
     git
     mercurial
     openssh
-    man # for testing `nix-* --help`
   ] ++ lib.optionals (doInstallCheck || enableManual) [
     jq # Also for custom mdBook preprocessor.
+  ] ++ lib.optionals enableManual [
+    man
   ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump
   ;
 
@@ -235,6 +239,7 @@ in {
     gtest
     rapidcheck
   ] ++ lib.optional stdenv.isLinux libseccomp
+    ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox
     ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
     # There have been issues building these dependencies
     ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))

packaging/hydra.nix

@@ -116,15 +116,10 @@ in
     self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
     self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
   ];
-  installerScriptForGHA = installScriptFor [
-    # Native
-    self.hydraJobs.binaryTarball."x86_64-linux"
-    self.hydraJobs.binaryTarball."aarch64-darwin"
-    # Cross
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
-  ];
+
+  installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix {
+    tarballs = [ self.hydraJobs.binaryTarball.${system} ];
+  });
 
   # docker image with Nix inside
   dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);

scripts/bigsur-nixbld-user-migration.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-((NEW_NIX_FIRST_BUILD_UID=301))
+((NEW_NIX_FIRST_BUILD_UID=351))
 
 id_available(){
 	dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null

scripts/binary-tarball.nix

@@ -65,7 +65,7 @@ runCommand "nix-binary-tarball-${version}" env ''
   fn=$out/$dir.tar.xz
   mkdir -p $out/nix-support
   echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
-  tar cvfJ $fn \
+  tar cfJ $fn \
     --owner=0 --group=0 --mode=u+rw,uga+r \
     --mtime='1970-01-01' \
     --absolute-names \

scripts/build-checks

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+system=$(nix eval --raw --impure --expr builtins.currentSystem)
+nix eval --json ".#checks.$system" --apply builtins.attrNames | \
+  jq -r '.[]' | \
+  xargs -P0 -I '{}' sh -c "nix build -L .#checks.$system.{} || { echo 'FAILED: \033[0;31mnix build -L .#checks.$system.{}\\033[0m'; kill 0; }"

scripts/create-darwin-volume.sh

@@ -463,7 +463,7 @@ EOF
 
     EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" <<EOF
 :a
-UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners
+UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,nosuid,noatime,owners
 .
 :x
 EOF

scripts/install-darwin-multi-user.sh

@@ -4,7 +4,17 @@ set -eu
 set -o pipefail
 
 # System specific settings
-export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}"
+# Notes:
+# - up to macOS Big Sur we used the same GID/UIDs as Linux (30000:30001-32)
+# - we changed UID to 301 because Big Sur updates failed into recovery mode
+#   we're targeting the 200-400 UID range for role users mentioned in the
+#   usage note for sysadminctl
+# - we changed UID to 351 because Sequoia now uses UIDs 300-304 for its own
+#   daemon users
+# - we changed GID to 350 alongside above just because it hides the nixbld
+#   group from the Users & Groups settings panel :)
+export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-351}"
+export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-350}"
 export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
 
 readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist

scripts/install-multi-user.sh

@@ -23,10 +23,10 @@ readonly RED='\033[31m'
 # installer allows overriding build user count to speed up installation
 # as creating each user takes non-trivial amount of time on macos
 readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
-readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 readonly NIX_BUILD_GROUP_NAME="nixbld"
 # each system specific installer must set these:
 #   NIX_FIRST_BUILD_UID
+#   NIX_BUILD_GROUP_ID
 #   NIX_BUILD_USER_NAME_TEMPLATE
 # Please don't change this. We don't support it, because the
 # default shell profile that comes with Nix doesn't support it.
@@ -530,9 +530,7 @@ It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
 with the UID $primary_group_id. This script can't really handle
 that right now, so I'm going to give up.
 
-You can fix this by editing this script and changing the
-NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
-to $primary_group_id and re-run.
+You can export NIX_BUILD_GROUP_ID=$primary_group_id and re-run.
 EOF
         else
             row "            Exists" "Yes"

scripts/install-systemd-multi-user.sh

@@ -5,6 +5,7 @@ set -o pipefail
 
 # System specific settings
 export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
+export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
 export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
 
 readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
@@ -95,6 +96,9 @@ poly_configure_nix_daemon_service() {
     if [ -e /run/systemd/system ]; then
         task "Setting up the nix-daemon systemd service"
 
+        _sudo "to create parent of the nix-daemon tmpfiles config" \
+              mkdir -p "$(dirname "$TMPFILES_DEST")"
+
         _sudo "to create the nix-daemon tmpfiles config" \
               ln -sfn "/nix/var/nix/profiles/default$TMPFILES_SRC" "$TMPFILES_DEST"
 

scripts/nix-profile.sh.in

@@ -1,5 +1,5 @@
 # This file is tested by tests/installer/default.nix.
-if [ -n "$HOME" ] && [ -n "$USER" ]; then
+if [ -n "${HOME-}" ] && [ -n "${USER-}" ]; then
 
     # Set up the per-user profile.
 

scripts/prepare-installer-for-github-actions

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
-script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
-installerHash=$(echo "$script" | cut -b12-43 -)
+nix build -L ".#installerScriptForGHA" ".#binaryTarball"
 
-installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
-
-echo "::set-output name=installerURL::$installerURL"
+mkdir -p out
+cp ./result/install "out/install"
+name="$(basename "$(realpath ./result-1)")"
+# everything before the first dash
+cp -r ./result-1 "out/${name%%-*}"

scripts/sequoia-nixbld-user-migration.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# stock path to avoid unexpected command versions
+PATH="$(/usr/bin/getconf PATH)"
+
+((NEW_NIX_FIRST_BUILD_UID=351))
+((TEMP_NIX_FIRST_BUILD_UID=31000))
+
+nix_user_n() {
+	printf "_nixbld%d" "$1"
+}
+
+id_unavailable(){
+	dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null
+}
+
+any_nixbld(){
+	dscl . list /Users UniqueID | grep -E '\b_nixbld' >/dev/null
+}
+
+dsclattr() {
+	dscl . -read "$1" | awk "/$2/ { print \$2 }"
+}
+
+re_create_nixbld_user(){
+	local name uid
+
+	name="$1"
+	uid="$2"
+	gid="$3"
+
+	sudo /usr/bin/dscl . -create "/Users/$name" "UniqueID" "$uid"
+	sudo /usr/bin/dscl . -create "/Users/$name" "IsHidden" "1"
+	sudo /usr/bin/dscl . -create "/Users/$name" "NFSHomeDirectory" "/var/empty"
+	sudo /usr/bin/dscl . -create "/Users/$name" "RealName" "Nix build user $name"
+	sudo /usr/bin/dscl . -create "/Users/$name" "UserShell" "/sbin/nologin"
+	sudo /usr/bin/dscl . -create "/Users/$name" "PrimaryGroupID" "$gid"
+}
+
+hit_id_cap(){
+	echo "We've hit UID 400 without placing all of your users :("
+	echo "You should use the commands in this script as a starting"
+	echo "point to review your UID-space and manually move the"
+	echo "remaining users (or delete them, if you don't need them)."
+}
+
+# evacuate the role-uid space to simplify final placement logic
+temporarily_move_existing_nixbld_uids(){
+	local name uid next_id user_n
+
+	((next_id=TEMP_NIX_FIRST_BUILD_UID))
+
+	echo ""
+	echo "Step 1: move existing _nixbld users out of the destination UID range."
+
+	while read -r name uid; do
+		# iterate for a clean ID
+		while id_unavailable "$next_id"; do
+			((next_id++))
+			# We really want to get these all placed, but I guess there's
+			# some risk we iterate forever--so we'll give up after 9k uids.
+			if ((next_id >= 40000)); then
+				echo "We've hit UID 40000 without temporarily placing all of your users :("
+				echo "You should use the commands in this script as a starting"
+				echo "point to review your UID-space and manually move the"
+				echo "remaining users to any open UID over 1000."
+				exit 1
+			fi
+		done
+		sudo dscl . -create "/Users/$name" UniqueID "$next_id"
+		echo "   Temporarily moved $name from uid $uid -> $next_id"
+
+	done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2)
+}
+
+change_nixbld_uids(){
+	local existing_gid name next_id user_n
+
+	((next_id=NEW_NIX_FIRST_BUILD_UID))
+	((user_n=1))
+	name="$(nix_user_n "$user_n")"
+	existing_gid="$(dsclattr "/Groups/nixbld" "PrimaryGroupID")"
+
+	# we know that we have *some* nixbld users, but macOS may have
+	# already clobbered the first few users if this system has been
+	# upgraded
+
+	echo ""
+	echo "Step 2: re-create missing early _nixbld# users."
+
+	until dscl . read "/Users/$name" &>/dev/null; do
+		# iterate for a clean ID
+		while id_unavailable "$next_id"; do
+			((next_id++))
+			if ((next_id >= 400)); then
+				hit_id_cap
+				exit 1
+			fi
+		done
+
+		re_create_nixbld_user "$name" "$next_id" "$existing_gid"
+		echo "      $name was missing; created with uid: $next_id"
+
+		((user_n++))
+		name="$(nix_user_n "$user_n")"
+	done
+
+	echo ""
+	echo "Step 3: relocate remaining _nixbld# UIDs to $next_id+"
+
+	# start at first _nixbld# not re-created above and increment
+	# until _nixbld<n> doesn't exist
+	while dscl . read "/Users/$name" &>/dev/null; do
+		# iterate for a clean ID
+		while id_unavailable "$next_id"; do
+			((next_id++))
+			if ((next_id >= 400)); then
+				hit_id_cap
+				exit 1
+			fi
+		done
+
+		sudo dscl . -create "/Users/$name" UniqueID "$next_id"
+		echo "      $name migrated to uid: $next_id"
+
+		((user_n++))
+		name="$(nix_user_n "$user_n")"
+	done
+
+	if ((user_n == 1)); then
+		echo "Didn't find _nixbld1. Perhaps you have single-user Nix?"
+		exit 1
+	else
+		echo "Migrated $((user_n - 1)) users. If you want to double-check, try:"
+		echo "dscl . list /Users UniqueID | grep _nixbld | sort -n -k2"
+	fi
+}
+needs_migration(){
+	local name uid next_id user_n
+
+	((next_id=NEW_NIX_FIRST_BUILD_UID))
+	((user_n=1))
+
+	while read -r name uid; do
+		expected_name="$(nix_user_n "$user_n")"
+		if [[ "$expected_name" != "$name" ]]; then
+			return 0
+		fi
+		if [[ "$next_id" != "$uid" ]]; then
+			return 0
+		fi
+		((next_id++))
+		((user_n++))
+	done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2)
+	return 1
+}
+
+
+if any_nixbld; then
+	if needs_migration; then
+		echo "Attempting to migrate _nixbld users."
+		temporarily_move_existing_nixbld_uids
+		change_nixbld_uids
+	else
+		echo "_nixbld users already appear to be migrated."
+	fi
+else
+	echo "Didn't find any _nixbld users. Perhaps you have single-user Nix?"
+	exit 1
+fi

scripts/serve-installer-for-github-actions

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+if [[ ! -d out ]]; then
+  echo "run prepare-installer-for-github-actions first"
+  exit 1
+fi
+cd out
+PORT=${PORT:-8126}
+nohup python -m http.server "$PORT" >/dev/null 2>&1 &
+pid=$!
+
+while ! curl -s "http://localhost:$PORT"; do
+  sleep 1
+  if ! kill -0 $pid; then
+    echo "Failed to start http server"
+    exit 1
+  fi
+done
+
+echo 'To install nix, run the following command:'
+echo "sh <(curl http://localhost:$PORT/install) --tarball-url-prefix http://localhost:$PORT"

src/libcmd/common-eval-args.cc

@@ -171,7 +171,9 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto accessor = fetchers::downloadTarball(
-            EvalSettings::resolvePseudoUrl(s)).accessor;
+            state.store,
+            state.fetchSettings,
+            EvalSettings::resolvePseudoUrl(s));
         auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }

src/libcmd/installable-derived-path.cc

@@ -32,16 +32,6 @@ InstallableDerivedPath InstallableDerivedPath::parse(
         // store path.
         [&](const ExtendedOutputsSpec::Default &) -> DerivedPath {
             auto storePath = store->followLinksToStorePath(prefix);
-            // Remove this prior to stabilizing the new CLI.
-            if (storePath.isDerivation()) {
-                auto oldDerivedPath = DerivedPath::Built {
-                    .drvPath = makeConstantStorePathRef(storePath),
-                    .outputs = OutputsSpec::All { },
-                };
-                warn(
-                    "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'",
-                    oldDerivedPath.to_string(*store));
-            };
             return DerivedPath::Opaque {
                 .path = std::move(storePath),
             };

src/libcmd/installables.cc

@@ -86,7 +86,7 @@ MixFlakeOptions::MixFlakeOptions()
 
     > **DEPRECATED**
     >
-    > Use [`--no-use-registries`](#opt-no-use-registries) instead.
+    > Use [`--no-use-registries`](@docroot@/command-ref/conf-file.md#conf-use-registries) instead.
         )",
         .category = category,
         .handler = {[&]() {

src/libcmd/markdown.cc

@@ -16,13 +16,25 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown)
 {
     int windowWidth = getWindowSize().second;
 
-    struct lowdown_opts opts
-    {
-        .type = LOWDOWN_TERM,
-        .maxdepth = 20,
+#if HAVE_LOWDOWN_1_4
+    struct lowdown_opts_term opts_term {
         .cols = (size_t) std::max(windowWidth - 5, 60),
         .hmargin = 0,
         .vmargin = 0,
+    };
+#endif
+    struct lowdown_opts opts
+    {
+        .type = LOWDOWN_TERM,
+#if HAVE_LOWDOWN_1_4
+        .term = opts_term,
+#endif
+        .maxdepth = 20,
+#if !HAVE_LOWDOWN_1_4
+        .cols = (size_t) std::max(windowWidth - 5, 60),
+        .hmargin = 0,
+        .vmargin = 0,
+#endif
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
         .oflags = LOWDOWN_TERM_NOLINK,
     };

src/libcmd/meson.build

@@ -38,6 +38,8 @@ deps_public += nlohmann_json
 lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown'))
 deps_private += lowdown
 configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
+# The API changed slightly around terminal initialization.
+configdata.set('HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int())
 
 readline_flavor = get_option('readline-flavor')
 if readline_flavor == 'editline'

src/libexpr-c/nix_api_expr.h

@@ -14,6 +14,16 @@
 #include "nix_api_util.h"
 #include <stddef.h>
 
+#ifndef __has_c_attribute
+#  define __has_c_attribute(x) 0
+#endif
+
+#if __has_c_attribute(deprecated)
+#  define NIX_DEPRECATED(msg) [[deprecated(msg)]]
+#else
+#  define NIX_DEPRECATED(msg)
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -45,7 +55,7 @@ typedef struct EvalState EvalState; // nix::EvalState
  * @see nix_value_incref, nix_value_decref
  */
 typedef struct nix_value nix_value;
-[[deprecated("use nix_value instead")]] typedef nix_value Value;
+NIX_DEPRECATED("use nix_value instead") typedef nix_value Value;
 
 // Function prototypes
 /**

src/libexpr/eval-gc.cc

@@ -32,122 +32,6 @@ static void * oomHandler(size_t requested)
     throw std::bad_alloc();
 }
 
-class BoehmGCStackAllocator : public StackAllocator
-{
-    boost::coroutines2::protected_fixedsize_stack stack{
-        // We allocate 8 MB, the default max stack size on NixOS.
-        // A smaller stack might be quicker to allocate but reduces the stack
-        // depth available for source filter expressions etc.
-        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))};
-
-    // This is specific to boost::coroutines2::protected_fixedsize_stack.
-    // The stack protection page is included in sctx.size, so we have to
-    // subtract one page size from the stack size.
-    std::size_t pfss_usable_stack_size(boost::context::stack_context & sctx)
-    {
-        return sctx.size - boost::context::stack_traits::page_size();
-    }
-
-public:
-    boost::context::stack_context allocate() override
-    {
-        auto sctx = stack.allocate();
-
-        // Stacks generally start at a high address and grow to lower addresses.
-        // Architectures that do the opposite are rare; in fact so rare that
-        // boost_routine does not implement it.
-        // So we subtract the stack size.
-        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        return sctx;
-    }
-
-    void deallocate(boost::context::stack_context sctx) override
-    {
-        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        stack.deallocate(sctx);
-    }
-};
-
-static BoehmGCStackAllocator boehmGCStackAllocator;
-
-/**
- * When a thread goes into a coroutine, we lose its original sp until
- * control flow returns to the thread.
- * While in the coroutine, the sp points outside the thread stack,
- * so we can detect this and push the entire thread stack instead,
- * as an approximation.
- * The coroutine's stack is covered by `BoehmGCStackAllocator`.
- * This is not an optimal solution, because the garbage is scanned when a
- * coroutine is active, for both the coroutine and the original thread stack.
- * However, the implementation is quite lean, and usually we don't have active
- * coroutines during evaluation, so this is acceptable.
- */
-void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id)
-{
-    void *& sp = *sp_ptr;
-    auto pthread_id = reinterpret_cast<pthread_t>(_pthread_id);
-#  ifndef __APPLE__
-    pthread_attr_t pattr;
-#  endif
-    size_t osStackSize;
-    // The low address of the stack, which grows down.
-    void * osStackLimit;
-    void * osStackBase;
-
-#  ifdef __APPLE__
-    osStackSize = pthread_get_stacksize_np(pthread_id);
-    osStackLimit = pthread_get_stackaddr_np(pthread_id);
-#  else
-    if (pthread_attr_init(&pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_init failed");
-    }
-#    ifdef HAVE_PTHREAD_GETATTR_NP
-    if (pthread_getattr_np(pthread_id, &pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_getattr_np failed");
-    }
-#    elif HAVE_PTHREAD_ATTR_GET_NP
-    if (!pthread_attr_init(&pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_init failed");
-    }
-    if (!pthread_attr_get_np(pthread_id, &pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed");
-    }
-#    else
-#      error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`"
-#    endif
-    if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed");
-    }
-    if (pthread_attr_destroy(&pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed");
-    }
-#  endif
-    osStackBase = (char *) osStackLimit + osStackSize;
-    // NOTE: We assume the stack grows down, as it does on all architectures we support.
-    //       Architectures that grow the stack up are rare.
-    if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack
-        sp = osStackLimit;
-    }
-}
-
-/* Disable GC while this object lives. Used by CoroutineContext.
- *
- * Boehm keeps a count of GC_disable() and GC_enable() calls,
- * and only enables GC when the count matches.
- */
-class BoehmDisableGC
-{
-public:
-    BoehmDisableGC()
-    {
-        GC_disable();
-    };
-    ~BoehmDisableGC()
-    {
-        GC_enable();
-    };
-};
-
 static inline void initGCReal()
 {
     /* Initialise the Boehm garbage collector. */
@@ -168,24 +52,6 @@ static inline void initGCReal()
 
     GC_set_oom_fn(oomHandler);
 
-    StackAllocator::defaultAllocator = &boehmGCStackAllocator;
-
-// TODO: Remove __APPLE__ condition.
-//       Comment suggests an implementation that works on darwin and windows
-//       https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196
-#  if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__)
-    GC_set_sp_corrector(&fixupBoehmStackPointer);
-
-    if (!GC_get_sp_corrector()) {
-        printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines");
-        /* Used to disable GC when entering coroutines on macOS */
-        create_coro_gc_hook = []() -> std::shared_ptr<void> { return std::make_shared<BoehmDisableGC>(); };
-    }
-#  else
-#    warning \
-        "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later."
-#  endif
-
     /* Set the initial heap size to something fairly big (25% of
        physical RAM, up to a maximum of 384 MiB) so that in most cases
        we don't need to garbage collect at all.  (Collection has a

src/libexpr/eval.cc

@@ -379,6 +379,16 @@ void EvalState::allowPath(const StorePath & storePath)
         rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath)));
 }
 
+void EvalState::allowClosure(const StorePath & storePath)
+{
+    if (!rootFS.dynamic_pointer_cast<AllowListSourceAccessor>()) return;
+
+    StorePathSet closure;
+    store->computeFSClosure(storePath, closure);
+    for (auto & p : closure)
+        allowPath(p);
+}
+
 void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
 {
     allowPath(storePath);
@@ -428,7 +438,7 @@ void EvalState::checkURI(const std::string & uri)
 
     /* If the URI is a path, then check it against allowedPaths as
        well. */
-    if (hasPrefix(uri, "/")) {
+    if (isAbsolute(uri)) {
         if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
             rootFS2->checkAccess(CanonPath(uri));
         return;
@@ -3083,7 +3093,9 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
     if (EvalSettings::isPseudoUrl(value)) {
         try {
             auto accessor = fetchers::downloadTarball(
-                EvalSettings::resolvePseudoUrl(value)).accessor;
+                store,
+                fetchSettings,
+                EvalSettings::resolvePseudoUrl(value));
             auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
             return finish(store->toRealPath(storePath));
         } catch (Error & e) {
@@ -3111,10 +3123,7 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
             allowPath(path);
             if (store->isInStore(path)) {
                 try {
-                    StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(path).first, closure);
-                    for (auto & p : closure)
-                        allowPath(p);
+                    allowClosure(store->toStorePath(path).first);
                 } catch (InvalidPath &) { }
             }
         }

src/libexpr/eval.hh

@@ -392,6 +392,11 @@ public:
      */
     void allowPath(const StorePath & storePath);
 
+    /**
+     * Allow access to the closure of a store path.
+     */
+    void allowClosure(const StorePath & storePath);
+
     /**
      * Allow access to a store path and return it as a string.
      */

src/libexpr/parser-state.hh

@@ -86,7 +86,8 @@ struct ParserState
 
     void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos);
     void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos);
-    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos);
+    void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
+    void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def);
     Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {});
     Expr * stripIndentation(const PosIdx pos,
         std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
@@ -110,75 +111,101 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre
     });
 }
 
-inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos)
+inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc)
 {
     AttrPath::iterator i;
     // All attrpaths have at least one attr
     assert(!attrPath.empty());
+    auto pos = at(loc);
     // Checking attrPath validity.
     // ===========================
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
+        ExprAttrs * nested;
         if (i->symbol) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
-                if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
-                    ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
-                    if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
-                    attrs = attrs2;
-                } else
+                nested = dynamic_cast<ExprAttrs *>(j->second.e);
+                if (!nested) {
+                    attrPath.erase(i + 1, attrPath.end());
                     dupAttr(attrPath, pos, j->second.pos);
+                }
             } else {
-                ExprAttrs * nested = new ExprAttrs;
+                nested = new ExprAttrs;
                 attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
-                attrs = nested;
             }
         } else {
-            ExprAttrs *nested = new ExprAttrs;
+            nested = new ExprAttrs;
             attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
-            attrs = nested;
         }
+        attrs = nested;
     }
     // Expr insertion.
     // ==========================
     if (i->symbol) {
-        ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
-        if (j != attrs->attrs.end()) {
-            // This attr path is already defined. However, if both
-            // e and the expr pointed by the attr path are two attribute sets,
-            // we want to merge them.
-            // Otherwise, throw an error.
-            auto ae = dynamic_cast<ExprAttrs *>(e);
-            auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
-            if (jAttrs && ae) {
-                if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                    jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-                for (auto & ad : ae->attrs) {
-                    auto j2 = jAttrs->attrs.find(ad.first);
-                    if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
-                        dupAttr(ad.first, j2->second.pos, ad.second.pos);
-                    jAttrs->attrs.emplace(ad.first, ad.second);
-                    if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
-                        auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
-                        auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
-                        from.displ += jAttrs->inheritFromExprs->size();
-                    }
-                }
-                jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
-                if (ae->inheritFromExprs) {
-                    jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
-                        ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
-                }
-            } else {
-                dupAttr(attrPath, pos, j->second.pos);
-            }
-        } else {
-            // This attr path is not defined. Let's create it.
-            attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos));
-            e->setName(i->symbol);
-        }
+        addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
     } else {
         attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
     }
+
+    auto it = lexerState.positionToDocComment.find(pos);
+    if (it != lexerState.positionToDocComment.end()) {
+        e->setDocComment(it->second);
+        lexerState.positionToDocComment.emplace(at(exprLoc), it->second);
+    }
+}
+
+/**
+ * Precondition: attrPath is used for error messages and should already contain
+ * symbol as its last element.
+ */
+inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
+{
+    ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
+    if (j != attrs->attrs.end()) {
+        // This attr path is already defined. However, if both
+        // e and the expr pointed by the attr path are two attribute sets,
+        // we want to merge them.
+        // Otherwise, throw an error.
+        auto ae = dynamic_cast<ExprAttrs *>(def.e);
+        auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
+
+        // N.B. In a world in which we are less bound by our past mistakes, we
+        // would also test that jAttrs and ae are not recursive. The effect of
+        // not doing so is that any `rec` marker on ae is discarded, and any
+        // `rec` marker on jAttrs will apply to the attributes in ae.
+        // See https://github.com/NixOS/nix/issues/9020.
+        if (jAttrs && ae) {
+            if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
+                jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+            for (auto & ad : ae->attrs) {
+                if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
+                    auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
+                    auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
+                    from.displ += jAttrs->inheritFromExprs->size();
+                }
+                attrPath.emplace_back(AttrName(ad.first));
+                addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
+                attrPath.pop_back();
+            }
+            ae->attrs.clear();
+            jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(),
+                std::make_move_iterator(ae->dynamicAttrs.begin()),
+                std::make_move_iterator(ae->dynamicAttrs.end()));
+            ae->dynamicAttrs.clear();
+            if (ae->inheritFromExprs) {
+                jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
+                    std::make_move_iterator(ae->inheritFromExprs->begin()),
+                    std::make_move_iterator(ae->inheritFromExprs->end()));
+                ae->inheritFromExprs = nullptr;
+            }
+        } else {
+            dupAttr(attrPath, def.pos, j->second.pos);
+        }
+    } else {
+        // This attr path is not defined. Let's create it.
+        attrs->attrs.emplace(symbol, def);
+        def.e->setName(symbol);
+    }
 }
 
 inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg)

src/libexpr/parser.y

@@ -180,22 +180,22 @@ expr_function
       $$ = me;
       SET_DOC_POS(me, @1);
     }
-  | '{' formals '}' ':' expr_function
-    { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5);
+  | '{' formals '}' ':' expr_function[body]
+    { auto me = new ExprLambda(CUR_POS, state->validateFormals($formals), $body);
       $$ = me;
       SET_DOC_POS(me, @1);
     }
-  | '{' formals '}' '@' ID ':' expr_function
+  | '{' formals '}' '@' ID ':' expr_function[body]
     {
-      auto arg = state->symbols.create($5);
-      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7);
+      auto arg = state->symbols.create($ID);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body);
       $$ = me;
       SET_DOC_POS(me, @1);
     }
-  | ID '@' '{' formals '}' ':' expr_function
+  | ID '@' '{' formals '}' ':' expr_function[body]
     {
-      auto arg = state->symbols.create($1);
-      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7);
+      auto arg = state->symbols.create($ID);
+      auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body);
       $$ = me;
       SET_DOC_POS(me, @1);
     }
@@ -364,50 +364,39 @@ ind_string_parts
   ;
 
 binds
-  : binds attrpath '=' expr ';' {
-      $$ = $1;
-
-      auto pos = state->at(@2);
-      auto exprPos = state->at(@4);
-      {
-        auto it = state->lexerState.positionToDocComment.find(pos);
-        if (it != state->lexerState.positionToDocComment.end()) {
-          $4->setDocComment(it->second);
-          state->lexerState.positionToDocComment.emplace(exprPos, it->second);
-        }
-      }
-
-      state->addAttr($$, std::move(*$2), $4, pos);
-      delete $2;
+  : binds[accum] attrpath '=' expr ';' {
+      $$ = $accum;
+      state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
+      delete $attrpath;
     }
-  | binds INHERIT attrs ';'
-    { $$ = $1;
-      for (auto & [i, iPos] : *$3) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
+  | binds[accum] INHERIT attrs ';'
+    { $$ = $accum;
+      for (auto & [i, iPos] : *$attrs) {
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
       }
-      delete $3;
+      delete $attrs;
     }
-  | binds INHERIT '(' expr ')' attrs ';'
-    { $$ = $1;
-      if (!$$->inheritFromExprs)
-          $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
-      $$->inheritFromExprs->push_back($4);
-      auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
-      for (auto & [i, iPos] : *$6) {
-          if ($$->attrs.find(i.symbol) != $$->attrs.end())
-              state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(
+  | binds[accum] INHERIT '(' expr ')' attrs ';'
+    { $$ = $accum;
+      if (!$accum->inheritFromExprs)
+          $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+      $accum->inheritFromExprs->push_back($expr);
+      auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
+      for (auto & [i, iPos] : *$attrs) {
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(
                   new ExprSelect(iPos, from, i.symbol),
                   iPos,
                   ExprAttrs::AttrDef::Kind::InheritedFrom));
       }
-      delete $6;
+      delete $attrs;
     }
   | { $$ = new ExprAttrs(state->at(@0)); }
   ;
@@ -468,10 +457,10 @@ expr_list
   ;
 
 formals
-  : formal ',' formals
-    { $$ = $3; $$->formals.emplace_back(*$1); delete $1; }
+  : formal ',' formals[accum]
+    { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; }
   | formal
-    { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; }
+    { $$ = new Formals; $$->formals.emplace_back(*$formal); $$->ellipsis = false; delete $formal; }
   |
     { $$ = new Formals; $$->ellipsis = false; }
   | ELLIPSIS

src/libexpr/primops.cc

@@ -113,11 +113,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS
     if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
 
     if (isIFD) {
-        for (auto & outputPath : outputsToCopyAndAllow) {
-            /* Add the output of this derivations to the allowed
-            paths. */
-            allowPath(outputPath);
-        }
+        /* Allow access to the output closures of this derivation. */
+        for (auto & outputPath : outputsToCopyAndAllow)
+            allowClosure(outputPath);
     }
 
     return res;
@@ -941,6 +939,9 @@ static RegisterPrimOp primop_tryEval({
       `let e = { x = throw ""; }; in
       (builtins.tryEval (builtins.deepSeq e e)).success` will be
       `false`.
+
+      `tryEval` intentionally does not return the error message, because that risks bringing non-determinism into the evaluation result, and it would become very difficult to improve error reporting without breaking existing expressions.
+      Instead, use [`builtins.addErrorContext`](@docroot@/language/builtins.md#builtins-addErrorContext) to add context to the error message, and use a Nix unit testing tool for testing.
     )",
     .fun = prim_tryEval,
 });
@@ -3136,7 +3137,11 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg
         std::optional<ListBuilder> list;
     };
 
+#if HAVE_BOEHMGC
+    std::map<Symbol, Item, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Item>>> attrsSeen;
+#else
     std::map<Symbol, Item> attrsSeen;
+#endif
 
     state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith");
     state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith");
@@ -3993,7 +3998,7 @@ static RegisterPrimOp primop_toString({
 });
 
 /* `substring start len str' returns the substring of `str' starting
-   at character position `min(start, stringLength str)' inclusive and
+   at byte position `min(start, stringLength str)' inclusive and
    ending at `min(start + len, stringLength str)'.  `start' must be
    non-negative. */
 static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@@ -4026,7 +4031,7 @@ static RegisterPrimOp primop_substring({
     .name = "__substring",
     .args = {"start", "len", "s"},
     .doc = R"(
-      Return the substring of *s* from character position *start*
+      Return the substring of *s* from byte position *start*
       (zero-based) up to but not including *start + len*. If *start* is
       greater than the length of the string, an empty string is returned.
       If *start + len* lies beyond the end of the string or *len* is `-1`,

src/libexpr/primops/context.cc

@@ -86,7 +86,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency({
 
       This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies).
 
-      This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context,
+      This is unsafe because it allows us to "forget" store objects we would have otherwise referred to with the string context,
       whereas Nix normally tracks all dependencies consistently.
       Safe operations "grow" but never "shrink" string contexts.
       [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things).

src/libexpr/primops/fetchTree.cc

@@ -240,7 +240,7 @@ static RegisterPrimOp primop_fetchTree({
       The following source types and associated input attributes are supported.
 
       <!-- TODO: It would be soooo much more predictable to work with (and
-      document) if `fetchTree` was a curried call with the first paramter for
+      document) if `fetchTree` was a curried call with the first parameter for
       `type` or an attribute like `builtins.fetchTree.git`! -->
 
       - `"file"`
@@ -501,7 +501,11 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     //       https://github.com/NixOS/nix/issues/4313
     auto storePath =
         unpack
-        ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
+        ? fetchToStore(
+            *state.store,
+            fetchers::downloadTarball(state.store, state.fetchSettings, *url),
+            FetchMode::Copy,
+            name)
         : fetchers::downloadFile(state.store, *url, name).storePath;
 
     if (expectedHash) {

src/libexpr/value-to-json.cc

@@ -108,7 +108,11 @@ json printValueAsJSON(EvalState & state, bool strict,
 void printValueAsJSON(EvalState & state, bool strict,
     Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore)
 {
-    str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
+    try {
+        str << printValueAsJSON(state, strict, v, pos, context, copyToStore);
+    } catch (nlohmann::json::exception & e) {
+        throw JSONSerializationError("JSON serialization error: %s", e.what());
+    }
 }
 
 json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict,

src/libexpr/value-to-json.hh

@@ -16,4 +16,7 @@ nlohmann::json printValueAsJSON(EvalState & state, bool strict,
 void printValueAsJSON(EvalState & state, bool strict,
     Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true);
 
+
+MakeError(JSONSerializationError, Error);
+
 }

src/libfetchers/cache.cc

@@ -36,7 +36,7 @@ struct CacheImpl : Cache
     {
         auto state(_state.lock());
 
-        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v2.sqlite";
+        auto dbPath = getCacheDir() + "/nix/fetcher-cache-v3.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);

src/libfetchers/git-utils.cc

@@ -159,6 +159,31 @@ static Object peelToTreeOrBlob(git_object * obj)
         return peelObject<Object>(obj, GIT_OBJECT_TREE);
 }
 
+static void initRepoAtomically(std::filesystem::path &path, bool bare) {
+    if (pathExists(path.string())) return;
+
+    Path tmpDir = createTempDir(std::filesystem::path(path).parent_path());
+    AutoDelete delTmpDir(tmpDir, true);
+    Repository tmpRepo;
+
+    if (git_repository_init(Setter(tmpRepo), tmpDir.c_str(), bare))
+        throw Error("creating Git repository %s: %s", path, git_error_last()->message);
+    try {
+        std::filesystem::rename(tmpDir, path);
+    } catch (std::filesystem::filesystem_error & e) {
+        // Someone may race us to create the repository.
+        if (e.code() == std::errc::file_exists
+            // `path` may be attempted to be deleted by s::f::rename, in which case the code is:
+            || e.code() == std::errc::directory_not_empty) {
+            return;
+        }
+        else
+            throw SysError("moving temporary git repository from %s to %s", tmpDir, path);
+    }
+    // we successfully moved the repository, so the temporary directory no longer exists.
+    delTmpDir.cancel();
+}
+
 struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 {
     /** Location of the repository on disk. */
@@ -170,13 +195,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     {
         initLibGit2();
 
-        if (pathExists(path.string())) {
-            if (git_repository_open(Setter(repo), path.string().c_str()))
-                throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
-        } else {
-            if (git_repository_init(Setter(repo), path.string().c_str(), bare))
-                throw Error("creating Git repository '%s': %s", path, git_error_last()->message);
-        }
+        initRepoAtomically(path, bare);
+        if (git_repository_open(Setter(repo), path.string().c_str()))
+            throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
+
     }
 
     operator git_repository * ()
@@ -460,7 +482,13 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         std::string re = R"(Good "git" signature for \* with .* key SHA256:[)";
         for (const fetchers::PublicKey & k : publicKeys){
             // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally
-            auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "=");
+            std::string keyDecoded;
+            try {
+                keyDecoded = base64Decode(k.key);
+            } catch (Error & e) {
+                e.addTrace({}, "while decoding public key '%s' used for git signature", k.key);
+            }
+            auto fingerprint = trim(hashString(HashAlgorithm::SHA256, keyDecoded).to_string(nix::HashFormat::Base64, false), "=");
             auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" );
             re += "(" + escaped_fingerprint + ")";
         }
@@ -601,12 +629,16 @@ struct GitSourceAccessor : SourceAccessor
         return readBlob(path, true);
     }
 
-    Hash getSubmoduleRev(const CanonPath & path)
+    /**
+     * If `path` exists and is a submodule, return its
+     * revision. Otherwise return nothing.
+     */
+    std::optional<Hash> getSubmoduleRev(const CanonPath & path)
     {
-        auto entry = need(path);
+        auto entry = lookup(path);
 
-        if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
-            throw Error("'%s' is not a submodule", showPath(path));
+        if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
+            return std::nullopt;
 
         return toHash(*git_tree_entry_id(entry));
     }
@@ -827,8 +859,24 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
     void pushBuilder(std::string name)
     {
+        const git_tree_entry * entry;
+        Tree prevTree = nullptr;
+
+        if (!pendingDirs.empty() &&
+            (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str())))
+        {
+            /* Clone a tree that we've already finished. This happens
+               if a tarball has directory entries that are not
+               contiguous. */
+            if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+                throw Error("parent of '%s' is not a directory", name);
+
+            if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(prevTree), *repo, entry))
+                throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
+        }
+
         git_treebuilder * b;
-        if (git_treebuilder_new(&b, *repo, nullptr))
+        if (git_treebuilder_new(&b, *repo, prevTree.get()))
             throw Error("creating a tree builder: %s", git_error_last()->message);
         pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) });
     };
@@ -1074,8 +1122,10 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     auto rawAccessor = getRawAccessor(rev);
 
     for (auto & submodule : parseSubmodules(pathTemp)) {
-        auto rev = rawAccessor->getSubmoduleRev(submodule.path);
-        result.push_back({std::move(submodule), rev});
+        /* Filter out .gitmodules entries that don't exist or are not
+           submodules. */
+        if (auto rev = rawAccessor->getSubmoduleRev(submodule.path))
+            result.push_back({std::move(submodule), *rev});
     }
 
     return result;

src/libfetchers/git.cc

@@ -425,7 +425,26 @@ struct GitInputScheme : InputScheme
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
         repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
-        repoInfo.url = repoInfo.isLocal ? url.path : url.base;
+        //
+        // FIXME: here we turn a possibly relative path into an absolute path.
+        // This allows relative git flake inputs to be resolved against the
+        // **current working directory** (as in POSIX), which tends to work out
+        // ok in the context of flakes, but is the wrong behavior,
+        // as it should resolve against the flake.nix base directory instead.
+        //
+        // See: https://discourse.nixos.org/t/57783 and #9708
+        //
+        if (repoInfo.isLocal) {
+            if (!isAbsolute(url.path)) {
+                warn(
+                    "Fetching Git repository '%s', which uses a path relative to the current directory. "
+                    "This is not supported and will stop working in a future release. "
+                    "See https://github.com/NixOS/nix/issues/12281 for details.",
+                    url.to_string());
+            }
+            repoInfo.url = std::filesystem::absolute(url.path).string();
+        } else
+            repoInfo.url = url.to_string();
 
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.
@@ -584,9 +603,10 @@ struct GitInputScheme : InputScheme
                 }
 
                 try {
-                    setWriteTime(localRefFile, now, now);
+                    if (!input.getRev())
+                        setWriteTime(localRefFile, now, now);
                 } catch (Error & e) {
-                    warn("could not update mtime for file '%s': %s", localRefFile, e.msg());
+                    warn("could not update mtime for file '%s': %s", localRefFile, e.info().msg);
                 }
                 if (!originalRef && !storeCachedHead(repoInfo.url, ref))
                     warn("could not update cached head '%s' for '%s'", ref, repoInfo.url);

src/libfetchers/path.cc

@@ -96,7 +96,7 @@ struct PathInputScheme : InputScheme
     std::optional<std::string> isRelative(const Input & input) const
     {
         auto path = getStrAttr(input.attrs, "path");
-        if (hasPrefix(path, "/"))
+        if (isAbsolute(path))
             return std::nullopt;
         else
             return path;

src/libfetchers/registry.cc

@@ -156,7 +156,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, re
             return std::make_shared<Registry>(settings, Registry::Global); // empty registry
         }
 
-        if (!hasPrefix(path, "/")) {
+        if (!isAbsolute(path)) {
             auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");

src/libfetchers/tarball.cc

@@ -90,6 +90,7 @@ DownloadFileResult downloadFile(
     /* Cache metadata for all URLs in the redirect chain. */
     for (auto & url : res.urls) {
         key.second.insert_or_assign("url", url);
+        assert(!res.urls.empty());
         infoAttrs.insert_or_assign("url", *res.urls.rbegin());
         getCache()->upsert(key, *store, infoAttrs, *storePath);
     }
@@ -102,7 +103,7 @@ DownloadFileResult downloadFile(
     };
 }
 
-DownloadTarballResult downloadTarball(
+static DownloadTarballResult downloadTarball_(
     const std::string & url,
     const Headers & headers)
 {
@@ -202,6 +203,22 @@ DownloadTarballResult downloadTarball(
     return attrsToResult(infoAttrs);
 }
 
+ref<SourceAccessor> downloadTarball(
+    ref<Store> store,
+    const Settings & settings,
+    const std::string & url)
+{
+    /* Go through Input::getAccessor() to ensure that the resulting
+       accessor has a fingerprint. */
+    fetchers::Attrs attrs;
+    attrs.insert_or_assign("type", "tarball");
+    attrs.insert_or_assign("url", url);
+
+    auto input = Input::fromAttrs(settings, std::move(attrs));
+
+    return input.getAccessor(store).first;
+}
+
 // An input scheme corresponding to a curl-downloadable resource.
 struct CurlInputScheme : InputScheme
 {
@@ -353,7 +370,7 @@ struct TarballInputScheme : CurlInputScheme
     {
         auto input(_input);
 
-        auto result = downloadTarball(getStrAttr(input.attrs, "url"), {});
+        auto result = downloadTarball_(getStrAttr(input.attrs, "url"), {});
 
         result.accessor->setPathDisplay("«" + input.to_string() + "»");
 

src/libfetchers/tarball.hh

@@ -14,6 +14,8 @@ struct SourceAccessor;
 
 namespace nix::fetchers {
 
+struct Settings;
+
 struct DownloadFileResult
 {
     StorePath storePath;
@@ -40,8 +42,9 @@ struct DownloadTarballResult
  * Download and import a tarball into the Git cache. The result is the
  * Git tree hash of the root directory.
  */
-DownloadTarballResult downloadTarball(
-    const std::string & url,
-    const Headers & headers = {});
+ref<SourceAccessor> downloadTarball(
+    ref<Store> store,
+    const Settings & settings,
+    const std::string & url);
 
 }

src/libflake/flake/flakeref.cc

@@ -175,7 +175,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
         }
 
     } else {
-        if (!hasPrefix(path, "/"))
+        if (!isAbsolute(path))
             throw BadURL("flake reference '%s' is not an absolute path", url);
         path = canonPath(path + "/" + getOr(query, "dir", ""));
     }

src/libmain/common-args.cc

@@ -17,7 +17,9 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
         .shortName = 'v',
         .description = "Increase the logging verbosity level.",
         .category = loggingCategory,
-        .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
+        .handler = {[]() {
+            verbosity = (Verbosity) std::min<std::underlying_type_t<Verbosity>>(verbosity + 1, lvlVomit);
+        }},
     });
 
     addFlag({

src/libstore/build/derivation-goal.cc

@@ -1158,7 +1158,7 @@ HookReply DerivationGoal::tryBuildHook()
                     throw;
                 }
             }();
-            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
+            if (handleJSONLogMessage(s, worker.act, worker.hook->activities, "the build hook", true))
                 ;
             else if (s.substr(0, 2) == "# ") {
                 reply = s.substr(2);
@@ -1343,9 +1343,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data)
     if (hook && fd == hook->fromHook.readSide.get()) {
         for (auto c : data)
             if (c == '\n') {
-                auto json = parseJSONMessage(currentHookLine);
+                auto json = parseJSONMessage(currentHookLine, "the derivation builder");
                 if (json) {
-                    auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true);
+                    auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true);
                     // ensure that logs from a builder using `ssh-ng://` as protocol
                     // are also available to `nix log`.
                     if (s && !isWrittenToLog && logSink) {
@@ -1387,7 +1387,7 @@ void DerivationGoal::handleEOF(Descriptor fd)
 
 void DerivationGoal::flushLine()
 {
-    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false))
+    if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false))
         ;
 
     else {

src/libstore/build/substitution-goal.cc

@@ -145,8 +145,10 @@ Goal::Co PathSubstitutionGoal::init()
     /* None left.  Terminate this goal and let someone else deal
        with it. */
 
-    worker.failedSubstitutions++;
-    worker.updateProgress();
+    if (substituterFailed) {
+        worker.failedSubstitutions++;
+        worker.updateProgress();
+    }
 
     /* Hack: don't indicate failure if there were no substituters.
        In that case the calling derivation should just do a
@@ -158,7 +160,7 @@ Goal::Co PathSubstitutionGoal::init()
 }
 
 
-Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool& substituterFailed)
+Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed)
 {
     trace("all references realised");
 
@@ -181,7 +183,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
     /* Make sure that we are allowed to start a substitution.  Note that even
        if maxSubstitutionJobs == 0, we still allow a substituter to run. This
        prevents infinite waiting. */
-    if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
+    while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
         worker.waitForBuildSlot(shared_from_this());
         co_await Suspend{};
     }

src/libstore/build/substitution-goal.hh

@@ -66,7 +66,7 @@ public:
      */
     Co init() override;
     Co gotInfo();
-    Co tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool& substituterFailed);
+    Co tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed);
     Co finished();
 
     /**

src/libstore/build/worker.cc

@@ -184,13 +184,13 @@ void Worker::wakeUp(GoalPtr goal)
 }
 
 
-unsigned Worker::getNrLocalBuilds()
+size_t Worker::getNrLocalBuilds()
 {
     return nrLocalBuilds;
 }
 
 
-unsigned Worker::getNrSubstitutions()
+size_t Worker::getNrSubstitutions()
 {
     return nrSubstitutions;
 }

src/libstore/build/worker.hh

@@ -92,12 +92,12 @@ private:
      * Number of build slots occupied.  This includes local builds but does not
      * include substitutions or remote builds via the build hook.
      */
-    unsigned int nrLocalBuilds;
+    size_t nrLocalBuilds;
 
     /**
      * Number of substitution slots occupied.
      */
-    unsigned int nrSubstitutions;
+    size_t nrSubstitutions;
 
     /**
      * Maps used to prevent multiple instantiations of a goal for the
@@ -235,12 +235,12 @@ public:
      * Return the number of local build processes currently running (but not
      * remote builds via the build hook).
      */
-    unsigned int getNrLocalBuilds();
+    size_t getNrLocalBuilds();
 
     /**
      * Return the number of substitution processes currently running.
      */
-    unsigned int getNrSubstitutions();
+    size_t getNrSubstitutions();
 
     /**
      * Registers a running child process.  `inBuildSlot` means that

src/libstore/builtins.hh

@@ -9,7 +9,8 @@ namespace nix {
 void builtinFetchurl(
     const BasicDerivation & drv,
     const std::map<std::string, Path> & outputs,
-    const std::string & netrcData);
+    const std::string & netrcData,
+    const std::string & caFileData);
 
 void builtinUnpackChannel(
     const BasicDerivation & drv,

src/libstore/builtins/fetchurl.cc

@@ -9,7 +9,8 @@ namespace nix {
 void builtinFetchurl(
     const BasicDerivation & drv,
     const std::map<std::string, Path> & outputs,
-    const std::string & netrcData)
+    const std::string & netrcData,
+    const std::string & caFileData)
 {
     /* Make the host's netrc data available. Too bad curl requires
        this to be stored in a file. It would be nice if we could just
@@ -19,6 +20,9 @@ void builtinFetchurl(
         writeFile(settings.netrcFile, netrcData, 0600);
     }
 
+    settings.caFile = "ca-certificates.crt";
+    writeFile(settings.caFile, caFileData, 0600);
+
     auto out = get(drv.outputs, "out");
     if (!out)
         throw Error("'builtin:fetchurl' requires an 'out' output");
@@ -38,10 +42,7 @@ void builtinFetchurl(
 
         auto source = sinkToSource([&](Sink & sink) {
 
-            /* No need to do TLS verification, because we check the hash of
-               the result anyway. */
             FileTransferRequest request(url);
-            request.verifyTLS = false;
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(

src/libstore/builtins/unpack-channel.cc

@@ -3,31 +3,53 @@
 
 namespace nix {
 
+namespace fs { using namespace std::filesystem; }
+
 void builtinUnpackChannel(
     const BasicDerivation & drv,
     const std::map<std::string, Path> & outputs)
 {
-    auto getAttr = [&](const std::string & name) {
+    auto getAttr = [&](const std::string & name) -> const std::string & {
         auto i = drv.env.find(name);
         if (i == drv.env.end()) throw Error("attribute '%s' missing", name);
         return i->second;
     };
 
-    auto out = outputs.at("out");
-    auto channelName = getAttr("channelName");
-    auto src = getAttr("src");
+    fs::path out{outputs.at("out")};
+    auto & channelName = getAttr("channelName");
+    auto & src = getAttr("src");
 
-    createDirs(out);
+    if (fs::path{channelName}.filename().string() != channelName) {
+        throw Error("channelName is not allowed to contain filesystem separators, got %1%", channelName);
+    }
+
+    try {
+        fs::create_directories(out);
+    } catch (fs::filesystem_error &) {
+        throw SysError("creating directory '%1%'", out.string());
+    }
 
     unpackTarfile(src, out);
 
-    auto entries = std::filesystem::directory_iterator{out};
-    auto fileName = entries->path().string();
-    auto fileCount = std::distance(std::filesystem::begin(entries), std::filesystem::end(entries));
+    size_t fileCount;
+    std::string fileName;
+    try {
+        auto entries = fs::directory_iterator{out};
+        fileName = entries->path().string();
+        fileCount = std::distance(fs::begin(entries), fs::end(entries));
+    } catch (fs::filesystem_error &) {
+        throw SysError("failed to read directory %1%", out.string());
+    }
 
     if (fileCount != 1)
         throw Error("channel tarball '%s' contains more than one file", src);
-    std::filesystem::rename(fileName, (out + "/" + channelName));
+
+    auto target = out / channelName;
+    try {
+        fs::rename(fileName, target);
+    } catch (fs::filesystem_error &) {
+        throw SysError("failed to rename %1% to %2%", fileName, target.string());
+    }
 }
 
 }

src/libstore/daemon.cc

@@ -402,6 +402,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
+                // FIXME: this means that if there is an error
+                // half-way through, the client will keep sending
+                // data, since we haven't sent it the error yet.
                 auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr);
                 FramedSource source(conn.from);
                 FileSerialisationMethod dumpMethod;

src/libstore/filetransfer.cc

@@ -754,12 +754,17 @@ struct curlFileTransfer : public FileTransfer
 
                 S3Helper s3Helper(profile, region, scheme, endpoint);
 
+                Activity act(*logger, lvlTalkative, actFileTransfer,
+                    fmt("downloading '%s'", request.uri),
+                    {request.uri}, request.parentAct);
+
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
                 FileTransferResult res;
                 if (!s3Res.data)
-                    throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri);
+                    throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri);
                 res.data = std::move(*s3Res.data);
+                res.urls.push_back(request.uri);
                 callback(std::move(res));
 #else
                 throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri);

src/libstore/gc.cc

@@ -4,6 +4,7 @@
 #include "finally.hh"
 #include "unix-domain-socket.hh"
 #include "signals.hh"
+#include "posix-fs-canonicalise.hh"
 
 #if !defined(__linux__)
 // For shelling out to lsof
@@ -763,13 +764,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 }
             }
         }
-
         for (auto & path : topoSortPaths(visited)) {
             if (!dead.insert(path).second) continue;
             if (shouldDelete) {
-                invalidatePathChecked(path);
-                deleteFromStore(path.to_string());
-                referrersCache.erase(path);
+                try {
+                    invalidatePathChecked(path);
+                    deleteFromStore(path.to_string());
+                    referrersCache.erase(path);
+                } catch (PathInUse &e) {
+                    // If we end up here, it's likely a new occurence
+                    // of https://github.com/NixOS/nix/issues/11923
+                    printError("BUG: %s", e.what());
+                }
             }
         }
     };

src/libstore/http-binary-cache-store.cc

@@ -169,28 +169,29 @@ protected:
     {
         try {
             checkEnabled();
+
+            auto request(makeRequest(path));
+
+            auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
+            getFileTransfer()->enqueueFileTransfer(request,
+                {[callbackPtr, this](std::future<FileTransferResult> result) {
+                    try {
+                        (*callbackPtr)(std::move(result.get().data));
+                    } catch (FileTransferError & e) {
+                        if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
+                            return (*callbackPtr)({});
+                        maybeDisable();
+                        callbackPtr->rethrow();
+                    } catch (...) {
+                        callbackPtr->rethrow();
+                    }
+            }});
+
         } catch (...) {
             callback.rethrow();
             return;
         }
-
-        auto request(makeRequest(path));
-
-        auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
-
-        getFileTransfer()->enqueueFileTransfer(request,
-            {[callbackPtr, this](std::future<FileTransferResult> result) {
-                try {
-                    (*callbackPtr)(std::move(result.get().data));
-                } catch (FileTransferError & e) {
-                    if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
-                        return (*callbackPtr)({});
-                    maybeDisable();
-                    callbackPtr->rethrow();
-                } catch (...) {
-                    callbackPtr->rethrow();
-                }
-            }});
     }
 
     /**

src/libstore/local-overlay-store.md

@@ -4,7 +4,7 @@ R"(
 
 This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short).
 Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store].
-("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.)
+("almost" because while the upper filesystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.)
 To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly).
 
 ### Conceptual parts of a local overlay store
@@ -77,13 +77,13 @@ The parts of a local overlay store are as follows:
 
     The lower store directory and upper layer directory are combined via OverlayFS to create this directory.
     Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first.
-    Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings.
+    Nix can, however, optionally check that the OverlayFS mount settings appear as expected, matching Nix's own settings.
 
   - **Upper SQLite database**:
 
     > Not directly specified.
     > The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting.
-    > It is is always `${state}/db`.
+    > It is always `${state}/db`.
 
     This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta.
     The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer.

src/libstore/local.mk

@@ -59,7 +59,7 @@ NIX_ROOT =
 endif
 
 # Prefix all but `NIX_STORE_DIR`, since we aren't doing a local store
-# yet so a "logical" store dir that is the same as unix is prefered.
+# yet so a "logical" store dir that is the same as unix is preferred.
 #
 # Also, it keeps the unit tests working.
 

src/libstore/machines.cc

@@ -159,8 +159,9 @@ static Machine parseBuilderLine(const std::set<std::string> & defaultSystems, co
         const auto & str = tokens[fieldIndex];
         try {
             base64Decode(str);
-        } catch (const Error & e) {
-            throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what());
+        } catch (FormatError & e) {
+            e.addTrace({}, "while parsing machine specification at a column #%lu in a row: '%s'", fieldIndex, line);
+            throw;
         }
         return str;
     };

src/libstore/meson.build

@@ -68,6 +68,11 @@ has_acl_support = cxx.has_header('sys/xattr.h') \
   and cxx.has_function('lremovexattr')
 configdata.set('HAVE_ACL_SUPPORT', has_acl_support.to_int())
 
+if host_machine.system() == 'darwin'
+  sandbox = cxx.find_library('sandbox')
+  deps_other += [sandbox]
+endif
+
 subdir('build-utils-meson/threads')
 
 boost = dependency(

src/libstore/package.nix

@@ -7,6 +7,7 @@
 , ninja
 , pkg-config
 , unixtools
+, darwin
 
 , nix-util
 , boost
@@ -65,6 +66,7 @@ mkMesonDerivation (finalAttrs: {
     sqlite
   ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp
     # There have been issues building these dependencies
+    ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox
     ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
       aws-sdk-cpp
   ;

src/libstore/remote-store-connection.hh

@@ -49,7 +49,7 @@ struct RemoteStore::ConnectionHandle
     RemoteStore::Connection & operator * () { return *handle; }
     RemoteStore::Connection * operator -> () { return &*handle; }
 
-    void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
+    void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true);
 
     void withFramedSink(std::function<void(Sink & sink)> fun);
 };

src/libstore/remote-store.cc

@@ -153,9 +153,9 @@ RemoteStore::ConnectionHandle::~ConnectionHandle()
     }
 }
 
-void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush)
+void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush, bool block)
 {
-    handle->processStderr(&daemonException, sink, source, flush);
+    handle->processStderr(&daemonException, sink, source, flush, block);
 }
 
 
@@ -534,14 +534,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
 
 
 void RemoteStore::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
     auto source = sinkToSource([&](Sink & sink) {
         sink << pathsToCopy.size();
-        for (auto & [pathInfo, pathSource] : pathsToCopy) {
+        // Reverse, so we can release memory at the original start
+        std::reverse(pathsToCopy.begin(), pathsToCopy.end());
+        while (!pathsToCopy.empty()) {
+            auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {
                      .to = sink,
@@ -549,6 +552,7 @@ void RemoteStore::addMultipleToStore(
                  },
                  pathInfo);
             pathSource->drainInto(sink);
+            pathsToCopy.pop_back();
         }
     });
 
@@ -926,43 +930,17 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sin
 {
     (*this)->to.flush();
 
-    std::exception_ptr ex;
-
-    /* Handle log messages / exceptions from the remote on a separate
-       thread. */
-    std::thread stderrThread([&]()
     {
-        try {
-            ReceiveInterrupts receiveInterrupts;
-            processStderr(nullptr, nullptr, false);
-        } catch (...) {
-            ex = std::current_exception();
-        }
-    });
-
-    Finally joinStderrThread([&]()
-    {
-        if (stderrThread.joinable()) {
-            stderrThread.join();
-            if (ex) {
-                try {
-                    std::rethrow_exception(ex);
-                } catch (...) {
-                    ignoreException();
-                }
-            }
-        }
-    });
-
-    {
-        FramedSink sink((*this)->to, ex);
+        FramedSink sink((*this)->to, [&]() {
+            /* Periodically process stderr messages and exceptions
+               from the daemon. */
+            processStderr(nullptr, nullptr, false, false);
+        });
         fun(sink);
         sink.flush();
     }
 
-    stderrThread.join();
-    if (ex)
-        std::rethrow_exception(ex);
+    processStderr(nullptr, nullptr, false);
 }
 
 }

src/libstore/remote-store.hh

@@ -102,7 +102,7 @@ public:
         CheckSigsFlag checkSigs) override;
 
     void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair,
         CheckSigsFlag checkSigs) override;

src/libstore/s3-binary-cache-store.cc

@@ -9,6 +9,7 @@
 #include "globals.hh"
 #include "compression.hh"
 #include "filetransfer.hh"
+#include "signals.hh"
 
 #include <aws/core/Aws.h>
 #include <aws/core/VersionConfig.h>
@@ -117,6 +118,7 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy
 {
     bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, long attemptedRetries) const override
     {
+        checkInterrupt();
         auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries);
         if (retry)
             printError("AWS error '%s' (%s), will retry in %d ms",
@@ -220,8 +222,6 @@ std::string S3BinaryCacheStoreConfig::doc()
 
 struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore
 {
-    std::string bucketName;
-
     Stats stats;
 
     S3Helper s3Helper;

src/libstore/ssh.cc

@@ -6,6 +6,16 @@
 
 namespace nix {
 
+static std::string parsePublicHostKey(std::string_view host, std::string_view sshPublicHostKey)
+{
+    try {
+        return base64Decode(sshPublicHostKey);
+    } catch (Error & e) {
+        e.addTrace({}, "while decoding ssh public host key for host '%s'", host);
+        throw;
+    }
+}
+
 SSHMaster::SSHMaster(
     std::string_view host,
     std::string_view keyFile,
@@ -14,7 +24,7 @@ SSHMaster::SSHMaster(
     : host(host)
     , fakeSSH(host == "localhost")
     , keyFile(keyFile)
-    , sshPublicHostKey(sshPublicHostKey)
+    , sshPublicHostKey(parsePublicHostKey(host, sshPublicHostKey))
     , useMaster(useMaster && !fakeSSH)
     , compress(compress)
     , logFD(logFD)
@@ -38,7 +48,7 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
         std::filesystem::path fileName = state->tmpDir->path() / "host-key";
         auto p = host.rfind("@");
         std::string thost = p != std::string::npos ? std::string(host, p + 1) : host;
-        writeFile(fileName.string(), thost + " " + base64Decode(sshPublicHostKey) + "\n");
+        writeFile(fileName.string(), thost + " " + sshPublicHostKey + "\n");
         args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName.string()});
     }
     if (compress)

src/libstore/ssh.hh

@@ -14,6 +14,9 @@ private:
     const std::string host;
     bool fakeSSH;
     const std::string keyFile;
+    /**
+     * Raw bytes, not Base64 encoding.
+     */
     const std::string sshPublicHostKey;
     const bool useMaster;
     const bool compress;

src/libstore/store-api.cc

@@ -210,18 +210,20 @@ StorePath Store::addToStore(
         fsm = FileSerialisationMethod::NixArchive;
         break;
     }
-    auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(path, sink, fsm, filter);
+    std::optional<StorePath> storePath;
+    auto sink = sourceToSink([&](Source & source) {
+        LengthSource lengthSource(source);
+        storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair);
+        if (lengthSource.total >= settings.warnLargePathThreshold)
+            warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total));
     });
-    LengthSource lengthSource(*source);
-    auto storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair);
-    if (lengthSource.total >= settings.warnLargePathThreshold)
-        warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total));
-    return storePath;
+    dumpPath(path, *sink, fsm, filter);
+    sink->finish();
+    return storePath.value();
 }
 
 void Store::addMultipleToStore(
-    PathsSource & pathsToCopy,
+    PathsSource && pathsToCopy,
     Activity & act,
     RepairFlag repair,
     CheckSigsFlag checkSigs)
@@ -244,9 +246,7 @@ void Store::addMultipleToStore(
         act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
     };
 
-    ThreadPool pool;
-
-    processGraph<StorePath>(pool,
+    processGraph<StorePath>(
         storePathsToAdd,
 
         [&](const StorePath & path) {
@@ -820,14 +820,25 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m
     auto doQuery = [&](const StorePath & path) {
         checkInterrupt();
         queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) {
-            auto state(state_.lock());
+            bool exists = false;
+            std::exception_ptr newExc{};
+
             try {
                 auto info = fut.get();
-                state->valid.insert(path);
+                exists = true;
             } catch (InvalidPath &) {
             } catch (...) {
-                state->exc = std::current_exception();
+                newExc = std::current_exception();
             }
+
+            auto state(state_.lock());
+
+            if (exists)
+                state->valid.insert(path);
+
+            if (newExc)
+                state->exc = newExc;
+
             assert(state->left);
             if (!--state->left)
                 wakeup.notify_one();
@@ -1015,12 +1026,10 @@ std::map<StorePath, StorePath> copyPaths(
     }
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
 
-    ThreadPool pool;
-
     try {
         // Copy the realisation closure
         processGraph<Realisation>(
-            pool, Realisation::closure(srcStore, toplevelRealisations),
+            Realisation::closure(srcStore, toplevelRealisations),
             [&](const Realisation & current) -> std::set<Realisation> {
                 std::set<Realisation> children;
                 for (const auto & [drvOutput, _] : current.dependentRealisations) {
@@ -1129,7 +1138,7 @@ std::map<StorePath, StorePath> copyPaths(
         pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
     }
 
-    dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
+    dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);
 
     return pathsMap;
 }

src/libstore/store-api.hh

@@ -425,7 +425,7 @@ public:
         CheckSigsFlag checkSigs = CheckSigs);
 
     virtual void addMultipleToStore(
-        PathsSource & pathsToCopy,
+        PathsSource && pathsToCopy,
         Activity & act,
         RepairFlag repair = NoRepair,
         CheckSigsFlag checkSigs = CheckSigs);

src/libstore/unix/build/local-derivation-goal.cc

@@ -58,6 +58,10 @@
 #if __APPLE__
 #include <spawn.h>
 #include <sys/sysctl.h>
+#include <sandbox.h>
+
+/* This definition is undocumented but depended upon by all major browsers. */
+extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf);
 #endif
 
 #include <pwd.h>
@@ -1746,13 +1750,20 @@ void LocalDerivationGoal::runChild()
 
         bool setUser = true;
 
-        /* Make the contents of netrc available to builtin:fetchurl
-           (which may run under a different uid and/or in a sandbox). */
+        /* Make the contents of netrc and the CA certificate bundle
+           available to builtin:fetchurl (which may run under a
+           different uid and/or in a sandbox). */
         std::string netrcData;
-        try {
-            if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
-                netrcData = readFile(settings.netrcFile);
-        } catch (SystemError &) { }
+        std::string caFileData;
+        if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") {
+           try {
+               netrcData = readFile(settings.netrcFile);
+           } catch (SystemError &) { }
+
+           try {
+               caFileData = readFile(settings.caFile);
+           } catch (SystemError &) { }
+        }
 
 #if __linux__
         if (useChroot) {
@@ -1955,7 +1966,7 @@ void LocalDerivationGoal::runChild()
             if (chdir(chrootRootDir.c_str()) == -1)
                 throw SysError("cannot change directory to '%1%'", chrootRootDir);
 
-            if (mkdir("real-root", 0) == -1)
+            if (mkdir("real-root", 0500) == -1)
                 throw SysError("cannot create real-root directory");
 
             if (pivot_root(".", "real-root") == -1)
@@ -2027,154 +2038,130 @@ void LocalDerivationGoal::runChild()
                 throw SysError("setuid failed");
         }
 
-        /* Fill in the arguments. */
-        Strings args;
-
-        std::string builder = "invalid";
-
-        if (drv->isBuiltin()) {
-            ;
-        }
 #if __APPLE__
-        else {
-            /* This has to appear before import statements. */
-            std::string sandboxProfile = "(version 1)\n";
+        /* This has to appear before import statements. */
+        std::string sandboxProfile = "(version 1)\n";
 
-            if (useChroot) {
+        if (useChroot) {
 
-                /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
-                PathSet ancestry;
+            /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
+            PathSet ancestry;
 
-                /* We build the ancestry before adding all inputPaths to the store because we know they'll
-                   all have the same parents (the store), and there might be lots of inputs. This isn't
-                   particularly efficient... I doubt it'll be a bottleneck in practice */
-                for (auto & i : pathsInChroot) {
-                    Path cur = i.first;
-                    while (cur.compare("/") != 0) {
-                        cur = dirOf(cur);
-                        ancestry.insert(cur);
-                    }
-                }
-
-                /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost
-                   path component this time, since it's typically /nix/store and we care about that. */
-                Path cur = worker.store.storeDir;
+            /* We build the ancestry before adding all inputPaths to the store because we know they'll
+               all have the same parents (the store), and there might be lots of inputs. This isn't
+               particularly efficient... I doubt it'll be a bottleneck in practice */
+            for (auto & i : pathsInChroot) {
+                Path cur = i.first;
                 while (cur.compare("/") != 0) {
-                    ancestry.insert(cur);
                     cur = dirOf(cur);
+                    ancestry.insert(cur);
                 }
+            }
 
-                /* Add all our input paths to the chroot */
-                for (auto & i : inputPaths) {
-                    auto p = worker.store.printStorePath(i);
-                    pathsInChroot[p] = p;
-                }
+            /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost
+               path component this time, since it's typically /nix/store and we care about that. */
+            Path cur = worker.store.storeDir;
+            while (cur.compare("/") != 0) {
+                ancestry.insert(cur);
+                cur = dirOf(cur);
+            }
 
-                /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
-                if (settings.darwinLogSandboxViolations) {
-                    sandboxProfile += "(deny default)\n";
-                } else {
-                    sandboxProfile += "(deny default (with no-log))\n";
-                }
+            /* Add all our input paths to the chroot */
+            for (auto & i : inputPaths) {
+                auto p = worker.store.printStorePath(i);
+                pathsInChroot[p] = p;
+            }
 
-                sandboxProfile +=
-                    #include "sandbox-defaults.sb"
-                    ;
-
-                if (!derivationType->isSandboxed())
-                    sandboxProfile +=
-                        #include "sandbox-network.sb"
-                        ;
-
-                /* Add the output paths we'll use at build-time to the chroot */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & [_, path] : scratchOutputs)
-                    sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
-                sandboxProfile += ")\n";
-
-                /* Our inputs (transitive dependencies and any impurities computed above)
-
-                   without file-write* allowed, access() incorrectly returns EPERM
-                 */
-                sandboxProfile += "(allow file-read* file-write* process-exec\n";
-                for (auto & i : pathsInChroot) {
-                    if (i.first != i.second.source)
-                        throw Error(
-                            "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
-                            i.first, i.second.source);
-
-                    std::string path = i.first;
-                    auto optSt = maybeLstat(path.c_str());
-                    if (!optSt) {
-                        if (i.second.optional)
-                            continue;
-                        throw SysError("getting attributes of required path '%s", path);
-                    }
-                    if (S_ISDIR(optSt->st_mode))
-                        sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
-                    else
-                        sandboxProfile += fmt("\t(literal \"%s\")\n", path);
-                }
-                sandboxProfile += ")\n";
-
-                /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
-                sandboxProfile += "(allow file-read*\n";
-                for (auto & i : ancestry) {
-                    sandboxProfile += fmt("\t(literal \"%s\")\n", i);
-                }
-                sandboxProfile += ")\n";
-
-                sandboxProfile += additionalSandboxProfile;
-            } else
-                sandboxProfile +=
-                    #include "sandbox-minimal.sb"
-                    ;
-
-            debug("Generated sandbox profile:");
-            debug(sandboxProfile);
-
-            Path sandboxFile = tmpDir + "/.sandbox.sb";
-
-            writeFile(sandboxFile, sandboxProfile);
-
-            bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
-            /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
-            Path globalTmpDir = canonPath(defaultTempDir(), true);
-
-            /* They don't like trailing slashes on subpath directives */
-            while (!globalTmpDir.empty() && globalTmpDir.back() == '/')
-                globalTmpDir.pop_back();
-
-            if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
-                builder = "/usr/bin/sandbox-exec";
-                args.push_back("sandbox-exec");
-                args.push_back("-f");
-                args.push_back(sandboxFile);
-                args.push_back("-D");
-                args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
-                if (allowLocalNetworking) {
-                    args.push_back("-D");
-                    args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));
-                }
-                args.push_back(drv->builder);
+            /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
+            if (settings.darwinLogSandboxViolations) {
+                sandboxProfile += "(deny default)\n";
             } else {
-                builder = drv->builder;
-                args.push_back(std::string(baseNameOf(drv->builder)));
+                sandboxProfile += "(deny default (with no-log))\n";
+            }
+
+            sandboxProfile +=
+                #include "sandbox-defaults.sb"
+                ;
+
+            if (!derivationType->isSandboxed())
+                sandboxProfile +=
+                    #include "sandbox-network.sb"
+                    ;
+
+            /* Add the output paths we'll use at build-time to the chroot */
+            sandboxProfile += "(allow file-read* file-write* process-exec\n";
+            for (auto & [_, path] : scratchOutputs)
+                sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
+
+            sandboxProfile += ")\n";
+
+            /* Our inputs (transitive dependencies and any impurities computed above)
+
+               without file-write* allowed, access() incorrectly returns EPERM
+             */
+            sandboxProfile += "(allow file-read* file-write* process-exec\n";
+            for (auto & i : pathsInChroot) {
+                if (i.first != i.second.source)
+                    throw Error(
+                        "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
+                        i.first, i.second.source);
+
+                std::string path = i.first;
+                auto optSt = maybeLstat(path.c_str());
+                if (!optSt) {
+                    if (i.second.optional)
+                        continue;
+                    throw SysError("getting attributes of required path '%s", path);
+                }
+                if (S_ISDIR(optSt->st_mode))
+                    sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
+                else
+                    sandboxProfile += fmt("\t(literal \"%s\")\n", path);
+            }
+            sandboxProfile += ")\n";
+
+            /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
+            sandboxProfile += "(allow file-read*\n";
+            for (auto & i : ancestry) {
+                sandboxProfile += fmt("\t(literal \"%s\")\n", i);
+            }
+            sandboxProfile += ")\n";
+
+            sandboxProfile += additionalSandboxProfile;
+        } else
+            sandboxProfile +=
+                #include "sandbox-minimal.sb"
+                ;
+
+        debug("Generated sandbox profile:");
+        debug(sandboxProfile);
+
+        bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
+
+        /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
+            to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
+        Path globalTmpDir = canonPath(defaultTempDir(), true);
+
+        /* They don't like trailing slashes on subpath directives */
+        while (!globalTmpDir.empty() && globalTmpDir.back() == '/')
+            globalTmpDir.pop_back();
+
+        if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
+            Strings sandboxArgs;
+            sandboxArgs.push_back("_GLOBAL_TMP_DIR");
+            sandboxArgs.push_back(globalTmpDir);
+            if (allowLocalNetworking) {
+                sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING");
+                sandboxArgs.push_back("1");
+            }
+            char * sandbox_errbuf = nullptr;
+            if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) {
+                writeFull(STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)"));
+                _exit(1);
             }
         }
-#else
-        else {
-            builder = drv->builder;
-            args.push_back(std::string(baseNameOf(drv->builder)));
-        }
 #endif
 
-        for (auto & i : drv->args)
-            args.push_back(rewriteStrings(i, inputRewrites));
-
         /* Indicate that we managed to set up the build environment. */
         writeFull(STDERR_FILENO, std::string("\2\n"));
 
@@ -2191,7 +2178,7 @@ void LocalDerivationGoal::runChild()
                         worker.store.printStorePath(scratchOutputs.at(e.first)));
 
                 if (drv->builder == "builtin:fetchurl")
-                    builtinFetchurl(*drv, outputs, netrcData);
+                    builtinFetchurl(*drv, outputs, netrcData, caFileData);
                 else if (drv->builder == "builtin:buildenv")
                     builtinBuildenv(*drv, outputs);
                 else if (drv->builder == "builtin:unpack-channel")
@@ -2205,6 +2192,14 @@ void LocalDerivationGoal::runChild()
             }
         }
 
+        // Now builder is not builtin
+
+        Strings args;
+        args.push_back(std::string(baseNameOf(drv->builder)));
+
+        for (auto & i : drv->args)
+            args.push_back(rewriteStrings(i, inputRewrites));
+
 #if __APPLE__
         posix_spawnattr_t attrp;
 
@@ -2226,9 +2221,9 @@ void LocalDerivationGoal::runChild()
             posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
         }
 
-        posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+        posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 #else
-        execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+        execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 #endif
 
         throw SysError("executing '%1%'", drv->builder);
@@ -3000,6 +2995,7 @@ void LocalDerivationGoal::deleteTmpDir(bool force)
            might have privileged stuff (like a copy of netrc). */
         if (settings.keepFailed && !force && !drv->isBuiltin()) {
             printError("note: keeping build directory '%s'", tmpDir);
+            chmod(topTmpDir.c_str(), 0755);
             chmod(tmpDir.c_str(), 0755);
         }
         else

src/libstore/unix/build/sandbox-defaults.sb

@@ -49,6 +49,7 @@ R""(
 (if (param "_ALLOW_LOCAL_NETWORKING")
     (begin
       (allow network* (remote ip "localhost:*"))
+      (allow network-inbound (local ip "*:*")) ; required to bind and listen
 
       ; Allow access to /etc/resolv.conf (which is a symlink to
       ; /private/var/run/resolv.conf).

src/libstore/worker-protocol-connection.cc

@@ -32,7 +32,8 @@ static Logger::Fields readFields(Source & from)
     return fields;
 }
 
-std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * source, bool flush)
+std::exception_ptr
+WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * source, bool flush, bool block)
 {
     if (flush)
         to.flush();
@@ -41,6 +42,9 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink
 
     while (true) {
 
+        if (!block && !from.hasData())
+            break;
+
         auto msg = readNum<uint64_t>(from);
 
         if (msg == STDERR_WRITE) {
@@ -95,8 +99,10 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink
             logger->result(act, type, fields);
         }
 
-        else if (msg == STDERR_LAST)
+        else if (msg == STDERR_LAST) {
+            assert(block);
             break;
+        }
 
         else
             throw Error("got unknown message type %x from Nix daemon", msg);
@@ -130,9 +136,10 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink
     }
 }
 
-void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, Sink * sink, Source * source, bool flush)
+void WorkerProto::BasicClientConnection::processStderr(
+    bool * daemonException, Sink * sink, Source * source, bool flush, bool block)
 {
-    auto ex = processStderrReturn(sink, source, flush);
+    auto ex = processStderrReturn(sink, source, flush, block);
     if (ex) {
         *daemonException = true;
         std::rethrow_exception(ex);

src/libstore/worker-protocol-connection.hh

@@ -70,9 +70,10 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection
 
     virtual void closeWrite() = 0;
 
-    std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true);
+    std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true);
 
-    void processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true);
+    void
+    processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true);
 
     /**
      * Establishes connection, negotiating version.