Merge pull request #14942 from NixOS/backport-14778-to-2.31-maintenance

[Backport 2.31-maintenance] test: add shebangs to shell.nix test scripts

.github/actions/install-nix-action/action.yaml

@@ -4,15 +4,29 @@ inputs:
   dogfood:
     description: "Whether to use Nix installed from the latest artifact from master branch"
     required: true # Be explicit about the fact that we are using unreleased artifacts
+  experimental-installer:
+    description: "Whether to use the experimental installer to install Nix"
+    default: false
+  experimental-installer-version:
+    description: "Version of the experimental installer to use. If `latest`, the newest artifact from the default branch is used."
+    # TODO: This should probably be pinned to a release after https://github.com/NixOS/experimental-nix-installer/pull/49 lands in one
+    default: "latest"
   extra_nix_config:
     description: "Gets appended to `/etc/nix/nix.conf` if passed."
   install_url:
     description: "URL of the Nix installer"
     required: false
     default: "https://releases.nixos.org/nix/nix-2.30.2/install"
+  tarball_url:
+    description: "URL of the Nix tarball to use with the experimental installer"
+    required: false
   github_token:
     description: "Github token"
     required: true
+  use_cache:
+    description: "Whether to setup magic-nix-cache"
+    default: true
+    required: false
 runs:
   using: "composite"
   steps:
@@ -37,14 +51,81 @@ runs:
 
         gh run download "$RUN_ID" --repo "$DOGFOOD_REPO" -n "$INSTALLER_ARTIFACT" -D "$INSTALLER_DOWNLOAD_DIR"
         echo "installer-path=file://$INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT"
+        TARBALL_PATH="$(find "$INSTALLER_DOWNLOAD_DIR" -name 'nix*.tar.xz' -print | head -n 1)"
+        echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
 
         echo "::notice ::Dogfooding Nix installer from master (https://github.com/$DOGFOOD_REPO/actions/runs/$RUN_ID)"
       env:
         GH_TOKEN: ${{ inputs.github_token }}
         DOGFOOD_REPO: "NixOS/nix"
+    - name: "Gather system info for experimental installer"
+      shell: bash
+      if: ${{ inputs.experimental-installer == 'true' }}
+      run: |
+        echo "::notice Using experimental installer from $EXPERIMENTAL_INSTALLER_REPO (https://github.com/$EXPERIMENTAL_INSTALLER_REPO)"
+
+        if [ "$RUNNER_OS" == "Linux" ]; then
+          EXPERIMENTAL_INSTALLER_SYSTEM="linux"
+          echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV"
+        elif [ "$RUNNER_OS" == "macOS" ]; then
+          EXPERIMENTAL_INSTALLER_SYSTEM="darwin"
+          echo "EXPERIMENTAL_INSTALLER_SYSTEM=$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV"
+        else
+          echo "::error ::Unsupported RUNNER_OS: $RUNNER_OS"
+          exit 1
+        fi
+
+        if [ "$RUNNER_ARCH" == "X64" ]; then
+          EXPERIMENTAL_INSTALLER_ARCH=x86_64
+          echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV"
+        elif [ "$RUNNER_ARCH" == "ARM64" ]; then
+          EXPERIMENTAL_INSTALLER_ARCH=aarch64
+          echo "EXPERIMENTAL_INSTALLER_ARCH=$EXPERIMENTAL_INSTALLER_ARCH" >> "$GITHUB_ENV"
+        else
+          echo "::error ::Unsupported RUNNER_ARCH: $RUNNER_ARCH"
+          exit 1
+        fi
+
+        echo "EXPERIMENTAL_INSTALLER_ARTIFACT=nix-installer-$EXPERIMENTAL_INSTALLER_ARCH-$EXPERIMENTAL_INSTALLER_SYSTEM" >> "$GITHUB_ENV"
+      env:
+        EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer"
+    - name: "Download latest experimental installer"
+      shell: bash
+      id: download-latest-experimental-installer
+      if: ${{ inputs.experimental-installer == 'true' && inputs.experimental-installer-version == 'latest' }}
+      run: |
+        RUN_ID=$(gh run list --repo "$EXPERIMENTAL_INSTALLER_REPO" --workflow ci.yml --branch main --status success --json databaseId --jq ".[0].databaseId")
+
+        EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR="$GITHUB_WORKSPACE/$EXPERIMENTAL_INSTALLER_ARTIFACT"
+        mkdir -p "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR"
+
+        gh run download "$RUN_ID" --repo "$EXPERIMENTAL_INSTALLER_REPO" -n "$EXPERIMENTAL_INSTALLER_ARTIFACT" -D "$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR"
+        # Executable permissions are lost in artifacts
+        find $EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR -type f -exec chmod +x {} +
+        echo "installer-path=$EXPERIMENTAL_INSTALLER_DOWNLOAD_DIR" >> "$GITHUB_OUTPUT"
+      env:
+        GH_TOKEN: ${{ inputs.github_token }}
+        EXPERIMENTAL_INSTALLER_REPO: "NixOS/experimental-nix-installer"
     - uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1
+      if: ${{ inputs.experimental-installer != 'true' }}
       with:
         # Ternary operator in GHA: https://www.github.com/actions/runner/issues/409#issuecomment-752775072
         install_url: ${{ inputs.dogfood == 'true' && format('{0}/install', steps.download-nix-installer.outputs.installer-path) || inputs.install_url }}
         install_options: ${{ inputs.dogfood == 'true' && format('--tarball-url-prefix {0}', steps.download-nix-installer.outputs.installer-path) || '' }}
         extra_nix_config: ${{ inputs.extra_nix_config }}
+    - uses: DeterminateSystems/nix-installer-action@786fff0690178f1234e4e1fe9b536e94f5433196 # v20
+      if: ${{ inputs.experimental-installer == 'true' }}
+      with:
+        diagnostic-endpoint: ""
+        # TODO: It'd be nice to use `artifacts.nixos.org` for both of these, maybe through an `/experimental-installer/latest` endpoint? or `/commit/<hash>`?
+        local-root: ${{ inputs.experimental-installer-version == 'latest' && steps.download-latest-experimental-installer.outputs.installer-path || '' }}
+        source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
+        nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
+        extra-conf: ${{ inputs.extra_nix_config }}
+    - uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
+      if: ${{ inputs.use_cache == 'true' }}
+      with:
+        diagnostic-endpoint: ''
+        use-flakehub: false
+        use-gha-cache: true
+        source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main

.github/workflows/ci.yml

@@ -27,6 +27,7 @@ jobs:
         extra_nix_config:
           experimental-features = nix-command flakes
         github_token: ${{ secrets.GITHUB_TOKEN }}
+        use_cache: false
     - run: nix flake show --all-systems --json
 
   tests:
@@ -65,7 +66,6 @@ jobs:
         dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
         # The sandbox would otherwise be disabled by default on Darwin
         extra_nix_config: "sandbox = true"
-    - uses: DeterminateSystems/magic-nix-cache-action@main
     # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user:
     # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces
     - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
@@ -140,78 +140,20 @@ jobs:
     - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
     - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
 
-  # Steps to test CI automation in your own fork.
-  # 1. Sign-up for https://hub.docker.com/
-  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
-  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
-  check_secrets:
-    permissions:
-      contents: none
-    name: Check presence of secrets
-    runs-on: ubuntu-24.04
-    outputs:
-      docker: ${{ steps.secret.outputs.docker }}
-    steps:
-      - name: Check for DockerHub secrets
-        id: secret
-        env:
-          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
-
   docker_push_image:
-    needs: [tests, vm_tests, check_secrets]
+    name: Push docker image to DockerHub and GHCR
+    needs: [tests, vm_tests]
+    if: github.event_name == 'push' && github.ref_name == 'master'
+    uses: ./.github/workflows/docker-push.yml
+    with:
+      ref: ${{ github.sha }}
+      is_master: true
     permissions:
       contents: read
       packages: write
-    if: >-
-      needs.check_secrets.outputs.docker == 'true' &&
-      github.event_name == 'push' &&
-      github.ref_name == 'master'
-    runs-on: ubuntu-24.04
-    steps:
-    - uses: actions/checkout@v5
-      with:
-        fetch-depth: 0
-    - uses: cachix/install-nix-action@v31
-      with:
-        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
-    - uses: DeterminateSystems/magic-nix-cache-action@main
-    - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
-    - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L
-    - run: docker load -i ./result/image.tar.gz
-    - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
-    - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
-    # We'll deploy the newly built image to both Docker Hub and Github Container Registry.
-    #
-    # Push to Docker Hub first
-    - name: Login to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKERHUB_USERNAME }}
-        password: ${{ secrets.DOCKERHUB_TOKEN }}
-    - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
-    - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
-    # Push to GitHub Container Registry as well
-    - name: Login to GitHub Container Registry
-      uses: docker/login-action@v3
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Push image
-      run: |
-        IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
-        # Change all uppercase to lowercase
-        IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-
-        docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
-        docker tag nix:$NIX_VERSION $IMAGE_ID:latest
-        docker push $IMAGE_ID:$NIX_VERSION
-        docker push $IMAGE_ID:latest
-        # deprecated 2024-02-24
-        docker tag nix:$NIX_VERSION $IMAGE_ID:master
-        docker push $IMAGE_ID:master
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   vm_tests:
     runs-on: ubuntu-24.04
@@ -254,7 +196,6 @@ jobs:
           extra_nix_config:
             experimental-features = nix-command flakes
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
 
   profile_build:
@@ -275,7 +216,6 @@ jobs:
         extra_nix_config: |
           experimental-features = flakes nix-command ca-derivations impure-derivations
           max-jobs = 1
-    - uses: DeterminateSystems/magic-nix-cache-action@main
     - run: |
         nix build -L --file ./ci/gha/profile-build buildTimeReport --out-link build-time-report.md
         cat build-time-report.md >> $GITHUB_STEP_SUMMARY

.github/workflows/docker-push.yml

@@ -0,0 +1,101 @@
+name: "Push Docker Image"
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: "Git ref to build the docker image from"
+        required: true
+        type: string
+      is_master:
+        description: "Whether run from master branch"
+        required: true
+        type: boolean
+    secrets:
+      DOCKERHUB_USERNAME:
+        required: true
+      DOCKERHUB_TOKEN:
+        required: true
+
+permissions: {}
+
+jobs:
+  # Steps to test CI automation in your own fork.
+  # 1. Sign-up for https://hub.docker.com/
+  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
+  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
+  check_secrets:
+    permissions:
+      contents: none
+    name: Check presence of secrets
+    runs-on: ubuntu-24.04
+    outputs:
+      docker: ${{ steps.secret.outputs.docker }}
+    steps:
+      - name: Check for DockerHub secrets
+        id: secret
+        env:
+          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
+        run: |
+          echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT
+
+  push:
+    name: Push docker image to DockerHub and GHCR
+    needs: [check_secrets]
+    permissions:
+      contents: read
+      packages: write
+    if: needs.check_secrets.outputs.docker == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      with:
+        fetch-depth: 0
+        ref: ${{ inputs.ref }}
+    - uses: ./.github/actions/install-nix-action
+      with:
+        dogfood: false
+        extra_nix_config: |
+          experimental-features = flakes nix-command
+    - run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
+    - run: nix build .#dockerImage -L
+    - run: docker load -i ./result/image.tar.gz
+    # We'll deploy the newly built image to both Docker Hub and Github Container Registry.
+    #
+    # Push to Docker Hub first
+    - name: Login to Docker Hub
+      uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Push to Docker Hub
+      env:
+        IS_MASTER: ${{ inputs.is_master }}
+        DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_USERNAME }}/nix
+      run: |
+        docker tag nix:$NIX_VERSION $DOCKERHUB_REPO:$NIX_VERSION
+        docker push $DOCKERHUB_REPO:$NIX_VERSION
+        if [ "$IS_MASTER" = "true" ]; then
+          docker tag nix:$NIX_VERSION $DOCKERHUB_REPO:master
+          docker push $DOCKERHUB_REPO:master
+        fi
+    # Push to GitHub Container Registry as well
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Push to GHCR
+      env:
+        IS_MASTER: ${{ inputs.is_master }}
+      run: |
+        IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
+        IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+        docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
+        docker push $IMAGE_ID:$NIX_VERSION
+        if [ "$IS_MASTER" = "true" ]; then
+          docker tag nix:$NIX_VERSION $IMAGE_ID:master
+          docker push $IMAGE_ID:master
+        fi

.github/workflows/upload-release.yml

@@ -0,0 +1,69 @@
+name: Upload Release
+on:
+  workflow_dispatch:
+    inputs:
+      eval_id:
+        description: "Hydra evaluation ID"
+        required: true
+        type: number
+      is_latest:
+        description: "Mark as latest release"
+        required: false
+        type: boolean
+        default: false
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+jobs:
+  release:
+    runs-on: ubuntu-24.04
+    environment: releases
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: ./.github/actions/install-nix-action
+        with:
+          dogfood: false # Use stable version
+          use_cache: false # Don't want any cache injection shenanigans
+          extra_nix_config: |
+            experimental-features = nix-command flakes
+      - name: Set NIX_PATH from flake input
+        run: |
+          NIXPKGS_PATH=$(nix build --inputs-from .# nixpkgs#path --print-out-paths --no-link)
+          # Shebangs with perl have issues. Pin nixpkgs this way. nix shell should maybe
+          # get the same uberhack that nix-shell has to support it.
+          echo "NIX_PATH=nixpkgs=$NIXPKGS_PATH" >> "$GITHUB_ENV"
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
+        with:
+          role-to-assume: "arn:aws:iam::080433136561:role/nix-release"
+          role-session-name: nix-release-oidc-${{ github.run_id }}
+          aws-region: eu-west-1
+      - name: Login to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload release
+        run: |
+          ./maintainers/upload-release.pl \
+            ${{ inputs.eval_id }} \
+            --skip-git
+        env:
+          IS_LATEST: ${{ inputs.is_latest && '1' || '' }}
+      - name: Push to GHCR
+        run: |
+          DOCKER_OWNER="ghcr.io/$(echo '${{ github.repository_owner }}' | tr '[A-Z]' '[a-z]')/nix"
+          ./maintainers/upload-release.pl \
+            ${{ inputs.eval_id }} \
+            --skip-git \
+            --skip-s3 \
+            --docker-owner "$DOCKER_OWNER"
+        env:
+          IS_LATEST: ${{ inputs.is_latest && '1' || '' }}

.version

@@ -1 +1 @@
-2.31.0
+2.31.3

doc/manual/anchors.jq

@@ -24,8 +24,15 @@ def map_contents_recursively(transformer):
 def process_command:
     .[0] as $context |
     .[1] as $body |
-    $body + {
-        sections: $body.sections | map(map_contents_recursively(if $context.renderer == "html" then transform_anchors_html else transform_anchors_strip end)),
-    };
+    # mdbook 0.5.x uses 'items' instead of 'sections'
+    if $body.items then
+        $body + {
+            items: $body.items | map(map_contents_recursively(if $context.renderer == "html" then transform_anchors_html else transform_anchors_strip end)),
+        }
+    else
+        $body + {
+            sections: $body.sections | map(map_contents_recursively(if $context.renderer == "html" then transform_anchors_html else transform_anchors_strip end)),
+        }
+    end;
 
 process_command

doc/manual/book.toml.in

@@ -23,12 +23,3 @@ renderers = ["html"]
 command = "jq --from-file ./anchors.jq"
 
 [output.markdown]
-
-[output.linkcheck]
-# no Internet during the build (in the sandbox)
-follow-web-links = false
-
-# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in
-# excessive "Potential incomplete link" warnings. No other kind of warning was
-# produced at the time of writing.
-warning-policy = "ignore"

doc/manual/generate-store-types.nix

@@ -24,9 +24,9 @@ let
     in
     concatStringsSep "\n" (map showEntry storesList);
 
-  "index.md" =
-    replaceStrings [ "@store-types@" ] [ index ]
-      (readFile ./source/store/types/index.md.in);
+  "index.md" = replaceStrings [ "@store-types@" ] [ index ] (
+    readFile ./source/store/types/index.md.in
+  );
 
   tableOfContents =
     let

doc/manual/package.nix

@@ -6,7 +6,6 @@
   ninja,
   lowdown-unsandboxed,
   mdbook,
-  mdbook-linkcheck,
   jq,
   python3,
   rsync,
@@ -51,7 +50,6 @@ mkMesonDerivation (finalAttrs: {
     ninja
     (lib.getBin lowdown-unsandboxed)
     mdbook
-    mdbook-linkcheck
     jq
     python3
     rsync

doc/manual/rl-next/shorter-build-dir-names.md

@@ -0,0 +1,6 @@
+---
+synopsis: "Temporary build directories no longer include derivation names"
+prs: [13839]
+---
+
+Temporary build directories created during derivation builds no longer include the derivation name in their path to avoid build failures when the derivation name is too long. This change ensures predictable prefix lengths for build directories under `/nix/var/nix/builds`.

doc/manual/source/store/building.md

@@ -12,10 +12,11 @@
 
 The [`builder`](./derivation/index.md#builder) is executed as follows:
 
-- A temporary directory is created under the directory specified by
-  `TMPDIR` (default `/tmp`) where the build will take place. The
+- A temporary directory is created where the build will take place. The
   current directory is changed to this directory.
 
+  See the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting for more information.
+
 - The environment is cleared and set to the derivation attributes, as
   specified above.
 

doc/manual/substitute.py

@@ -41,6 +41,10 @@ def recursive_replace(data: dict[str, t.Any], book_root: Path, search_path: Path
             return data | dict(
                 sections = [recursive_replace(section, book_root, search_path) for section in sections],
             )
+        case {'items': items}:
+            return data | dict(
+                items = [recursive_replace(item, book_root, search_path) for item in items],
+            )
         case {'Chapter': chapter}:
             path_to_chapter = Path(chapter['path'])
             chapter_content = chapter['content']

flake.lock

@@ -63,16 +63,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1755442223,
-        "narHash": "sha256-VtMQg02B3kt1oejwwrGn50U9Xbjgzfbb5TV5Wtx8dKI=",
+        "lastModified": 1761597516,
+        "narHash": "sha256-wxX7u6D2rpkJLWkZ2E932SIvDJW8+ON/0Yy8+a5vsDU=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cd32a774ac52caaa03bcfc9e7591ac8c18617ced",
+        "rev": "daf6dc47aa4b44791372d6139ab7b25269184d55",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-25.05-small",
+        "ref": "nixos-25.05",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,7 +1,7 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
 
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
@@ -32,7 +32,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux32BitSystems = [ "i686-linux" ];
       linux64BitSystems = [

maintainers/keys/158A6F530EA202E5F651611314FAEA63448E1DF9.asc

@@ -0,0 +1,110 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGPtMiwBEAC0sFZW2QW/OaDjKm5zGRpDvHXDsMIUtlHfoi5ce8pocC63W05o
+FSXbUZjZ1VfYO8lT8DFANCzTkiXYaZx0cPRG2pVY4AOQZDNFt5XrAyvw496XCAIM
+DTYGFLjCqgjPt9RUFEy4MyHPJTEpB0x3rXgT4ILNu9vsj9Q0vttps7SpbZ3Ldq5H
+o/BBbLW77q/vNjpYzCbBIXF7ycUGpnNv9Go/WuiDnrBMcyxh+8kjjIHB5cxZSnjJ
+DUv681+m83v+gLZQGX/jexQrrf5JpS0X9qEnhGLrNUDhtyv5ud3Je4EfamkjLVVC
+RlNLofgflOCsl/tP80i+K7S1QdKhUALxuJ6H0prYUflGBDxDyC8XYuJ62TT0OUpa
+vJvgwVlCq8/jq+ykYQXlbuBVOzi5wAuI4l3+HqreSQYPSiwe+6N590Zbafdv1fvN
+WFtZKCTGMqfyaaAnppioH9/+NWkI2AQxaYVasYM/JEYvY9pJgA7alh51jHW4JglP
+ErypKfBKPKJID0QENqYoa3bDDCihuNWhgQf9dxzPlj2ckd35Zb6w4DfuSmtjaa9D
+o0jZVY1JbFuxBqP09+saVPrxLHgmPxjcdzPGQQtAqdO2vyJXNEGLFMoVEZPNaLo3
+QmcIJnT7oSck+4vGfOYtWUHXQynu/Tnwsv2XkA/uyw8HNe+RRMqv/apnzQARAQAB
+tCdTZXJnZWkgWmltbWVybWFuIDxzZXJnZWlAemltbWVybWFuLmZvbz6JAlEEEwEK
+ADsCGwEFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQQVim9TDqIC5fZRYRMU+upj
+RI4d+QUCaUbsaAIZAQAKCRAU+upjRI4d+VdDD/492HRaJ/8V7R7VUzkafmb2Hb28
+SLf7oiB8Uq9I7SukiDEaIT1fUhquYWQ9KWpPRNR1TX6ApXnIeuJRMGFoDVIRnmnr
+cKnYYXfqqc81VxIyKvaumB7KWbS7G4Nbor8AH1ouOOOMMS50OTJOWQA4A26inIuG
+n+7L8MeS5aT+3uNKDoTKsidC47vnaxNMcke1taPfbfo7vn69PsRCM/g9/7TQYU8b
+6xp+pM9Ao9nJneRk2YCpsGYRrWTpaik0DFKnfpPKJM/yunhtGLF2IYAp3l1mvHPK
+nnzo92zjpQuZwazEIK+23V1vRT4IjM2BewbJPAzf2/UuxEjjgNQm0tOtH2JhFNeB
+VM0BVrGxWrrwrsmv6lWghTtBc6zRWyHrj/rpjtVQNmeKYrHWJeXwVz1rqgGPmB2N
+k0MZD1UjHHhEs1Cntn7yLmxTPztRJCtR+euRu81Uo2NAvrMJ4xsDjaM0LeLTnzjV
+9AsPjD188dOFyz7VExZum4+XaaEJ41FIPLEqU3U0GAa6stEy0ylSlIN4x9aiXXVW
+xfzHHchS5jK6QAjuZxN8t01GRactNylINRf7uoTECFZTtXNqfeuk0HQxBH0LuKVE
+0PxJbcNI4mVWw1KgTJ8PUVC1IXP3sEpqPdJOYiRXgnpcS26fWOBu0aQ4mhxiaJhr
+/zBfrkLEqp20TdNDZLkCDQRj7TM1ARAAt73xO24curnHTTgXkkVMMRzcMLx3Mb1a
+2FuddxC5hzTpEpw01L91UBrXVJEg9K2KAwP5CtCLgPCqXr47Tm7krvHxWwBksgY/
+6aHRsoPQfCFUZHc0aiO+C4NCzR+aEeGKn66Oc1Hq9oUTpDgiBWhsuEPiyA1OSGF0
+4L0jeTCqfm68kWp4PIK9yuugkdDsoyj6TonuMsb3V5ctHLqop9KH+eHSkUTPo+Lk
++bxaeAOJ1UfbohgbRbrYKAfsaghhOMDH3R1w2pvtUJz+sDbuQsiPFTqbxsXDTFws
+H4N/AQCYnnvOhqEek2sOEZ19bJXt5UrAr10mX4PGmAkWqE1JWBxpOKG3BXSGOTu1
+3dFhQfPMK+PmvUrs0kcWQr53K/aRUdKKhIfTcMfkqYTGPK5HclHph24WjXj3QFFA
+SjksQTdm6486ZmLZK4CTbAFOPfTF/aWg8gu9v4ihdq6lqHNNXxv2xBAChcd59H7p
+D5zy9z8SpwWR9V5JDmlF6HWIIau1c6lSsQq1xHvYM8EuPe03vJvor+2u/cn5zYF1
+5ZxAuPI2i5vtavg1s8ZGAAogJ9dVcP36LdJfL9quXWvmovkd//qHIepBB+l/zQio
+ZRDZlIcfV3Xycaqsb5OqHGARHE0097koipMt5y/iXlqG4Ruue6Idb8bW96EKpaWj
+kKy/iNfQfQMAEQEAAYkEcgQYAQoAJgIbAhYhBBWKb1MOogLl9lFhExT66mNEjh35
+BQJpRuz3BQkJHCDCAkDBdCAEGQEKAB0WIQRK3hK0WyJ4BicGpcmpsLVXymMjJQUC
+Y+0zNQAKCRCpsLVXymMjJbdSD/9+f1FOOeGDAJI6Duo5fsWnf4xJJdtQtDbz6d2A
+SeDapxeJ3zWfKBD0wu5sISEa0uiWsYSmLtsa2SqVAKHlEaMGRR+tkBMPQ+rvgI4c
+62YjGTgm+IPd+NFIn+ixFU1hpinTh+KhUEoeOwWCvKs9nZfSG9vkienfiG0bBxo2
+zrvBzXA50x5hbUL+ghKu/AVfN9qZDwh30O4KZTwk4g4cM9SeaQa4YvHYIS3IEhDZ
+hGybwrrqV9cs92ln4IJw9WCy9QReBNrdeFgC4+3ziUp1QsG3RvqrtuMttwBVC1Z3
+bj5QjLLOREhhodfvk98t9yVkragObb4rGrLo1mWuF0c4mJGvXwnrqhCMvzv4M+0T
+Zdrmw6YpGkGOaOPghVuwoTtqSAkl+zFWIJS89jidvkYG3EqKAkgLKog/TQReCq13
+HWrF8cMck+Rf2K8k26q/RNZaA9ZUKjLExzz8lsWmd2C7rvkGLrlxnzxz0gGyNR3Q
+KK74vcPhqeABt2GSkHtEXZFFA9IVVzwlRWK3e0S+mVQnZVjNL+cBPn3/hZHMLesB
+CucyYZv+DxvT+JkYXBkGSw4s3hpABqGym7gdPUIa0q4rbBFG6xP5sLLBG4yru8vV
+2dyCMmFqRuxpT49uNfyQ6Vj+dobN6qHnP/9NwfzOixXYBHXR6LBqb/M+iCiJaaIn
+uiRLHwkQFPrqY0SOHfkQpA//W51vj8meuz7snRO+vZFcjLneFFzqfh1Jdz8IqDpO
+CkI5pBJmi8e0oSe6r68MkahiQLlYPwm7d+sjHvJhPWipNKWq/uwCgBs+Ac1lpPXR
+MwLbrZukcLMYlLmb2MrCKmjcMt0BZsZKBNYL3a3X9nHgwXdeqFYS4WQDMCCc09lz
+9YqfdoEsqRO4qN7D0hFqnwjOzb34ixZ6UO8a8ekY9QKxAgWc9fJWGMg6Pjdg4qsK
+nqymOIAdGVOJdoRM46wKGVBvbsF2gNfQU4XyzgJo5vHGFwJm6EoSnODlL5e2wsQh
+uN1oqBt/8ef/plloMEqVBweUBATqSqjRF6IhhYJvWVuQHQL1p1vnV9FebiVj34ir
+Z8ID+o0AnTJcclbUcDwannGJ0cuDcPhk/v/ahVuoMERCi12qnMBo5B/e6Omyh1yB
+4pbf4GATGGQipDQG75eC/kP2GQEqJP5WYN0Ar8Le/AA/2xyL7upW0yIByyXCwGEb
+JRwEgU3+bPyu58bFt8Pftit6J7rA3oBVVMOPrYH5eZwRaj5m2RptwKGL6BfHnhNv
+ZqmCq9EBGX6L1NI0xHMjEFfXJ8jU01XdfG8nCqkwqsHwslXLhqjJphfHcx89YwbV
+/15GCuURAv1cKe/7277sOhcvP/QpQqSWgvYExHw8PeFJcTYtF2NrRgNwcQsWS1Rj
+gXa5Ag0EY+0zcAEQANC5N6kSfezuucAgi+X3BD+MT37mxQyvICSggEJf1LDSmy0+
+bnvD7setL8CP9etTA2fcVNYKI1oboMyhoCnsRP2jDdv1iXOI/hZg4wSb/D1yUkae
+fUpxv3Wuci2QKavH2MfraDD7BFMbsQeMcHtn4Rk216T6jndZHnzT1Ih7iX0XeQPb
+li5fojOiZssgWAVT4HPXFCJB6lI35Hjp35oRYwrtMmu5INinZ79n9h1igGtt1ItZ
+b7rQKNd772Jxcn4UU71ovORSL/xT5i5sxZ+evQOxkpqUAokMOFaoHcOXLmA1NsFv
+yryXHK4Ioq9ap2jKlLTWkJWjua9JZ4AmKhbvT8X4ELxIKSCAdJKAWP8ZHbXNu5MD
+aznyzZQLxSO7uFvu356De75mI5iohZNj5wB5Wju71pBiorTKVj4+iJ4e+xVIzFdG
+hFC0DehNcl2t9w/y8qHwIQ1yUAjXHLXq0/2jsVeH6bU5q/MsgvUP1jcFe0eyOpxy
+CDvyFdzZFbI57TnB/fvcZTRZ5ewXMFpH8gzuoFzAjUAP95UjYKgaGdrNPNIy28Ii
+4zhvdghei2+n9jgiMfcGQg8lyfH5yF0vWWWynX0KcJsRwEZoL2EauVdwq4PcYOoU
+pQFhpcreCjD4LdZ4yRU4InbhcUogXjrQ9Dz01TbPmQD5b5iso21bCEFBXrhzABEB
+AAGJAjwEGAEKACYCGwwWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwg
+hwAKCRAU+upjRI4d+X/XD/sH5xvHPfTJq52v8weFmB52up+DzqG2lyhGdoUQ1Muw
+dRDLTLXLJrFdfpoOo7/j4Scr0rdc7/dpCn0DLcPuCoPxu+SkjEnVehFmZrGSv7Ga
+x9dHr3DBh42fdlX/U/EnDuyosY0JU1gNF2/6FIA+bTTOFE3RxfN906RjslYQDjMZ
+UAlSeLYHOZofdltI0YIr32vrxgdWQGZXPxU4XusDUc0z163OO+TGg7iUNWFZP5Qj
+ubM7e0YbDX0NPIshk8us99YJmrWnhaix1/W5ryO3DXiGaQ7XFi9u7QofRqvRIctg
+QXavdepkzJow9V9qpMECAJePIuICq7rm+xy+njjbuF436W7390bfVBwRr+FPADsl
+jgQP4KvY5rykss30kheom8wNEbveWkhH5oTfH9b7O4KXJfpfJzrlgOWp2BD9JL8t
+/M4HvFXTr2a75H/QbHK5OFrZeGATuv9OTxv7EZvnrPXU+DYTFldpu7TrNNqKCoj3
+ZyXmc3Hhg5kskDhfHJppaeOayuhMOpT3ud1MFzROY5SLVIH8rBR12KUgsCUYQcGs
+Iy0+0QvEGkjb4cAH1NK3VlbqVNsy1RmqRt2B28R2ueewDfTOoqkzt4MmzLqTdnAx
+mTqmHmkEKhEf3K4MRNUPO2yieUg2COk5l6x9HhAnoxxeOZrTmcMsPY/UViG2HEPm
+ybkCDQRj7TRDARAA9DZuKdfKq4Bs2+NwxC0aplljWOl8VIsEVg+Q8agD7/HU6/b6
+Dry0njtWybn2x6Axf/nUdeOC01Fi1lmht/fpj6mRkgAvd/V6P10xnsUoykPSDSTh
+P25MFFGW3JAA82bwdJ4AJpEQvTZG2nTb3237vlBiI1qHQrac8GYkju2O4UfySRN6
+7cyi7bMf2pjWBBOEhaNy4b6CMDsb32P/N5J7sTE/TXgrS+u4ITIgjzSrkUkh5Z+B
+8QVRa7xPIDZJdvZWTEXWu5fgRPZvxbr154GIkWJkFzlDoB1UcO56/uzRUuKhEV6o
+HW3LMUuWdPMjpHpq8hrL0G2rDniJFUtbDFzHdZK1LUU3T2BJM8rjI3D/euph+IDT
+27vl5qo72zCYE/iKzx4FMLZcQvx1kUAxkPX8l+dzZEwKeRIIpFDxQvatRtl+z0bM
+jbkpDb+Yjv66sC4dYRpgTTGX6rok0PWHR3IxDNzyf2j8zQ4LFJ+rVBM1GjGSt6mG
+j9TeL8CVeiSp4SuJ7I/FJVPHsKb50m+BDzeB31qTydNqh2kKr0DVAUa+TUsCr7e0
+OYr8WE2adJcRXIW0qw50xXF+W7/05GqSCVD0dpeOUdBTQTsSkQmM3/0hcj9aVo9e
+UDCM9RF0WRqiDAoHzJFfg+ztamkQI5HO6CklC4Ok22qrHRf6HDNYSuT6QFkAEQEA
+AYkCPQQYAQoAJwMbIAQWIQQVim9TDqIC5fZRYRMU+upjRI4d+QUCaUbs9wUJCRwf
+tAAKCRAU+upjRI4d+Y+cD/9yllG6uo934pcHNsVppZBfREFwSc8ywlbosCuSVpay
+PjSqgrWwDrnqrsk0F2kUdC6rR3BIcXbn+lA9KqylH+cCXAJCkh8EDq6TlQ7Lt5EV
+w1U0MAMXOyxPwDymQ/BO+iDyjXWkRRYgbF5XiFhCfGeuKyhkhACisAgNZ1uA1P5k
+0SJYc14YfEhQkB46Y20SpfVHRsQ46FyNB6GHbmTmfoO8La8VTh++7GBdh85HfvkG
+VNQ3wpi5oXsOLN9+MJOezc0XsW2LQsKQj1/J7QKzGh+lxN5cemsA5aqPzh8dyxeT
+0lYRFp4AHkimqGUomVpRkbegMIPxXqOE+ZAmsddErw0UtmrKxcmMptOJwNgYzEgu
+++2vtqerL/NYp+wsdcWaBjCz2F3NiwHgNli7NSB/FPwucZZ5gN5C4SnmeFzrGdHg
+Oy+tQUN6ayQKljHeBO7CjMlsFNo/dcVrEMa1ShxBMqlj/6ivoEhktLz0Nru4FwNU
+xE5SJYDYfpjD7Ws8y4LoXgWXjFHrMO6N9GzqLN/e8LT7I+w4ps2MrgJ8QSrelmQ3
+rjkxp3uWp5v2lqy4rLfpi9iB6zIAeoN2eU1yOM9joxOYMxKYaYeYyP1Mm90wFol8
+LcTSaN+tVniPddBiL6zvsGBEMbCR9XN3EQ+mErbuw5ovWBOCrr+dvN3FxvD11y4J
+7w==
+=mXYP
+-----END PGP PUBLIC KEY BLOCK-----

maintainers/keys/B541D55301270E0BCF15CA5D8170B4726D7198DE.asc

@@ -0,0 +1,51 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQENBFZu2zwBCADfatenjH3cvhlU6AeInvp4R0JmPBG942aghFj1Qh57smRcO5Bv
+y9mqrX3UDdmVvu58V3k1k9/GzPnAG1t+c7ohdymv/AMuNY4pE2sfxx7bX+mncTHX
+5wthipn8kTNm4WjREjCJM1Bm5sozzEZetED3+0/dWlnHl8b38evnLsD+WbSrDPVp
+o6M6Eg9IfMwTfcXzdmLmSnGolBWDQ9i1a0x0r3o+sDW5UTnr7jVP+zILcnOZ1Ewl
+Rn9OJ4Qg3ULM7WTMDYpKH4BO7RLR3aJgmsFAHp17vgUnzzFBZ10MCS3UOyUNoyph
+xo3belf7Q9nrHcSNbqSeQuBnW/vafAZUreAlABEBAAG0IkVlbGNvIERvbHN0cmEg
+PGVkb2xzdHJhQGdtYWlsLmNvbT6JATwEEwEIACYCGyMHCwkIBwMCAQYVCAIJCgsE
+FgIDAQIeAQIXgAUCVm7etAIZAQAKCRCBcLRybXGY3q51B/96qt41tmcDSzrj/UTl
+O6rErfW5zFvVsJTZ95Duwu87t/DVhw5lKBQcjALqVddufw1nMzyN/tSOMVDW8xe4
+wMEdcU4+QAMzNX80enuyinsw1glxfLcK0+VbTvqNIfw0sG3MjPqNs6cK2VRfMHK4
+paJjytBVICszNX9TfjLyIpKKoSSo1vqnT47LDZ5GIMy7l9Cs2sO/rqQHSPcR79yz
+8m8tbHpDDEMZmJeklckKP2QoiqnHiIvlisDxLclYnUmNaPdaN/f++qZz5Yqvu1n+
+sNUBA5eLaZH64Uy2SwtABxO3JPJ8nQ2+SFZ7ocFm4Gcdv4aM+Ura9S6fvM91tEJp
+yAQOiQE5BBMBCAAjBQJWbts8AhsjBwsJCAcDAgEGFQgCCQoLBBYCAwECHgECF4AA
+CgkQgXC0cm1xmN6sIAgAielxO8zJREqEkA2xudg/o4e9ZlNZ3X1NvY8OzJH/qlB2
+SmwKqwifhtbC1K0uavXA7eaxdtd2zrI+Yq7IooUyv7juMjHTZhLcFbR5iVkQ4Mfp
+JmeHXJ/ChYKxD5mMj/C3WbCZ91oCSNZ6Iyi5fvQj/691OC4q+y/2NEUcOI8D8cw8
+XKHbKtceFYc+nZmdOv3ZZrNTSN/kszGViNNLKgnpPdDVPtLp+vjXtbmitiFG2HL/
+WfbJ+3Gh2Yr1Vy3O9dWKH++e1AmIv7WWqmUjRFVpqC/wr7/BLaScWT8WKF5vkshU
+gq8Ez1/cuizsgs3wQIZWgXKQK5njvwnbKg+Zmh/uGbQmRWVsY28gRG9sc3RyYSA8
+ZWVsY28uZG9sc3RyYUB0d2VhZy5pbz6JAU4EEwEIADgWIQS1QdVTAScOC88Vyl2B
+cLRybXGY3gUCXELt4gIbIwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCBcLRy
+bXGY3ujFCADfS5D1xHU8KH6TpqgssSggYVq62Wwn/Ga+4XPPetM+ajcXagyH6SwB
+mxlHICcnv9xC93ryiTI10P1ADJl+aBsI66wEdHBU+ty4RTDy4JZNUPtmRCk9LhSc
+mtUO3ry/wtWkRLdJxP49hg7BbQvWoU0M6WODp7SJjPKPWNX64mzHBeOuy+DqGCbM
+lpGNCvW8ahU/ewbm7+xwWmzqLDoWzXjHsdF4QdzMVM/vkAgWEP4y0wEqFASzIYaR
+GNEkBWU4OQVq5Bdm9+wWWAgsbM0FJAQl0GDqnz4QxWzxxCAAXdbh9F5ffafWYsA9
+bise4ZQLkvYo6iUnrcFm4dtZbT8iL3gptCtFZWxjbyBEb2xzdHJhIDxlZWxjby5k
+b2xzdHJhQGxvZ2ljYmxveC5jb20+iQE5BBMBCAAjBQJWbt6nAhsjBwsJCAcDAgEG
+FQgCCQoLBBYCAwECHgECF4AACgkQgXC0cm1xmN4b/wf8DApMV/jSPEpibekrUPQu
+Ye3Z8cxBQuRm/nOPowtPEH/ShAevrCdRiob2nuEZWNoqZ2e5/+6ud07Hs9bslvco
+cDv1jeY1dof1idxfKhH3kfSpuD2XJhuzQBxBqOrIlCS/rdnW+Y9wOGD7+bs9QpcA
+IyAeQGLLkfggAxaGYQ2Aev8pS7i3a/+lOWbFhcTe02I49KemCOJqBorG5FfILLNr
+DjO3EoutNGpuz6rZvc/BlymphWBoAdUmxgoObr7NYWgw9pI8WeE6C7bbSOO7p5aQ
+spWXU7Hm17DkzsVDpaJlyClllqK+DdKza5oWlBMe/P02jD3Y+0P/2rCCyQQwmH3D
+RbkBDQRWbts8AQgA0g556xc08dH5YNEjbCwEt1j+XoRnV4+GfbSJIXOl9joIgzRC
+4IaijvL8+4biWvX7HiybfvBKto0XB1AWLZRC3jWKX5p74I77UAcrD+VQ/roWQqlJ
+BKbiQMlRYEsj/5Xnf72G90IP4DAFKvNl+rLChe+jUySA91BCtrYoP75Sw1BE9Cyz
+xEtm4WUzKAJdXI+ZTBttA2Nbqy+GSuzBs7fSKDwREJaZmVrosvmns+pQVG4WPWf4
+0l4mPguDQmZ9wSWZvBDkpG7AgHYDRYRGkMbAGsVfc6cScN2VsSTa6cbeeAEowKxM
+qx9RbY3WOq6aKAm0qDvow1nl7WwXwe8K0wQxfQARAQABiQEfBBgBCAAJBQJWbts8
+AhsMAAoJEIFwtHJtcZjeuAAH/0YNz2Qe1IAEO5oqEZNFOccL4KxVPrBhWUen83/b
+C6PjOnOqv6q5ztAcms88WIKxBlfzIfq+dzJcbKVS/H7TEXgcaC+7EYW8sJVEsipN
+BtEZ3LQNJ5coDjm7WZygniah1lfXNuiritAXduK5FWNNndqGArEaeZ8Shzdo/Uyi
+b9lOsBIL6xc2ZcnX5f+rTu02LCEtEb0FwCycZLEWYf8hG4k8uttIOZOC+CLk/k8d
+kBmPikMwUVTTV0CdT1cemQKdTaoAaK+kurF6FYXwcnjhRlHrisSt/tVMEwTw4LUM
+3MYf6qfjjvE4HlDwZal8th7ccoQp/flfJIuRv85xCcKK+PI=
+=u5cX
+-----END PGP PUBLIC KEY BLOCK-----

maintainers/keys/README.md

@@ -0,0 +1,13 @@
+# Maintainer GPG Keys
+
+Release tags are signed by members of the [Nix maintainer team](https://nixos.org/community/teams/nix/) as part of the [release process](../release-process.md). This directory contains the public GPG keys used for signing.
+
+## Keys
+
+- **Eelco Dolstra**
+  GPG Fingerprint: `B541 D553 0127 0E0B CF15 CA5D 8170 B472 6D71 98DE`
+
+- **Sergei Zimmerman**
+  GPG Fingerprint: [`158A 6F53 0EA2 02E5 F651 6113 14FA EA63 448E 1DF9`](https://keys.openpgp.org/vks/v1/by-fingerprint/158A6F530EA202E5F651611314FAEA63448E1DF9)
+
+<!-- TODO: Add keys for other Nix team members -->

maintainers/release-process.md

@@ -5,11 +5,11 @@
 The release process is intended to create the following for each
 release:
 
-* A Git tag
+* A signed Git tag (public keys in `maintainers/keys/`)
 
 * Binary tarballs in https://releases.nixos.org/?prefix=nix/
 
-* Docker images
+* Docker images (arm64 and amd64 variants, uploaded to DockerHub and GHCR)
 
 * Closures in https://cache.nixos.org
 
@@ -98,21 +98,17 @@ release:
   evaluation ID (e.g. `1780832` in
   `https://hydra.nixos.org/eval/1780832`).
 
-* Tag the release and upload the release artifacts to
-  [`releases.nixos.org`](https://releases.nixos.org/) and [Docker Hub](https://hub.docker.com/):
+* Tag the release:
 
   ```console
-  $ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
+  $ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
   ```
 
   Note: `IS_LATEST=1` causes the `latest-release` branch to be
   force-updated. This is used by the `nixos.org` website to get the
   [latest Nix manual](https://nixos.org/manual/nixpkgs/unstable/).
 
-  TODO: This script requires the right AWS credentials. Document.
-
-  TODO: This script currently requires a
-  `/home/eelco/Dev/nix-pristine`.
+* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
 
   TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/
 
@@ -177,16 +173,18 @@ release:
 * Wait for the desired evaluation of the maintenance jobset to finish
   building.
 
-* Run
+* Tag the release
 
   ```console
-  $ IS_LATEST=1 ./maintainers/upload-release.pl <EVAL-ID>
+  $ IS_LATEST=1 ./maintainers/upload-release.pl --skip-docker --skip-s3 --project-root $PWD <EVAL-ID>
   ```
 
   Omit `IS_LATEST=1` when creating a point release that is not on the
   most recent stable branch. This prevents `nixos.org` to going back
   to an older release.
 
+* Trigger the [`upload-release.yml` workflow](https://github.com/NixOS/nix/actions/workflows/upload-release.yml) via `workflow_dispatch` trigger. At the top click `Run workflow` -> select the current release branch from `Use workflow from` -> fill in `Hydra evaluation ID` with `<EVAL-ID>` value from previous steps -> click `Run workflow`. Wait for the run to be approved by `NixOS/nix-team` (or bypass checks if warranted). Wait for the workflow to succeed.
+
 * Bump the version number of the release branch as above (e.g. to
   `2.12.2`).
 

maintainers/upload-release.pl

@@ -1,7 +1,8 @@
 #! /usr/bin/env nix-shell
-#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1
+#! nix-shell -i perl -p awscli2 perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 perlPackages.GetoptLongDescriptive gnupg1
 
 use strict;
+use Getopt::Long::Descriptive;
 use Data::Dumper;
 use File::Basename;
 use File::Path;
@@ -13,7 +14,30 @@ use Net::Amazon::S3;
 
 delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning
 
-my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n";
+my ($opt, $usage) = describe_options(
+    '%c %o <eval-id>',
+    [ 'skip-docker',      'Skip Docker image upload' ],
+    [ 'skip-git',         'Skip Git tagging' ],
+    [ 'skip-s3',          'Skip S3 upload' ],
+    [ 'docker-owner=s',   'Docker image owner', { default => 'nixos/nix' } ],
+    [ 'project-root=s',   'Pristine git repository path' ],
+    [ 's3-endpoint=s',    'Custom S3 endpoint' ],
+    [ 's3-host=s',        'S3 host', { default => 's3-eu-west-1.amazonaws.com' } ],
+    [],
+    [ 'help|h',           'Show this help message', { shortcircuit => 1 } ],
+    [],
+    [ 'Environment variables:' ],
+    [ 'AWS_ACCESS_KEY_ID' ],
+    [ 'AWS_SECRET_ACCESS_KEY' ],
+    [ 'AWS_SESSION_TOKEN   For OIDC' ],
+    [ 'IS_LATEST           Set to "1" to mark as latest release' ],
+);
+
+print($usage->text), exit if $opt->help;
+
+my $evalId = $ARGV[0] or do { print STDERR $usage->text; exit 1 };
+
+die "--project-root is required unless --skip-git is specified\n" unless $opt->skip_git || $opt->project_root;
 
 my $releasesBucketName = "nix-releases";
 my $channelsBucketName = "nix-channels";
@@ -62,25 +86,38 @@ File::Path::make_path($narCache);
 my $binaryCache = "https://cache.nixos.org/?local-nar-cache=$narCache";
 
 # S3 setup.
-my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given.";
-my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given.";
+my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'};
+my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'};
+my $aws_session_token = $ENV{'AWS_SESSION_TOKEN'};
 
-my $s3 = Net::Amazon::S3->new(
-    { aws_access_key_id     => $aws_access_key_id,
-      aws_secret_access_key => $aws_secret_access_key,
-      retry                 => 1,
-      host                  => "s3-eu-west-1.amazonaws.com",
-    });
+my ($s3, $releasesBucket, $s3_channels, $channelsBucket);
 
-my $releasesBucket = $s3->bucket($releasesBucketName) or die;
+unless ($opt->skip_s3) {
+    $aws_access_key_id or die "No AWS_ACCESS_KEY_ID given.";
+    $aws_secret_access_key or die "No AWS_SECRET_ACCESS_KEY given.";
 
-my $s3_us = Net::Amazon::S3->new(
-    { aws_access_key_id     => $aws_access_key_id,
-      aws_secret_access_key => $aws_secret_access_key,
-      retry                 => 1,
-    });
+    $s3 = Net::Amazon::S3->new(
+        { aws_access_key_id     => $aws_access_key_id,
+          aws_secret_access_key => $aws_secret_access_key,
+          $aws_session_token ? (aws_session_token => $aws_session_token) : (),
+          retry                 => 1,
+          host                  => $opt->s3_host,
+          secure                => ($opt->s3_endpoint && $opt->s3_endpoint =~ /^http:/) ? 0 : 1,
+        });
 
-my $channelsBucket = $s3_us->bucket($channelsBucketName) or die;
+    $releasesBucket = $s3->bucket($releasesBucketName) or die;
+
+    $s3_channels = Net::Amazon::S3->new(
+        { aws_access_key_id     => $aws_access_key_id,
+          aws_secret_access_key => $aws_secret_access_key,
+          $aws_session_token ? (aws_session_token => $aws_session_token) : (),
+          retry                 => 1,
+          $opt->s3_endpoint ? (host => $opt->s3_host) : (),
+          $opt->s3_endpoint ? (secure => ($opt->s3_endpoint =~ /^http:/) ? 0 : 1) : (),
+        });
+
+    $channelsBucket = $s3_channels->bucket($channelsBucketName) or die;
+}
 
 sub getStorePath {
     my ($jobName, $output) = @_;
@@ -115,11 +152,12 @@ sub copyManual {
         File::Path::remove_tree("$tmpDir/manual.tmp", {safe => 1});
     }
 
-    system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
+    my $awsEndpoint = $opt->s3_endpoint ? "--endpoint-url " . $opt->s3_endpoint : "";
+    system("aws $awsEndpoint s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0
         or die "syncing manual to S3\n";
 }
 
-copyManual;
+copyManual unless $opt->skip_s3;
 
 sub downloadFile {
     my ($jobName, $productNr, $dstName) = @_;
@@ -158,30 +196,12 @@ sub downloadFile {
     return $sha256_expected;
 }
 
-downloadFile("binaryTarball.i686-linux", "1");
-downloadFile("binaryTarball.x86_64-linux", "1");
-downloadFile("binaryTarball.aarch64-linux", "1");
-downloadFile("binaryTarball.x86_64-darwin", "1");
-downloadFile("binaryTarball.aarch64-darwin", "1");
-eval {
-    downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
-};
-warn "$@" if $@;
-eval {
-    downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
-};
-warn "$@" if $@;
-eval {
-    downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
-};
-warn "$@" if $@;
-downloadFile("installerScript", "1");
-
-# Upload docker images to dockerhub.
+# Upload docker images.
 my $dockerManifest = "";
 my $dockerManifestLatest = "";
 my $haveDocker = 0;
 
+unless ($opt->skip_docker) {
 for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
     my $system = $platforms->[0];
     my $dockerPlatform = $platforms->[1];
@@ -195,8 +215,8 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
     print STDERR "loading docker image for $dockerPlatform...\n";
     system("docker load -i $tmpDir/$fn") == 0 or die;
 
-    my $tag = "nixos/nix:$version-$dockerPlatform";
-    my $latestTag = "nixos/nix:latest-$dockerPlatform";
+    my $tag = $opt->docker_owner . ":$version-$dockerPlatform";
+    my $latestTag = $opt->docker_owner . ":latest-$dockerPlatform";
 
     print STDERR "tagging $version docker image for $dockerPlatform...\n";
     system("docker tag nix:$version $tag") == 0 or die;
@@ -219,68 +239,94 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) {
 }
 
 if ($haveDocker) {
+    my $dockerOwner = $opt->docker_owner;
     print STDERR "creating multi-platform docker manifest...\n";
-    system("docker manifest rm nixos/nix:$version");
-    system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die;
+    system("docker manifest rm $dockerOwner:$version");
+    system("docker manifest create $dockerOwner:$version $dockerManifest") == 0 or die;
     if ($isLatest) {
         print STDERR "creating latest multi-platform docker manifest...\n";
-        system("docker manifest rm nixos/nix:latest");
-        system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die;
+        system("docker manifest rm $dockerOwner:latest");
+        system("docker manifest create $dockerOwner:latest $dockerManifestLatest") == 0 or die;
     }
 
     print STDERR "pushing multi-platform docker manifest...\n";
-    system("docker manifest push nixos/nix:$version") == 0 or die;
+    system("docker manifest push $dockerOwner:$version") == 0 or die;
 
     if ($isLatest) {
         print STDERR "pushing latest multi-platform docker manifest...\n";
-        system("docker manifest push nixos/nix:latest") == 0 or die;
+        system("docker manifest push $dockerOwner:latest") == 0 or die;
     }
 }
+}
 
-# Upload nix-fallback-paths.nix.
-write_file("$tmpDir/fallback-paths.nix",
-    "{\n" .
-    "  x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
-    "  i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
-    "  aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
-    "  riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
-    "  x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
-    "  aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
-    "}\n");
 
 # Upload release files to S3.
-for my $fn (glob "$tmpDir/*") {
-    my $name = basename($fn);
-    next if $name eq "manual";
-    my $dstKey = "$releaseDir/" . $name;
-    unless (defined $releasesBucket->head_key($dstKey)) {
-        print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
+unless ($opt->skip_s3) {
+    downloadFile("binaryTarball.i686-linux", "1");
+    downloadFile("binaryTarball.x86_64-linux", "1");
+    downloadFile("binaryTarball.aarch64-linux", "1");
+    downloadFile("binaryTarball.x86_64-darwin", "1");
+    downloadFile("binaryTarball.aarch64-darwin", "1");
+    eval {
+        downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1");
+    };
+    warn "$@" if $@;
+    eval {
+        downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1");
+    };
+    warn "$@" if $@;
+    eval {
+        downloadFile("binaryTarballCross.x86_64-linux.riscv64-unknown-linux-gnu", "1");
+    };
+    warn "$@" if $@;
+    downloadFile("installerScript", "1");
 
-        my $configuration = ();
-        $configuration->{content_type} = "application/octet-stream";
+    # Upload nix-fallback-paths.nix.
+    write_file("$tmpDir/fallback-paths.nix",
+        "{\n" .
+        "  x86_64-linux = \"" . getStorePath("build.nix-everything.x86_64-linux") . "\";\n" .
+        "  i686-linux = \"" . getStorePath("build.nix-everything.i686-linux") . "\";\n" .
+        "  aarch64-linux = \"" . getStorePath("build.nix-everything.aarch64-linux") . "\";\n" .
+        "  riscv64-linux = \"" . getStorePath("buildCross.nix-everything.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" .
+        "  x86_64-darwin = \"" . getStorePath("build.nix-everything.x86_64-darwin") . "\";\n" .
+        "  aarch64-darwin = \"" . getStorePath("build.nix-everything.aarch64-darwin") . "\";\n" .
+        "}\n");
 
-        if ($fn =~ /.sha256|install|\.nix$/) {
-            $configuration->{content_type} = "text/plain";
+    for my $fn (glob "$tmpDir/*") {
+        my $name = basename($fn);
+        next if $name eq "manual";
+        my $dstKey = "$releaseDir/" . $name;
+        unless (defined $releasesBucket->head_key($dstKey)) {
+            print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n";
+
+            my $configuration = ();
+            $configuration->{content_type} = "application/octet-stream";
+
+            if ($fn =~ /.sha256|install|\.nix$/) {
+                $configuration->{content_type} = "text/plain";
+            }
+
+            $releasesBucket->add_key_filename($dstKey, $fn, $configuration)
+                or die $releasesBucket->err . ": " . $releasesBucket->errstr;
         }
-
-        $releasesBucket->add_key_filename($dstKey, $fn, $configuration)
-            or die $releasesBucket->err . ": " . $releasesBucket->errstr;
     }
+
+    # Update the "latest" symlink.
+    $channelsBucket->add_key(
+        "nix-latest/install", "",
+        { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
+        or die $channelsBucket->err . ": " . $channelsBucket->errstr
+        if $isLatest;
 }
 
-# Update the "latest" symlink.
-$channelsBucket->add_key(
-    "nix-latest/install", "",
-    { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" })
-    or die $channelsBucket->err . ": " . $channelsBucket->errstr
-    if $isLatest;
-
 # Tag the release in Git.
-chdir("/home/eelco/Dev/nix-pristine") or die;
-system("git remote update origin") == 0 or die;
-system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-system("git push --tags") == 0 or die;
-system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
+unless ($opt->skip_git) {
+    chdir($opt->project_root) or die "Cannot chdir to " . $opt->project_root . ": $!";
+    system("git remote update origin") == 0 or die;
+    system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
+    system("git push origin refs/tags/$version") == 0 or die;
+    system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest;
+}
 
 File::Path::remove_tree($narCache, {safe => 1});
 File::Path::remove_tree($tmpDir, {safe => 1});

packaging/dependencies.nix

@@ -10,27 +10,8 @@
   stdenv,
 }:
 
-let
-  prevStdenv = stdenv;
-in
-
 let
   inherit (pkgs) lib;
-
-  stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 then darwinStdenv else prevStdenv;
-
-  # Fix the following error with the default x86_64-darwin SDK:
-  #
-  #     error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer
-  #
-  # Despite the use of the 10.13 deployment target here, the aligned
-  # allocation function Clang uses with this setting actually works
-  # all the way back to 10.6.
-  # NOTE: this is not just a version constraint, but a request to make Darwin
-  #       provide this version level of support. Removing this minimum version
-  #       request will regress the above error.
-  darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; };
-
 in
 scope: {
   inherit stdenv;
@@ -73,7 +54,7 @@ scope: {
     nativeBuildInputs = prevAttrs.nativeBuildInputs ++ [ pkgs.buildPackages.bmake ];
     postInstall =
       lib.replaceStrings [ "lowdown.so.1" "lowdown.1.dylib" ] [ "lowdown.so.2" "lowdown.2.dylib" ]
-        prevAttrs.postInstall;
+        (prevAttrs.postInstall or "");
   });
 
   # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.

src/libcmd/installables.cc

@@ -178,10 +178,16 @@ MixFlakeOptions::MixFlakeOptions()
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
                 if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
+                    fetchers::Attrs extraAttrs;
+
+                    if (!input3->lockedRef.subdir.empty()) {
+                        extraAttrs["dir"] = input3->lockedRef.subdir;
+                    }
+
                     overrideRegistry(
                         fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
-                        {});
+                        extraAttrs);
                 }
             }
         }},

src/libcmd/repl.cc

@@ -669,7 +669,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                 ss << "No documentation found.\n\n";
             }
 
-            auto markdown = toView(ss);
+            auto markdown = ss.view();
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
 
         } else

src/libexpr/eval-profiler.cc

@@ -324,7 +324,7 @@ void SampleStack::saveProfile()
             std::visit([&](auto && info) { info.symbolize(state, os, posCache); }, pos);
         }
         os << " " << count;
-        writeLine(profileFd.get(), std::move(os).str());
+        writeLine(profileFd.get(), os.str());
         /* Clear ostringstream. */
         os.str("");
         os.clear();

src/libexpr/eval.cc

@@ -649,7 +649,7 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
             .name = name,
             .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
             .args = {},
-            .doc = makeImmutableString(toView(s)), // NOTE: memory leak when compiled without GC
+            .doc = makeImmutableString(s.view()), // NOTE: memory leak when compiled without GC
         };
     }
     if (isFunctor(v)) {
@@ -1845,7 +1845,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
     if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
         std::ostringstream out;
         cond->show(state.symbols, out);
-        auto exprStr = toView(out);
+        auto exprStr = out.view();
 
         if (auto eq = dynamic_cast<ExprOpEq *>(cond)) {
             try {

src/libexpr/meson.build

@@ -40,7 +40,10 @@ endforeach
 
 boost = dependency(
   'boost',
-  modules : [ 'container', 'context' ],
+  modules : [
+    'container',
+    'context',
+  ],
   include_type : 'system',
 )
 # boost is a public dependency, but not a pkg-config dependency unfortunately, so we
@@ -71,6 +74,12 @@ toml11 = dependency(
   method : 'cmake',
   include_type : 'system',
 )
+
+configdata_priv.set(
+  'HAVE_TOML11_4',
+  toml11.version().version_compare('>= 4.0.0').to_int(),
+)
+
 deps_other += toml11
 
 config_priv_h = configure_file(

src/libexpr/primops.cc

@@ -1939,8 +1939,13 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
         NixStringContext context;
         auto path = state.coerceToString(
             pos, *args[0], context, "while evaluating the first argument passed to 'builtins.dirOf'", false, false);
-        auto dir = dirOf(*path);
-        v.mkString(dir, context);
+        auto pos = path->rfind('/');
+        if (pos == path->npos)
+            v.mkStringMove(".", context);
+        else if (pos == 0)
+            v.mkStringMove("/", context);
+        else
+            v.mkString(path->substr(0, pos), context);
     }
 }
 
@@ -2350,7 +2355,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value ** args, Value
     std::ostringstream out;
     NixStringContext context;
     printValueAsXML(state, true, false, *args[0], out, context, pos);
-    v.mkString(toView(out), context);
+    v.mkString(out.view(), context);
 }
 
 static RegisterPrimOp primop_toXML({
@@ -2458,7 +2463,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value ** args, Valu
     std::ostringstream out;
     NixStringContext context;
     printValueAsJSON(state, true, *args[0], pos, out, context);
-    v.mkString(toView(out), context);
+    v.mkString(out.view(), context);
 }
 
 static RegisterPrimOp primop_toJSON({

src/libexpr/primops/fetchClosure.cc

@@ -185,7 +185,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
             {.msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
              .pos = state.positions[pos]});
 
-    auto parsedURL = parseURL(*fromStoreUrl);
+    auto parsedURL = parseURL(*fromStoreUrl, /*lenient=*/true);
 
     if (parsedURL.scheme != "http" && parsedURL.scheme != "https"
         && !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))

src/libexpr/primops/fromTOML.cc

@@ -1,78 +1,145 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
 
+#include "expr-config-private.hh"
+
 #include <sstream>
 
 #include <toml.hpp>
 
 namespace nix {
 
+#if HAVE_TOML11_4
+
+/**
+ * This is what toml11 < 4.0 did when choosing the subsecond precision.
+ * TOML 1.0.0 spec doesn't define how sub-millisecond ranges should be handled and calls it
+ * implementation defined behavior. For a lack of a better choice we stick with what older versions
+ * of toml11 did [1].
+ *
+ * [1]: https://github.com/ToruNiina/toml11/blob/dcfe39a783a94e8d52c885e5883a6fbb21529019/toml/datetime.hpp#L282
+ */
+static size_t normalizeSubsecondPrecision(toml::local_time lt)
+{
+    auto millis = lt.millisecond;
+    auto micros = lt.microsecond;
+    auto nanos = lt.nanosecond;
+    if (millis != 0 || micros != 0 || nanos != 0) {
+        if (micros != 0 || nanos != 0) {
+            if (nanos != 0)
+                return 9;
+            return 6;
+        }
+        return 3;
+    }
+    return 0;
+}
+
+/**
+ * Normalize date/time formats to serialize to the same strings as versions prior to toml11 4.0.
+ *
+ * Several things to consider:
+ *
+ * 1. Sub-millisecond range is represented the same way as in toml11 versions prior to 4.0. Precisioun is rounded
+ *    towards the next multiple of 3 or capped at 9 digits.
+ * 2. Seconds must be specified. This may become optional in (yet unreleased) TOML 1.1.0, but 1.0.0 defined local time
+ *    in terms of RFC3339 [1].
+ * 3. date-time separator (`t`, `T` or space ` `) is canonicalized to an upper T. This is compliant with RFC3339
+ *    [1] 5.6:
+ *    > Applications that generate this format SHOULD use upper case letters.
+ *
+ * [1]: https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+ */
+static void normalizeDatetimeFormat(toml::value & t)
+{
+    if (t.is_local_datetime()) {
+        auto & ldt = t.as_local_datetime();
+        t.as_local_datetime_fmt() = {
+            .delimiter = toml::datetime_delimiter_kind::upper_T,
+            // https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+            .has_seconds = true, // Mandated by TOML 1.0.0
+            .subsecond_precision = normalizeSubsecondPrecision(ldt.time),
+        };
+        return;
+    }
+
+    if (t.is_offset_datetime()) {
+        auto & odt = t.as_offset_datetime();
+        t.as_offset_datetime_fmt() = {
+            .delimiter = toml::datetime_delimiter_kind::upper_T,
+            // https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+            .has_seconds = true, // Mandated by TOML 1.0.0
+            .subsecond_precision = normalizeSubsecondPrecision(odt.time),
+        };
+        return;
+    }
+
+    if (t.is_local_time()) {
+        auto & lt = t.as_local_time();
+        t.as_local_time_fmt() = {
+            .has_seconds = true, // Mandated by TOML 1.0.0
+            .subsecond_precision = normalizeSubsecondPrecision(lt),
+        };
+        return;
+    }
+}
+
+#endif
+
 static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Value & val)
 {
     auto toml = state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.fromTOML");
 
     std::istringstream tomlStream(std::string{toml});
 
-    std::function<void(Value &, toml::value)> visit;
-
-    visit = [&](Value & v, toml::value t) {
+    auto visit = [&](auto & self, Value & v, toml::value t) -> void {
         switch (t.type()) {
         case toml::value_t::table: {
             auto table = toml::get<toml::table>(t);
-
-            size_t size = 0;
-            for (auto & i : table) {
-                (void) i;
-                size++;
-            }
-
-            auto attrs = state.buildBindings(size);
+            auto attrs = state.buildBindings(table.size());
 
             for (auto & elem : table) {
                 forceNoNullByte(elem.first);
-                visit(attrs.alloc(elem.first), elem.second);
+                self(self, attrs.alloc(elem.first), elem.second);
             }
 
             v.mkAttrs(attrs);
         } break;
-            ;
         case toml::value_t::array: {
             auto array = toml::get<std::vector<toml::value>>(t);
 
             auto list = state.buildList(array.size());
             for (const auto & [n, v] : enumerate(list))
-                visit(*(v = state.allocValue()), array[n]);
+                self(self, *(v = state.allocValue()), array[n]);
             v.mkList(list);
         } break;
-            ;
         case toml::value_t::boolean:
             v.mkBool(toml::get<bool>(t));
             break;
-            ;
         case toml::value_t::integer:
             v.mkInt(toml::get<int64_t>(t));
             break;
-            ;
         case toml::value_t::floating:
             v.mkFloat(toml::get<NixFloat>(t));
             break;
-            ;
         case toml::value_t::string: {
             auto s = toml::get<std::string_view>(t);
             forceNoNullByte(s);
             v.mkString(s);
         } break;
-            ;
         case toml::value_t::local_datetime:
         case toml::value_t::offset_datetime:
         case toml::value_t::local_date:
         case toml::value_t::local_time: {
             if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) {
+#if HAVE_TOML11_4
+                normalizeDatetimeFormat(t);
+#endif
                 auto attrs = state.buildBindings(2);
                 attrs.alloc("_type").mkString("timestamp");
                 std::ostringstream s;
                 s << t;
-                auto str = toView(s);
+                auto str = s.view();
                 forceNoNullByte(str);
                 attrs.alloc("value").mkString(str);
                 v.mkAttrs(attrs);
@@ -80,16 +147,24 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 throw std::runtime_error("Dates and times are not supported");
             }
         } break;
-            ;
         case toml::value_t::empty:
             v.mkNull();
             break;
-            ;
         }
     };
 
     try {
-        visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */));
+        visit(
+            visit,
+            val,
+            toml::parse(
+                tomlStream,
+                "fromTOML" /* the "filename" */
+#if HAVE_TOML11_4
+                ,
+                toml::spec::v(1, 0, 0) // Be explicit that we are parsing TOML 1.0.0 without extensions
+#endif
+                ));
     } catch (std::exception & e) { // TODO: toml::syntax_error
         state.error<EvalError>("while parsing TOML: %s", e.what()).atPos(pos).debugThrow();
     }

src/libexpr/print.cc

@@ -460,7 +460,7 @@ private:
 
                 std::ostringstream s;
                 s << state.positions[v.lambda().fun->pos];
-                output << " @ " << filterANSIEscapes(toView(s));
+                output << " @ " << filterANSIEscapes(s.view());
             }
         } else if (v.isPrimOp()) {
             if (v.primOp())

src/libfetchers-tests/git-utils.cc

@@ -175,6 +175,12 @@ TEST_F(GitUtilsTest, peel_reference)
 
 TEST(GitUtils, isLegalRefName)
 {
+    ASSERT_TRUE(isLegalRefName("A/b"));
+    ASSERT_TRUE(isLegalRefName("AaA/b"));
+    ASSERT_TRUE(isLegalRefName("FOO/BAR/BAZ"));
+    ASSERT_TRUE(isLegalRefName("HEAD"));
+    ASSERT_TRUE(isLegalRefName("refs/tags/1.2.3"));
+    ASSERT_TRUE(isLegalRefName("refs/heads/master"));
     ASSERT_TRUE(isLegalRefName("foox"));
     ASSERT_TRUE(isLegalRefName("1337"));
     ASSERT_TRUE(isLegalRefName("foo.baz"));

src/libfetchers/git-utils.cc

@@ -12,6 +12,7 @@
 
 #include <git2/attr.h>
 #include <git2/blob.h>
+#include <git2/branch.h>
 #include <git2/commit.h>
 #include <git2/config.h>
 #include <git2/describe.h>
@@ -28,6 +29,7 @@
 #include <git2/submodule.h>
 #include <git2/sys/odb_backend.h>
 #include <git2/sys/mempack.h>
+#include <git2/tag.h>
 #include <git2/tree.h>
 
 #include <iostream>
@@ -1311,63 +1313,33 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path &
     return workdirInfo;
 }
 
-/**
- * Checks that the git reference is valid and normalizes slash '/' sequences.
- *
- * Accepts shorthand references (one-level refnames are allowed).
- */
-bool isValidRefNameAllowNormalizations(const std::string & refName)
-{
-    /* Unfortunately libgit2 doesn't expose the limit in headers, but its internal
-       limit is also 1024. */
-    std::array<char, 1024> normalizedRefBuffer;
-
-    /* It would be nice to have a better API like git_reference_name_is_valid, but
-     * with GIT_REFERENCE_FORMAT_REFSPEC_SHORTHAND flag. libgit2 uses it internally
-     * but doesn't expose it in public headers [1].
-     * [1]:
-     * https://github.com/libgit2/libgit2/blob/9d5f1bacc23594c2ba324c8f0d41b88bf0e9ef04/src/libgit2/refs.c#L1362-L1365
-     */
-
-    auto res = git_reference_normalize_name(
-        normalizedRefBuffer.data(),
-        normalizedRefBuffer.size(),
-        refName.c_str(),
-        GIT_REFERENCE_FORMAT_ALLOW_ONELEVEL | GIT_REFERENCE_FORMAT_REFSPEC_SHORTHAND);
-
-    return res == 0;
-}
-
 bool isLegalRefName(const std::string & refName)
 {
     initLibGit2();
 
-    /* Since `git_reference_normalize_name` is the best API libgit2 has for verifying
-     * reference names with shorthands (see comment in normalizeRefName), we need to
-     * ensure that exceptions to the validity checks imposed by normalization [1] are checked
-     * explicitly.
-     * [1]: https://git-scm.com/docs/git-check-ref-format#Documentation/git-check-ref-format.txt---normalize
-     */
-
     /* Check for cases that don't get rejected by libgit2.
      * FIXME: libgit2 should reject this. */
     if (refName == "@")
         return false;
 
-    /* Leading slashes and consecutive slashes are stripped during normalizatiton. */
-    if (refName.starts_with('/') || refName.find("//") != refName.npos)
-        return false;
-
-    /* Refer to libgit2. */
-    if (!isValidRefNameAllowNormalizations(refName))
-        return false;
-
     /* libgit2 doesn't barf on DEL symbol.
      * FIXME: libgit2 should reject this. */
     if (refName.find('\177') != refName.npos)
         return false;
 
-    return true;
+    for (auto * func : {
+             git_reference_name_is_valid,
+             git_branch_name_is_valid,
+             git_tag_name_is_valid,
+         }) {
+        int valid = 0;
+        if (func(&valid, refName.c_str()))
+            throw Error("checking git reference '%s': %s", refName, git_error_last()->message);
+        if (valid)
+            return true;
+    }
+
+    return false;
 }
 
 } // namespace nix

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -158,9 +158,12 @@ struct Setter
 };
 
 /**
- * Checks that the git reference is valid and normalized.
+ * Checks that the string can be a valid git reference, branch or tag name.
+ * Accepts shorthand references (one-level refnames are allowed), pseudorefs
+ * like `HEAD`.
  *
- * Accepts shorthand references (one-level refnames are allowed).
+ * @note This is a coarse test to make sure that the refname is at least something
+ * that Git can make sense of.
  */
 bool isLegalRefName(const std::string & refName);
 

src/libfetchers/include/nix/fetchers/input-cache.hh

@@ -11,6 +11,7 @@ struct InputCache
         ref<SourceAccessor> accessor;
         Input resolvedInput;
         Input lockedInput;
+        Attrs extraAttrs;
     };
 
     CachedResult getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
@@ -19,6 +20,7 @@ struct InputCache
     {
         Input lockedInput;
         ref<SourceAccessor> accessor;
+        Attrs extraAttrs;
     };
 
     virtual std::optional<CachedInput> lookup(const Input & originalInput) const = 0;

src/libfetchers/input-cache.cc

@@ -22,7 +22,8 @@ InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegist
                 fetched = lookup(resolvedInput);
                 if (!fetched) {
                     auto [accessor, lockedInput] = resolvedInput.getAccessor(store);
-                    fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
+                    fetched.emplace(
+                        CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs});
                 }
                 upsert(resolvedInput, *fetched);
             } else {
@@ -36,7 +37,7 @@ InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegist
 
     debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedInput.to_string());
 
-    return {fetched->accessor, resolvedInput, fetched->lockedInput};
+    return {fetched->accessor, resolvedInput, fetched->lockedInput, fetched->extraAttrs};
 }
 
 struct InputCacheImpl : InputCache

src/libfetchers/tarball.cc

@@ -115,7 +115,7 @@ static DownloadTarballResult downloadTarball_(
     // it is not in fact a tarball.
     if (url.rfind("file://", 0) == 0) {
         // Remove "file://" prefix to get the local file path
-        std::string localPath = url.substr(7);
+        std::string localPath = percentDecode(url.substr(7));
         if (!std::filesystem::exists(localPath)) {
             throw Error("tarball '%s' does not exist.", localPath);
         }

src/libflake-tests/url-name.cc

@@ -13,8 +13,9 @@ TEST(getNameFromURL, getNameFromURL)
     ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.Hello")), "Hello");
     ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "mylaptop");
     ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop");
-    ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex");
-    ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj");
+    ASSERT_EQ(
+        getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man", /*lenient=*/true)), "complex");
+    ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*", /*lenient=*/true)), "myproj");
     ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#defaultPackage.x86_64-linux")), "myproj");
 
     ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello");
@@ -80,6 +81,6 @@ TEST(getNameFromURL, getNameFromURL)
     ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt);
     ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt);
     ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt);
-    ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*")), std::nullopt);
+    ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*", /*lenient=*/true)), std::nullopt);
 }
 } // namespace nix

src/libflake/flake.cc

@@ -340,8 +340,9 @@ static Flake getFlake(
     // Fetch a lazy tree first.
     auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
 
-    auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir);
-    auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir);
+    auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
+    auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
+    auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), subdir);
 
     // Parse/eval flake.nix to get at the input.self attributes.
     auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {cachedInput.accessor}, lockRootAttrPath);

src/libflake/flakeref.cc

@@ -80,9 +80,10 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
 
     std::smatch match;
     auto succeeds = std::regex_match(url, match, pathFlakeRegex);
-    assert(succeeds);
+    if (!succeeds)
+        throw Error("invalid flakeref '%s'", url);
     auto path = match[1].str();
-    auto query = decodeQuery(match[3]);
+    auto query = decodeQuery(match[3].str(), /*lenient=*/true);
     auto fragment = percentDecode(match[5].str());
 
     if (baseDir) {
@@ -210,7 +211,7 @@ std::optional<std::pair<FlakeRef, std::string>> parseURLFlakeRef(
     bool isFlake)
 {
     try {
-        auto parsed = parseURL(url);
+        auto parsed = parseURL(url, /*lenient=*/true);
         if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file") && !isAbsolute(parsed.path))
             parsed.path = absPath(parsed.path, *baseDir);
         return fromParsedURL(fetchSettings, std::move(parsed), isFlake);
@@ -289,7 +290,7 @@ FlakeRef FlakeRef::canonicalize() const
        filtering the `dir` query parameter from the URL. */
     if (auto url = fetchers::maybeGetStrAttr(flakeRef.input.attrs, "url")) {
         try {
-            auto parsed = parseURL(*url);
+            auto parsed = parseURL(*url, /*lenient=*/true);
             if (auto dir2 = get(parsed.query, "dir")) {
                 if (flakeRef.subdir != "" && flakeRef.subdir == *dir2)
                     parsed.query.erase("dir");

src/libmain/progress-bar.cc

@@ -183,7 +183,7 @@ public:
         std::ostringstream oss;
         showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
-        log(*state, ei.level, toView(oss));
+        log(*state, ei.level, oss.view());
     }
 
     void log(State & state, Verbosity lvl, std::string_view s)

src/libmain/shared.cc

@@ -320,34 +320,29 @@ int handleExceptions(const std::string & programName, std::function<void()> fun)
     std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
     try {
         try {
-            try {
-                fun();
-            } catch (...) {
-                /* Subtle: we have to make sure that any `interrupted'
-                   condition is discharged before we reach printMsg()
-                   below, since otherwise it will throw an (uncaught)
-                   exception. */
-                setInterruptThrown();
-                throw;
-            }
-        } catch (Exit & e) {
-            return e.status;
-        } catch (UsageError & e) {
-            logError(e.info());
-            printError("Try '%1% --help' for more information.", programName);
-            return 1;
-        } catch (BaseError & e) {
-            logError(e.info());
-            return e.info().status;
-        } catch (std::bad_alloc & e) {
-            printError(error + "out of memory");
-            return 1;
-        } catch (std::exception & e) {
-            printError(error + e.what());
-            return 1;
+            fun();
+        } catch (...) {
+            /* Subtle: we have to make sure that any `interrupted'
+               condition is discharged before we reach printMsg()
+               below, since otherwise it will throw an (uncaught)
+               exception. */
+            setInterruptThrown();
+            throw;
         }
-    } catch (...) {
-        /* In case logger also throws just give up. */
+    } catch (Exit & e) {
+        return e.status;
+    } catch (UsageError & e) {
+        logError(e.info());
+        printError("Try '%1% --help' for more information.", programName);
+        return 1;
+    } catch (BaseError & e) {
+        logError(e.info());
+        return e.info().status;
+    } catch (std::bad_alloc & e) {
+        printError(error + "out of memory");
+        return 1;
+    } catch (std::exception & e) {
+        printError(error + e.what());
         return 1;
     }
 

src/libstore-tests/data/store-reference/daemon_shorthand.txt

@@ -0,0 +1 @@
+daemon

src/libstore-tests/data/store-reference/local_shorthand_3.txt

@@ -0,0 +1 @@
+local

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_1.txt

@@ -0,0 +1 @@
+ssh://::1

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_2.txt

@@ -0,0 +1 @@
+ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_3.txt

@@ -0,0 +1 @@
+ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e?a=b&c=d

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_4.txt

@@ -0,0 +1 @@
+ssh://userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%eth0]?a=b&c=d

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_5.txt

@@ -0,0 +1 @@
+ssh://userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%25eth0]?a=b&c=d

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_6.txt

@@ -0,0 +1 @@
+ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%25?a=b&c=d

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_7.txt

@@ -0,0 +1 @@
+ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%eth0?a=b&c=d

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_8.txt

@@ -0,0 +1 @@
+ssh://fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%eth0?a=b&c=d

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_9.txt

@@ -0,0 +1 @@
+ssh://fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%eth0

src/libstore-tests/local-store.cc

@@ -33,4 +33,10 @@ TEST(LocalStore, constructConfig_rootPath)
     EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"});
 }
 
+TEST(LocalStore, constructConfig_to_string)
+{
+    LocalStoreConfig config{"local", "", {}};
+    EXPECT_EQ(config.getReference().render(), "local");
+}
+
 } // namespace nix

src/libstore-tests/store-reference.cc

@@ -107,6 +107,13 @@ URI_TEST_READ(local_shorthand_1, localExample_1)
 
 URI_TEST_READ(local_shorthand_2, localExample_2)
 
+URI_TEST(
+    local_shorthand_3,
+    (StoreReference{
+        .variant = StoreReference::Local{},
+        .params = {},
+    }))
+
 static StoreReference unixExample{
     .variant =
         StoreReference::Specified{
@@ -134,4 +141,106 @@ URI_TEST(
         .params = {},
     }))
 
+URI_TEST(
+    daemon_shorthand,
+    (StoreReference{
+        .variant = StoreReference::Daemon{},
+        .params = {},
+    }))
+
+static StoreReference sshLoopbackIPv6{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "[::1]",
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_1, sshLoopbackIPv6)
+
+static StoreReference sshIPv6AuthorityWithUserinfo{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e]",
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_2, sshIPv6AuthorityWithUserinfo)
+
+static StoreReference sshIPv6AuthorityWithUserinfoAndParams{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e]",
+        },
+    .params =
+        {
+            {"a", "b"},
+            {"c", "d"},
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_3, sshIPv6AuthorityWithUserinfoAndParams)
+
+static const StoreReference sshIPv6AuthorityWithUserinfoAndParamsAndZoneId{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%25eth0]",
+        },
+    .params =
+        {
+            {"a", "b"},
+            {"c", "d"},
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_4, sshIPv6AuthorityWithUserinfoAndParamsAndZoneId)
+URI_TEST_READ(ssh_unbracketed_ipv6_5, sshIPv6AuthorityWithUserinfoAndParamsAndZoneId)
+
+static const StoreReference sshIPv6AuthorityWithUserinfoAndParamsAndZoneIdTricky{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%2525]",
+        },
+    .params =
+        {
+            {"a", "b"},
+            {"c", "d"},
+        },
+};
+
+// Non-standard syntax where the IPv6 literal appears without brackets. In
+// this case don't considering %25 to be a pct-encoded % and just take it as a
+// literal value. 25 is a perfectly legal ZoneId value in theory.
+URI_TEST_READ(ssh_unbracketed_ipv6_6, sshIPv6AuthorityWithUserinfoAndParamsAndZoneIdTricky)
+URI_TEST_READ(ssh_unbracketed_ipv6_7, sshIPv6AuthorityWithUserinfoAndParamsAndZoneId)
+
+static const StoreReference sshIPv6AuthorityWithParamsAndZoneId{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%25eth0]",
+        },
+    .params =
+        {
+            {"a", "b"},
+            {"c", "d"},
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_8, sshIPv6AuthorityWithParamsAndZoneId)
+
+static const StoreReference sshIPv6AuthorityWithZoneId{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e%25eth0]",
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_9, sshIPv6AuthorityWithZoneId)
+
 } // namespace nix

src/libstore-tests/uds-remote-store.cc

@@ -16,4 +16,10 @@ TEST(UDSRemoteStore, constructConfigWrongScheme)
     EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError);
 }
 
+TEST(UDSRemoteStore, constructConfig_to_string)
+{
+    UDSRemoteStoreConfig config{"unix", "", {}};
+    EXPECT_EQ(config.getReference().render(), "daemon");
+}
+
 } // namespace nix

src/libstore/build/derivation-goal.cc

@@ -182,8 +182,16 @@ Goal::Co DerivationGoal::haveDerivation()
             }
         }
 
-        if (buildResult.success())
-            assert(buildResult.builtOutputs.count(wantedOutput) > 0);
+        if (buildResult.success()) {
+            /* If the wanted output is not in builtOutputs (e.g., because it
+               was already valid and therefore not re-registered), we need to
+               add it ourselves to ensure we return the correct information. */
+            if (buildResult.builtOutputs.count(wantedOutput) == 0) {
+                debug(
+                    "BUG! wanted output '%s' not in builtOutputs, working around by adding it manually", wantedOutput);
+                buildResult.builtOutputs = {{wantedOutput, assertPathValidity()}};
+            }
+        }
     }
 
     co_return amDone(g->exitCode, g->ex);
@@ -272,9 +280,10 @@ Goal::Co DerivationGoal::repairClosure()
                 bmRepair));
     }
 
+    bool haveWaitees = !waitees.empty();
     co_await await(std::move(waitees));
 
-    if (!waitees.empty()) {
+    if (haveWaitees) {
         trace("closure repaired");
         if (nrFailed > 0)
             throw Error(

src/libstore/daemon.cc

@@ -102,7 +102,7 @@ struct TunnelLogger : public Logger
         showErrorInfo(oss, ei, false);
 
         StringSink buf;
-        buf << STDERR_NEXT << toView(oss);
+        buf << STDERR_NEXT << oss.view();
         enqueueMsg(buf.s);
     }
 

src/libstore/derivation-options.cc

@@ -99,6 +99,17 @@ DerivationOptions DerivationOptions::fromStructuredAttrs(
     return fromStructuredAttrs(env, parsed ? &*parsed : nullptr);
 }
 
+static void flatten(const nlohmann::json & value, StringSet & res)
+{
+    if (value.is_array())
+        for (auto & v : value)
+            flatten(v, res);
+    else if (value.is_string())
+        res.insert(value);
+    else
+        throw Error("'exportReferencesGraph' value is not an array or a string");
+}
+
 DerivationOptions
 DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
 {
@@ -219,12 +230,9 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
                     if (!e || !e->is_object())
                         return ret;
                     for (auto & [key, value] : getObject(*e)) {
-                        if (value.is_array())
-                            ret.insert_or_assign(key, value);
-                        else if (value.is_string())
-                            ret.insert_or_assign(key, StringSet{value});
-                        else
-                            throw Error("'exportReferencesGraph' value is not an array or a string");
+                        StringSet ss;
+                        flatten(value, ss);
+                        ret.insert_or_assign(key, std::move(ss));
                     }
                 } else {
                     auto s = getOr(env, "exportReferencesGraph", "");

src/libstore/gc.cc

@@ -21,6 +21,7 @@
 #include <random>
 
 #include <climits>
+#include <thread>
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/stat.h>

src/libstore/include/nix/store/filetransfer.hh

@@ -30,9 +30,17 @@ struct FileTransferSettings : Config
         )",
         {"binary-caches-parallel-connections"}};
 
+    /* Do not set this too low. On glibc, getaddrinfo() contains fallback code
+       paths that deal with ill-behaved DNS servers. Setting this too low
+       prevents some fallbacks from occurring.
+
+       See description of options timeout, single-request, single-request-reopen
+       in resolv.conf(5). Also see https://github.com/NixOS/nix/pull/13985 for
+       details on the interaction between getaddrinfo(3) behavior and libcurl
+       CURLOPT_CONNECTTIMEOUT. */
     Setting<unsigned long> connectTimeout{
         this,
-        5,
+        15,
         "connect-timeout",
         R"(
           The timeout (in seconds) for establishing connections in the

src/libstore/include/nix/store/globals.hh

@@ -794,6 +794,8 @@ public:
         "build-dir",
         R"(
             Override the `build-dir` store setting for all stores that have this setting.
+
+            See also the per-store [`build-dir`](@docroot@/store/types/local-store.md#store-local-store-build-dir) setting.
         )"};
 
     Setting<PathSet> allowedImpureHostPrefixes{

src/libstore/include/nix/store/local-store.hh

@@ -61,6 +61,8 @@ private:
             > `build-dir` must not be set to a world-writable directory.
             > Placing temporary build directories in a world-writable place allows other users to access or modify build data that is currently in use.
             > This alone is merely an impurity, but combined with another factor this has allowed malicious derivations to escape the build sandbox.
+
+            See also the global [`build-dir`](@docroot@/command-ref/conf-file.md#conf-build-dir) setting.
         )"};
 public:
     Path getBuildDir() const;

src/libstore/include/nix/store/restricted-store.hh

@@ -52,7 +52,21 @@ struct RestrictionContext
      * Add 'path' to the set of paths that may be referenced by the
      * outputs, and make it appear in the sandbox.
      */
-    virtual void addDependency(const StorePath & path) = 0;
+    void addDependency(const StorePath & path)
+    {
+        if (isAllowed(path))
+            return;
+        addDependencyImpl(path);
+    }
+
+protected:
+
+    /**
+     * This is the underlying implementation to be defined. The caller
+     * will ensure that this is only called on newly added dependencies,
+     * and that idempotent calls are a no-op.
+     */
+    virtual void addDependencyImpl(const StorePath & path) = 0;
 };
 
 /**

src/libstore/include/nix/store/store-reference.hh

@@ -64,7 +64,29 @@ struct StoreReference
         auto operator<=>(const Specified & rhs) const = default;
     };
 
-    typedef std::variant<Auto, Specified> Variant;
+    /**
+     * Special case for `daemon` to avoid normalization.
+     */
+    struct Daemon : Specified
+    {
+        Daemon()
+            : Specified({.scheme = "unix"})
+        {
+        }
+    };
+
+    /**
+     * Special case for `local` to avoid normalization.
+     */
+    struct Local : Specified
+    {
+        Local()
+            : Specified({.scheme = "local"})
+        {
+        }
+    };
+
+    typedef std::variant<Auto, Specified, Daemon, Local> Variant;
 
     Variant variant;
 

src/libstore/local-store.cc

@@ -456,12 +456,17 @@ LocalStore::~LocalStore()
 
 StoreReference LocalStoreConfig::getReference() const
 {
+    auto params = getQueryParams();
+    /* Back-compatibility kludge. Tools like nix-output-monitor expect 'local'
+       and can't parse 'local://'. */
+    if (params.empty())
+        return {.variant = StoreReference::Local{}};
     return {
         .variant =
             StoreReference::Specified{
                 .scheme = *uriSchemes().begin(),
             },
-        .params = getQueryParams(),
+        .params = std::move(params),
     };
 }
 
@@ -1388,7 +1393,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : DirectoryIterator{linksDir}) {
             checkInterrupt();
             auto name = link.path().filename();
-            printMsg(lvlTalkative, "checking contents of '%s'", name);
+            printMsg(lvlTalkative, "checking contents of %s", name);
             PosixSourceAccessor accessor;
             std::string hash = hashPath(
                                    PosixSourceAccessor::createAtRoot(link.path()),
@@ -1396,10 +1401,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                                    HashAlgorithm::SHA256)
                                    .first.to_string(HashFormat::Nix32, false);
             if (hash != name.string()) {
-                printError("link '%s' was modified! expected hash '%s', got '%s'", link.path(), name, hash);
+                printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash);
                 if (repair) {
                     std::filesystem::remove(link.path());
-                    printInfo("removed link '%s'", link.path());
+                    printInfo("removed link %s", link.path());
                 } else {
                     errors = true;
                 }

src/libstore/meson.build

@@ -101,7 +101,12 @@ subdir('nix-meson-build-support/libatomic')
 
 boost = dependency(
   'boost',
-  modules : [ 'container', 'regex' ],
+  modules : [
+    'container',
+    # Shouldn't list, because can header-only, and Meson currently looks for libs
+    #'regex',
+    'url',
+  ],
   include_type : 'system',
 )
 # boost is a public dependency, but not a pkg-config dependency unfortunately, so we

src/libstore/optimise-store.cc

@@ -202,7 +202,7 @@ void LocalStore::optimisePath_(
                    full.  When that happens, it's fine to ignore it: we
                    just effectively disable deduplication of this
                    file.  */
-                printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
+                printInfo("cannot link %s to '%s': %s", linkPath, path, strerror(errno));
                 return;
             }
 
@@ -216,11 +216,11 @@ void LocalStore::optimisePath_(
     auto stLink = lstat(linkPath.string());
 
     if (st.st_ino == stLink.st_ino) {
-        debug("'%1%' is already linked to '%2%'", path, linkPath);
+        debug("'%1%' is already linked to %2%", path, linkPath);
         return;
     }
 
-    printMsg(lvlTalkative, "linking '%1%' to '%2%'", path, linkPath);
+    printMsg(lvlTalkative, "linking '%1%' to %2%", path, linkPath);
 
     /* Make the containing directory writable, but only if it's not
        the store itself (we don't want or need to mess with its
@@ -234,7 +234,7 @@ void LocalStore::optimisePath_(
        its timestamp back to 0. */
     MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : "");
 
-    std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", config->realStoreDir, getpid(), rand());
+    std::filesystem::path tempLink = makeTempPath(config->realStoreDir.get(), ".tmp-link");
 
     try {
         std::filesystem::create_hard_link(linkPath, tempLink);
@@ -245,7 +245,7 @@ void LocalStore::optimisePath_(
                systems).  This is likely to happen with empty files.
                Just shrug and ignore. */
             if (st.st_size)
-                printInfo("'%1%' has maximum number of links", linkPath);
+                printInfo("%1% has maximum number of links", linkPath);
             return;
         }
         throw;
@@ -255,14 +255,18 @@ void LocalStore::optimisePath_(
     try {
         std::filesystem::rename(tempLink, path);
     } catch (std::filesystem::filesystem_error & e) {
-        std::filesystem::remove(tempLink);
-        printError("unable to unlink '%1%'", tempLink);
+        {
+            std::error_code ec;
+            remove(tempLink, ec); /* Clean up after ourselves. */
+            if (ec)
+                printError("unable to unlink %1%: %2%", tempLink, ec.message());
+        }
         if (e.code() == std::errc::too_many_links) {
             /* Some filesystems generate too many links on the rename,
                rather than on the original link.  (Probably it
                temporarily increases the st_nlink field before
                decreasing it again.) */
-            debug("'%s' has reached maximum number of links", linkPath);
+            debug("%s has reached maximum number of links", linkPath);
             return;
         }
         throw;

src/libstore/package.nix

@@ -64,8 +64,6 @@ mkMesonLibrary (finalAttrs: {
     sqlite
   ]
   ++ lib.optional stdenv.hostPlatform.isLinux libseccomp
-  # There have been issues building these dependencies
-  ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox
   ++ lib.optional withAWS aws-sdk-cpp;
 
   propagatedBuildInputs = [

src/libstore/sqlite.cc

@@ -4,6 +4,10 @@
 #include "nix/util/url.hh"
 #include "nix/util/signals.hh"
 
+#ifdef __linux__
+#  include <sys/vfs.h>
+#endif
+
 #include <sqlite3.h>
 
 #include <atomic>
@@ -60,6 +64,28 @@ static void traceSQL(void * x, const char * sql)
 
 SQLite::SQLite(const std::filesystem::path & path, SQLiteOpenMode mode)
 {
+    // Work around a ZFS issue where SQLite's truncate() call on
+    // db.sqlite-shm can randomly take up to a few seconds. See
+    // https://github.com/openzfs/zfs/issues/14290#issuecomment-3074672917.
+    // Remove this workaround when a fix is widely installed, perhaps 2027? Candidate:
+    // https://github.com/search?q=repo%3Aopenzfs%2Fzfs+%22Linux%3A+zfs_putpage%3A+complete+async+page+writeback+immediately%22&type=commits
+#ifdef __linux__
+    try {
+        auto shmFile = path;
+        shmFile += "-shm";
+        AutoCloseFD fd = open(shmFile.string().c_str(), O_RDWR | O_CLOEXEC);
+        if (fd) {
+            struct statfs fs;
+            if (fstatfs(fd.get(), &fs))
+                throw SysError("statfs() on '%s'", shmFile);
+            if (fs.f_type == /* ZFS_SUPER_MAGIC */ 801189825 && fdatasync(fd.get()) != 0)
+                throw SysError("fsync() on '%s'", shmFile);
+        }
+    } catch (...) {
+        throw;
+    }
+#endif
+
     // useSQLiteWAL also indicates what virtual file system we need.  Using
     // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem
     // for Linux (WSL) where useSQLiteWAL should be false by default.

src/libstore/ssh.cc

@@ -78,7 +78,7 @@ SSHMaster::SSHMaster(
         oss << authority.host;
         return std::move(oss).str();
     }())
-    , fakeSSH(authority.host == "localhost")
+    , fakeSSH(authority.to_string() == "localhost")
     , keyFile(keyFile)
     , sshPublicHostKey(parsePublicHostKey(authority.host, sshPublicHostKey))
     , useMaster(useMaster && !fakeSSH)

src/libstore/store-api.cc

@@ -817,7 +817,13 @@ makeCopyPathMessage(const StoreConfig & srcCfg, const StoreConfig & dstCfg, std:
 
     auto isShorthand = [](const StoreReference & ref) {
         /* At this point StoreReference **must** be resolved. */
-        const auto & specified = std::get<StoreReference::Specified>(ref.variant);
+        const auto & specified = std::visit(
+            overloaded{
+                [](const StoreReference::Auto &) -> const StoreReference::Specified & { unreachable(); },
+                [](const StoreReference::Specified & specified) -> const StoreReference::Specified & {
+                    return specified;
+                }},
+            ref.variant);
         const auto & scheme = specified.scheme;
         return (scheme == "local" || scheme == "unix") && specified.authority.empty();
     };

src/libstore/store-reference.cc

@@ -1,11 +1,12 @@
-#include <regex>
-
 #include "nix/util/error.hh"
+#include "nix/util/split.hh"
 #include "nix/util/url.hh"
 #include "nix/store/store-reference.hh"
 #include "nix/util/file-system.hh"
 #include "nix/util/util.hh"
 
+#include <boost/url/ipv6_address.hpp>
+
 namespace nix {
 
 static bool isNonUriPath(const std::string & spec)
@@ -25,6 +26,8 @@ std::string StoreReference::render(bool withParams) const
     std::visit(
         overloaded{
             [&](const StoreReference::Auto &) { res = "auto"; },
+            [&](const StoreReference::Daemon &) { res = "daemon"; },
+            [&](const StoreReference::Local &) { res = "local"; },
             [&](const StoreReference::Specified & g) {
                 res = g.scheme;
                 res += "://";
@@ -41,11 +44,34 @@ std::string StoreReference::render(bool withParams) const
     return res;
 }
 
+namespace {
+
+struct SchemeAndAuthorityWithPath
+{
+    std::string_view scheme;
+    std::string_view authority;
+};
+
+} // namespace
+
+/**
+ * Return the 'scheme' and remove the '://' or ':' separator.
+ */
+static std::optional<SchemeAndAuthorityWithPath> splitSchemePrefixTo(std::string_view string)
+{
+    auto scheme = splitPrefixTo(string, ':');
+    if (!scheme)
+        return std::nullopt;
+
+    splitPrefix(string, "//");
+    return SchemeAndAuthorityWithPath{.scheme = *scheme, .authority = string};
+}
+
 StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
 {
     auto params = extraParams;
     try {
-        auto parsedUri = parseURL(uri);
+        auto parsedUri = parseURL(uri, /*lenient=*/true);
         params.insert(parsedUri.query.begin(), parsedUri.query.end());
 
         auto baseURI = parsedUri.authority.value_or(ParsedURL::Authority{}).to_string() + parsedUri.path;
@@ -68,21 +94,17 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen
                 .params = std::move(params),
             };
         } else if (baseURI == "daemon") {
+            if (params.empty())
+                return {.variant = Daemon{}};
             return {
-                .variant =
-                    Specified{
-                        .scheme = "unix",
-                        .authority = "",
-                    },
+                .variant = Specified{.scheme = "unix", .authority = ""},
                 .params = std::move(params),
             };
         } else if (baseURI == "local") {
+            if (params.empty())
+                return {.variant = Local{}};
             return {
-                .variant =
-                    Specified{
-                        .scheme = "local",
-                        .authority = "",
-                    },
+                .variant = Specified{.scheme = "local", .authority = ""},
                 .params = std::move(params),
             };
         } else if (isNonUriPath(baseURI)) {
@@ -94,6 +116,56 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen
                     },
                 .params = std::move(params),
             };
+        } else if (auto schemeAndAuthority = splitSchemePrefixTo(baseURI)) {
+            /* Back-compatibility shim to accept unbracketed IPv6 addresses after the scheme.
+             * Old versions of nix allowed that. Note that this is ambiguous and does not allow
+             * specifying the port number. For that the address must be bracketed, otherwise it's
+             * greedily assumed to be the part of the host address. */
+            auto authorityString = schemeAndAuthority->authority;
+            auto userinfo = splitPrefixTo(authorityString, '@');
+            /* Back-compat shim for ZoneId specifiers. Technically this isn't
+             * standard, but the expectation is this works with the old syntax
+             * for ZoneID specifiers. For the full story behind the fiasco that
+             * is ZoneID in URLs look at [^].
+             * [^]: https://datatracker.ietf.org/doc/html/draft-schinazi-httpbis-link-local-uri-bcp-03
+             */
+
+            /* Fish out the internals from inside square brackets. It might be that the pct-sign is unencoded and that's
+             * why we failed to parse it previously. */
+            if (authorityString.starts_with('[') && authorityString.ends_with(']')) {
+                authorityString.remove_prefix(1);
+                authorityString.remove_suffix(1);
+            }
+
+            auto maybeBeforePct = splitPrefixTo(authorityString, '%');
+            bool hasZoneId = maybeBeforePct.has_value();
+            auto maybeZoneId = hasZoneId ? std::optional{authorityString} : std::nullopt;
+
+            std::string_view maybeIpv6S = maybeBeforePct.value_or(authorityString);
+            auto maybeIpv6 = boost::urls::parse_ipv6_address(maybeIpv6S);
+
+            if (maybeIpv6) {
+                std::string fixedAuthority;
+                if (userinfo) {
+                    fixedAuthority += *userinfo;
+                    fixedAuthority += '@';
+                }
+                fixedAuthority += '[';
+                fixedAuthority += maybeIpv6S;
+                if (maybeZoneId) {
+                    fixedAuthority += "%25"; // pct-encoded percent character
+                    fixedAuthority += *maybeZoneId;
+                }
+                fixedAuthority += ']';
+                return {
+                    .variant =
+                        Specified{
+                            .scheme = std::string(schemeAndAuthority->scheme),
+                            .authority = fixedAuthority,
+                        },
+                    .params = std::move(params),
+                };
+            }
         }
     }
 
@@ -107,7 +179,7 @@ std::pair<std::string, StoreReference::Params> splitUriAndParams(const std::stri
     StoreReference::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
-        params = decodeQuery(uri.substr(q + 1));
+        params = decodeQuery(uri.substr(q + 1), /*lenient=*/true);
         uri = uri_.substr(0, q);
     }
     return {uri, params};

src/libstore/uds-remote-store.cc

@@ -57,15 +57,16 @@ UDSRemoteStore::UDSRemoteStore(ref<const Config> config)
 
 StoreReference UDSRemoteStoreConfig::getReference() const
 {
+    /* We specifically return "daemon" here instead of "unix://" or "unix://${path}"
+     * to be more compatible with older versions of nix. Some tooling out there
+     * tries hard to parse store references and it might not be able to handle "unix://". */
+    if (path == settings.nixDaemonSocketFile)
+        return {.variant = StoreReference::Daemon{}};
     return {
         .variant =
             StoreReference::Specified{
                 .scheme = *uriSchemes().begin(),
-                // We return the empty string when the path looks like the
-                // default path, but we could also just return the path
-                // verbatim always, to be robust to overall config changes
-                // at the cost of some verbosity.
-                .authority = path == settings.nixDaemonSocketFile ? "" : path,
+                .authority = path,
             },
     };
 }

src/libstore/unix/build/chroot-derivation-builder.cc

@@ -186,7 +186,7 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl
 
     std::pair<Path, Path> addDependencyPrep(const StorePath & path)
     {
-        DerivationBuilderImpl::addDependency(path);
+        DerivationBuilderImpl::addDependencyImpl(path);
 
         debug("materialising '%s' in the sandbox", store.printStorePath(path));
 

src/libstore/unix/build/derivation-builder.cc

@@ -284,7 +284,7 @@ public:
 
 protected:
 
-    void addDependency(const StorePath & path) override;
+    void addDependencyImpl(const StorePath & path) override;
 
     /**
      * Make a file owned by the builder.
@@ -652,17 +652,17 @@ static void handleChildException(bool sendException)
     }
 }
 
-static bool checkNotWorldWritable(std::filesystem::path path)
+static void checkNotWorldWritable(std::filesystem::path path)
 {
     while (true) {
         auto st = lstat(path);
         if (st.st_mode & S_IWOTH)
-            return false;
+            throw Error("Path %s is world-writable or a symlink. That's not allowed for security.", path);
         if (path == path.parent_path())
             break;
         path = path.parent_path();
     }
-    return true;
+    return;
 }
 
 void DerivationBuilderImpl::startBuilder()
@@ -700,13 +700,12 @@ void DerivationBuilderImpl::startBuilder()
 
     createDirs(buildDir);
 
-    if (buildUser && !checkNotWorldWritable(buildDir))
-        throw Error(
-            "Path %s or a parent directory is world-writable or a symlink. That's not allowed for security.", buildDir);
+    if (buildUser)
+        checkNotWorldWritable(buildDir);
 
     /* Create a temporary directory where the build will take
        place. */
-    topTmpDir = createTempDir(buildDir, "nix-build-" + std::string(drvPath.name()), 0700);
+    topTmpDir = createTempDir(buildDir, "nix", 0700);
     setBuildTmpDir();
     assert(!tmpDir.empty());
 
@@ -1167,11 +1166,8 @@ void DerivationBuilderImpl::stopDaemon()
     daemonSocket.close();
 }
 
-void DerivationBuilderImpl::addDependency(const StorePath & path)
+void DerivationBuilderImpl::addDependencyImpl(const StorePath & path)
 {
-    if (isAllowed(path))
-        return;
-
     addedPaths.insert(path);
 }
 

src/libstore/unix/build/linux-derivation-builder.cc

@@ -681,7 +681,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
         DerivationBuilderImpl::killSandbox(getStats);
     }
 
-    void addDependency(const StorePath & path) override
+    void addDependencyImpl(const StorePath & path) override
     {
         auto [source, target] = ChrootDerivationBuilder::addDependencyPrep(path);
 

src/libstore/unix/build/sandbox-network.sb

@@ -16,6 +16,7 @@ R""(
 
 ; Allow DNS lookups.
 (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder")))
+(allow mach-lookup (global-name "com.apple.SystemConfiguration.DNSConfiguration"))
 
 ; Allow access to trustd.
 (allow mach-lookup (global-name "com.apple.trustd"))

src/libutil-tests/archive.cc

@@ -0,0 +1,47 @@
+#include "nix/util/archive.hh"
+#include "nix/util/tests/characterization.hh"
+#include "nix/util/tests/gmock-matchers.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+namespace {
+
+class NarTest : public CharacterizationTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "nars";
+
+public:
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / (std::string(testStem) + ".nar");
+    }
+};
+
+class InvalidNarTest : public NarTest, public ::testing::WithParamInterface<std::tuple<std::string, std::string>>
+{};
+
+} // namespace
+
+TEST_P(InvalidNarTest, throwsErrorMessage)
+{
+    const auto & [name, message] = GetParam();
+    readTest(name, [&](const std::string & narContents) {
+        ASSERT_THAT(
+            [&]() {
+                StringSource source{narContents};
+                NullFileSystemObjectSink sink;
+                parseDump(sink, source);
+            },
+            ::testing::ThrowsMessage<SerialisationError>(testing::HasSubstrIgnoreANSIMatcher(message)));
+    });
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    NarTest,
+    InvalidNarTest,
+    ::testing::Values(
+        std::pair{"invalid-tag-instead-of-contents", "bad archive: expected tag 'contents', got 'AAAAAAAA'"}));
+
+} // namespace nix

src/libutil-tests/meson.build

@@ -44,6 +44,7 @@ config_priv_h = configure_file(
 subdir('nix-meson-build-support/common')
 
 sources = files(
+  'archive.cc',
   'args.cc',
   'base-n.cc',
   'canon-path.cc',

src/libutil-tests/url.cc

@@ -221,15 +221,20 @@ TEST(parseURL, parsedUrlsWithUnescapedChars)
      * 2. Unescaped spaces and quotes in query.
      */
     auto s = "http://www.example.org/file.tar.gz?query \"= 123\"#shevron^quote\"space ";
-    auto url = parseURL(s);
 
-    ASSERT_EQ(url.fragment, "shevron^quote\"space ");
+    /* Without leniency for back compat, this should throw. */
+    EXPECT_THROW(parseURL(s), Error);
+
+    /* With leniency for back compat, this should parse. */
+    auto url = parseURL(s, /*lenient=*/true);
+
+    EXPECT_EQ(url.fragment, "shevron^quote\"space ");
 
     auto query = StringMap{
         {"query \"", " 123\""},
     };
 
-    ASSERT_EQ(url.query, query);
+    EXPECT_EQ(url.query, query);
 }
 
 TEST(parseURL, parseFTPUrl)
@@ -268,6 +273,23 @@ TEST(parseURL, emptyStringIsInvalidURL)
     ASSERT_THROW(parseURL(""), Error);
 }
 
+TEST(parseURL, parsesHttpUrlWithEmptyPort)
+{
+    auto s = "http://www.example.org:/file.tar.gz?foo=bar";
+    auto parsed = parseURL(s);
+
+    ParsedURL expected{
+        .scheme = "http",
+        .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
+        .path = "/file.tar.gz",
+        .query = (StringMap) {{"foo", "bar"}},
+        .fragment = "",
+    };
+
+    ASSERT_EQ(parsed, expected);
+    ASSERT_EQ("http://www.example.org/file.tar.gz?foo=bar", parsed.to_string());
+}
+
 /* ----------------------------------------------------------------------------
  * decodeQuery
  * --------------------------------------------------------------------------*/

src/libutil/archive.cc

@@ -187,8 +187,10 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath
                 tag = getString();
             }
 
-            if (tag == "contents")
-                parseContents(crf, source);
+            if (tag != "contents")
+                throw badArchive("expected tag 'contents', got '%s'", tag);
+
+            parseContents(crf, source);
 
             expectTag(")");
         });

src/libutil/error.cc

@@ -6,6 +6,7 @@
 #include "nix/util/terminal.hh"
 #include "nix/util/position.hh"
 
+#include <cinttypes>
 #include <iostream>
 #include <optional>
 #include "nix/util/serialise.hh"
@@ -436,13 +437,19 @@ void panic(std::string_view msg)
     writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL);
     writeErr(msg);
     writeErr("\n");
-    abort();
+    std::terminate();
 }
 
-void panic(const char * file, int line, const char * func)
+void unreachable(std::source_location loc)
 {
     char buf[512];
-    int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line);
+    int n = snprintf(
+        buf,
+        sizeof(buf),
+        "Unexpected condition in %s at %s:%" PRIuLEAST32,
+        loc.function_name(),
+        loc.file_name(),
+        loc.line());
     if (n < 0)
         panic("Unexpected condition and could not format error message");
     panic(std::string_view(buf, std::min(static_cast<int>(sizeof(buf)), n)));

src/libutil/file-system.cc

@@ -730,11 +730,10 @@ Path makeTempPath(const Path & root, const Path & suffix)
 
 void createSymlink(const Path & target, const Path & link)
 {
-    try {
-        std::filesystem::create_symlink(target, link);
-    } catch (std::filesystem::filesystem_error & e) {
-        throw SysError("creating symlink '%1%' -> '%2%'", link, target);
-    }
+    std::error_code ec;
+    std::filesystem::create_symlink(target, link, ec);
+    if (ec)
+        throw SysError(ec.value(), "creating symlink '%1%' -> '%2%'", link, target);
 }
 
 void replaceSymlink(const std::filesystem::path & target, const std::filesystem::path & link)

src/libutil/hash.cc

@@ -135,7 +135,8 @@ static Hash parseLowLevel(std::string_view rest, HashAlgorithm algo, DecodeNameP
         e.addTrace({}, "While decoding hash '%s'", rest);
     }
     if (d.size() != res.hashSize)
-        throw BadHash("invalid %s hash '%s' %d %d", pair.encodingName, rest);
+        throw BadHash(
+            "invalid %s hash '%s', length %d != expected length %d", pair.encodingName, rest, d.size(), res.hashSize);
     assert(res.hashSize);
     memcpy(res.hash, d.data(), res.hashSize);
 

src/libutil/include/nix/util/error.hh

@@ -22,6 +22,7 @@
 #include <list>
 #include <memory>
 #include <optional>
+#include <utility>
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -299,23 +300,16 @@ using NativeSysError =
 void throwExceptionSelfCheck();
 
 /**
- * Print a message and abort().
+ * Print a message and std::terminate().
  */
 [[noreturn]]
 void panic(std::string_view msg);
 
 /**
- * Print a basic error message with source position and abort().
- * Use the unreachable() macro to call this.
- */
-[[noreturn]]
-void panic(const char * file, int line, const char * func);
-
-/**
- * Print a basic error message with source position and abort().
+ * Print a basic error message with source position and std::terminate().
  *
  * @note: This assumes that the logger is operational
  */
-#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__))
+[[gnu::noinline, gnu::cold, noreturn]] void unreachable(std::source_location loc = std::source_location::current());
 
 } // namespace nix

src/libutil/include/nix/util/file-system.hh

@@ -287,6 +287,10 @@ class AutoDelete
 public:
     AutoDelete();
     AutoDelete(const std::filesystem::path & p, bool recursive = true);
+    AutoDelete(AutoDelete &&) = delete;
+    AutoDelete(const AutoDelete &) = delete;
+    AutoDelete & operator=(AutoDelete &&) = delete;
+    AutoDelete & operator=(const AutoDelete &) = delete;
     ~AutoDelete();
 
     void cancel();

src/libutil/include/nix/util/strings.hh

@@ -12,11 +12,6 @@
 
 namespace nix {
 
-/*
- * workaround for unavailable view() method (C++20) of std::ostringstream under MacOS with clang-16
- */
-std::string_view toView(const std::ostringstream & os);
-
 /**
  * String tokenizer.
  *

src/libutil/include/nix/util/url.hh

@@ -96,14 +96,18 @@ MakeError(BadURL, Error);
 std::string percentDecode(std::string_view in);
 std::string percentEncode(std::string_view s, std::string_view keep = "");
 
-StringMap decodeQuery(const std::string & query);
+/**
+ * @param lenient @see parseURL
+ */
+StringMap decodeQuery(std::string_view query, bool lenient = false);
 
 std::string encodeQuery(const StringMap & query);
 
 /**
- * Parse a Nix URL into a ParsedURL.
+ * Parse a URL into a ParsedURL.
  *
- * Nix URI is mostly compliant with RFC3986, but with some deviations:
+ * @parm lenient Also allow some long-supported Nix URIs that are not quite compliant with RFC3986.
+ * Here are the deviations:
  * - Fragments can contain unescaped (not URL encoded) '^', '"' or space literals.
  * - Queries may contain unescaped '"' or spaces.
  *
@@ -111,7 +115,7 @@ std::string encodeQuery(const StringMap & query);
  *
  * @throws BadURL
  */
-ParsedURL parseURL(std::string_view url);
+ParsedURL parseURL(std::string_view url, bool lenient = false);
 
 /**
  * Although that’s not really standardized anywhere, an number of tools

src/libutil/logging.cc

@@ -121,7 +121,7 @@ public:
         std::ostringstream oss;
         showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
-        log(ei.level, toView(oss));
+        log(ei.level, oss.view());
     }
 
     void startActivity(

src/libutil/meson.build

@@ -57,7 +57,12 @@ deps_private += blake3
 
 boost = dependency(
   'boost',
-  modules : [ 'context', 'coroutine', 'iostreams', 'url' ],
+  modules : [
+    'context',
+    'coroutine',
+    'iostreams',
+    'url',
+  ],
   include_type : 'system',
   version : '>=1.82.0',
 )

src/libutil/strings.cc

@@ -8,23 +8,6 @@
 
 namespace nix {
 
-struct view_stringbuf : public std::stringbuf
-{
-    inline std::string_view toView()
-    {
-        auto begin = pbase();
-        return {begin, begin + pubseekoff(0, std::ios_base::cur, std::ios_base::out)};
-    }
-};
-
-__attribute__((no_sanitize("undefined"))) std::string_view toView(const std::ostringstream & os)
-{
-    /* Downcasting like this is very much undefined behavior, so we disable
-       UBSAN for this function. */
-    auto buf = static_cast<view_stringbuf *>(os.rdbuf());
-    return buf->toView();
-}
-
 template std::list<std::string> tokenizeString(std::string_view s, std::string_view separators);
 template StringSet tokenizeString(std::string_view s, std::string_view separators);
 template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators);

src/libutil/unix/include/nix/util/monitor-fd.hh

@@ -2,15 +2,18 @@
 ///@file
 
 #include <thread>
-#include <atomic>
+#include <cassert>
 
-#include <cstdlib>
 #include <poll.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <signal.h>
+#include <errno.h>
+
+#ifdef __APPLE__
+#  include <sys/types.h>
+#  include <sys/event.h>
+#endif
 
 #include "nix/util/signals.hh"
+#include "nix/util/file-descriptor.hh"
 
 namespace nix {
 
@@ -20,111 +23,113 @@ private:
     std::thread thread;
     Pipe notifyPipe;
 
+    void runThread(int watchFd, int notifyFd);
+
 public:
-    MonitorFdHup(int fd)
-    {
-        notifyPipe.create();
-        thread = std::thread([this, fd]() {
-            while (true) {
-                // There is a POSIX violation on macOS: you have to listen for
-                // at least POLLHUP to receive HUP events for a FD. POSIX says
-                // this is not so, and you should just receive them regardless.
-                // However, as of our testing on macOS 14.5, the events do not
-                // get delivered if in the all-bits-unset case, but do get
-                // delivered if `POLLHUP` is set.
-                //
-                // This bug filed as rdar://37537852
-                // (https://openradar.appspot.com/37537852).
-                //
-                // macOS's own man page
-                // (https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/poll.2.html)
-                // additionally says that `POLLHUP` is ignored as an input. It
-                // seems the likely order of events here was
-                //
-                // 1. macOS did not follow the POSIX spec
-                //
-                // 2. Somebody ninja-fixed this other spec violation to make
-                // sure `POLLHUP` was not forgotten about, even though they
-                // "fixed" this issue in a spec-non-compliant way. Whatever,
-                // we'll use the fix.
-                //
-                // Relevant code, current version, which shows the :
-                // https://github.com/apple-oss-distributions/xnu/blob/94d3b452840153a99b38a3a9659680b2a006908e/bsd/kern/sys_generic.c#L1751-L1758
-                //
-                // The `POLLHUP` detection was added in
-                // https://github.com/apple-oss-distributions/xnu/commit/e13b1fa57645afc8a7b2e7d868fe9845c6b08c40#diff-a5aa0b0e7f4d866ca417f60702689fc797e9cdfe33b601b05ccf43086c35d395R1468
-                // That means added in 2007 or earlier. Should be good enough
-                // for us.
-                short hangup_events =
-#ifdef __APPLE__
-                    POLLHUP
-#else
-                    0
-#endif
-                    ;
-
-                /* Wait indefinitely until a POLLHUP occurs. */
-                constexpr size_t num_fds = 2;
-                struct pollfd fds[num_fds] = {
-                    {
-                        .fd = fd,
-                        .events = hangup_events,
-                    },
-                    {
-                        .fd = notifyPipe.readSide.get(),
-                        .events = hangup_events,
-                    },
-                };
-
-                auto count = poll(fds, num_fds, -1);
-                if (count == -1) {
-                    if (errno == EINTR || errno == EAGAIN)
-                        continue;
-                    throw SysError("failed to poll() in MonitorFdHup");
-                }
-                /* This shouldn't happen, but can on macOS due to a bug.
-                   See rdar://37550628.
-
-                   This may eventually need a delay or further
-                   coordination with the main thread if spinning proves
-                   too harmful.
-                */
-                if (count == 0)
-                    continue;
-                if (fds[0].revents & POLLHUP) {
-                    unix::triggerInterrupt();
-                    break;
-                }
-                if (fds[1].revents & POLLHUP) {
-                    break;
-                }
-                // On macOS, (jade thinks that) it is possible (although not
-                // observed on macOS 14.5) that in some limited cases on buggy
-                // kernel versions, all the non-POLLHUP events for the socket
-                // get delivered.
-                //
-                // We could sleep to avoid pointlessly spinning a thread on
-                // those, but this opens up a different problem, which is that
-                // if do sleep, it will be longer before the daemon fork for a
-                // client exits. Imagine a sequential shell script, running Nix
-                // commands, each of which talk to the daemon. If the previous
-                // command registered a temp root, exits, and then the next
-                // command issues a delete request before the temp root is
-                // cleaned up, that delete request might fail.
-                //
-                // Not sleeping doesn't actually fix the race condition --- we
-                // would need to block on the old connections' tempt roots being
-                // cleaned up in in the new connection --- but it does make it
-                // much less likely.
-            }
-        });
-    };
+    MonitorFdHup(int fd);
 
     ~MonitorFdHup()
     {
+        // Close the write side to signal termination via POLLHUP
         notifyPipe.writeSide.close();
         thread.join();
     }
 };
 
+#ifdef __APPLE__
+/* This custom kqueue usage exists because Apple's poll implementation is
+ * broken and loses event subscriptions if EVFILT_READ fires without matching
+ * the requested `events` in the pollfd.
+ *
+ * We use EVFILT_READ, which causes some spurious wakeups (at most one per write
+ * from the client, in addition to the socket lifecycle events), because the
+ * alternate API, EVFILT_SOCK, doesn't work on pipes, which this is also used
+ * to monitor in certain situations.
+ *
+ * See (EVFILT_SOCK):
+ * https://github.com/netty/netty/blob/64bd2f4eb62c2fb906bc443a2aabf894c8b7dce9/transport-classes-kqueue/src/main/java/io/netty/channel/kqueue/AbstractKQueueChannel.java#L434
+ *
+ * See: https://git.lix.systems/lix-project/lix/issues/729
+ * Apple bug in poll(2): FB17447257, available at https://openradar.appspot.com/FB17447257
+ */
+inline void MonitorFdHup::runThread(int watchFd, int notifyFd)
+{
+    int kqResult = kqueue();
+    if (kqResult < 0) {
+        throw SysError("MonitorFdHup kqueue");
+    }
+    AutoCloseFD kq{kqResult};
+
+    std::array<struct kevent, 2> kevs;
+
+    // kj uses EVFILT_WRITE for this, but it seems that it causes more spurious
+    // wakeups in our case of doing blocking IO from another thread compared to
+    // EVFILT_READ.
+    //
+    // EVFILT_WRITE and EVFILT_READ (for sockets at least, where I am familiar
+    // with the internals) both go through a common filter which catches EOFs
+    // and generates spurious wakeups for either readable/writable events.
+    EV_SET(&kevs[0], watchFd, EVFILT_READ, EV_ADD | EV_ENABLE | EV_CLEAR, 0, 0, nullptr);
+    EV_SET(&kevs[1], notifyFd, EVFILT_READ, EV_ADD | EV_ENABLE | EV_CLEAR, 0, 0, nullptr);
+
+    int result = kevent(kq.get(), kevs.data(), kevs.size(), nullptr, 0, nullptr);
+    if (result < 0) {
+        throw SysError("MonitorFdHup kevent add");
+    }
+
+    while (true) {
+        struct kevent event;
+        int numEvents = kevent(kq.get(), nullptr, 0, &event, 1, nullptr);
+        if (numEvents < 0) {
+            throw SysError("MonitorFdHup kevent watch");
+        }
+
+        if (numEvents > 0 && (event.flags & EV_EOF)) {
+            if (event.ident == uintptr_t(watchFd)) {
+                unix::triggerInterrupt();
+            }
+            // Either watched fd or notify fd closed, exit
+            return;
+        }
+    }
+}
+#else
+inline void MonitorFdHup::runThread(int watchFd, int notifyFd)
+{
+    while (true) {
+        struct pollfd fds[2];
+        fds[0].fd = watchFd;
+        fds[0].events = 0; // POSIX: POLLHUP is always reported
+        fds[1].fd = notifyFd;
+        fds[1].events = 0;
+
+        auto count = poll(fds, 2, -1);
+        if (count == -1) {
+            if (errno == EINTR || errno == EAGAIN) {
+                continue;
+            } else {
+                throw SysError("in MonitorFdHup poll()");
+            }
+        }
+
+        if (fds[0].revents & POLLHUP) {
+            unix::triggerInterrupt();
+            break;
+        }
+
+        if (fds[1].revents & POLLHUP) {
+            // Notify pipe closed, exit thread
+            break;
+        }
+    }
+}
+#endif
+
+inline MonitorFdHup::MonitorFdHup(int fd)
+{
+    notifyPipe.create();
+    int notifyFd = notifyPipe.readSide.get();
+    thread = std::thread([this, fd, notifyFd]() { this->runThread(fd, notifyFd); });
+};
+
 } // namespace nix

src/libutil/unix/include/nix/util/signals-impl.hh

@@ -42,13 +42,6 @@ extern thread_local std::function<bool()> interruptCheck;
 
 void _interrupted();
 
-/**
- * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
- * necessarily match the current thread's mask.
- * See saveSignalMask() to set the saved mask to the current mask.
- */
-void setChildSignalMask(sigset_t * sigs);
-
 /**
  * Start a thread that handles various signals. Also block those signals
  * on the current thread (and thus any threads created by it).
@@ -60,8 +53,6 @@ void startSignalHandlerThread();
 /**
  * Saves the signal mask, which is the signal mask that nix will restore
  * before creating child processes.
- * See setChildSignalMask() to set an arbitrary signal mask instead of the
- * current mask.
  */
 void saveSignalMask();
 

src/libutil/unix/signals.cc

@@ -99,26 +99,6 @@ void unix::triggerInterrupt()
 static sigset_t savedSignalMask;
 static bool savedSignalMaskIsSet = false;
 
-void unix::setChildSignalMask(sigset_t * sigs)
-{
-    assert(sigs); // C style function, but think of sigs as a reference
-
-#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE) \
-    || (defined(_POSIX_SOURCE) && _POSIX_SOURCE)
-    sigemptyset(&savedSignalMask);
-    // There's no "assign" or "copy" function, so we rely on (math) idempotence
-    // of the or operator: a or a = a.
-    sigorset(&savedSignalMask, sigs, sigs);
-#else
-    // Without sigorset, our best bet is to assume that sigset_t is a type that
-    // can be assigned directly, such as is the case for a sigset_t defined as
-    // an integer type.
-    savedSignalMask = *sigs;
-#endif
-
-    savedSignalMaskIsSet = true;
-}
-
 void unix::saveSignalMask()
 {
     if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))

src/libutil/url.cc

@@ -33,7 +33,7 @@ ParsedURL::Authority ParsedURL::Authority::parse(std::string_view encodedAuthori
     }();
 
     auto port = [&]() -> std::optional<uint16_t> {
-        if (!parsed->has_port())
+        if (!parsed->has_port() || parsed->port() == "")
             return std::nullopt;
         /* If the port number is non-zero and representable. */
         if (auto portNumber = parsed->port_number())
@@ -108,41 +108,48 @@ static std::string percentEncodeCharSet(std::string_view s, auto charSet)
     return res;
 }
 
-ParsedURL parseURL(std::string_view url)
+ParsedURL parseURL(std::string_view url, bool lenient)
 try {
     /* Account for several non-standard properties of nix urls (for back-compat):
      *  - Allow unescaped spaces ' ' and '"' characters in queries.
      *  - Allow '"', ' ' and '^' characters in the fragment component.
      * We could write our own grammar for this, but fixing it up here seems
      * more concise, since the deviation is rather minor.
+     *
+     * If `!lenient` don't bother initializing, because we can just
+     * parse `url` directly`.
      */
-    std::string fixedEncodedUrl = [&]() {
-        std::string fixed;
-        std::string_view view = url;
+    std::string fixedEncodedUrl;
 
-        if (auto beforeQuery = splitPrefixTo(view, '?')) {
-            fixed += *beforeQuery;
-            fixed += '?';
-            auto fragmentStart = view.find('#');
-            auto queryView = view.substr(0, fragmentStart);
-            auto fixedQuery = percentEncodeCharSet(queryView, extraAllowedCharsInQuery);
-            fixed += fixedQuery;
-            view.remove_prefix(std::min(fragmentStart, view.size()));
-        }
+    if (lenient) {
+        fixedEncodedUrl = [&] {
+            std::string fixed;
+            std::string_view view = url;
 
-        if (auto beforeFragment = splitPrefixTo(view, '#')) {
-            fixed += *beforeFragment;
-            fixed += '#';
-            auto fixedFragment = percentEncodeCharSet(view, extraAllowedCharsInFragment);
-            fixed += fixedFragment;
+            if (auto beforeQuery = splitPrefixTo(view, '?')) {
+                fixed += *beforeQuery;
+                fixed += '?';
+                auto fragmentStart = view.find('#');
+                auto queryView = view.substr(0, fragmentStart);
+                auto fixedQuery = percentEncodeCharSet(queryView, extraAllowedCharsInQuery);
+                fixed += fixedQuery;
+                view.remove_prefix(std::min(fragmentStart, view.size()));
+            }
+
+            if (auto beforeFragment = splitPrefixTo(view, '#')) {
+                fixed += *beforeFragment;
+                fixed += '#';
+                auto fixedFragment = percentEncodeCharSet(view, extraAllowedCharsInFragment);
+                fixed += fixedFragment;
+                return fixed;
+            }
+
+            fixed += view;
             return fixed;
-        }
+        }();
+    }
 
-        fixed += view;
-        return fixed;
-    }();
-
-    auto urlView = boost::urls::url_view(fixedEncodedUrl);
+    auto urlView = boost::urls::url_view(lenient ? fixedEncodedUrl : url);
 
     if (!urlView.has_scheme())
         throw BadURL("'%s' doesn't have a scheme", url);
@@ -179,7 +186,7 @@ try {
         .scheme = scheme,
         .authority = authority,
         .path = path,
-        .query = decodeQuery(std::string(query)),
+        .query = decodeQuery(query, lenient),
         .fragment = fragment,
     };
 } catch (boost::system::system_error & e) {
@@ -201,14 +208,17 @@ std::string percentEncode(std::string_view s, std::string_view keep)
         s, [keep](char c) { return boost::urls::unreserved_chars(c) || keep.find(c) != keep.npos; });
 }
 
-StringMap decodeQuery(const std::string & query)
+StringMap decodeQuery(std::string_view query, bool lenient)
 try {
-    /* For back-compat unescaped characters are allowed. */
-    auto fixedEncodedQuery = percentEncodeCharSet(query, extraAllowedCharsInQuery);
+    /* When `lenient = true`, for back-compat unescaped characters are allowed. */
+    std::string fixedEncodedQuery;
+    if (lenient) {
+        fixedEncodedQuery = percentEncodeCharSet(query, extraAllowedCharsInQuery);
+    }
 
     StringMap result;
 
-    auto encodedQuery = boost::urls::params_encoded_view(fixedEncodedQuery);
+    auto encodedQuery = boost::urls::params_encoded_view(lenient ? fixedEncodedQuery : query);
     for (auto && [key, value, value_specified] : encodedQuery) {
         if (!value_specified) {
             warn("dubious URI query '%s' is missing equal sign '%s', ignoring", std::string_view(key), "=");

src/nix/config-check.cc

@@ -100,7 +100,7 @@ struct CmdConfigCheck : StoreCommand
             ss << "Multiple versions of nix found in PATH:\n";
             for (auto & dir : dirs)
                 ss << "  " << dir << "\n";
-            return checkFail(toView(ss));
+            return checkFail(ss.view());
         }
 
         return checkPass("PATH contains only one nix version.");
@@ -143,7 +143,7 @@ struct CmdConfigCheck : StoreCommand
             for (auto & dir : dirs)
                 ss << "  " << dir << "\n";
             ss << "\n";
-            return checkFail(toView(ss));
+            return checkFail(ss.view());
         }
 
         return checkPass("All profiles are gcroots.");
@@ -162,7 +162,7 @@ struct CmdConfigCheck : StoreCommand
                << "sync with the daemon.\n\n"
                << "Client protocol: " << formatProtocol(clientProto) << "\n"
                << "Store protocol: " << formatProtocol(storeProto) << "\n\n";
-            return checkFail(toView(ss));
+            return checkFail(ss.view());
         }
 
         return checkPass("Client protocol matches store protocol.");

src/nix/develop.cc

@@ -647,7 +647,7 @@ struct CmdDevelop : Common, MixEnvironment
                 nixpkgs = i->nixpkgsFlakeRef();
 
             auto bashInstallable = make_ref<InstallableFlake>(
-                this,
+                nullptr, //< Don't barf when the command is run with --arg/--argstr
                 state,
                 std::move(nixpkgs),
                 "bashInteractive",

src/nix/main.cc

@@ -226,8 +226,8 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
 
     auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand));
 
-    evalSettings.restrictEval = false;
-    evalSettings.pureEval = false;
+    evalSettings.restrictEval = true;
+    evalSettings.pureEval = true;
     EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings);
 
     auto vGenerateManpage = state.allocValue();

src/nix/nix-build/nix-build.cc

@@ -285,10 +285,10 @@ static void main_nix_build(int argc, char ** argv)
                         execArgs,
                         interpreter,
                         escapeShellArgAlways(script),
-                        toView(joined));
+                        joined.view());
             } else {
                 envCommand =
-                    fmt("exec %1% %2% %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), toView(joined));
+                    fmt("exec %1% %2% %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), joined.view());
             }
         }
 

src/nix/nix-env/user-env.cc

@@ -108,7 +108,7 @@ bool createUserEnv(
     auto manifestFile = ({
         std::ostringstream str;
         printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits<int>::max());
-        StringSource source{toView(str)};
+        StringSource source{str.view()};
         state.store->addToStoreFromDump(
             source,
             "env-manifest.nix",