libutil: Make computeClosure async

This makes computeFSClosure much faster in practice, because the graph is now traversed in an async BFS manner, while
previously it serialised on each single path info query. For something like

rm ~/.cache/nix/binary-cache-v7.sqlite && nix path-info --store https://cache.nixos.org --recursive /nix/store/7zz3zmv2a0ssmgqlfhy4rsb6ii6z475a-stdenv-linux.drv

The difference is huge:

(Before)

Command being timed: "nix path-info --store https://cache.nixos.org --recursive /nix/store/7zz3zmv2a0ssmgqlfhy4rsb6ii6z475a-stdenv-linux.drv"
        User time (seconds): 0.07
        System time (seconds): 0.06
        Percent of CPU this job got: 1%
        Elapsed (wall clock) time (h:mm:ss or m:ss): 0:08.88

(After)

Command being timed: "build/src/nix/nix path-info --store https://cache.nixos.org --recursive /nix/store/7zz3zmv2a0ssmgqlfhy4rsb6ii6z475a-stdenv-linux.drv"
        User time (seconds): 0.07
        System time (seconds): 0.04
        Percent of CPU this job got: 22%
        Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.49

Basically 20x speedup with just processing stuff in an async manner.

Note that for now we have rather ad-hoc event loop just for
computeClosure. This seems perfectly fine for now, but in the future
we could extended it and possibly have a global even loop with multiple
threads handling it.

For now we have basic rate limiting <= 64 coroutines in flight which is
smaller than CURLMOPT_MAX_CONCURRENT_STREAMS (100 by default).

.github/workflows/backport.yml

@@ -26,7 +26,7 @@ jobs:
           # required to find all branches
           fetch-depth: 0
       - name: Create backport PRs
-        uses: korthout/backport-action@d07416681cab29bf2661702f925f020aaa962997 # v3.4.1
+        uses: korthout/backport-action@c656f5d5851037b2b38fb5db2691a03fa229e3b2 # v4.0.1
         id: backport
         with:
           # Config README: https://github.com/korthout/backport-action#backport-action

.github/workflows/ci.yml

@@ -125,13 +125,13 @@ jobs:
         cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
       if: ${{ matrix.instrumented }}
     - name: Upload coverage reports
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: coverage-reports
         path: coverage-reports/
       if: ${{ matrix.instrumented }}
     - name: Upload installer tarball
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: installer-${{matrix.os}}
         path: out/*
@@ -164,7 +164,7 @@ jobs:
     steps:
     - uses: actions/checkout@v6
     - name: Download installer tarball
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v7
       with:
         name: installer-${{matrix.os}}
         path: out
@@ -174,7 +174,7 @@ jobs:
         echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
         TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
         echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
-    - uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
+    - uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
       if: ${{ !matrix.experimental-installer }}
       with:
         install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}

.version

@@ -1 +1 @@
-2.33.0
+2.34.0

doc/manual/meson.build

@@ -26,7 +26,6 @@ bash = find_program('bash', native : true)
 # HTML manual dependencies (conditional)
 if get_option('html-manual')
   mdbook = find_program('mdbook', native : true)
-  rsync = find_program('rsync', required : true, native : true)
 endif
 
 pymod = import('python')
@@ -126,7 +125,12 @@ if get_option('html-manual')
           @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
           @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
           sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
-          @4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
+          # Copy source to build directory, excluding the build directory itself
+          # (which is present when built as an individual component).
+          # Use tar with --dereference to copy symlink targets (e.g., JSON examples from tests).
+          (cd @CURRENT_SOURCE_DIR@ && find . -mindepth 1 -maxdepth 1 ! -name build | tar -c --dereference -T - -f -) | (cd @2@ && tar -xf -)
+          chmod -R u+w @2@
+          find @2@ -name '*.drv' -delete
           (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
           rm -rf @2@/manual
           mv @2@/html @2@/manual
@@ -138,7 +142,6 @@ if get_option('html-manual')
         mdbook.full_path(),
         meson.current_build_dir(),
         meson.project_version(),
-        rsync.full_path(),
       ),
     ],
     input : [

doc/manual/package.nix

@@ -10,7 +10,6 @@
   mdbook,
   jq,
   python3,
-  rsync,
   nix-cli,
   changelog-d,
   json-schema-for-humans,
@@ -54,6 +53,8 @@ mkMesonDerivation (finalAttrs: {
         ../../src/libstore-tests/data/nar-info
         ../../src/libstore-tests/data/build-result
         ../../src/libstore-tests/data/dummy-store
+        # For derivation examples referenced by symlinks in doc/manual/source/protocols/json/schema/
+        ../../tests/functional/derivation
         # Too many different types of files to filter for now
         ../../doc/manual
         ./.
@@ -90,7 +91,6 @@ mkMesonDerivation (finalAttrs: {
   ]
   ++ lib.optionals buildHtmlManual [
     mdbook
-    rsync
     json-schema-for-humans
   ]
   ++ lib.optionals (!officialRelease && buildHtmlManual) [

doc/manual/rl-next/channels-subdomain.md

@@ -1,9 +0,0 @@
----
-synopsis: Channel URLs migrated to channels.nixos.org subdomain
-prs: [14518]
-issues: [14517]
----
-
-Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix.
-
-The subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.

doc/manual/rl-next/json-format-changes.md

@@ -1,88 +0,0 @@
----
-synopsis: "JSON format changes for store path info and derivations"
-prs: []
-issues: []
----
-
-JSON formats for store path info and derivations have been updated with new versions and structured fields.
-
-## Store Path Info JSON
-
-`nix path-info --json` now requires a `--json-format` flag to specify the output format version.
-Using `--json` without `--json-format` is deprecated and will become an error in a future release.
-For now, it defaults to version 1 with a warning, for a smoother migration.
-
-### Version 1 (`--json-format 1`)
-
-This is the legacy format, preserved for backwards compatibility:
-
-- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
-- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
-- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
-- Now includes `"storeDir"` field at the top level
-
-### Version 2 (`--json-format 2`)
-
-The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
-
-- **Nested structure with top-level metadata**:
-
-  The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
-
-  ```json
-  {
-    "version": 2,
-    "storeDir": "/nix/store",
-    "info": { ... }
-  }
-  ```
-
-  The map from store bath base names to store object info is nested under the `info` field.
-
-- **Store path base names instead of full paths**:
-
-  Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
-  Combined with `storeDir`, the full path can be reconstructed.
-
-- **Structured `ca` field**:
-
-  Content address is now a structured JSON object instead of a string:
-
-  - Old: `"ca": "fixed:r:sha256:1abc..."`
-  - New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
-  - Still `null` values for input-addressed store objects
-
-  The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
-
-Nix currently only produces, and doesn't consume this format.
-
-Additionally the following field is added to both formats.
-(The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.)
-
-- **`version` field**:
-
-  All store path info JSON now includes `"version": <1|2>`.
-
-- **`storeDir` field**:
-
-  Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
-
-## Derivation JSON (Version 4)
-
-The derivation JSON format has been updated from version 3 to version 4:
-
-- **Restructured inputs**:
-
-  Inputs are now nested under an `inputs` object:
-
-  - Old: `"inputSrcs": [...], "inputDrvs": {...}`
-  - New: `"inputs": {"srcs": [...], "drvs": {...}}`
-
-- **Consistent content addresses**:
-
-  Fixed content-addressed outputs now use structured JSON format.
-  This is the same format as `ca` in store path info (after the new version).
-
-Version 3 and earlier formats are *not* accepted when reading.
-
-**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.

doc/manual/rl-next/libcurl-pausing.md

@@ -1,12 +0,0 @@
----
-synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
-prs: [14614]
-issues: [11728]
----
-
-The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
-[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
-
-This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
-
-Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).

doc/manual/rl-next/repl-interrupt.md

@@ -1,8 +0,0 @@
----
-synopsis: Interrupting REPL commands works more than once
-issues: [13481]
----
-
-Previously, this only worked once per REPL session; further attempts would be ignored.
-This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
-This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).

doc/manual/rl-next/s3-curl-implementation.md

@@ -1,40 +0,0 @@
----
-synopsis: "Improved S3 binary cache support via HTTP"
-prs: [13752, 13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 14330, 14333, 14335, 14336, 14337, 14350, 14356, 14357, 14374, 14375, 14376, 14377, 14391, 14393, 14420, 14421]
-issues: [13084, 12671, 11748, 12403]
----
-
-S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native
-AWS SigV4 authentication instead of the AWS C++ SDK, providing significant
-improvements:
-
-- **Reduced memory usage**: Eliminates memory buffering issues that caused
-  segfaults with large files
-- **Fixed upload reliability**: Resolves AWS SDK chunking errors
-  (`InvalidChunkSizeError`)
-- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full
-  `aws-cpp-sdk`, reducing build complexity
-
-The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential
-management.
-
-All existing S3 URL formats and parameters remain supported, however the store
-settings for configuring multipart uploads have changed:
-
-- **`multipart-upload`** (default: `false`): Enable multipart uploads for large
-  files. When enabled, files exceeding the multipart threshold will be uploaded
-  in multiple parts.
-
-- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using
-  multipart uploads. Files smaller than this will use regular PUT requests.
-  Only takes effect when `multipart-upload` is enabled.
-
-- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart
-  uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes
-  reduce the number of requests but use more memory.
-
-- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
-
-Note that this change also means Nix now supports S3 binary cache stores even
-if built without `aws-crt-cpp`, but only for public buckets which do not
-require authentication.

doc/manual/rl-next/s3-object-versioning.md

@@ -1,14 +0,0 @@
----
-synopsis: "S3 URLs now support object versioning via versionId parameter"
-prs: [14274]
-issues: [13955]
----
-
-S3 URLs now support a `versionId` query parameter to fetch specific versions
-of objects from S3 buckets with versioning enabled. This allows pinning to
-exact object versions for reproducibility and protection against unexpected
-changes:
-
-```
-s3://bucket/key?region=us-east-1&versionId=abc123def456
-```

doc/manual/rl-next/s3-storage-class.md

@@ -1,21 +0,0 @@
----
-synopsis: "S3 binary cache stores now support storage class configuration"
-prs: [14464]
-issues: [7015]
----
-
-S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
-
-Example usage:
-
-```bash
-# Use Glacier storage for long-term archival
-nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
-
-# Use Intelligent Tiering for automatic cost optimization
-nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
-```
-
-The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
-
-See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.

doc/manual/source/SUMMARY.md.in

@@ -151,6 +151,7 @@
   - [Contributing](development/contributing.md)
 - [Releases](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md)
   - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md)
   - [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md)
   - [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md)

doc/manual/source/development/building.md

@@ -3,6 +3,10 @@
 This section provides some notes on how to start hacking on Nix.
 To get the latest version of Nix from GitHub:
 
+> **Note**
+>
+> When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo.
+
 ```console
 $ git clone https://github.com/NixOS/nix.git
 $ cd nix

doc/manual/source/development/debugging.md

@@ -6,14 +6,7 @@ Additionally, see [Testing Nix](./testing.md) for further instructions on how to
 
 ## Building Nix with Debug Symbols
 
-In the development shell, set the `mesonBuildType` environment variable to `debug` before configuring the build:
-
-```console
-[nix-shell]$ export mesonBuildType=debugoptimized
-```
-
-Then, proceed to build Nix as described in [Building Nix](./building.md).
-This will build Nix with debug symbols, which are essential for effective debugging.
+In the development shell, `mesonBuildType` is set automatically to `debugoptimized`. This builds Nix with debug symbols, which are essential for effective debugging.
 
 It is also possible to build without optimization for faster build:
 

doc/manual/source/protocols/json/build-trace-entry.md

@@ -1,27 +1,21 @@
-{{#include build-trace-entry-v1-fixed.md}}
+{{#include build-trace-entry-v2-fixed.md}}
 
 ## Examples
 
 ### Simple build trace entry
 
 ```json
-{{#include schema/build-trace-entry-v1/simple.json}}
-```
-
-### Build trace entry with dependencies
-
-```json
-{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}}
+{{#include schema/build-trace-entry-v2/simple.json}}
 ```
 
 ### Build trace entry with signature
 
 ```json
-{{#include schema/build-trace-entry-v1/with-signature.json}}
+{{#include schema/build-trace-entry-v2/with-signature.json}}
 ```
 
 <!--
 ## Raw Schema
 
-[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v1.json)
--->
+[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v2.json)
+-->

doc/manual/source/protocols/json/meson.build

@@ -17,7 +17,7 @@ schemas = [
   'derivation-v4',
   'derivation-options-v1',
   'deriving-path-v1',
-  'build-trace-entry-v1',
+  'build-trace-entry-v2',
   'build-result-v1',
   'store-v1',
 ]

doc/manual/source/protocols/json/schema/build-result-v1.yaml

@@ -83,7 +83,7 @@ properties:
         description: |
           A mapping from output names to their build trace entries.
         additionalProperties:
-          "$ref": "build-trace-entry-v1.yaml"
+          "$ref": "build-trace-entry-v2.yaml"
 
   failure:
     type: object

doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml

@@ -1,5 +1,5 @@
 "$schema": "http://json-schema.org/draft-04/schema"
-"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json"
 title: Build Trace Entry
 description: |
   A record of a successful build outcome for a specific derivation output.
@@ -11,10 +11,17 @@ description: |
   > This JSON format is currently
   > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
   > and subject to change.
+
+  Verision history:
+
+  - Version 1: Original format
+
+  - Version 2: Remove `dependentRealisations`
+
+type: object
 required:
   - id
   - outPath
-  - dependentRealisations
   - signatures
 allOf:
   - "$ref": "#/$defs/key"
@@ -22,9 +29,11 @@ allOf:
 properties:
   id: {}
   outPath: {}
-  dependentRealisations: {}
   signatures: {}
-additionalProperties: false
+additionalProperties:
+  dependentRealisations:
+    description: deprecated field
+    type: object
 
 "$defs":
   key:
@@ -60,7 +69,6 @@ additionalProperties: false
     type: object
     required:
       - outPath
-      - dependentRealisations
       - signatures
     properties:
       outPath:
@@ -69,19 +77,6 @@ additionalProperties: false
         description: |
           The path to the store object that resulted from building this derivation for the given output name.
 
-      dependentRealisations:
-        type: object
-        title: Underlying Base Build Trace
-        description: |
-          This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
-
-          Keys are derivation output IDs (same format as the main `id` field).
-          Values are the store paths that those dependencies resolved to.
-
-          As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
-          This is the set of base build trace entries that this derived build trace is derived from.
-          (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
-
         patternProperties:
           "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
             "$ref": "store-path-v1.yaml"

doc/manual/source/protocols/json/schema/store-v1.yaml

@@ -70,7 +70,7 @@ properties:
       "^[A-Za-z0-9+/]{43}=$":
         type: object
         additionalProperties:
-          "$ref": "./build-trace-entry-v1.yaml#/$defs/value"
+          "$ref": "./build-trace-entry-v2.yaml#/$defs/value"
     additionalProperties: false
 
 "$defs":

doc/manual/source/release-notes/rl-2.33.md

@@ -0,0 +1,281 @@
+# Release 2.33.0 (2025-12-09)
+
+## New features
+
+- New command `nix registry resolve` [#14595](https://github.com/NixOS/nix/pull/14595)
+
+  This command looks up a flake registry input name and returns the flakeref it resolves to.
+
+  For example, looking up Nixpkgs:
+
+  ```
+  $ nix registry resolve nixpkgs
+  github:NixOS/nixpkgs/nixpkgs-unstable
+  ```
+
+  Upstreamed from [Determinate Nix 3.14.0](https://github.com/DeterminateSystems/nix-src/pull/273).
+
+- `nix flake clone` supports all input types [#14581](https://github.com/NixOS/nix/pull/14581)
+
+  `nix flake clone` now supports arbitrary input types. In particular, this allows you to clone tarball flakes, such as flakes on FlakeHub.
+
+  Upstreamed from [Determinate Nix 3.12.0](https://github.com/DeterminateSystems/nix-src/pull/229).
+
+## Performance improvements
+
+- Git fetcher computes `revCount`s using multiple threads [#14462](https://github.com/NixOS/nix/pull/14462)
+
+  When using Git repositories with a long history, calculating the `revCount` attribute can take a long time. Nix now computes `revCount` using multiple threads, making it much faster (e.g. 9.1s to 3.7s for Nixpkgs).
+
+  Note that if you don't need `revCount`, you can disable it altogether by setting the flake input attribute `shallow = true`.
+
+  Upstreamed from [Determinate Nix 3.12.2](https://github.com/DeterminateSystems/nix-src/pull/245).
+
+- `builtins.stringLength` now runs in constant time [#14442](https://github.com/NixOS/nix/pull/14442)
+
+  The internal representation of strings has been replaced with a size-prefixed Pascal style string. Previously Nix stored strings as a NUL-terminated array of bytes, necessitating a linear scan to calculate the length.
+
+- Uploads to `http://` and `https://` binary cache stores now run in constant memory [#14390](https://github.com/NixOS/nix/pull/14390)
+
+  Nix used to buffer the whole compressed NAR contents in memory. It now reads it in a streaming fashion.
+
+- Channel URLs migrated to channels.nixos.org subdomain [#14517](https://github.com/NixOS/nix/issues/14517) [#14518](https://github.com/NixOS/nix/pull/14518)
+
+  Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix. This subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.
+
+- Fix `download buffer is full; consider increasing the 'download-buffer-size' setting` warning [#11728](https://github.com/NixOS/nix/issues/11728) [#14614](https://github.com/NixOS/nix/pull/14614)
+
+  The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
+  [libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
+
+  This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
+
+  Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).
+
+- Significantly improve tarball unpacking performance [#14689](https://github.com/NixOS/nix/pull/14689) [#14696](https://github.com/NixOS/nix/pull/14696) [#10683](https://github.com/NixOS/nix/issues/10683) [#11098](https://github.com/NixOS/nix/issues/11098)
+
+  Nix uses a content-addressed cache backed by libgit2 for deduplicating files fetched via `fetchTarball` and `github`, `tarball` flake inputs. Its usage has been significantly optimised to reduce the amount of I/O operations that are performed. For a typical nixpkgs source tarball this results in 200 times fewer system calls on Linux. In combination with libcurl pausing this alleviates performance regressions stemming from the tarball cache.
+
+- Already valid derivations are no longer copied to the store [#14219](https://github.com/NixOS/nix/pull/14219)
+
+  This results in a modest speedup when using the Nix daemon.
+
+- `nix nar ls` and `nix nar cat` are significantly faster and no longer buffer the whole NAR in memory [#14273](https://github.com/NixOS/nix/pull/14273) [#14732](https://github.com/NixOS/nix/pull/14732)
+
+## S3 improvements
+
+- Improved S3 binary cache support via HTTP [#11748](https://github.com/NixOS/nix/issues/11748) [#12403](https://github.com/NixOS/nix/issues/12403) [#12671](https://github.com/NixOS/nix/issues/12671) [#13084](https://github.com/NixOS/nix/issues/13084) [#13752](https://github.com/NixOS/nix/pull/13752) [#13823](https://github.com/NixOS/nix/pull/13823) [#14026](https://github.com/NixOS/nix/pull/14026) [#14120](https://github.com/NixOS/nix/pull/14120) [#14131](https://github.com/NixOS/nix/pull/14131) [#14135](https://github.com/NixOS/nix/pull/14135) [#14144](https://github.com/NixOS/nix/pull/14144) [#14170](https://github.com/NixOS/nix/pull/14170) [#14190](https://github.com/NixOS/nix/pull/14190) [#14198](https://github.com/NixOS/nix/pull/14198) [#14206](https://github.com/NixOS/nix/pull/14206) [#14209](https://github.com/NixOS/nix/pull/14209) [#14222](https://github.com/NixOS/nix/pull/14222) [#14223](https://github.com/NixOS/nix/pull/14223) [#14330](https://github.com/NixOS/nix/pull/14330) [#14333](https://github.com/NixOS/nix/pull/14333) [#14335](https://github.com/NixOS/nix/pull/14335) [#14336](https://github.com/NixOS/nix/pull/14336) [#14337](https://github.com/NixOS/nix/pull/14337) [#14350](https://github.com/NixOS/nix/pull/14350) [#14356](https://github.com/NixOS/nix/pull/14356) [#14357](https://github.com/NixOS/nix/pull/14357) [#14374](https://github.com/NixOS/nix/pull/14374) [#14375](https://github.com/NixOS/nix/pull/14375) [#14376](https://github.com/NixOS/nix/pull/14376) [#14377](https://github.com/NixOS/nix/pull/14377) [#14391](https://github.com/NixOS/nix/pull/14391) [#14393](https://github.com/NixOS/nix/pull/14393) [#14420](https://github.com/NixOS/nix/pull/14420) [#14421](https://github.com/NixOS/nix/pull/14421)
+
+  S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native AWS SigV4 authentication instead of the AWS C++ SDK, providing significant improvements:
+
+  - **Reduced memory usage**: Eliminates memory buffering issues that caused segfaults with large files
+  - **Fixed upload reliability**: Resolves AWS SDK chunking errors (`InvalidChunkSizeError`)
+  - **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full `aws-cpp-sdk`, reducing build complexity
+
+  The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential management.
+
+  All existing S3 URL formats and parameters remain supported, however the store settings for configuring multipart uploads have changed:
+
+  - **`multipart-upload`** (default: `false`): Enable multipart uploads for large files. When enabled, files exceeding the multipart threshold will be uploaded in multiple parts.
+
+  - **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using multipart uploads. Files smaller than this will use regular PUT requests. Only takes effect when `multipart-upload` is enabled.
+
+  - **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the number of requests but use more memory.
+
+  - **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
+
+  Note that this change also means Nix now supports S3 binary cache stores even if built without `aws-crt-cpp`, but only for public buckets which do not require authentication.
+
+- S3 URLs now support object versioning via `versionId` parameter [#13955](https://github.com/NixOS/nix/issues/13955) [#14274](https://github.com/NixOS/nix/pull/14274)
+
+  S3 URLs now support a `versionId` query parameter to fetch specific versions
+  of objects from S3 buckets with versioning enabled. This allows pinning to
+  exact object versions for reproducibility and protection against unexpected
+  changes:
+
+  ```
+  s3://bucket/key?region=us-east-1&versionId=abc123def456
+  ```
+
+- S3 binary cache stores now support storage class configuration [#7015](https://github.com/NixOS/nix/issues/7015) [#14464](https://github.com/NixOS/nix/pull/14464)
+
+  S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
+
+  Example usage:
+
+  ```bash
+  # Use Glacier storage for long-term archival
+  nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
+
+  # Use Intelligent Tiering for automatic cost optimization
+  nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
+  ```
+
+  The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
+
+  See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.
+
+
+## Store path info JSON format changes
+
+The JSON format emitted by `nix path-info --json` has been updated to a new version with improved structure.
+
+To maintain compatibility, `nix path-info --json` now requires a `--json-format` flag to specify the output format version.
+Using `--json` without `--json-format` is deprecated and will become an error in a future release.
+For now, it defaults to version 1 with a warning, for a smoother migration.
+
+### Version 1 (`--json-format 1`)
+
+This is the legacy format, preserved for backwards compatibility:
+
+- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
+- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
+- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
+- Now includes `"storeDir"` field at the top level
+
+### Version 2 (`--json-format 2`)
+
+The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
+
+- **Nested structure with top-level metadata**:
+
+  The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
+
+  ```json
+  {
+    "version": 2,
+    "storeDir": "/nix/store",
+    "info": { ... }
+  }
+  ```
+
+  The map from store path base names to store object info is nested under the `info` field.
+
+- **Store path base names instead of full paths**:
+
+  Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
+  Combined with `storeDir`, the full path can be reconstructed.
+
+- **Structured `ca` field**:
+
+  Content address is now a structured JSON object instead of a string:
+
+  - Old: `"ca": "fixed:r:sha256:1abc..."`
+  - New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
+  - Still `null` values for input-addressed store objects
+
+  The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
+
+Additionally the following fields are added to both formats:
+
+  - **`version` field**:
+
+    All store path info JSON now includes `"version": <1|2>`. The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.
+
+  - **`storeDir` field**:
+
+    Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
+
+## Derivation JSON format changes
+
+The derivation JSON format has been updated from version 3 to version 4:
+
+- **Nested structure with top-level metadata**:
+
+  The output of `nix derivation show` is now wrapped in an object with `version` and `derivations` fields:
+
+  ```json
+  {
+    "version": 4,
+    "derivations": { ... }
+  }
+  ```
+
+  The map from derivation paths to derivation info is nested under the `derivations` field.
+
+  This matches the structure used for `nix path-info --json --json-format 2`, and likewise brings this command into compliance with the JSON guidelines.
+
+- **Restructured inputs**:
+
+  Inputs are now nested under an `inputs` object:
+
+  - Old: `"inputSrcs": [...], "inputDrvs": {...}`
+  - New: `"inputs": {"srcs": [...], "drvs": {...}}`
+
+- **Consistent content addresses**:
+
+  Fixed content-addressed outputs now use structured JSON format.
+  This is the same format as `ca` in store path info (after the new version).
+
+Version 3 and earlier formats are *not* accepted when reading.
+
+**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.
+
+## Miscellaneous changes
+
+- Git fetcher: Restore progress indication [#14487](https://github.com/NixOS/nix/pull/14487)
+
+  Nix used to feel "stuck" while it was cloning large repositories. Nix now shows Git's native progress indicator while fetching.
+
+  Upstreamed from [Determinate Nix 3.13.0](https://github.com/DeterminateSystems/nix-src/pull/250).
+
+- Interrupting REPL commands works more than once [#13481](https://github.com/NixOS/nix/issues/13481)
+
+  Previously, this only worked once per REPL session; further attempts would be ignored.
+  This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
+  This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).
+
+- NAR unpacking code has been rewritten to make use of dirfd-based `openat` and `openat2` system calls when available [#14597](https://github.com/NixOS/nix/pull/14597)
+
+- Dynamic size unit rendering [#14423](https://github.com/NixOS/nix/pull/14423) [#14364](https://github.com/NixOS/nix/pull/14364)
+
+  Various commands and the progress bar now use dynamically determined size units instead
+  of always using `MiB`. For example, the progress bar now reports download status like:
+
+  ```
+  [1/196/197 copied (773.7 MiB/2.1 GiB), 172.4/421.5 MiB DL]
+  ```
+
+  Instead of:
+
+  ```
+  [1/196/197 copied (773.7/2147.3 MiB), 172.4/421.5 MiB DL]
+  ```
+
+## Contributors
+
+This release was made possible by the following 33 contributors:
+
+- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and)
+- jonhermansen [**(@jonhermansen)**](https://github.com/jonhermansen)
+- Arnout Engelen [**(@raboof)**](https://github.com/raboof)
+- Jean-François Roche [**(@jfroche)**](https://github.com/jfroche)
+- tomberek [**(@tomberek)**](https://github.com/tomberek)
+- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra)
+- Marcel [**(@MarcelCoding)**](https://github.com/MarcelCoding)
+- David McFarland [**(@corngood)**](https://github.com/corngood)
+- Soumyadip Sarkar [**(@neuralsorcerer)**](https://github.com/neuralsorcerer)
+- Cole Helbling [**(@cole-h)**](https://github.com/cole-h)
+- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314)
+- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy)
+- Alex Auvolat [**(@Alexis211)**](https://github.com/Alexis211)
+- edef [**(@edef1c)**](https://github.com/edef1c)
+- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium)
+- Vinayak Goyal [**(@vinayakankugoyal)**](https://github.com/vinayakankugoyal)
+- Graham Dennis [**(@GrahamDennis)**](https://github.com/GrahamDennis)
+- Aspen Smith [**(@glittershark)**](https://github.com/glittershark)
+- Jens Petersen [**(@juhp)**](https://github.com/juhp)
+- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault)
+- Peter Bynum [**(@pkpbynum)**](https://github.com/pkpbynum)
+- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92)
+- Alex Decious [**(@adeci)**](https://github.com/adeci)
+- Matthieu Coudron [**(@teto)**](https://github.com/teto)
+- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar)
+- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii)
+- Seth Flynn [**(@getchoo)**](https://github.com/getchoo)
+- Robert Hensing [**(@roberth)**](https://github.com/roberth)
+- Vladimir Panteleev [**(@CyberShadow)**](https://github.com/CyberShadow)
+- bryango [**(@bryango)**](https://github.com/bryango)
+- Henry [**(@cootshk)**](https://github.com/cootshk)
+- Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg)
+- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)

maintainers/data/release-credits-email-to-handle.json

@@ -224,5 +224,25 @@
     "42688647+netadr@users.noreply.github.com": "netadr",
     "matej.urbas@gmail.com": "urbas",
     "ethanalexevans@gmail.com": "ethanavatar",
-    "greg.marti@gmail.com": "gmarti"
+    "greg.marti@gmail.com": "gmarti",
+    "arnout@bzzt.net": "raboof",
+    "vinayakankugoyal@gmail.com": "vinayakankugoyal",
+    "Radvendii@users.noreply.github.com": "Radvendii",
+    "jon@jh86.org": "jonhermansen",
+    "edef@edef.eu": "edef1c",
+    "pkpbynum@gmail.com": "pkpbynum",
+    "886074+teto@users.noreply.github.com": "teto",
+    "alex@adnab.me": "Alexis211",
+    "root@gws.fyi": "glittershark",
+    "me@m4rc3l.de": "MarcelCoding",
+    "taeer.bar-yam@bevuta.com": "Radvendii",
+    "martin.joerg@gmail.com": "mjoerg",
+    "git@cy.md": "CyberShadow",
+    "cootshk@duck.com": "cootshk",
+    "adam@dinwoodie.org": "me-and",
+    "domen@cachix.org": "domenkozar",
+    "alex.decious@gmail.com": "adeci",
+    "soumya.papanvk18@gmail.com": "neuralsorcerer",
+    "gdennis@anduril.com": null,
+    "graham.dennis@gmail.com": "GrahamDennis"
 }

maintainers/data/release-credits-handle-to-name.json

@@ -196,5 +196,21 @@
     "gmarti": "Gr\u00e9gory Marti",
     "lovesegfault": "Bernardo Meurer",
     "EphraimSiegfried": "Ephraim Siegfried",
-    "hgl": "Glen Huang"
+    "hgl": "Glen Huang",
+    "mjoerg": "Martin Joerg",
+    "Alexis211": "Alex Auvolat",
+    "domenkozar": "Domen Ko\u017ear",
+    "edef1c": "edef",
+    "cootshk": "Henry",
+    "raboof": "Arnout Engelen",
+    "pkpbynum": "Peter Bynum",
+    "glittershark": "Aspen Smith",
+    "MarcelCoding": "Marcel",
+    "teto": "Matthieu Coudron",
+    "jonhermansen": null,
+    "neuralsorcerer": "Soumyadip Sarkar",
+    "adeci": "Alex Decious",
+    "vinayakankugoyal": "Vinayak Goyal",
+    "me-and": "Adam Dinwoodie",
+    "GrahamDennis": "Graham Dennis"
 }

packaging/dev-shell.nix

@@ -148,6 +148,15 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
     isInternal =
       dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
 
+    activeComponentNames = lib.listToAttrs (
+      map (c: {
+        name = c.pname or c.name;
+        value = null;
+      }) activeComponents
+    );
+
+    isActiveComponent = name: activeComponentNames ? ${name};
+
   in
   {
     pname = "shell-for-nix";
@@ -190,27 +199,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
           }
         );
 
-      small =
-        (finalAttrs.finalPackage.withActiveComponents (
-          c:
-          lib.intersectAttrs (lib.genAttrs [
-            "nix-cli"
-            "nix-util-tests"
-            "nix-store-tests"
-            "nix-expr-tests"
-            "nix-fetchers-tests"
-            "nix-flake-tests"
-            "nix-functional-tests"
-            "nix-perl-bindings"
-          ] (_: null)) c
-        )).overrideAttrs
-          (o: {
-            mesonFlags = o.mesonFlags ++ [
-              # TODO: infer from activeComponents or vice versa
-              "-Dkaitai-struct-checks=false"
-              "-Djson-schema-checks=false"
-            ];
-          });
+      small = finalAttrs.finalPackage.withActiveComponents (
+        c:
+        lib.intersectAttrs (lib.genAttrs [
+          "nix-cli"
+          "nix-util-tests"
+          "nix-store-tests"
+          "nix-expr-tests"
+          "nix-fetchers-tests"
+          "nix-flake-tests"
+          "nix-functional-tests"
+          "nix-perl-bindings"
+        ] (_: null)) c
+      );
     };
 
     # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
@@ -258,10 +259,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
 
     # We use this shell with the local checkout, not unpackPhase.
     src = null;
+
     # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html
     # Remove when gdb fix is rolled out everywhere.
     separateDebugInfo = false;
 
+    mesonBuildType = "debugoptimized";
+
     env = {
       # For `make format`, to work without installing pre-commit
       _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml"
@@ -275,21 +279,33 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
 
     dontUseCmakeConfigure = true;
 
-    mesonFlags =
-      map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
-      ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
-      ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
-      ++ lib.optionals havePerl (
-        map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
-      )
-      ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
-      ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
+    mesonFlags = [
+      (lib.mesonBool "kaitai-struct-checks" (isActiveComponent "nix-kaitai-struct-checks"))
+      (lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks"))
+    ]
+    ++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
+    ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
+    ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
+    ++ lib.optionals havePerl (
+      map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
+    )
+    ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
+    ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
 
     nativeBuildInputs =
       let
         inputs =
           dedupByString (v: "${v}") (
-            lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents)
+            lib.filter (x: !isInternal x) (
+              lib.lists.concatMap (
+                # Nix manual has a build-time dependency on nix, but we
+                # don't want to do a native build just to enter the ross
+                # dev shell.
+                #
+                # TODO: think of a more principled fix for this.
+                c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs
+              ) activeComponents
+            )
           )
           ++ lib.optional (
             !buildCanExecuteHost
@@ -305,8 +321,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
             pkgs.buildPackages.nixfmt-rfc-style
             pkgs.buildPackages.shellcheck
             pkgs.buildPackages.include-what-you-use
-            pkgs.buildPackages.gdb
           ]
+          ++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
           ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
             lib.hiPrio pkgs.buildPackages.clang-tools
           )
@@ -322,13 +338,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
       )
     );
 
-    buildInputs = [
-      pkgs.gbenchmark
-    ]
-    ++ dedupByString (v: "${v}") (
-      lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
-    )
-    ++ lib.optional havePerl pkgs.perl;
+    buildInputs =
+      # TODO change Nixpkgs to mark gbenchmark as building on Windows
+      lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
+      ++ dedupByString (v: "${v}") (
+        lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
+      )
+      ++ lib.optional havePerl pkgs.perl;
 
     propagatedBuildInputs = dedupByString (v: "${v}") (
       lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)

src/json-schema-checks/meson.build

@@ -62,9 +62,11 @@ schemas = [
   },
   {
     'stem' : 'build-trace-entry',
-    'schema' : schema_dir / 'build-trace-entry-v1.yaml',
+    'schema' : schema_dir / 'build-trace-entry-v2.yaml',
     'files' : [
       'simple.json',
+      # The field is no longer supported, but we want to show that we
+      # ignore it during parsing.
       'with-dependent-realisations.json',
       'with-signature.json',
     ],

src/libexpr/eval.cc

@@ -741,6 +741,11 @@ public:
         inDebugger = true;
     }
 
+    DebuggerGuard(DebuggerGuard &&) = delete;
+    DebuggerGuard(const DebuggerGuard &) = delete;
+    DebuggerGuard & operator=(DebuggerGuard &&) = delete;
+    DebuggerGuard & operator=(const DebuggerGuard &) = delete;
+
     ~DebuggerGuard()
     {
         inDebugger = false;

src/libexpr/include/nix/expr/attr-set.hh

@@ -107,6 +107,8 @@ private:
     Bindings & operator=(const Bindings &) = delete;
     Bindings & operator=(Bindings &&) = delete;
 
+    ~Bindings() = default;
+
     friend class BindingsBuilder;
 
     /**

src/libexpr/include/nix/expr/value.hh

@@ -164,8 +164,6 @@ public:
     Value ** elems;
     ListBuilder(EvalMemory & mem, size_t size);
 
-    // NOTE: Can be noexcept because we are just copying integral values and
-    // raw pointers.
     ListBuilder(ListBuilder && x) noexcept
         : size(x.size)
         , inlineElems{x.inlineElems[0], x.inlineElems[1]}
@@ -173,6 +171,11 @@ public:
     {
     }
 
+    ListBuilder(const ListBuilder &) = delete;
+    ListBuilder & operator=(ListBuilder &&) = delete;
+    ListBuilder & operator=(const ListBuilder &) = delete;
+    ~ListBuilder() = default;
+
     Value *& operator[](size_t n)
     {
         return elems[n];

src/libexpr/primops.cc

@@ -4061,6 +4061,8 @@ static RegisterPrimOp primop_sort({
 
         1. Transitivity
 
+        If a is less than b and b is less than c, then it follows that a is less than c.
+
         ```nix
         comparator a b && comparator b c -> comparator a c
         ```
@@ -4073,9 +4075,23 @@ static RegisterPrimOp primop_sort({
 
         1. Transitivity of equivalence
 
+        First, two values a and b are considered equivalent with respect to the comparator if:
+
+        ```
+        !comparator a b && !comparator b a
+        ```
+
+        In other words, neither is considered "less than" the other.
+
+        Transitivity of equivalence means:
+
+        If a is equivalent to b, and b is equivalent to c, then a must also be equivalent to c.
+
         ```nix
-        let equiv = a: b: (!comparator a b && !comparator b a); in
-        equiv a b && equiv b c -> equiv a c
+        let
+          equiv = x: y: (!comparator x y && !comparator y x);
+        in
+          equiv a b && equiv b c -> equiv a c
         ```
 
       If the *comparator* violates any of these properties, then `builtins.sort`

src/libfetchers-tests/git-utils.cc

@@ -48,7 +48,7 @@ public:
 
     ref<GitRepo> openRepo()
     {
-        return GitRepo::openRepo(tmpDir, true, false);
+        return GitRepo::openRepo(tmpDir, {.create = true});
     }
 
     std::string getRepoName() const

src/libfetchers/git-utils.cc

@@ -203,16 +203,19 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
 
 } // extern "C"
 
-static void initRepoAtomically(std::filesystem::path & path, bool bare)
+static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options)
 {
     if (pathExists(path.string()))
         return;
 
+    if (!options.create)
+        throw Error("Git repository %s does not exist.", path);
+
     std::filesystem::path tmpDir = createTempDir(path.parent_path());
     AutoDelete delTmpDir(tmpDir, true);
     Repository tmpRepo;
 
-    if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), bare))
+    if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
         throw Error("creating Git repository %s: %s", path, git_error_last()->message);
     try {
         std::filesystem::rename(tmpDir, path);
@@ -234,7 +237,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     /** Location of the repository on disk. */
     std::filesystem::path path;
 
-    bool bare;
+    Options options;
 
     /**
      * libgit2 repository. Note that new objects are not written to disk,
@@ -255,18 +258,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
      */
     git_odb_backend * packBackend = nullptr;
 
-    GitRepoImpl(std::filesystem::path _path, bool create, bool bare, bool packfilesOnly = false)
+    GitRepoImpl(std::filesystem::path _path, Options _options)
         : path(std::move(_path))
-        , bare(bare)
+        , options(_options)
     {
         initLibGit2();
 
-        initRepoAtomically(path, bare);
+        initRepoAtomically(path, options);
         if (git_repository_open(Setter(repo), path.string().c_str()))
             throw Error("opening Git repository %s: %s", path, git_error_last()->message);
 
         ObjectDb odb;
-        if (packfilesOnly) {
+        if (options.packfilesOnly) {
             /* Create a fresh object database because by default the repo also
                loose object backends. We are not using any of those for the
                tarball cache, but libgit2 still does a bunch of unnecessary
@@ -295,7 +298,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         if (git_odb_add_backend(odb.get(), mempackBackend, 999))
             throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
 
-        if (packfilesOnly) {
+        if (options.packfilesOnly) {
             if (git_repository_set_odb(repo.get(), odb.get()))
                 throw Error("setting Git object database: %s", git_error_last()->message);
         }
@@ -366,7 +369,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     {
         // TODO: as an optimization, it would be nice to include `this` in the pool.
         return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
-            return make_ref<GitRepoImpl>(path, false, bare);
+            return make_ref<GitRepoImpl>(path, options);
         });
     }
 
@@ -712,9 +715,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 };
 
-ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare, bool packfilesOnly)
+ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Options options)
 {
-    return make_ref<GitRepoImpl>(path, create, bare, packfilesOnly);
+    return make_ref<GitRepoImpl>(path, options);
 }
 
 /**
@@ -1427,8 +1430,12 @@ namespace fetchers {
 
 ref<GitRepo> Settings::getTarballCache() const
 {
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
-    return GitRepo::openRepo(repoDir, /*create=*/true, /*bare=*/true, /*packfilesOnly=*/true);
+    /* v1: Had either only loose objects or thin packfiles referring to loose objects
+     * v2: Must have only packfiles with no loose objects. Should get repacked periodically
+     * for optimal packfiles.
+     */
+    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2";
+    return GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true});
 }
 
 } // namespace fetchers
@@ -1442,7 +1449,7 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path &
         if (i != cache->end())
             return i->second;
     }
-    auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo();
+    auto workdirInfo = GitRepo::openRepo(path, {})->getWorkdirInfo();
     _cache.lock()->emplace(path, workdirInfo);
     return workdirInfo;
 }

src/libfetchers/git.cc

@@ -637,11 +637,6 @@ struct GitInputScheme : InputScheme
                     url);
             }
 
-            // If we don't check here for the path existence, then we can give libgit2 any directory
-            // and it will initialize them as git directories.
-            if (!pathExists(path)) {
-                throw Error("The path '%s' does not exist.", path);
-            }
             repoInfo.location = std::filesystem::absolute(path);
         } else {
             if (url.scheme == "file")
@@ -703,7 +698,7 @@ struct GitInputScheme : InputScheme
         if (auto res = cache->lookup(key))
             return getIntAttr(*res, "lastModified");
 
-        auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev);
+        auto lastModified = GitRepo::openRepo(repoDir, {})->getLastModified(rev);
 
         cache->upsert(key, {{"lastModified", lastModified}});
 
@@ -726,7 +721,7 @@ struct GitInputScheme : InputScheme
         Activity act(
             *logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg()));
 
-        auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev);
+        auto revCount = GitRepo::openRepo(repoDir, {})->getRevCount(rev);
 
         cache->upsert(key, Attrs{{"revCount", revCount}});
 
@@ -737,7 +732,7 @@ struct GitInputScheme : InputScheme
     {
         auto head = std::visit(
             overloaded{
-                [&](const std::filesystem::path & path) { return GitRepo::openRepo(path)->getWorkdirRef(); },
+                [&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
                 [&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
             repoInfo.location);
         if (!head) {
@@ -795,7 +790,7 @@ struct GitInputScheme : InputScheme
         if (auto repoPath = repoInfo.getPath()) {
             repoDir = *repoPath;
             if (!input.getRev())
-                input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
+                input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir, {})->resolveRef(ref).gitRev());
         } else {
             auto repoUrl = std::get<ParsedURL>(repoInfo.location);
             std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
@@ -805,7 +800,7 @@ struct GitInputScheme : InputScheme
             std::filesystem::create_directories(cacheDir.parent_path());
             PathLocks cacheDirLock({cacheDir.string()});
 
-            auto repo = GitRepo::openRepo(cacheDir, true, true);
+            auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true});
 
             // We need to set the origin so resolving submodule URLs works
             repo->setRemote("origin", repoUrl.to_string());
@@ -876,7 +871,7 @@ struct GitInputScheme : InputScheme
             // the remainder
         }
 
-        auto repo = GitRepo::openRepo(repoDir);
+        auto repo = GitRepo::openRepo(repoDir, {});
 
         auto isShallow = repo->isShallow();
 
@@ -963,7 +958,7 @@ struct GitInputScheme : InputScheme
             for (auto & submodule : repoInfo.workdirInfo.submodules)
                 repoInfo.workdirInfo.files.insert(submodule.path);
 
-        auto repo = GitRepo::openRepo(repoPath, false, false);
+        auto repo = GitRepo::openRepo(repoPath, {});
 
         auto exportIgnore = getExportIgnoreAttr(input);
 
@@ -1003,7 +998,7 @@ struct GitInputScheme : InputScheme
         }
 
         if (!repoInfo.workdirInfo.isDirty) {
-            auto repo = GitRepo::openRepo(repoPath);
+            auto repo = GitRepo::openRepo(repoPath, {});
 
             if (auto ref = repo->getWorkdirRef())
                 input.attrs.insert_or_assign("ref", *ref);

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -32,8 +32,14 @@ struct GitRepo
 {
     virtual ~GitRepo() {}
 
-    static ref<GitRepo>
-    openRepo(const std::filesystem::path & path, bool create = false, bool bare = false, bool packfilesOnly = false);
+    struct Options
+    {
+        bool create = false;
+        bool bare = false;
+        bool packfilesOnly = false;
+    };
+
+    static ref<GitRepo> openRepo(const std::filesystem::path & path, Options options);
 
     virtual uint64_t getRevCount(const Hash & rev) = 0;
 

src/libstore-test-support/include/nix/store/tests/protocol.hh

@@ -88,25 +88,38 @@ public:
     }
 };
 
-#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    TEST_F(FIXTURE, NAME##_read)                                                     \
-    {                                                                                \
-        readProtoTest(STEM, VERSION, VALUE);                                         \
-    }                                                                                \
-    TEST_F(FIXTURE, NAME##_write)                                                    \
-    {                                                                                \
-        writeProtoTest(STEM, VERSION, VALUE);                                        \
+#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_read)                                                          \
+    {                                                                                     \
+        readProtoTest(STEM, VERSION, VALUE);                                              \
     }
 
-#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
-    VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    TEST_F(FIXTURE, NAME##_json_read)                                            \
-    {                                                                            \
-        readJsonTest(STEM, VALUE);                                               \
-    }                                                                            \
-    TEST_F(FIXTURE, NAME##_json_write)                                           \
-    {                                                                            \
-        writeJsonTest(STEM, VALUE);                                              \
+#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_write)                                                          \
+    {                                                                                      \
+        writeProtoTest(STEM, VERSION, VALUE);                                              \
     }
 
+#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)  \
+    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
+
+#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
+    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_json_read)                                                 \
+    {                                                                                 \
+        readJsonTest(STEM, VALUE);                                                    \
+    }
+
+#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_json_write)                                                 \
+    {                                                                                  \
+        writeJsonTest(STEM, VALUE);                                                    \
+    }
+
+#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)  \
+    VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
+
 } // namespace nix

src/libstore-tests/common-protocol.cc

@@ -47,24 +47,30 @@ public:
     }
 };
 
-#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
-    TEST_F(CommonProtoTest, NAME##_read)         \
-    {                                            \
-        readProtoTest(STEM, VALUE);              \
-    }                                            \
-    TEST_F(CommonProtoTest, NAME##_write)        \
-    {                                            \
-        writeProtoTest(STEM, VALUE);             \
-    }                                            \
-    TEST_F(CommonProtoTest, NAME##_json_read)    \
-    {                                            \
-        readJsonTest(STEM, VALUE);               \
-    }                                            \
-    TEST_F(CommonProtoTest, NAME##_json_write)   \
-    {                                            \
-        writeJsonTest(STEM, VALUE);              \
+#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
+    TEST_F(CommonProtoTest, NAME##_read)              \
+    {                                                 \
+        readProtoTest(STEM, VALUE);                   \
+    }                                                 \
+    TEST_F(CommonProtoTest, NAME##_json_read)         \
+    {                                                 \
+        readJsonTest(STEM, VALUE);                    \
     }
 
+#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
+    TEST_F(CommonProtoTest, NAME##_write)              \
+    {                                                  \
+        writeProtoTest(STEM, VALUE);                   \
+    }                                                  \
+    TEST_F(CommonProtoTest, NAME##_json_write)         \
+    {                                                  \
+        writeJsonTest(STEM, VALUE);                    \
+    }
+
+#define CHARACTERIZATION_TEST(NAME, STEM, VALUE)  \
+    READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
+    WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE)
+
 CHARACTERIZATION_TEST(
     string,
     "string",
@@ -141,7 +147,7 @@ CHARACTERIZATION_TEST(
         },
     }))
 
-CHARACTERIZATION_TEST(
+READ_CHARACTERIZATION_TEST(
     realisation_with_deps,
     "realisation-with-deps",
     (std::tuple<Realisation>{
@@ -149,16 +155,6 @@ CHARACTERIZATION_TEST(
             {
                 .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
                 .signatures = {"asdf", "qwer"},
-                .dependentRealisations =
-                    {
-                        {
-                            DrvOutput{
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "quux",
-                            },
-                            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
-                        },
-                    },
             },
             {
                 .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),

src/libstore-tests/data/worker-substitution/ca-drv/store-after.json

@@ -0,0 +1,58 @@
+{
+  "buildTrace": {
+    "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
+      "contents": {
+        "contents": "I am the output of a CA derivation",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+        "narSize": 152,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {
+    "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "test-ca-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/ca-drv/store-before.json

@@ -0,0 +1,27 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {
+    "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "test-ca-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/ca-drv/substituter.json

@@ -0,0 +1,39 @@
+{
+  "buildTrace": {
+    "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
+      "contents": {
+        "contents": "I am the output of a CA derivation",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+        "narSize": 152,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/issue-11928/store-after.json

@@ -0,0 +1,83 @@
+{
+  "buildTrace": {
+    "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
+      "contents": {
+        "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+        "narSize": 232,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {
+    "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {
+          "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+            "dynamicOutputs": {},
+            "outputs": [
+              "out"
+            ]
+          }
+        },
+        "srcs": []
+      },
+      "name": "root-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    },
+    "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "dep-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/issue-11928/store-before.json

@@ -0,0 +1,52 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {
+    "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {
+          "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+            "dynamicOutputs": {},
+            "outputs": [
+              "out"
+            ]
+          }
+        },
+        "srcs": []
+      },
+      "name": "root-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    },
+    "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "dep-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/issue-11928/substituter.json

@@ -0,0 +1,68 @@
+{
+  "buildTrace": {
+    "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
+        "signatures": []
+      }
+    },
+    "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
+      "contents": {
+        "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+        "narSize": 232,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    },
+    "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": {
+      "contents": {
+        "contents": "I am the dependency output",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
+        "narSize": 144,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/single/substituter.json

@@ -0,0 +1,31 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": {
+      "contents": {
+        "contents": "Hello, world!",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
+        "narSize": 128,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/with-dep/store.json

@@ -0,0 +1,55 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
+      "contents": {
+        "contents": "I am a dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+        "narSize": 136,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    },
+    "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
+      "contents": {
+        "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+        "narSize": 184,
+        "references": [
+          "4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
+        ],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/with-dep/substituter.json

@@ -0,0 +1,55 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
+      "contents": {
+        "contents": "I am a dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+        "narSize": 136,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    },
+    "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
+      "contents": {
+        "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+        "narSize": 184,
+        "references": [
+          "4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
+        ],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/derivation-parser-bench.cc

@@ -2,7 +2,7 @@
 #include "nix/store/derivations.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/experimental-features.hh"
-#include "nix/util/environment-variables.hh"
+#include "nix/util/tests/test-data.hh"
 #include "nix/store/store-open.hh"
 #include <fstream>
 #include <sstream>
@@ -50,11 +50,7 @@ static void BM_UnparseRealDerivationFile(benchmark::State & state, const std::st
 }
 
 // Register benchmarks for actual test derivation files if they exist
-BENCHMARK_CAPTURE(
-    BM_ParseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
-BENCHMARK_CAPTURE(
-    BM_ParseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
-BENCHMARK_CAPTURE(
-    BM_UnparseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
-BENCHMARK_CAPTURE(
-    BM_UnparseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
+BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
+BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());
+BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
+BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());

src/libstore-tests/meson.build

@@ -85,6 +85,7 @@ sources = files(
   'store-reference.cc',
   'uds-remote-store.cc',
   'worker-protocol.cc',
+  'worker-substitution.cc',
   'write-derivation.cc',
 )
 
@@ -123,6 +124,7 @@ if get_option('benchmarks')
     'bench-main.cc',
     'derivation-parser-bench.cc',
     'ref-scan-bench.cc',
+    'register-valid-paths-bench.cc',
   )
 
   benchmark_exe = executable(

src/libstore-tests/nix_api_store.cc

@@ -8,6 +8,7 @@
 #include "nix/store/tests/nix_api_store.hh"
 #include "nix/store/globals.hh"
 #include "nix/util/tests/string_callback.hh"
+#include "nix/util/tests/test-data.hh"
 #include "nix/util/url.hh"
 
 #include "store-tests-config.hh"
@@ -302,7 +303,7 @@ public:
 
         store = open_local_store();
 
-        std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+        std::filesystem::path unitTestData = nix::getUnitTestData();
         std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
         std::stringstream buffer;
         buffer << t.rdbuf();
@@ -357,7 +358,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
 
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
@@ -404,7 +405,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
     buffer << t.rdbuf();
@@ -449,7 +450,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
     buffer << t.rdbuf();
@@ -494,7 +495,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
     buffer << t.rdbuf();
@@ -870,7 +871,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagati
  */
 static std::string load_json_from_test_data(const char * filename)
 {
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / filename};
     std::stringstream buffer;
     buffer << t.rdbuf();

src/libstore-tests/realisation.cc

@@ -44,54 +44,45 @@ TEST_P(RealisationJsonTest, to_json)
     writeJsonTest(name, value);
 }
 
+Realisation simple{
+    {
+        .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
+    },
+    {
+        .drvHash = Hash::parseExplicitFormatUnprefixed(
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+            HashAlgorithm::SHA256,
+            HashFormat::Base16),
+        .outputName = "foo",
+    },
+};
+
 INSTANTIATE_TEST_SUITE_P(
     RealisationJSON,
     RealisationJsonTest,
-    ([] {
-        Realisation simple{
-            {
-                .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
-            },
-            {
-                .drvHash = Hash::parseExplicitFormatUnprefixed(
-                    "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
-                    HashAlgorithm::SHA256,
-                    HashFormat::Base16),
-                .outputName = "foo",
-            },
-        };
-        return ::testing::Values(
-            std::pair{
-                "simple",
-                simple,
-            },
-            std::pair{
-                "with-signature",
-                [&] {
-                    auto r = simple;
-                    // FIXME actually sign properly
-                    r.signatures = {"asdfasdfasdf"};
-                    return r;
-                }()},
-            std::pair{
-                "with-dependent-realisations",
-                [&] {
-                    auto r = simple;
-                    r.dependentRealisations = {{
-                        {
-                            .drvHash = Hash::parseExplicitFormatUnprefixed(
-                                "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
-                                HashAlgorithm::SHA256,
-                                HashFormat::Base16),
-                            .outputName = "foo",
-                        },
-                        StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
-                    }};
-                    return r;
-                }(),
-            });
-    }
+    ::testing::Values(
+        std::pair{
+            "simple",
+            simple,
+        },
+        std::pair{
+            "with-signature",
+            [&] {
+                auto r = simple;
+                // FIXME actually sign properly
+                r.signatures = {"asdfasdfasdf"};
+                return r;
+            }(),
+        }));
 
-     ()));
+/**
+ * We no longer have a notion of "dependent realisations", but we still
+ * want to parse old realisation files. So make this just be a read test
+ * (no write direction), accordingly.
+ */
+TEST_F(RealisationTest, dependent_realisations_from_json)
+{
+    readJsonTest("with-dependent-realisations", simple);
+}
 
 } // namespace nix

src/libstore-tests/register-valid-paths-bench.cc

@@ -0,0 +1,79 @@
+#include <benchmark/benchmark.h>
+
+#include "nix/store/derivations.hh"
+#include "nix/store/local-store.hh"
+#include "nix/store/store-open.hh"
+#include "nix/util/file-system.hh"
+#include "nix/util/hash.hh"
+#include "nix/util/tests/test-data.hh"
+
+#ifndef _WIN32
+
+#  include <filesystem>
+#  include <fstream>
+
+using namespace nix;
+
+static void BM_RegisterValidPathsDerivations(benchmark::State & state)
+{
+    const int derivationCount = state.range(0);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+
+        auto tmpRoot = createTempDir();
+        auto realStoreDir = tmpRoot / "nix/store";
+        std::filesystem::create_directories(realStoreDir);
+
+        std::shared_ptr<Store> store = openStore(fmt("local?root=%s", tmpRoot.string()));
+        auto localStore = std::dynamic_pointer_cast<LocalStore>(store);
+        if (!localStore)
+            throw Error("expected local store");
+
+        ValidPathInfos infos;
+        for (int i = 0; i < derivationCount; ++i) {
+            std::string drvName = fmt("register-valid-paths-bench-%d", i);
+            auto drvPath = StorePath::random(drvName + ".drv");
+
+            Derivation drv;
+            drv.name = drvName;
+            drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}});
+            drv.platform = "x86_64-linux";
+            drv.builder = "foo";
+            drv.env["out"] = "";
+            drv.fillInOutputPaths(*localStore);
+
+            auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false);
+
+            /* Create an on-disk store object without registering it
+               in the SQLite DB. LocalFSStore::getFSAccessor(path, false)
+               allows reading store objects based on their filesystem
+               presence alone. */
+            std::ofstream out(realStoreDir / std::string(drvPath.to_string()), std::ios::binary);
+            out.write(drvContents.data(), drvContents.size());
+            if (!out)
+                throw SysError("writing derivation to store");
+
+            ValidPathInfo info{drvPath, UnkeyedValidPathInfo(*localStore, Hash::dummy)};
+            info.narSize = drvContents.size();
+
+            infos.emplace(drvPath, std::move(info));
+        }
+
+        state.ResumeTiming();
+
+        localStore->registerValidPaths(infos);
+
+        state.PauseTiming();
+        localStore.reset();
+        store.reset();
+        std::filesystem::remove_all(tmpRoot);
+        state.ResumeTiming();
+    }
+
+    state.SetItemsProcessed(state.iterations() * derivationCount);
+}
+
+BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10);
+
+#endif

src/libstore-tests/serve-protocol.cc

@@ -118,7 +118,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         },
     }))
 
-VERSIONED_CHARACTERIZATION_TEST(
+VERSIONED_READ_CHARACTERIZATION_TEST(
     ServeProtoTest,
     realisation_with_deps,
     "realisation-with-deps",
@@ -128,16 +128,6 @@ VERSIONED_CHARACTERIZATION_TEST(
             {
                 .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
                 .signatures = {"asdf", "qwer"},
-                .dependentRealisations =
-                    {
-                        {
-                            DrvOutput{
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "quux",
-                            },
-                            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
-                        },
-                    },
             },
             {
                 .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),

src/libstore-tests/worker-protocol.cc

@@ -171,7 +171,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         },
     }))
 
-VERSIONED_CHARACTERIZATION_TEST(
+VERSIONED_READ_CHARACTERIZATION_TEST(
     WorkerProtoTest,
     realisation_with_deps,
     "realisation-with-deps",
@@ -181,16 +181,6 @@ VERSIONED_CHARACTERIZATION_TEST(
             {
                 .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
                 .signatures = {"asdf", "qwer"},
-                .dependentRealisations =
-                    {
-                        {
-                            DrvOutput{
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "quux",
-                            },
-                            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
-                        },
-                    },
             },
             {
                 .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),

src/libstore-tests/worker-substitution.cc

@@ -0,0 +1,450 @@
+#include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
+
+#include "nix/store/build/worker.hh"
+#include "nix/store/derivations.hh"
+#include "nix/store/dummy-store-impl.hh"
+#include "nix/store/globals.hh"
+#include "nix/util/memory-source-accessor.hh"
+
+#include "nix/store/tests/libstore.hh"
+#include "nix/util/tests/json-characterization.hh"
+
+namespace nix {
+
+class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest<ref<DummyStore>>
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution";
+
+protected:
+    ref<DummyStore> dummyStore;
+    ref<DummyStore> substituter;
+
+    WorkerSubstitutionTest()
+        : LibStoreTest([] {
+            auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
+            config->readOnly = false;
+            return config->openDummyStore();
+        }())
+        , dummyStore(store.dynamic_pointer_cast<DummyStore>())
+        , substituter([] {
+            auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
+            config->readOnly = false;
+            config->isTrusted = true;
+            return config->openDummyStore();
+        }())
+    {
+    }
+
+public:
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+
+    static void SetUpTestSuite()
+    {
+        initLibStore(false);
+    }
+};
+
+TEST_F(WorkerSubstitutionTest, singleStoreObject)
+{
+    // Add a store path to the substituter
+    auto pathInSubstituter = substituter->addToStore(
+        "hello",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "Hello, world!",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Snapshot the substituter (has one store object)
+    checkpointJson("single/substituter", substituter);
+
+    // Snapshot the destination store before (should be empty)
+    checkpointJson("../dummy-store/empty", dummyStore);
+
+    // The path should not exist in the destination store yet
+    ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituerAsStore = substituter;
+    worker.getSubstituters = [substituerAsStore]() -> std::list<ref<Store>> { return {substituerAsStore}; };
+
+    // Create a substitution goal for the path
+    auto goal = worker.makePathSubstitutionGoal(pathInSubstituter);
+
+    // Run the worker with -j0 semantics (no local builds, only substitution)
+    // The worker.run() takes a set of goals
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after (should match the substituter)
+    checkpointJson("single/substituter", dummyStore);
+
+    // The path should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter));
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+}
+
+TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject)
+{
+    // First, add a dependency store path to the substituter
+    auto dependencyPath = substituter->addToStore(
+        "dependency",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I am a dependency",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Now add a store path that references the dependency
+    auto mainPath = substituter->addToStore(
+        "main",
+        SourcePath{
+            [&] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                // Include a reference to the dependency path in the contents
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I depend on " + substituter->printStorePath(dependencyPath),
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256,
+        StorePathSet{dependencyPath});
+
+    // Snapshot the substituter (has two store objects)
+    checkpointJson("with-dep/substituter", substituter);
+
+    // Snapshot the destination store before (should be empty)
+    checkpointJson("../dummy-store/empty", dummyStore);
+
+    // Neither path should exist in the destination store yet
+    ASSERT_FALSE(dummyStore->isValidPath(dependencyPath));
+    ASSERT_FALSE(dummyStore->isValidPath(mainPath));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituterAsStore = substituter;
+    worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
+
+    // Create a substitution goal for the main path only
+    // The worker should automatically substitute the dependency as well
+    auto goal = worker.makePathSubstitutionGoal(mainPath);
+
+    // Run the worker
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after (should match the substituter)
+    checkpointJson("with-dep/substituter", dummyStore);
+
+    // Both paths should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(dependencyPath));
+    ASSERT_TRUE(dummyStore->isValidPath(mainPath));
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+}
+
+TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
+{
+    // Enable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
+
+    // Create a CA floating output derivation
+    Derivation drv;
+    drv.name = "test-ca-drv";
+    drv.outputs = {
+        {
+            "out",
+            DerivationOutput{DerivationOutput::CAFloating{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hashAlgo = HashAlgorithm::SHA256,
+            }},
+        },
+    };
+
+    // Write the derivation to the destination store
+    auto drvPath = writeDerivation(*dummyStore, drv);
+
+    // Snapshot the destination store before
+    checkpointJson("ca-drv/store-before", dummyStore);
+
+    // Compute the hash modulo of the derivation
+    // For CA floating derivations, the kind is Deferred since outputs aren't known until build
+    auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
+    ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
+    auto drvHash = hashModulo.hashes.at("out");
+
+    // Create the output store object
+    auto outputPath = substituter->addToStore(
+        "test-ca-drv-out",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I am the output of a CA derivation",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Add the realisation (build trace) to the substituter
+    substituter->buildTrace.insert_or_assign(
+        drvHash,
+        std::map<std::string, UnkeyedRealisation>{
+            {
+                "out",
+                UnkeyedRealisation{
+                    .outPath = outputPath,
+                },
+            },
+        });
+
+    // Snapshot the substituter
+    checkpointJson("ca-drv/substituter", substituter);
+
+    // The realisation should not exist in the destination store yet
+    DrvOutput drvOutput{drvHash, "out"};
+    ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituterAsStore = substituter;
+    worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
+
+    // Create a derivation goal for the CA derivation output
+    // The worker should substitute the output rather than building
+    auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true);
+
+    // Run the worker
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after
+    checkpointJson("ca-drv/store-after", dummyStore);
+
+    // The output path should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(outputPath));
+
+    // The realisation should now exist in the destination store
+    auto realisation = dummyStore->queryRealisation(drvOutput);
+    ASSERT_TRUE(realisation);
+    ASSERT_EQ(realisation->outPath, outputPath);
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+
+    // Disable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "");
+}
+
+/**
+ * Test for issue #11928: substituting a CA derivation output should not
+ * require fetching the output of an input derivation when that output
+ * is not referenced.
+ */
+TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
+{
+    // Enable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
+
+    // Create the dependency CA floating derivation
+    Derivation depDrv;
+    depDrv.name = "dep-drv";
+    depDrv.outputs = {
+        {
+            "out",
+            DerivationOutput{DerivationOutput::CAFloating{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hashAlgo = HashAlgorithm::SHA256,
+            }},
+        },
+    };
+
+    // Write the dependency derivation to the destination store
+    auto depDrvPath = writeDerivation(*dummyStore, depDrv);
+
+    // Compute the hash modulo for the dependency derivation
+    auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
+    ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
+    auto depDrvHash = depHashModulo.hashes.at("out");
+
+    // Create the output store object for the dependency in the substituter
+    auto depOutputPath = substituter->addToStore(
+        "dep-drv-out",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I am the dependency output",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Add the realisation for the dependency to the substituter
+    substituter->buildTrace.insert_or_assign(
+        depDrvHash,
+        std::map<std::string, UnkeyedRealisation>{
+            {
+                "out",
+                UnkeyedRealisation{
+                    .outPath = depOutputPath,
+                },
+            },
+        });
+
+    // Create the root CA floating derivation that depends on depDrv
+    Derivation rootDrv;
+    rootDrv.name = "root-drv";
+    rootDrv.outputs = {
+        {
+            "out",
+            DerivationOutput{DerivationOutput::CAFloating{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hashAlgo = HashAlgorithm::SHA256,
+            }},
+        },
+    };
+    // Add the dependency derivation as an input
+    rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
+
+    // Write the root derivation to the destination store
+    auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
+
+    // Snapshot the destination store before
+    checkpointJson("issue-11928/store-before", dummyStore);
+
+    // Compute the hash modulo for the root derivation
+    auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
+    ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
+    auto rootDrvHash = rootHashModulo.hashes.at("out");
+
+    // Create the output store object for the root derivation
+    // Note: it does NOT reference the dependency's output
+    auto rootOutputPath = substituter->addToStore(
+        "root-drv-out",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents =
+                        "I am the root output. "
+                        "I don't reference anything because the other derivation's output is just needed at build time.",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // The DrvOutputs for both derivations
+    DrvOutput depDrvOutput{depDrvHash, "out"};
+    DrvOutput rootDrvOutput{rootDrvHash, "out"};
+
+    // Add the realisation for the root derivation to the substituter
+    substituter->buildTrace.insert_or_assign(
+        rootDrvHash,
+        std::map<std::string, UnkeyedRealisation>{
+            {
+                "out",
+                UnkeyedRealisation{
+                    .outPath = rootOutputPath,
+                },
+            },
+        });
+
+    // Snapshot the substituter
+    // Note: it has realisations for both drvs, but only the root's output store object
+    checkpointJson("issue-11928/substituter", substituter);
+
+    // The realisations should not exist in the destination store yet
+    ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput));
+    ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituterAsStore = substituter;
+    worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
+
+    // Create a derivation goal for the root derivation output
+    // The worker should substitute the output rather than building
+    auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false);
+
+    // Run the worker
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after
+    checkpointJson("issue-11928/store-after", dummyStore);
+
+    // The root output path should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath));
+
+    // The root realisation should now exist in the destination store
+    auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput);
+    ASSERT_TRUE(rootRealisation);
+    ASSERT_EQ(rootRealisation->outPath, rootOutputPath);
+
+    // #11928: The dependency's REALISATION should be fetched, because
+    // it is needed to resolve the underlying derivation. Currently the
+    // realisation is not fetched (bug). Once fixed: Change
+    // depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the
+    // ASSERT_EQ
+    auto depRealisation = dummyStore->queryRealisation(depDrvOutput);
+    ASSERT_FALSE(depRealisation);
+    // ASSERT_EQ(depRealisation->outPath, depOutputPath);
+
+    // The dependency's OUTPUT is correctly not fetched (not referenced by root output)
+    ASSERT_FALSE(dummyStore->isValidPath(depOutputPath));
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+
+    // Disable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "");
+}
+
+} // namespace nix

src/libstore/aws-creds.cc

@@ -4,15 +4,15 @@
 
 #  include <aws/crt/Types.h>
 #  include "nix/store/s3-url.hh"
-#  include "nix/util/finally.hh"
 #  include "nix/util/logging.hh"
-#  include "nix/util/url.hh"
-#  include "nix/util/util.hh"
 
 #  include <aws/crt/Api.h>
 #  include <aws/crt/auth/Credentials.h>
 #  include <aws/crt/io/Bootstrap.h>
 
+// C library headers for SSO provider support
+#  include <aws/auth/credentials.h>
+
 #  include <boost/unordered/concurrent_flat_map.hpp>
 
 #  include <chrono>
@@ -30,6 +30,46 @@ AwsAuthError::AwsAuthError(int errorCode)
 
 namespace {
 
+/**
+ * Helper function to wrap a C credentials provider in the C++ interface.
+ * This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
+ */
+static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createWrappedProvider(
+    aws_credentials_provider * rawProvider, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
+{
+    if (rawProvider == nullptr) {
+        return nullptr;
+    }
+
+    auto provider = Aws::Crt::MakeShared<Aws::Crt::Auth::CredentialsProvider>(allocator, rawProvider, allocator);
+    return std::static_pointer_cast<Aws::Crt::Auth::ICredentialsProvider>(provider);
+}
+
+/**
+ * Create an SSO credentials provider using the C library directly.
+ * The C++ wrapper doesn't expose SSO, so we call the C library and wrap the result.
+ * Returns nullptr if SSO provider creation fails (e.g., profile doesn't have SSO config).
+ */
+static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createSSOProvider(
+    const std::string & profileName,
+    Aws::Crt::Io::ClientBootstrap * bootstrap,
+    Aws::Crt::Io::TlsContext * tlsContext,
+    Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
+{
+    aws_credentials_provider_sso_options options;
+    AWS_ZERO_STRUCT(options);
+
+    options.bootstrap = bootstrap->GetUnderlyingHandle();
+    options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr;
+    if (!profileName.empty()) {
+        options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str());
+    }
+
+    // Create the SSO provider - will return nullptr if SSO isn't configured for this profile
+    // createWrappedProvider handles nullptr gracefully
+    return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator);
+}
+
 static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
 {
     if (!provider || !provider->IsValid()) {
@@ -91,6 +131,22 @@ public:
             logLevel = Aws::Crt::LogLevel::Warn;
         }
         apiHandle.InitializeLogging(logLevel, stderr);
+
+        // Create a shared TLS context for SSO (required for HTTPS connections)
+        auto allocator = Aws::Crt::ApiAllocator();
+        auto tlsCtxOptions = Aws::Crt::Io::TlsContextOptions::InitDefaultClient(allocator);
+        tlsContext =
+            std::make_shared<Aws::Crt::Io::TlsContext>(tlsCtxOptions, Aws::Crt::Io::TlsMode::CLIENT, allocator);
+        if (!tlsContext || !*tlsContext) {
+            warn("failed to create TLS context for AWS SSO; SSO authentication will be unavailable");
+            tlsContext = nullptr;
+        }
+
+        // Get bootstrap (lives as long as apiHandle)
+        bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
+        if (!bootstrap) {
+            throw AwsAuthError("failed to create AWS client bootstrap");
+        }
     }
 
     AwsCredentials getCredentialsRaw(const std::string & profile);
@@ -111,6 +167,8 @@ public:
 
 private:
     Aws::Crt::ApiHandle apiHandle;
+    std::shared_ptr<Aws::Crt::Io::TlsContext> tlsContext;
+    Aws::Crt::Io::ClientBootstrap * bootstrap;
     boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
         credentialProviderCache;
 };
@@ -118,23 +176,58 @@ private:
 std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
 AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
 {
-    debug(
-        "[pid=%d] creating new AWS credential provider for profile '%s'",
-        getpid(),
-        profile.empty() ? "(default)" : profile.c_str());
+    // profileDisplayName is only used for debug logging - SDK uses its default profile
+    // when ProfileNameOverride is not set
+    const char * profileDisplayName = profile.empty() ? "(default)" : profile.c_str();
 
-    if (profile.empty()) {
-        Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
-        config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
-        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
+    debug("[pid=%d] creating new AWS credential provider for profile '%s'", getpid(), profileDisplayName);
+
+    // Build a custom credential chain: Environment → SSO → Profile → IMDS
+    // This works for both default and named profiles, ensuring consistent behavior
+    // including SSO support and proper TLS context for STS-based role assumption.
+    Aws::Crt::Auth::CredentialsProviderChainConfig chainConfig;
+    auto allocator = Aws::Crt::ApiAllocator();
+
+    auto addProviderToChain = [&](std::string_view name, auto createProvider) {
+        if (auto provider = createProvider()) {
+            chainConfig.Providers.push_back(provider);
+            debug("Added AWS %s Credential Provider to chain for profile '%s'", name, profileDisplayName);
+        } else {
+            debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName);
+        }
+    };
+
+    // 1. Environment variables (highest priority)
+    addProviderToChain("Environment", [&]() {
+        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderEnvironment(allocator);
+    });
+
+    // 2. SSO provider (try it, will fail gracefully if not configured)
+    if (tlsContext) {
+        addProviderToChain("SSO", [&]() { return createSSOProvider(profile, bootstrap, tlsContext.get(), allocator); });
+    } else {
+        debug("Skipped AWS SSO Credential Provider for profile '%s': TLS context unavailable", profileDisplayName);
     }
 
-    Aws::Crt::Auth::CredentialsProviderProfileConfig config;
-    config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
-    // This is safe because the underlying C library will copy this string
-    // c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
-    config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
-    return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
+    // 3. Profile provider (for static credentials and role_arn/source_profile with STS)
+    addProviderToChain("Profile", [&]() {
+        Aws::Crt::Auth::CredentialsProviderProfileConfig profileConfig;
+        profileConfig.Bootstrap = bootstrap;
+        profileConfig.TlsContext = tlsContext.get();
+        if (!profile.empty()) {
+            profileConfig.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
+        }
+        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(profileConfig, allocator);
+    });
+
+    // 4. IMDS provider (for EC2 instances, lowest priority)
+    addProviderToChain("IMDS", [&]() {
+        Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig;
+        imdsConfig.Bootstrap = bootstrap;
+        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator);
+    });
+
+    return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator);
 }
 
 AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)

src/libstore/build/derivation-building-goal.cc

@@ -71,14 +71,6 @@ void DerivationBuildingGoal::killChild()
 #endif
 }
 
-void DerivationBuildingGoal::timedOut(Error && ex)
-{
-    killChild();
-    // We're not inside a coroutine, hence we can't use co_return here.
-    // Thus we ignore the return value.
-    [[maybe_unused]] Done _ = doneFailure({BuildResult::Failure::TimedOut, std::move(ex)});
-}
-
 std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv)
 {
     std::string msg;
@@ -443,7 +435,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
     if (useHook) {
         buildResult.startTime = time(0); // inexact
         started();
-        co_await Suspend{};
+
+        while (true) {
+            auto event = co_await WaitForChildEvent{};
+            if (auto * output = std::get_if<ChildOutput>(&event)) {
+                co_await processChildOutput(output->fd, output->data);
+            } else if (std::get_if<ChildEOF>(&event)) {
+                if (!currentLogLine.empty())
+                    flushLine();
+                break;
+            } else if (auto * timeout = std::get_if<TimedOut>(&event)) {
+                killChild();
+                co_return doneFailure(std::move(*timeout));
+            }
+        }
 
 #ifndef _WIN32
         assert(hook);
@@ -664,7 +669,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
     worker.childStarted(shared_from_this(), {builderOut}, true, true);
 
     started();
-    co_await Suspend{};
+
+    while (true) {
+        auto event = co_await WaitForChildEvent{};
+        if (auto * output = std::get_if<ChildOutput>(&event)) {
+            co_await processChildOutput(output->fd, output->data);
+        } else if (std::get_if<ChildEOF>(&event)) {
+            if (!currentLogLine.empty())
+                flushLine();
+            break;
+        } else if (auto * timeout = std::get_if<TimedOut>(&event)) {
+            killChild();
+            co_return doneFailure(std::move(*timeout));
+        }
+    }
 
     trace("build done");
 
@@ -970,7 +988,7 @@ Path DerivationBuildingGoal::openLogFile()
     logFileSink = std::make_shared<FdSink>(fdLogFile.get());
 
     if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
+        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
     else
         logSink = logFileSink;
 
@@ -997,7 +1015,7 @@ bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
 #endif
 }
 
-void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view data)
+Goal::Co DerivationBuildingGoal::processChildOutput(Descriptor fd, std::string_view data)
 {
     // local & `ssh://`-builds are dealt with here.
     auto isWrittenToLog = isReadDesc(fd);
@@ -1005,14 +1023,11 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
             killChild();
-            // We're not inside a coroutine, hence we can't use co_return here.
-            // Thus we ignore the return value.
-            [[maybe_unused]] Done _ = doneFailure(BuildError(
+            co_return doneFailure(BuildError(
                 BuildResult::Failure::LogLimitExceeded,
                 "%s killed after writing more than %d bytes of log output",
                 getName(),
                 settings.maxLogSize));
-            return;
         }
 
         for (auto c : data)
@@ -1065,13 +1080,7 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
                 currentHookLine += c;
     }
 #endif
-}
-
-void DerivationBuildingGoal::handleEOF(Descriptor fd)
-{
-    if (!currentLogLine.empty())
-        flushLine();
-    worker.wakeUp(shared_from_this());
+    co_return Return{};
 }
 
 void DerivationBuildingGoal::flushLine()

src/libstore/build/derivation-goal.cc

@@ -1,4 +1,5 @@
 #include "nix/store/build/derivation-goal.hh"
+#include "nix/store/build/drv-output-substitution-goal.hh"
 #include "nix/store/build/derivation-building-goal.hh"
 #include "nix/store/build/derivation-resolution-goal.hh"
 #ifndef _WIN32 // TODO enable build hook on Windows
@@ -100,9 +101,24 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
            through substitutes.  If that doesn't work, we'll build
            them. */
         if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
-            if (!checkResult)
-                waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput})));
-            else {
+            if (!checkResult) {
+                DrvOutput id{outputHash, wantedOutput};
+                auto g = worker.makeDrvOutputSubstitutionGoal(id);
+                waitees.insert(g);
+                co_await await(std::move(waitees));
+
+                if (nrFailed == 0) {
+                    waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath)));
+                    co_await await(std::move(waitees));
+
+                    trace("output path substituted");
+
+                    if (nrFailed == 0)
+                        worker.store.registerDrvOutput({*g->outputInfo, id});
+                    else
+                        debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
+                }
+            } else {
                 auto * cap = getDerivationCA(*drv);
                 waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
                     checkResult->first.outPath,
@@ -210,11 +226,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
                         .outputName = wantedOutput,
                     }};
                 newRealisation.signatures.clear();
-                if (!drv->type().isFixed()) {
-                    auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
-                    newRealisation.dependentRealisations =
-                        drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
-                }
                 worker.store.signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
             }

src/libstore/build/drv-output-substitution-goal.cc

@@ -3,8 +3,6 @@
 #include "nix/store/build/worker.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/util/callback.hh"
-#include "nix/store/store-open.hh"
-#include "nix/store/globals.hh"
 
 namespace nix {
 
@@ -21,11 +19,11 @@ Goal::Co DrvOutputSubstitutionGoal::init()
     trace("init");
 
     /* If the derivation already exists, we’re done */
-    if (worker.store.queryRealisation(id)) {
+    if ((outputInfo = worker.store.queryRealisation(id))) {
         co_return amDone(ecSuccess);
     }
 
-    auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+    auto subs = worker.getSubstituters();
 
     bool substituterFailed = false;
 
@@ -66,16 +64,19 @@ Goal::Co DrvOutputSubstitutionGoal::init()
             true,
             false);
 
-        co_await Suspend{};
+        while (true) {
+            auto event = co_await WaitForChildEvent{};
+            if (std::get_if<ChildOutput>(&event)) {
+                // Doesn't process child output
+            } else if (std::get_if<ChildEOF>(&event)) {
+                break;
+            } else if (std::get_if<TimedOut>(&event)) {
+                unreachable();
+            }
+        }
 
         worker.childTerminated(this);
 
-        /*
-         * The realisation corresponding to the given output id.
-         * Will be filled once we can get it.
-         */
-        std::shared_ptr<const UnkeyedRealisation> outputInfo;
-
         try {
             outputInfo = promise->get_future().get();
         } catch (std::exception & e) {
@@ -86,45 +87,6 @@ Goal::Co DrvOutputSubstitutionGoal::init()
         if (!outputInfo)
             continue;
 
-        bool failed = false;
-
-        Goals waitees;
-
-        for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
-            if (depId != id) {
-                if (auto localOutputInfo = worker.store.queryRealisation(depId);
-                    localOutputInfo && localOutputInfo->outPath != depPath) {
-                    warn(
-                        "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
-                        "Local:  %s\n"
-                        "Remote: %s",
-                        sub->config.getHumanReadableURI(),
-                        depId.to_string(),
-                        worker.store.printStorePath(localOutputInfo->outPath),
-                        worker.store.printStorePath(depPath));
-                    failed = true;
-                    break;
-                }
-                waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
-            }
-        }
-
-        if (failed)
-            continue;
-
-        waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
-
-        co_await await(std::move(waitees));
-
-        trace("output path substituted");
-
-        if (nrFailed > 0) {
-            debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
-            co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
-        }
-
-        worker.store.registerDrvOutput({*outputInfo, id});
-
         trace("finished");
         co_return amDone(ecSuccess);
     }
@@ -149,9 +111,4 @@ std::string DrvOutputSubstitutionGoal::key()
     return "a$" + std::string(id.to_string());
 }
 
-void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
-{
-    worker.wakeUp(shared_from_this());
-}
-
 } // namespace nix

src/libstore/build/goal.cc

@@ -4,8 +4,58 @@
 
 namespace nix {
 
+TimedOut::TimedOut(time_t maxDuration)
+    : BuildError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration)
+    , maxDuration(maxDuration)
+{
+}
+
 using Co = nix::Goal::Co;
 using promise_type = nix::Goal::promise_type;
+using ChildEvents = decltype(promise_type::childEvents);
+
+void ChildEvents::pushChildEvent(ChildOutput event)
+{
+    if (childTimeout)
+        return; // Already timed out, ignore
+    childOutputs.push(std::move(event));
+}
+
+void ChildEvents::pushChildEvent(ChildEOF event)
+{
+    if (childTimeout)
+        return; // Already timed out, ignore
+    assert(!childEOF);
+    childEOF = std::move(event);
+}
+
+void ChildEvents::pushChildEvent(TimedOut event)
+{
+    // Timeout is immediate - flush pending events
+    childOutputs = {};
+    childEOF.reset();
+    childTimeout = std::move(event);
+}
+
+bool ChildEvents::hasChildEvent() const
+{
+    return !childOutputs.empty() || childEOF || childTimeout;
+}
+
+Goal::ChildEvent ChildEvents::popChildEvent()
+{
+    if (!childOutputs.empty()) {
+        auto event = std::move(childOutputs.front());
+        childOutputs.pop();
+        return event;
+    }
+    if (childEOF)
+        return *std::exchange(childEOF, std::nullopt);
+    if (childTimeout)
+        return *std::exchange(childTimeout, std::nullopt);
+    unreachable();
+}
+
 using handle_type = nix::Goal::handle_type;
 using Suspend = nix::Goal::Suspend;
 
@@ -206,6 +256,27 @@ void Goal::work()
     assert(top_co || exitCode != ecBusy);
 }
 
+void Goal::handleChildOutput(Descriptor fd, std::string_view data)
+{
+    assert(top_co);
+    top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}});
+    worker.wakeUp(shared_from_this());
+}
+
+void Goal::handleEOF(Descriptor fd)
+{
+    assert(top_co);
+    top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd});
+    worker.wakeUp(shared_from_this());
+}
+
+void Goal::timedOut(TimedOut && ex)
+{
+    assert(top_co);
+    top_co->handle.promise().childEvents.pushChildEvent(std::move(ex));
+    worker.wakeUp(shared_from_this());
+}
+
 Goal::Co Goal::yield()
 {
     worker.wakeUp(shared_from_this());

src/libstore/build/substitution-goal.cc

@@ -1,5 +1,4 @@
 #include "nix/store/build/worker.hh"
-#include "nix/store/store-open.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/store/nar-info.hh"
 #include "nix/util/finally.hh"
@@ -60,7 +59,7 @@ Goal::Co PathSubstitutionGoal::init()
         throw Error(
             "cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
 
-    auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+    auto subs = worker.getSubstituters();
 
     bool substituterFailed = false;
     std::optional<Error> lastStoresException = std::nullopt;
@@ -258,7 +257,16 @@ Goal::Co PathSubstitutionGoal::tryToRun(
         true,
         false);
 
-    co_await Suspend{};
+    while (true) {
+        auto event = co_await WaitForChildEvent{};
+        if (std::get_if<ChildOutput>(&event)) {
+            // Substitution doesn't process child output
+        } else if (std::get_if<ChildEOF>(&event)) {
+            break;
+        } else if (std::get_if<TimedOut>(&event)) {
+            unreachable(); // Substitution doesn't use timeouts
+        }
+    }
 
     trace("substitute finished");
 
@@ -310,11 +318,6 @@ Goal::Co PathSubstitutionGoal::tryToRun(
     co_return doneSuccess(BuildResult::Success::Substituted);
 }
 
-void PathSubstitutionGoal::handleEOF(Descriptor fd)
-{
-    worker.wakeUp(shared_from_this());
-}
-
 void PathSubstitutionGoal::cleanup()
 {
     try {

src/libstore/build/worker.cc

@@ -1,5 +1,6 @@
 #include "nix/store/local-store.hh"
 #include "nix/store/machines.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/build/worker.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/store/build/drv-output-substitution-goal.hh"
@@ -21,6 +22,7 @@ Worker::Worker(Store & store, Store & evalStore)
     , actSubstitutions(*logger, actCopyPaths)
     , store(store)
     , evalStore(evalStore)
+    , getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
 {
     nrLocalBuilds = 0;
     nrSubstitutions = 0;
@@ -479,14 +481,13 @@ void Worker::waitForInput()
 
         if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts
             && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) {
-            goal->timedOut(
-                Error("%1% timed out after %2% seconds of silence", goal->getName(), settings.maxSilentTime));
+            goal->timedOut(TimedOut(settings.maxSilentTime));
         }
 
         else if (
             goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts
             && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) {
-            goal->timedOut(Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout));
+            goal->timedOut(TimedOut(settings.buildTimeout));
         }
     }
 

src/libstore/derivation-options.cc

@@ -341,8 +341,12 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
 
                 if (parsed) {
                     auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
-                    if (!e || !e->is_object())
+                    if (!e)
                         return ret;
+                    if (!e->is_object()) {
+                        warn("'exportReferencesGraph' in structured attrs is not a JSON object, ignoring");
+                        return ret;
+                    }
                     for (auto & [key, storePathsJson] : getObject(*e)) {
                         StringSet ss;
                         flatten(storePathsJson, ss);

src/libstore/derivations.cc

@@ -1489,8 +1489,6 @@ adl_serializer<DerivationOutput>::from_json(const json & _json, const Experiment
     }
 }
 
-static unsigned constexpr expectedJsonVersionDerivation = 4;
-
 void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
 {
     res = nlohmann::json::object();

src/libstore/filetransfer.cc

@@ -30,8 +30,6 @@
 #include <thread>
 #include <regex>
 
-using namespace std::string_literals;
-
 namespace nix {
 
 const unsigned int RETRY_TIME_MS_DEFAULT = 250;
@@ -41,9 +39,27 @@ FileTransferSettings fileTransferSettings;
 
 static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
 
+namespace {
+
+using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>;
+using curlMulti = std::unique_ptr<::CURLM, decltype([](::CURLM * multi) { ::curl_multi_cleanup(multi); })>;
+
+struct curlMultiError : Error
+{
+    ::CURLMcode code;
+
+    curlMultiError(::CURLMcode code)
+        : Error{"unexpected curl multi error: %s", ::curl_multi_strerror(code)}
+    {
+        assert(code != CURLM_OK);
+    }
+};
+
+} // namespace
+
 struct curlFileTransfer : public FileTransfer
 {
-    CURLM * curlm = 0;
+    curlMulti curlm;
 
     std::random_device rd;
     std::mt19937 mt19937;
@@ -59,8 +75,9 @@ struct curlFileTransfer : public FileTransfer
         CURL * req = 0;
         // buffer to accompany the `req` above
         char errbuf[CURL_ERROR_SIZE];
-        bool active = false; // whether the handle has been added to the multi object
-        bool paused = false; // whether the request has been paused previously
+        bool active = false;   // whether the handle has been added to the multi object
+        bool paused = false;   // whether the request has been paused previously
+        bool enqueued = false; // whether the request has been added the incoming queue
         std::string statusMsg;
 
         unsigned int attempt = 0;
@@ -69,7 +86,7 @@ struct curlFileTransfer : public FileTransfer
            has been reached. */
         std::chrono::steady_clock::time_point embargo;
 
-        struct curl_slist * requestHeaders = 0;
+        curlSList requestHeaders;
 
         std::string encoding;
 
@@ -92,6 +109,15 @@ struct curlFileTransfer : public FileTransfer
             return httpStatus;
         }
 
+        void appendHeaders(const std::string & header)
+        {
+            curlSList tmpSList = curlSList(::curl_slist_append(requestHeaders.get(), requireCString(header)));
+            if (!tmpSList)
+                throw std::bad_alloc();
+            requestHeaders.release();
+            requestHeaders = std::move(tmpSList);
+        }
+
         TransferItem(
             curlFileTransfer & fileTransfer,
             const FileTransferRequest & request,
@@ -131,13 +157,13 @@ struct curlFileTransfer : public FileTransfer
         {
             result.urls.push_back(request.uri.to_string());
 
-            requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
+            appendHeaders("Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
             if (!request.expectedETag.empty())
-                requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
+                appendHeaders("If-None-Match: " + request.expectedETag);
             if (!request.mimeType.empty())
-                requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
+                appendHeaders("Content-Type: " + request.mimeType);
             for (auto it = request.headers.begin(); it != request.headers.end(); ++it) {
-                requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
+                appendHeaders(fmt("%s: %s", it->first, it->second));
             }
         }
 
@@ -145,13 +171,11 @@ struct curlFileTransfer : public FileTransfer
         {
             if (req) {
                 if (active)
-                    curl_multi_remove_handle(fileTransfer.curlm, req);
+                    curl_multi_remove_handle(fileTransfer.curlm.get(), req);
                 curl_easy_cleanup(req);
             }
-            if (requestHeaders)
-                curl_slist_free_all(requestHeaders);
             try {
-                if (!done)
+                if (!done && enqueued)
                     fail(FileTransferError(
                         Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri));
             } catch (...) {
@@ -348,7 +372,7 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
         }
 
-#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+#if !defined(_WIN32)
         static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose)
         {
             unix::closeOnExec(curlfd);
@@ -411,15 +435,11 @@ struct curlFileTransfer : public FileTransfer
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion
                  + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : ""))
                     .c_str());
-#if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
-#endif
-#if LIBCURL_VERSION_NUM >= 0x072f00
             if (fileTransferSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             else
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
-#endif
             curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
             curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
@@ -429,7 +449,7 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
             curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
 
-            curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
+            curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders.get());
 
             if (settings.downloadSpeed.get() > 0)
                 curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
@@ -459,10 +479,9 @@ struct curlFileTransfer : public FileTransfer
             if (settings.caFile != "")
                 curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
 
-#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+#if !defined(_WIN32)
             curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
 #endif
-
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
@@ -694,13 +713,6 @@ struct curlFileTransfer : public FileTransfer
 
     Sync<State> state_;
 
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-    /* We can't use a std::condition_variable to wake up the curl
-       thread, because it only monitors file descriptors. So use a
-       pipe instead. */
-    Pipe wakeupPipe;
-#endif
-
     std::thread workerThread;
 
     curlFileTransfer()
@@ -709,43 +721,35 @@ struct curlFileTransfer : public FileTransfer
         static std::once_flag globalInit;
         std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
 
-        curlm = curl_multi_init();
+        curlm = curlMulti(curl_multi_init());
 
-#if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0
-        curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
-#endif
-#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
-        curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
-#endif
-
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        wakeupPipe.create();
-        fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK);
-#endif
+        curl_multi_setopt(curlm.get(), CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
+        curl_multi_setopt(curlm.get(), CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
 
         workerThread = std::thread([&]() { workerThreadEntry(); });
     }
 
     ~curlFileTransfer()
     {
-        stopWorkerThread();
-
+        try {
+            stopWorkerThread();
+        } catch (...) {
+            ignoreExceptionInDestructor();
+        }
         workerThread.join();
-
-        if (curlm)
-            curl_multi_cleanup(curlm);
     }
 
     void stopWorkerThread()
     {
         /* Signal the worker thread to exit. */
-        {
-            auto state(state_.lock());
-            state->quit();
-        }
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ", false);
-#endif
+        state_.lock()->quit();
+        wakeupMulti();
+    }
+
+    void wakeupMulti()
+    {
+        if (auto ec = ::curl_multi_wakeup(curlm.get()))
+            throw curlMultiError(ec);
     }
 
     void workerThreadMain()
@@ -775,32 +779,25 @@ struct curlFileTransfer : public FileTransfer
 
             /* Let curl do its thing. */
             int running;
-            CURLMcode mc = curl_multi_perform(curlm, &running);
+            CURLMcode mc = curl_multi_perform(curlm.get(), &running);
             if (mc != CURLM_OK)
                 throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
 
             /* Set the promises of any finished requests. */
             CURLMsg * msg;
             int left;
-            while ((msg = curl_multi_info_read(curlm, &left))) {
+            while ((msg = curl_multi_info_read(curlm.get(), &left))) {
                 if (msg->msg == CURLMSG_DONE) {
                     auto i = items.find(msg->easy_handle);
                     assert(i != items.end());
                     i->second->finish(msg->data.result);
-                    curl_multi_remove_handle(curlm, i->second->req);
+                    curl_multi_remove_handle(curlm.get(), i->second->req);
                     i->second->active = false;
                     items.erase(i);
                 }
             }
 
             /* Wait for activity, including wakeup events. */
-            int numfds = 0;
-            struct curl_waitfd extraFDs[1];
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-            extraFDs[0].fd = wakeupPipe.readSide.get();
-            extraFDs[0].events = CURL_WAIT_POLLIN;
-            extraFDs[0].revents = 0;
-#endif
             long maxSleepTimeMs = items.empty() ? 10000 : 100;
             auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point()
                                    ? std::max(
@@ -809,23 +806,14 @@ struct curlFileTransfer : public FileTransfer
                                              nextWakeup - std::chrono::steady_clock::now())
                                              .count())
                                    : maxSleepTimeMs;
-            vomit("download thread waiting for %d ms", sleepTimeMs);
-            mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
+
+            int numfds = 0;
+            mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)
-                throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
+                throw curlMultiError(mc);
 
             nextWakeup = std::chrono::steady_clock::time_point();
 
-            /* Add new curl requests from the incoming requests queue,
-               except for requests that are embargoed (waiting for a
-               retry timeout to expire). */
-            if (extraFDs[0].revents & CURL_WAIT_POLLIN) {
-                char buf[1024];
-                auto res = read(extraFDs[0].fd, buf, sizeof(buf));
-                if (res == -1 && errno != EINTR)
-                    throw SysError("reading curl wakeup socket");
-            }
-
             std::vector<std::shared_ptr<TransferItem>> incoming;
             auto now = std::chrono::steady_clock::now();
 
@@ -848,7 +836,7 @@ struct curlFileTransfer : public FileTransfer
             for (auto & item : incoming) {
                 debug("starting %s of %s", item->request.noun(), item->request.uri);
                 item->init();
-                curl_multi_add_handle(curlm, item->req);
+                curl_multi_add_handle(curlm.get(), item->req);
                 item->active = true;
                 items[item->req] = item;
             }
@@ -898,11 +886,10 @@ struct curlFileTransfer : public FileTransfer
             if (state->isQuitting())
                 throw nix::Error("cannot enqueue download request because the download thread is shutting down");
             state->incoming.push(item);
+            item->enqueued = true; /* Now any exceptions should be reported via the callback. */
         }
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ");
-#endif
 
+        wakeupMulti();
         return ItemHandle(static_cast<Item &>(*item));
     }
 
@@ -922,9 +909,7 @@ struct curlFileTransfer : public FileTransfer
     {
         auto state(state_.lock());
         state->unpause.push_back(std::move(item));
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ");
-#endif
+        wakeupMulti();
     }
 
     void unpauseTransfer(ItemHandle handle) override

src/libstore/include/nix/store/build/derivation-builder.hh

@@ -79,7 +79,7 @@ struct DerivationBuilderParams
      */
     const StorePathSet & inputPaths;
 
-    const std::map<std::string, InitialOutput> & initialOutputs;
+    const std::map<std::string, InitialOutput> initialOutputs;
 
     const BuildMode & buildMode;
 

src/libstore/include/nix/store/build/derivation-building-goal.hh

@@ -100,8 +100,6 @@ private:
 
     std::map<ActivityId, Activity> builderActivities;
 
-    void timedOut(Error && ex) override;
-
     std::string key() override;
 
     /**
@@ -129,10 +127,10 @@ private:
     bool isReadDesc(Descriptor fd);
 
     /**
-     * Callback used by the worker to write to the log.
+     * Process output from a child process.
      */
-    void handleChildOutput(Descriptor fd, std::string_view data) override;
-    void handleEOF(Descriptor fd) override;
+    Co processChildOutput(Descriptor fd, std::string_view data);
+
     void flushLine();
 
     /**

src/libstore/include/nix/store/build/derivation-goal.hh

@@ -52,11 +52,6 @@ struct DerivationGoal : public Goal
         bool storeDerivation);
     ~DerivationGoal() = default;
 
-    void timedOut(Error && ex) override
-    {
-        unreachable();
-    };
-
     std::string key() override;
 
     JobCategory jobCategory() const override

src/libstore/include/nix/store/build/derivation-resolution-goal.hh

@@ -43,8 +43,6 @@ struct DerivationResolutionGoal : public Goal
      */
     std::unique_ptr<std::pair<StorePath, BasicDerivation>> resolvedDrv;
 
-    void timedOut(Error && ex) override {}
-
 private:
 
     /**

src/libstore/include/nix/store/build/derivation-trampoline-goal.hh

@@ -109,8 +109,6 @@ struct DerivationTrampolineGoal : public Goal
 
     virtual ~DerivationTrampolineGoal();
 
-    void timedOut(Error && ex) override {}
-
     std::string key() override;
 
     JobCategory jobCategory() const override

src/libstore/include/nix/store/build/drv-output-substitution-goal.hh

@@ -14,11 +14,14 @@ namespace nix {
 class Worker;
 
 /**
- * Substitution of a derivation output.
- * This is done in three steps:
- * 1. Fetch the output info from a substituter
- * 2. Substitute the corresponding output path
- * 3. Register the output info
+ * Fetch a `Realisation` (drv ⨯ output name -> output path) from a
+ * substituter.
+ *
+ * If the output store object itself should also be substituted, that is
+ * the responsibility of the caller to do so.
+ *
+ * @todo rename this `BuidlTraceEntryGoal`, which will make sense
+ * especially once `Realisation` is renamed to `BuildTraceEntry`.
  */
 class DrvOutputSubstitutionGoal : public Goal
 {
@@ -31,17 +34,16 @@ class DrvOutputSubstitutionGoal : public Goal
 public:
     DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
 
+    /**
+     * The realisation corresponding to the given output id.
+     * Will be filled once we can get it.
+     */
+    std::shared_ptr<const UnkeyedRealisation> outputInfo;
+
     Co init();
 
-    void timedOut(Error && ex) override
-    {
-        unreachable();
-    };
-
     std::string key() override;
 
-    void handleEOF(Descriptor fd) override;
-
     JobCategory jobCategory() const override
     {
         return JobCategory::Substitution;

src/libstore/include/nix/store/build/goal.hh

@@ -5,9 +5,18 @@
 #include "nix/store/build-result.hh"
 
 #include <coroutine>
+#include <queue>
+#include <variant>
 
 namespace nix {
 
+struct TimedOut : BuildError
+{
+    time_t maxDuration;
+
+    TimedOut(time_t maxDuration);
+};
+
 /**
  * Forward definition.
  */
@@ -138,6 +147,29 @@ public:
         friend Goal;
     };
 
+    /**
+     * Event types for child process communication, delivered via coroutines.
+     */
+    struct ChildOutput
+    {
+        Descriptor fd;
+        std::string data;
+    };
+
+    struct ChildEOF
+    {
+        Descriptor fd;
+    };
+
+    using ChildEvent = std::variant<ChildOutput, ChildEOF, TimedOut>;
+
+    /**
+     * Tag type for `co_await`-ing child events.
+     * Returns a `ChildEvent` when resumed.
+     */
+    struct WaitForChildEvent
+    {};
+
     // forward declaration of promise_type, see below
     struct promise_type;
 
@@ -276,6 +308,28 @@ public:
          */
         bool alive = true;
 
+        class
+        {
+            /**
+             * Structured queue of child events:
+             * - outputs: stream of data from child
+             * - eof: optional end-of-stream marker
+             * - timeout: optional timeout that flushes/overrides other events
+             */
+            std::queue<ChildOutput> childOutputs;
+            std::optional<ChildEOF> childEOF;
+            std::optional<TimedOut> childTimeout;
+
+        public:
+
+            void pushChildEvent(ChildOutput event);
+            void pushChildEvent(ChildEOF event);
+            void pushChildEvent(TimedOut event);
+            bool hasChildEvent() const;
+            ChildEvent popChildEvent();
+
+        } childEvents;
+
         /**
          * The awaiter used by @ref final_suspend.
          */
@@ -369,13 +423,66 @@ public:
             return static_cast<Co &&>(co);
         }
 
+        /**
+         * Awaiter for @ref Suspend. Always suspends, but asserts
+         * there are no pending child events (those should be
+         * consumed first via @ref WaitForChildEvent).
+         */
+        struct SuspendAwaiter
+        {
+            promise_type & promise;
+
+            bool await_ready()
+            {
+                assert(!promise.childEvents.hasChildEvent());
+                return false;
+            }
+
+            void await_suspend(handle_type) {}
+
+            void await_resume() {}
+        };
+
         /**
          * Allows awaiting a @ref Suspend.
          * Always suspends.
          */
-        std::suspend_always await_transform(Suspend)
+        SuspendAwaiter await_transform(Suspend)
         {
-            return {};
+            return SuspendAwaiter{*this};
+        };
+
+        /**
+         * Awaiter for child events. Suspends and returns the
+         * pending child event when resumed.
+         */
+        struct ChildEventAwaiter
+        {
+            handle_type handle;
+
+            bool await_ready()
+            {
+                return handle && handle.promise().childEvents.hasChildEvent();
+            }
+
+            void await_suspend(handle_type h)
+            {
+                handle = h;
+            }
+
+            ChildEvent await_resume()
+            {
+                assert(handle);
+                return handle.promise().childEvents.popChildEvent();
+            }
+        };
+
+        /**
+         * Allows awaiting child events (output, EOF, timeout).
+         */
+        ChildEventAwaiter await_transform(WaitForChildEvent)
+        {
+            return ChildEventAwaiter{handle_type::from_promise(*this)};
         };
     };
 
@@ -432,15 +539,23 @@ public:
 
     void work();
 
-    virtual void handleChildOutput(Descriptor fd, std::string_view data)
-    {
-        unreachable();
-    }
+    /**
+     * Called by the worker when data is received from a child process.
+     * Stores the event and resumes the coroutine.
+     */
+    void handleChildOutput(Descriptor fd, std::string_view data);
 
-    virtual void handleEOF(Descriptor fd)
-    {
-        unreachable();
-    }
+    /**
+     * Called by the worker when EOF is received from a child process.
+     * Stores the event and resumes the coroutine.
+     */
+    void handleEOF(Descriptor fd);
+
+    /**
+     * Called by the worker when a build times out.
+     * Stores the event and resumes the coroutine.
+     */
+    void timedOut(TimedOut && ex);
 
     void trace(std::string_view s);
 
@@ -449,13 +564,6 @@ public:
         return name;
     }
 
-    /**
-     * Callback in case of a timeout.  It should wake up its waiters,
-     * get rid of any running child processes that are being monitored
-     * by the worker (important!), etc.
-     */
-    virtual void timedOut(Error && ex) = 0;
-
     /**
      * Used for comparisons. The order matters a bit for scheduling. We
      * want:

src/libstore/include/nix/store/build/substitution-goal.hh

@@ -53,11 +53,6 @@ public:
         std::optional<ContentAddress> ca = std::nullopt);
     ~PathSubstitutionGoal();
 
-    void timedOut(Error && ex) override
-    {
-        unreachable();
-    };
-
     std::string key() override
     {
         return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
@@ -72,12 +67,6 @@ public:
         StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed);
     Co finished();
 
-    /**
-     * Callback used by the worker to write to the log.
-     */
-    void handleChildOutput(Descriptor fd, std::string_view data) override {};
-    void handleEOF(Descriptor fd) override;
-
     /* Called by destructor, can't be overridden */
     void cleanup() override final;
 

src/libstore/include/nix/store/build/worker.hh

@@ -8,6 +8,7 @@
 #include "nix/store/realisation.hh"
 #include "nix/util/muxable-pipe.hh"
 
+#include <functional>
 #include <future>
 #include <thread>
 
@@ -171,6 +172,14 @@ public:
     Store & store;
     Store & evalStore;
 
+    /**
+     * Function to get the substituters to use for path substitution.
+     *
+     * Defaults to `getDefaultSubstituters`. This allows tests to
+     * inject custom substituters.
+     */
+    std::function<std::list<ref<Store>>()> getSubstituters;
+
 #ifndef _WIN32 // TODO Enable building on Windows
     std::unique_ptr<HookInstance> hook;
 #endif

src/libstore/include/nix/store/derivations.hh

@@ -586,6 +586,12 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
  */
 std::string hashPlaceholder(const OutputNameView outputName);
 
+/**
+ * The expected JSON version for derivation serialization.
+ * Used by `nix derivation show` and `nix derivation add`.
+ */
+constexpr unsigned expectedJsonVersionDerivation = 4;
+
 } // namespace nix
 
 JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput)

src/libstore/include/nix/store/globals.hh

@@ -1143,7 +1143,7 @@ public:
 
     Setting<std::string> netrcFile{
         this,
-        fmt("%s/%s", nixConfDir, "netrc"),
+        (nixConfDir / "netrc").string(),
         "netrc-file",
         R"(
           If set to an absolute path to a `netrc` file, Nix uses the HTTP

src/libstore/include/nix/store/local-store.hh

@@ -420,7 +420,7 @@ private:
 
     uint64_t queryValidPathId(State & state, const StorePath & path);
 
-    uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
+    uint64_t addValidPath(State & state, const ValidPathInfo & info);
 
     void invalidatePath(State & state, const StorePath & path);
 

src/libstore/include/nix/store/pathlocks.hh

@@ -33,6 +33,22 @@ private:
 public:
     PathLocks();
     PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
+
+    PathLocks(PathLocks && other) noexcept
+        : fds(std::exchange(other.fds, {}))
+        , deletePaths(other.deletePaths)
+    {
+    }
+
+    PathLocks & operator=(PathLocks && other) noexcept
+    {
+        fds = std::exchange(other.fds, {});
+        deletePaths = other.deletePaths;
+        return *this;
+    }
+
+    PathLocks(const PathLocks &) = delete;
+    PathLocks & operator=(const PathLocks &) = delete;
     bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
     ~PathLocks();
     void unlock();
@@ -45,6 +61,10 @@ struct FdLock
     bool acquired = false;
 
     FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
+    FdLock(const FdLock &) = delete;
+    FdLock & operator=(const FdLock &) = delete;
+    FdLock(FdLock &&) = delete;
+    FdLock & operator=(FdLock &&) = delete;
 
     ~FdLock()
     {

src/libstore/include/nix/store/realisation.hh

@@ -56,14 +56,6 @@ struct UnkeyedRealisation
 
     StringSet signatures;
 
-    /**
-     * The realisations that are required for the current one to be valid.
-     *
-     * When importing this realisation, the store will first check that all its
-     * dependencies exist, and map to the correct output path
-     */
-    std::map<DrvOutput, StorePath> dependentRealisations;
-
     std::string fingerprint(const DrvOutput & key) const;
 
     void sign(const DrvOutput & key, const Signer &);
@@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation
 
     bool isCompatibleWith(const UnkeyedRealisation & other) const;
 
-    static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
-
-    static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
-
     bool operator==(const Realisation &) const = default;
     auto operator<=>(const Realisation &) const = default;
 };
@@ -154,10 +142,6 @@ struct RealisedPath
      */
     const StorePath & path() const &;
 
-    void closure(Store & store, Set & ret) const;
-    static void closure(Store & store, const Set & startPaths, Set & ret);
-    Set closure(Store & store) const;
-
     bool operator==(const RealisedPath &) const = default;
     auto operator<=>(const RealisedPath &) const = default;
 };

src/libstore/include/nix/store/store-api.hh

@@ -1007,9 +1007,6 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashR
 
 const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 
-std::map<DrvOutput, StorePath>
-drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr);
-
 template<>
 struct json_avoids_null<TrustedFlag> : std::true_type
 {};

src/libstore/local-store.cc

@@ -110,8 +110,6 @@ struct LocalStore::State::Stmts
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
-    SQLiteStmt QueryRealisationReferences;
-    SQLiteStmt AddRealisationReference;
 };
 
 LocalStore::LocalStore(ref<const Config> config)
@@ -390,21 +388,6 @@ LocalStore::LocalStore(ref<const Config> config)
                     where drvPath = ?
                     ;
             )");
-        state->stmts->QueryRealisationReferences.create(
-            state->db,
-            R"(
-                select drvPath, outputName from Realisations
-                    join RealisationsRefs on realisationReference = Realisations.id
-                    where referrer = ?;
-            )");
-        state->stmts->AddRealisationReference.create(
-            state->db,
-            R"(
-                insert or replace into RealisationsRefs (referrer, realisationReference)
-                values (
-                    (select id from Realisations where drvPath = ? and outputName = ?),
-                    (select id from Realisations where drvPath = ? and outputName = ?));
-            )");
     }
 }
 
@@ -654,25 +637,6 @@ void LocalStore::registerDrvOutput(const Realisation & info)
                     concatStringsSep(" ", info.signatures))
                 .exec();
         }
-        for (auto & [outputId, depPath] : info.dependentRealisations) {
-            auto localRealisation = queryRealisationCore_(*state, outputId);
-            if (!localRealisation)
-                throw Error(
-                    "unable to register the derivation '%s' as it "
-                    "depends on the non existent '%s'",
-                    info.id.to_string(),
-                    outputId.to_string());
-            if (localRealisation->second.outPath != depPath)
-                throw Error(
-                    "unable to register the derivation '%s' as it "
-                    "depends on a realisation of '%s' that doesn’t"
-                    "match what we have locally",
-                    info.id.to_string(),
-                    outputId.to_string());
-            state->stmts->AddRealisationReference
-                .use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName)
-                .exec();
-        }
     });
 }
 
@@ -683,7 +647,7 @@ void LocalStore::cacheDrvOutputMapping(
         [&]() { state.stmts->AddDerivationOutput.use()(deriver)(outputName) (printStorePath(output)).exec(); });
 }
 
-uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs)
+uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info)
 {
     if (info.ca.has_value() && !info.isContentAddressed(*this))
         throw Error(
@@ -704,17 +668,16 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, boo
        efficiently query whether a path is an output of some
        derivation. */
     if (info.path.isDerivation()) {
-        auto drv = readInvalidDerivation(info.path);
+        auto parsedDrv = readInvalidDerivation(info.path);
 
         /* Verify that the output paths in the derivation are correct
            (i.e., follow the scheme for computing output paths from
            derivations).  Note that if this throws an error, then the
            DB transaction is rolled back, so the path validity
            registration above is undone. */
-        if (checkOutputs)
-            drv.checkInvariants(*this, info.path);
+        parsedDrv.checkInvariants(*this, info.path);
 
-        for (auto & i : drv.outputsAndOptPaths(*this)) {
+        for (auto & i : parsedDrv.outputsAndOptPaths(*this)) {
             /* Floating CA derivations have indeterminate output paths until
                they are built, so don't register anything in that case */
             if (i.second.second)
@@ -965,7 +928,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
             if (isValidPath_(*state, i.path))
                 updatePathInfo(*state, i);
             else
-                addValidPath(*state, i, false);
+                addValidPath(*state, i);
             paths.insert(i.path);
         }
 
@@ -975,15 +938,6 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
                 state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
         }
 
-        /* Check that the derivation outputs are correct.  We can't do
-           this in addValidPath() above, because the references might
-           not be valid yet. */
-        for (auto & [_, i] : infos)
-            if (i.path.isDerivation()) {
-                // FIXME: inefficient; we already loaded the derivation in addValidPath().
-                readInvalidDerivation(i.path).checkInvariants(*this, i.path);
-            }
-
         /* Do a topological sort of the paths.  This will throw an
            error if a cycle is detected and roll back the
            transaction.  Cycles can only occur when a derivation
@@ -1606,21 +1560,6 @@ std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(LocalStore
         return std::nullopt;
     auto [realisationDbId, res] = *maybeCore;
 
-    std::map<DrvOutput, StorePath> dependentRealisations;
-    auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId));
-    while (useRealisationRefs.next()) {
-        auto depId = DrvOutput{
-            Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
-            useRealisationRefs.getStr(1),
-        };
-        auto dependentRealisation = queryRealisationCore_(state, depId);
-        assert(dependentRealisation); // Enforced by the db schema
-        auto outputPath = dependentRealisation->second.outPath;
-        dependentRealisations.insert({depId, outputPath});
-    }
-
-    res.dependentRealisations = dependentRealisations;
-
     return {res};
 }
 

src/libstore/meson.build

@@ -39,28 +39,32 @@ deps_public_maybe_subproject = [
 ]
 subdir('nix-meson-build-support/subprojects')
 
-run_command(
-  'ln',
-  '-s',
-  meson.project_build_root() / '__nothing_link_target',
-  meson.project_build_root() / '__nothing_symlink',
-  # native doesn't allow dangling symlinks, which the tests require
-  env : {'MSYS' : 'winsymlinks:lnk'},
-  check : true,
-)
-can_link_symlink = run_command(
-  'ln',
-  meson.project_build_root() / '__nothing_symlink',
-  meson.project_build_root() / '__nothing_hardlink',
-  check : false,
-).returncode() == 0
-run_command(
-  'rm',
-  '-f',
-  meson.project_build_root() / '__nothing_symlink',
-  meson.project_build_root() / '__nothing_hardlink',
-  check : true,
-)
+can_link_symlink = false
+native_ln = find_program('ln', required : false, native : true)
+if native_ln.found()
+  run_command(
+    native_ln,
+    '-s',
+    meson.project_build_root() / '__nothing_link_target',
+    meson.project_build_root() / '__nothing_symlink',
+    # native doesn't allow dangling symlinks, which the tests require
+    env : {'MSYS' : 'winsymlinks:lnk'},
+    check : true,
+  )
+  can_link_symlink = run_command(
+    native_ln,
+    meson.project_build_root() / '__nothing_symlink',
+    meson.project_build_root() / '__nothing_hardlink',
+    check : false,
+  ).returncode() == 0
+  run_command(
+    'rm',
+    '-f',
+    meson.project_build_root() / '__nothing_symlink',
+    meson.project_build_root() / '__nothing_hardlink',
+    check : true,
+  )
+endif
 summary('can hardlink to symlink', can_link_symlink, bool_yn : true)
 configdata_priv.set('CAN_LINK_SYMLINK', can_link_symlink.to_int())
 
@@ -160,6 +164,8 @@ if s3_aws_auth.enabled()
   deps_other += aws_crt_cpp
   aws_c_common = cxx.find_library('aws-c-common', required : true)
   deps_other += aws_c_common
+  aws_c_auth = cxx.find_library('aws-c-auth', required : true)
+  deps_other += aws_c_auth
 endif
 
 configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())

src/libstore/misc.cc

@@ -23,9 +23,9 @@ void Store::computeFSClosure(
     bool includeOutputs,
     bool includeDerivers)
 {
-    std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
+    std::function<asio::awaitable<StorePathSet>(const StorePath & path)> queryDeps;
     if (flipDirection)
-        queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
+        queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
             StorePathSet res;
             StorePathSet referrers;
             queryReferrers(path, referrers);
@@ -41,12 +41,14 @@ void Store::computeFSClosure(
                 for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
                     if (maybeOutPath && isValidPath(*maybeOutPath))
                         res.insert(*maybeOutPath);
-            return res;
+            co_return res;
         };
     else
-        queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
+        queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
             StorePathSet res;
-            auto info = fut.get();
+            auto info = co_await callbackToAwaitable<ref<const ValidPathInfo>>(
+                [this, path](Callback<ref<const ValidPathInfo>> cb) { queryPathInfo(path, std::move(cb)); });
+
             for (auto & ref : info->references)
                 if (ref != path)
                     res.insert(ref);
@@ -58,25 +60,9 @@ void Store::computeFSClosure(
 
             if (includeDerivers && info->deriver && isValidPath(*info->deriver))
                 res.insert(*info->deriver);
-            return res;
+            co_return res;
         };
-
-    computeClosure<StorePath>(
-        startPaths,
-        paths_,
-        [&](const StorePath & path, std::function<void(std::promise<std::set<StorePath>> &)> processEdges) {
-            std::promise<std::set<StorePath>> promise;
-            std::function<void(std::future<ref<const ValidPathInfo>>)> getDependencies =
-                [&](std::future<ref<const ValidPathInfo>> fut) {
-                    try {
-                        promise.set_value(queryDeps(path, fut));
-                    } catch (...) {
-                        promise.set_exception(std::current_exception());
-                    }
-                };
-            queryPathInfo(path, getDependencies);
-            processEdges(promise);
-        });
+    computeClosure<StorePath>(startPaths, paths_, queryDeps);
 }
 
 void Store::computeFSClosure(
@@ -334,65 +320,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         result);
 }
 
-std::map<DrvOutput, StorePath>
-drvOutputReferences(const std::set<Realisation> & inputRealisations, const StorePathSet & pathReferences)
-{
-    std::map<DrvOutput, StorePath> res;
-
-    for (const auto & input : inputRealisations) {
-        if (pathReferences.count(input.outPath)) {
-            res.insert({input.id, input.outPath});
-        }
-    }
-
-    return res;
-}
-
-std::map<DrvOutput, StorePath>
-drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_)
-{
-    auto & evalStore = evalStore_ ? *evalStore_ : store;
-
-    std::set<Realisation> inputRealisations;
-
-    auto accumRealisations = [&](this auto & self,
-                                 const StorePath & inputDrv,
-                                 const DerivedPathMap<StringSet>::ChildNode & inputNode) -> void {
-        if (!inputNode.value.empty()) {
-            auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
-            for (const auto & outputName : inputNode.value) {
-                auto outputHash = get(outputHashes, outputName);
-                if (!outputHash)
-                    throw Error(
-                        "output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv));
-                DrvOutput key{*outputHash, outputName};
-                auto thisRealisation = store.queryRealisation(key);
-                if (!thisRealisation)
-                    throw Error(
-                        "output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv));
-                inputRealisations.insert({*thisRealisation, std::move(key)});
-            }
-        }
-        if (!inputNode.value.empty()) {
-            auto d = makeConstantStorePathRef(inputDrv);
-            for (const auto & [outputName, childNode] : inputNode.childMap) {
-                SingleDerivedPath next = SingleDerivedPath::Built{d, outputName};
-                self(
-                    // TODO deep resolutions for dynamic derivations, issue #8947, would go here.
-                    resolveDerivedPath(store, next, evalStore_),
-                    childNode);
-            }
-        }
-    };
-
-    for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
-        accumRealisations(inputDrv, inputNode);
-
-    auto info = store.queryPathInfo(outputPath);
-
-    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
-}
-
 OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
 {
     auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);

src/libstore/realisation.cc

@@ -1,6 +1,5 @@
 #include "nix/store/realisation.hh"
 #include "nix/store/store-api.hh"
-#include "nix/util/closure.hh"
 #include "nix/util/signature/local-keys.hh"
 #include "nix/util/json-utils.hh"
 #include <nlohmann/json.hpp>
@@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const
     return strHash() + "!" + outputName;
 }
 
-std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
-{
-    std::set<Realisation> res;
-    Realisation::closure(store, startOutputs, res);
-    return res;
-}
-
-void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
-{
-    auto getDeps = [&](const Realisation & current) -> std::set<Realisation> {
-        std::set<Realisation> res;
-        for (auto & [currentDep, _] : current.dependentRealisations) {
-            if (auto currentRealisation = store.queryRealisation(currentDep))
-                res.insert({*currentRealisation, currentDep});
-            else
-                throw Error("Unrealised derivation '%s'", currentDep.to_string());
-        }
-        return res;
-    };
-
-    computeClosure<Realisation>(
-        startOutputs,
-        res,
-        [&](const Realisation & current, std::function<void(std::promise<std::set<Realisation>> &)> processEdges) {
-            std::promise<std::set<Realisation>> promise;
-            try {
-                auto res = getDeps(current);
-                promise.set_value(res);
-            } catch (...) {
-                promise.set_exception(std::current_exception());
-            }
-            return processEdges(promise);
-        });
-}
-
 std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
 {
     nlohmann::json serialized = Realisation{*this, key};
@@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const &
 
 bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
 {
-    if (outPath == other.outPath) {
-        if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
-            warn(
-                "Encountered a realisation for '%s' with an empty set of "
-                "dependencies. This is likely an artifact from an older Nix. "
-                "I’ll try to fix the realisation if I can",
-                id.to_string());
-            return true;
-        } else if (dependentRealisations == other.dependentRealisations) {
-            return true;
-        }
-    }
-    return false;
-}
-
-void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret)
-{
-    // FIXME: This only builds the store-path closure, not the real realisation
-    // closure
-    StorePathSet initialStorePaths, pathsClosure;
-    for (auto & path : startPaths)
-        initialStorePaths.insert(path.path());
-    store.computeFSClosure(initialStorePaths, pathsClosure);
-    ret.insert(startPaths.begin(), startPaths.end());
-    ret.insert(pathsClosure.begin(), pathsClosure.end());
-}
-
-void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const
-{
-    RealisedPath::closure(store, {*this}, ret);
-}
-
-RealisedPath::Set RealisedPath::closure(Store & store) const
-{
-    RealisedPath::Set ret;
-    closure(store, ret);
-    return ret;
+    return outPath == other.outPath;
 }
 
 } // namespace nix
@@ -162,27 +90,19 @@ UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & js
     if (auto signaturesOpt = optionalValueAt(json, "signatures"))
         signatures = *signaturesOpt;
 
-    std::map<DrvOutput, StorePath> dependentRealisations;
-    if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations"))
-        for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies))
-            dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath});
-
     return UnkeyedRealisation{
         .outPath = valueAt(json, "outPath"),
         .signatures = signatures,
-        .dependentRealisations = dependentRealisations,
     };
 }
 
 void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
 {
-    auto jsonDependentRealisations = nlohmann::json::object();
-    for (auto & [depId, depOutPath] : r.dependentRealisations)
-        jsonDependentRealisations.emplace(depId.to_string(), depOutPath);
     json = {
         {"outPath", r.outPath},
         {"signatures", r.signatures},
-        {"dependentRealisations", jsonDependentRealisations},
+        // back-compat
+        {"dependentRealisations", json::object()},
     };
 }
 

src/libstore/restricted-store.cc

@@ -292,7 +292,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
     next->computeFSClosure(newPaths, closure);
     for (auto & path : closure)
         goal.addDependency(path);
-    for (auto & real : Realisation::closure(*next, newRealisations))
+    for (auto & real : newRealisations)
         goal.addedDrvOutputs.insert(real.id);
 
     return results;

src/libstore/store-api.cc

@@ -915,36 +915,21 @@ std::map<StorePath, StorePath> copyPaths(
     SubstituteFlag substitute)
 {
     StorePathSet storePaths;
-    std::set<Realisation> toplevelRealisations;
+    std::vector<const Realisation *> realisations;
     for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
             experimentalFeatureSettings.require(Xp::CaDerivations);
-            toplevelRealisations.insert(*realisation);
+            realisations.push_back(realisation);
         }
     }
 
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
 
     try {
-        // Copy the realisation closure
-        processGraph<Realisation>(
-            Realisation::closure(srcStore, toplevelRealisations),
-            [&](const Realisation & current) -> std::set<Realisation> {
-                std::set<Realisation> children;
-                for (const auto & [drvOutput, _] : current.dependentRealisations) {
-                    auto currentChild = srcStore.queryRealisation(drvOutput);
-                    if (!currentChild)
-                        throw Error(
-                            "incomplete realisation closure: '%s' is a "
-                            "dependency of '%s' but isn't registered",
-                            drvOutput.to_string(),
-                            current.id.to_string());
-                    children.insert({*currentChild, drvOutput});
-                }
-                return children;
-            },
-            [&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); });
+        // Copy the realisations. TODO batch this
+        for (const auto * realisation : realisations)
+            dstStore.registerDrvOutput(*realisation, checkSigs);
     } catch (MissingExperimentalFeature & e) {
         // Don't fail if the remote doesn't support CA derivations is it might
         // not be within our control to change that, and we might still want
@@ -1055,8 +1040,19 @@ void copyClosure(
     if (&srcStore == &dstStore)
         return;
 
-    RealisedPath::Set closure;
-    RealisedPath::closure(srcStore, paths, closure);
+    StorePathSet closure0;
+    for (auto & path : paths) {
+        if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
+            closure0.insert(opaquePath->path);
+        }
+    }
+
+    StorePathSet closure1;
+    srcStore.computeFSClosure(closure0, closure1);
+
+    RealisedPath::Set closure = paths;
+    for (auto && path : closure1)
+        closure.insert({std::move(path)});
 
     copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
 }

src/libutil-test-support/include/nix/util/tests/characterization.hh

@@ -4,20 +4,11 @@
 #include <gtest/gtest.h>
 
 #include "nix/util/types.hh"
-#include "nix/util/environment-variables.hh"
 #include "nix/util/file-system.hh"
+#include "nix/util/tests/test-data.hh"
 
 namespace nix {
 
-/**
- * The path to the unit test data directory. See the contributing guide
- * in the manual for further details.
- */
-static inline std::filesystem::path getUnitTestData()
-{
-    return getEnv("_NIX_TEST_UNIT_DATA").value();
-}
-
 /**
  * Whether we should update "golden masters" instead of running tests
  * against them. See the contributing guide in the manual for further

src/libutil-test-support/include/nix/util/tests/json-characterization.hh

@@ -83,6 +83,31 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go
     }
 }
 
+/**
+ * Specialization for when we need to do "JSON -> `ref<T>`" in one
+ * direction, but "`const T &` -> JSON" in the other direction.
+ */
+template<typename T>
+void checkpointJson(CharacterizationTest & test, PathView testStem, const ref<T> & got)
+{
+    using namespace nlohmann;
+
+    auto file = test.goldenMaster(Path{testStem} + ".json");
+
+    json gotJson = static_cast<json>(*got);
+
+    if (testAccept()) {
+        std::filesystem::create_directories(file.parent_path());
+        writeFile(file, gotJson.dump(2) + "\n");
+        ADD_FAILURE() << "Updating golden master " << file;
+    } else {
+        json expectedJson = json::parse(readFile(file));
+        ASSERT_EQ(gotJson, expectedJson);
+        ref<T> expected = adl_serializer<ref<T>>::from_json(expectedJson);
+        ASSERT_EQ(*got, *expected);
+    }
+}
+
 /**
  * Mixin class for writing characterization tests for `nlohmann::json`
  * conversions for a given type.

src/libutil-test-support/include/nix/util/tests/meson.build

@@ -10,4 +10,5 @@ headers = files(
   'json-characterization.hh',
   'nix_api_util.hh',
   'string_callback.hh',
+  'test-data.hh',
 )

src/libutil-test-support/include/nix/util/tests/test-data.hh

@@ -0,0 +1,24 @@
+#pragma once
+///@file
+
+#include <filesystem>
+#include "nix/util/environment-variables.hh"
+#include "nix/util/error.hh"
+
+namespace nix {
+
+/**
+ * The path to the unit test data directory. See the contributing guide
+ * in the manual for further details.
+ */
+static inline std::filesystem::path getUnitTestData()
+{
+    auto data = getEnv("_NIX_TEST_UNIT_DATA");
+    if (!data)
+        throw Error(
+            "_NIX_TEST_UNIT_DATA environment variable is not set. "
+            "Recommendation: use meson, example: 'meson test -C build --gdb'");
+    return std::filesystem::path(*data);
+}
+
+} // namespace nix

src/libutil-tests/closure.cc

@@ -20,10 +20,8 @@ TEST(closure, correctClosure)
     set<string> aClosure;
     set<string> expectedClosure = {"A", "B", "C", "F", "G"};
     computeClosure<string>(
-        {"A"}, aClosure, [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
-            promise<set<string>> promisedNodes;
-            promisedNodes.set_value(testGraph[currentNode]);
-            processEdges(promisedNodes);
+        {"A"}, aClosure, [&](const std::string & currentNode) -> asio::awaitable<std::set<std::string>> {
+            co_return testGraph[currentNode];
         });
 
     ASSERT_EQ(aClosure, expectedClosure);
@@ -37,31 +35,7 @@ TEST(closure, properlyHandlesDirectExceptions)
     set<string> aClosure;
     EXPECT_THROW(
         computeClosure<string>(
-            {"A"},
-            aClosure,
-            [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) { throw TestExn(); }),
-        TestExn);
-}
-
-TEST(closure, properlyHandlesExceptionsInPromise)
-{
-    struct TestExn
-    {};
-
-    set<string> aClosure;
-    EXPECT_THROW(
-        computeClosure<string>(
-            {"A"},
-            aClosure,
-            [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
-                promise<set<string>> promise;
-                try {
-                    throw TestExn();
-                } catch (...) {
-                    promise.set_exception(std::current_exception());
-                }
-                processEdges(promise);
-            }),
+            {"A"}, aClosure, [&](const std::string &) -> asio::awaitable<std::set<std::string>> { throw TestExn(); }),
         TestExn);
 }
 

src/libutil-tests/file-system.cc

@@ -388,14 +388,13 @@ TEST(openFileEnsureBeneathNoSymlinks, works)
 TEST(createAnonymousTempFile, works)
 {
     auto fd = createAnonymousTempFile();
-    auto fd_ = fromDescriptorReadOnly(fd.get());
     writeFull(fd.get(), "test");
-    lseek(fd_, 0, SEEK_SET);
+    lseek(fd.get(), 0, SEEK_SET);
     FdSource source{fd.get()};
     EXPECT_EQ(source.drain(), "test");
-    lseek(fd_, 0, SEEK_END);
+    lseek(fd.get(), 0, SEEK_END);
     writeFull(fd.get(), "test");
-    lseek(fd_, 0, SEEK_SET);
+    lseek(fd.get(), 0, SEEK_SET);
     EXPECT_EQ(source.drain(), "testtest");
 }
 
@@ -406,9 +405,8 @@ TEST(createAnonymousTempFile, works)
 TEST(FdSource, restartWorks)
 {
     auto fd = createAnonymousTempFile();
-    auto fd_ = fromDescriptorReadOnly(fd.get());
     writeFull(fd.get(), "hello world");
-    lseek(fd_, 0, SEEK_SET);
+    lseek(fd.get(), 0, SEEK_SET);
     FdSource source{fd.get()};
     EXPECT_EQ(source.drain(), "hello world");
     source.restart();
@@ -416,4 +414,11 @@ TEST(FdSource, restartWorks)
     EXPECT_EQ(source.drain(), "");
 }
 
+TEST(createTempDir, works)
+{
+    auto tmpDir = createTempDir();
+    nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
+    ASSERT_TRUE(std::filesystem::is_directory(tmpDir));
+}
+
 } // namespace nix

src/libutil-tests/meson.build

@@ -33,6 +33,9 @@ deps_private += rapidcheck
 gtest = dependency('gtest', main : true)
 deps_private += gtest
 
+gmock = dependency('gmock')
+deps_private += gmock
+
 configdata = configuration_data()
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 
@@ -72,6 +75,7 @@ sources = files(
   'position.cc',
   'processes.cc',
   'sort.cc',
+  'source-accessor.cc',
   'spawn.cc',
   'strings.cc',
   'suggestions.cc',

src/libutil-tests/source-accessor.cc

@@ -0,0 +1,140 @@
+#include "nix/util/fs-sink.hh"
+#include "nix/util/file-system.hh"
+#include "nix/util/processes.hh"
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <rapidcheck/gtest.h>
+
+namespace nix {
+
+MATCHER_P2(HasContents, path, expected, "")
+{
+    auto stat = arg->maybeLstat(path);
+    if (!stat) {
+        *result_listener << arg->showPath(path) << " does not exist";
+        return false;
+    }
+    if (stat->type != SourceAccessor::tRegular) {
+        *result_listener << arg->showPath(path) << " is not a regular file";
+        return false;
+    }
+    auto actual = arg->readFile(path);
+    if (actual != expected) {
+        *result_listener << arg->showPath(path) << " has contents " << ::testing::PrintToString(actual);
+        return false;
+    }
+    return true;
+}
+
+MATCHER_P2(HasSymlink, path, target, "")
+{
+    auto stat = arg->maybeLstat(path);
+    if (!stat) {
+        *result_listener << arg->showPath(path) << " does not exist";
+        return false;
+    }
+    if (stat->type != SourceAccessor::tSymlink) {
+        *result_listener << arg->showPath(path) << " is not a symlink";
+        return false;
+    }
+    auto actual = arg->readLink(path);
+    if (actual != target) {
+        *result_listener << arg->showPath(path) << " points to " << ::testing::PrintToString(actual);
+        return false;
+    }
+    return true;
+}
+
+MATCHER_P2(HasDirectory, path, dirents, "")
+{
+    auto stat = arg->maybeLstat(path);
+    if (!stat) {
+        *result_listener << arg->showPath(path) << " does not exist";
+        return false;
+    }
+    if (stat->type != SourceAccessor::tDirectory) {
+        *result_listener << arg->showPath(path) << " is not a directory";
+        return false;
+    }
+    auto actual = arg->readDirectory(path);
+    std::set<std::string> actualKeys, expectedKeys(dirents.begin(), dirents.end());
+    for (auto & [k, _] : actual)
+        actualKeys.insert(k);
+    if (actualKeys != expectedKeys) {
+        *result_listener << arg->showPath(path) << " has entries " << ::testing::PrintToString(actualKeys);
+        return false;
+    }
+    return true;
+}
+
+class FSSourceAccessorTest : public ::testing::Test
+{
+protected:
+    std::filesystem::path tmpDir;
+    std::unique_ptr<nix::AutoDelete> delTmpDir;
+
+    void SetUp() override
+    {
+        tmpDir = nix::createTempDir();
+        delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
+    }
+
+    void TearDown() override
+    {
+        delTmpDir.reset();
+    }
+};
+
+TEST_F(FSSourceAccessorTest, works)
+{
+    {
+        RestoreSink sink(false);
+        sink.dstPath = tmpDir;
+#ifndef _WIN32
+        sink.dirFd = openDirectory(tmpDir);
+#endif
+        sink.createDirectory(CanonPath("subdir"));
+        sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); });
+        sink.createRegularFile(CanonPath("subdir/file2"), [](CreateRegularFileSink & crf) { crf("content2"); });
+        sink.createSymlink(CanonPath("rootlink"), "target");
+        sink.createDirectory(CanonPath("a"));
+        sink.createSymlink(CanonPath("a/dirlink"), "../subdir");
+    }
+
+    EXPECT_THAT(makeFSSourceAccessor(tmpDir / "file1"), HasContents(CanonPath::root, "content1"));
+    EXPECT_THAT(makeFSSourceAccessor(tmpDir / "rootlink"), HasSymlink(CanonPath::root, "target"));
+    EXPECT_THAT(
+        makeFSSourceAccessor(tmpDir),
+        HasDirectory(CanonPath::root, std::set<std::string>{"file1", "subdir", "rootlink", "a"}));
+    EXPECT_THAT(makeFSSourceAccessor(tmpDir / "subdir"), HasDirectory(CanonPath::root, std::set<std::string>{"file2"}));
+
+    {
+        auto accessor = makeFSSourceAccessor(tmpDir);
+        EXPECT_THAT(accessor, HasContents(CanonPath("file1"), "content1"));
+        EXPECT_THAT(accessor, HasContents(CanonPath("subdir/file2"), "content2"));
+
+        EXPECT_TRUE(accessor->pathExists(CanonPath("file1")));
+        EXPECT_FALSE(accessor->pathExists(CanonPath("nonexistent")));
+
+        EXPECT_THROW(accessor->readFile(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
+        EXPECT_THROW(accessor->maybeLstat(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
+        EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), SymlinkNotAllowed);
+        EXPECT_THROW(accessor->pathExists(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
+    }
+
+    {
+        auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent");
+        EXPECT_FALSE(accessor->maybeLstat(CanonPath::root));
+        EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError);
+    }
+
+    {
+        auto accessor = makeFSSourceAccessor(tmpDir, true);
+        EXPECT_EQ(accessor->getLastModified(), 0);
+        accessor->maybeLstat(CanonPath("file1"));
+        EXPECT_GT(accessor->getLastModified(), 0);
+    }
+}
+
+} // namespace nix

src/libutil/compression.cc

@@ -69,24 +69,54 @@ struct ArchiveDecompressionSource : Source
     }
 };
 
+/* These strings are a part of the public API in store parameters and such. Do not change!
+   Happens to match enum names. */
+#define NIX_FOR_EACH_LA_ALGO(MACRO) \
+    MACRO(bzip2)                    \
+    MACRO(compress)                 \
+    MACRO(grzip)                    \
+    MACRO(gzip)                     \
+    MACRO(lrzip)                    \
+    MACRO(lz4)                      \
+    MACRO(lzip)                     \
+    MACRO(lzma)                     \
+    MACRO(lzop)                     \
+    MACRO(xz)                       \
+    MACRO(zstd)
+
 struct ArchiveCompressionSink : CompressionSink
 {
     Sink & nextSink;
     struct archive * archive;
 
-    ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
+    ArchiveCompressionSink(
+        Sink & nextSink, CompressionAlgo method, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
         : nextSink(nextSink)
     {
         archive = archive_write_new();
         if (!archive)
             throw Error("failed to initialize libarchive");
-        check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
+
+        auto [addFilter, format] = [method]() -> std::pair<int (*)(struct archive *), const char *> {
+            switch (method) {
+            case CompressionAlgo::none:
+            case CompressionAlgo::brotli:
+                unreachable();
+#define NIX_DEF_LA_ALGO_CASE(algo) \
+    case CompressionAlgo::algo:    \
+        return {archive_write_add_filter_##algo, #algo};
+                NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
+#undef NIX_DEF_LA_ALGO_CASE
+            }
+            unreachable();
+        }();
+
+        check(addFilter(archive), "couldn't initialize compression (%s)");
         check(archive_write_set_format_raw(archive));
         if (parallel)
-            check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
+            check(archive_write_set_filter_option(archive, format, "threads", "0"));
         if (level != COMPRESSION_LEVEL_DEFAULT)
-            check(archive_write_set_filter_option(
-                archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
+            check(archive_write_set_filter_option(archive, format, "compression-level", std::to_string(level).c_str()));
         // disable internal buffering
         check(archive_write_set_bytes_per_block(archive, 0));
         // disable output padding
@@ -289,19 +319,52 @@ struct BrotliCompressionSink : ChunkedCompressionSink
     }
 };
 
+/* Parses a *compression* method into the corresponding enum. This is only used
+   in the *compression* case and user interface. Content-Encoding should not use
+   these. */
+static CompressionAlgo parseNixCompressionAlgoString(std::string_view method)
+{
+    static const std::unordered_map<std::string_view, CompressionAlgo> lookupTable = {
+        {"none", CompressionAlgo::none},
+        {"br", CompressionAlgo::brotli},
+#define NIX_DEF_LA_ALGO_NAME(algo) {#algo, CompressionAlgo::algo},
+        NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_NAME)
+#undef NIX_DEF_LA_ALGO_NAME
+    };
+
+    if (auto it = lookupTable.find(method); it != lookupTable.end())
+        return it->second;
+
+    static const StringSet allNames = [&]() {
+        StringSet res;
+        for (auto & [name, _] : lookupTable)
+            res.emplace(name);
+        return res;
+    }();
+
+    throw UnknownCompressionMethod(
+        Suggestions::bestMatches(allNames, method), "unknown compression method '%s'", method);
+}
+
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
 {
-    std::vector<std::string> la_supports = {
-        "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
-    if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
-        return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
-    }
-    if (method == "none")
+    return makeCompressionSink(parseNixCompressionAlgoString(method), nextSink, parallel, level);
+}
+
+ref<CompressionSink> makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level)
+{
+    switch (method) {
+    case CompressionAlgo::none:
         return make_ref<NoneSink>(nextSink);
-    else if (method == "br")
+    case CompressionAlgo::brotli:
         return make_ref<BrotliCompressionSink>(nextSink);
-    else
-        throw UnknownCompressionMethod("unknown compression method '%s'", method);
+        /* Everything else is supported via libarchive. */
+#define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo:
+        NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
+        return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
+#undef NIX_DEF_LA_ALGO_CASE
+    }
+    unreachable();
 }
 
 std::string compress(const std::string & method, std::string_view in, const bool parallel, int level)

src/libutil/file-system.cc

@@ -115,9 +115,6 @@ Path canonPath(PathView path, bool resolveSymlinks)
     if (!isAbsolute(path))
         throw Error("not an absolute path: '%1%'", path);
 
-    // For Windows
-    auto rootName = std::filesystem::path{path}.root_name();
-
     /* This just exists because we cannot set the target of `remaining`
        (the callback parameter) directly to a newly-constructed string,
        since it is `std::string_view`. */
@@ -147,8 +144,6 @@ Path canonPath(PathView path, bool resolveSymlinks)
             }
         });
 
-    if (!rootName.empty())
-        ret = rootName.string() + std::move(ret);
     return ret;
 }
 
@@ -380,14 +375,6 @@ void syncParent(const Path & path)
     fd.fsync();
 }
 
-#ifdef __FreeBSD__
-#  define MOUNTEDPATHS_PARAM , std::set<Path> & mountedPaths
-#  define MOUNTEDPATHS_ARG , mountedPaths
-#else
-#  define MOUNTEDPATHS_PARAM
-#  define MOUNTEDPATHS_ARG
-#endif
-
 void recursiveSync(const Path & path)
 {
     /* If it's a file or symlink, just fsync and return. */
@@ -432,129 +419,6 @@ void recursiveSync(const Path & path)
     }
 }
 
-static void _deletePath(
-    Descriptor parentfd,
-    const std::filesystem::path & path,
-    uint64_t & bytesFreed,
-    std::exception_ptr & ex MOUNTEDPATHS_PARAM)
-{
-#ifndef _WIN32
-    checkInterrupt();
-
-#  ifdef __FreeBSD__
-    // In case of emergency (unmount fails for some reason) not recurse into mountpoints.
-    // This prevents us from tearing up the nullfs-mounted nix store.
-    if (mountedPaths.find(path) != mountedPaths.end()) {
-        return;
-    }
-#  endif
-
-    std::string name(path.filename());
-    assert(name != "." && name != ".." && !name.empty());
-
-    struct stat st;
-    if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
-        if (errno == ENOENT)
-            return;
-        throw SysError("getting status of %1%", path);
-    }
-
-    if (!S_ISDIR(st.st_mode)) {
-        /* We are about to delete a file. Will it likely free space? */
-
-        switch (st.st_nlink) {
-        /* Yes: last link. */
-        case 1:
-            bytesFreed += st.st_size;
-            break;
-        /* Maybe: yes, if 'auto-optimise-store' or manual optimisation
-           was performed. Instead of checking for real let's assume
-           it's an optimised file and space will be freed.
-
-           In worst case we will double count on freed space for files
-           with exactly two hardlinks for unoptimised packages.
-         */
-        case 2:
-            bytesFreed += st.st_size;
-            break;
-        /* No: 3+ links. */
-        default:
-            break;
-        }
-    }
-
-    if (S_ISDIR(st.st_mode)) {
-        /* Make the directory accessible. */
-        const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
-        if ((st.st_mode & PERM_MASK) != PERM_MASK) {
-            if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
-                throw SysError("chmod %1%", path);
-        }
-
-        int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
-        if (fd == -1)
-            throw SysError("opening directory %1%", path);
-        AutoCloseDir dir(fdopendir(fd));
-        if (!dir)
-            throw SysError("opening directory %1%", path);
-
-        struct dirent * dirent;
-        while (errno = 0, dirent = readdir(dir.get())) { /* sic */
-            checkInterrupt();
-            std::string childName = dirent->d_name;
-            if (childName == "." || childName == "..")
-                continue;
-            _deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG);
-        }
-        if (errno)
-            throw SysError("reading directory %1%", path);
-    }
-
-    int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
-    if (unlinkat(parentfd, name.c_str(), flags) == -1) {
-        if (errno == ENOENT)
-            return;
-        try {
-            throw SysError("cannot unlink %1%", path);
-        } catch (...) {
-            if (!ex)
-                ex = std::current_exception();
-            else
-                ignoreExceptionExceptInterrupt();
-        }
-    }
-#else
-    // TODO implement
-    throw UnimplementedError("_deletePath");
-#endif
-}
-
-static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM)
-{
-    assert(path.is_absolute());
-    assert(path.parent_path() != path);
-
-    AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY));
-    if (!dirfd) {
-        if (errno == ENOENT)
-            return;
-        throw SysError("opening directory %s", path.parent_path());
-    }
-
-    std::exception_ptr ex;
-
-    _deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG);
-
-    if (ex)
-        std::rethrow_exception(ex);
-}
-
-void deletePath(const std::filesystem::path & path)
-{
-    uint64_t dummy;
-    deletePath(path, dummy);
-}
-
 void createDir(const Path & path, mode_t mode)
 {
     if (mkdir(
@@ -577,25 +441,6 @@ void createDirs(const std::filesystem::path & path)
     }
 }
 
-void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
-{
-    // Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
-#ifdef __FreeBSD__
-    std::set<Path> mountedPaths;
-    struct statfs * mntbuf;
-    int count;
-    if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) {
-        throw SysError("getmntinfo");
-    }
-
-    for (int i = 0; i < count; i++) {
-        mountedPaths.emplace(mntbuf[i].f_mntonname);
-    }
-#endif
-    bytesFreed = 0;
-    _deletePath(path, bytesFreed MOUNTEDPATHS_ARG);
-}
-
 //////////////////////////////////////////////////////////////////////
 
 AutoDelete::AutoDelete()
@@ -672,11 +517,6 @@ void AutoUnmount::cancel()
 
 //////////////////////////////////////////////////////////////////////
 
-std::filesystem::path defaultTempDir()
-{
-    return getEnvNonEmpty("TMPDIR").value_or("/tmp");
-}
-
 std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode)
 {
     while (1) {
@@ -713,17 +553,27 @@ AutoCloseFD createAnonymousTempFile()
 {
     AutoCloseFD fd;
 #ifdef O_TMPFILE
-    fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR, S_IWUSR | S_IRUSR);
-    if (!fd)
-        throw SysError("creating anonymous temporary file");
-#else
+    static std::atomic_flag tmpfileUnsupported{};
+    if (!tmpfileUnsupported.test()) /* Try with O_TMPFILE first. */ {
+        /* Use O_EXCL, because the file is never supposed to be linked into filesystem. */
+        fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR | O_EXCL, S_IWUSR | S_IRUSR);
+        if (!fd) {
+            /* Not supported by the filesystem or the kernel. */
+            if (errno == EOPNOTSUPP || errno == EISDIR)
+                tmpfileUnsupported.test_and_set(); /* Set flag and fall through to createTempFile. */
+            else
+                throw SysError("creating anonymous temporary file");
+        } else {
+            return fd; /* Successfully created. */
+        }
+    }
+#endif
     auto [fd2, path] = createTempFile("nix-anonymous");
     if (!fd2)
         throw SysError("creating temporary file '%s'", path);
     fd = std::move(fd2);
-#  ifndef _WIN32
+#ifndef _WIN32
     unlink(requireCString(path)); /* We only care about the file descriptor. */
-#  endif
 #endif
     return fd;
 }

src/libutil/freebsd/include/nix/util/freebsd-jail.hh

@@ -11,6 +11,24 @@ class AutoRemoveJail
     bool del;
 public:
     AutoRemoveJail(int jid);
+    AutoRemoveJail(const AutoRemoveJail &) = delete;
+    AutoRemoveJail & operator=(const AutoRemoveJail &) = delete;
+
+    AutoRemoveJail(AutoRemoveJail && other) noexcept
+        : jid(other.jid)
+        , del(other.del)
+    {
+        other.cancel();
+    }
+
+    AutoRemoveJail & operator=(AutoRemoveJail && other) noexcept
+    {
+        jid = other.jid;
+        del = other.del;
+        other.cancel();
+        return *this;
+    }
+
     AutoRemoveJail();
     ~AutoRemoveJail();
     void cancel();

src/libutil/include/nix/util/async.hh

@@ -0,0 +1,68 @@
+#pragma once
+///@file
+
+#include "nix/util/callback.hh"
+#include "nix/util/ref.hh"
+#include "nix/util/signals.hh"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/post.hpp>
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/use_awaitable.hpp>
+#include <boost/asio/associated_cancellation_slot.hpp>
+
+#include <concepts>
+
+namespace nix {
+
+namespace asio = boost::asio;
+
+template<typename T, std::invocable<Callback<T>> F, typename CompletionToken>
+auto callbackToAwaitable(F && initiate, CompletionToken && token)
+{
+    return asio::async_initiate<CompletionToken, void(std::future<T>)>(
+        [initiate = std::forward<F>(initiate)](auto handler) mutable {
+            auto executor = asio::get_associated_executor(handler);
+            auto done = std::make_shared<std::atomic<bool>>(false);
+            auto h = std::make_shared<decltype(handler)>(std::move(handler));
+
+            if (auto slot = asio::get_associated_cancellation_slot(*h); slot.is_connected()) {
+                std::weak_ptr wh = h; /* To handle the cyclic ownership. */
+                std::weak_ptr wdone = done;
+                slot.assign([executor, wh, wdone](asio::cancellation_type /*don't care*/) {
+                    auto h = wh.lock();
+                    auto done = wdone.lock();
+                    if (!h || !done || done->exchange(true))
+                        return; /* Gracefully die. */
+                    /* Doesn't need to be kept alive for get_future() since it shares the ownership. */
+                    std::promise<T> p;
+                    p.set_exception(std::make_exception_ptr(Interrupted("interrupted by user")));
+                    asio::post(executor, [h, fut = p.get_future()]() mutable { std::move (*h)(std::move(fut)); });
+                });
+            }
+
+            initiate(Callback<T>([executor, done, h](std::future<T> fut) mutable {
+                if (done->exchange(true))
+                    /* Early return for cooperative cancellation. The callback has been caller
+                       later than we've been cancelled. In practice we'll get an error, the handler
+                       has already been posted by the cancellation handler. */
+                    return;
+                asio::post(executor, [h, fut = std::move(fut)]() mutable { std::move (*h)(std::move(fut)); });
+            }));
+        },
+        std::forward<CompletionToken>(token));
+}
+
+/**
+ * Convert a completion handler callback into a stackless coroutine. The
+ * callback can be invoked on any thread and the completion handler will be
+ * marshalled to the coroutines executer.
+ */
+template<typename T, std::invocable<Callback<T>> F>
+asio::awaitable<T> callbackToAwaitable(F && initiate)
+{
+    auto fut = co_await callbackToAwaitable<T>(std::forward<F>(initiate), asio::use_awaitable);
+    co_return fut.get();
+}
+
+} // namespace nix

src/libutil/include/nix/util/closure.hh

@@ -1,73 +1,167 @@
 #pragma once
 ///@file
 
-#include <set>
-#include <future>
-#include "nix/util/sync.hh"
+#include "nix/util/async.hh"
+#include "nix/util/ref.hh"
 
-using std::set;
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/post.hpp>
+#include <boost/asio/strand.hpp>
+#include <boost/asio/detached.hpp>
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/co_spawn.hpp>
+#include <boost/asio/bind_executor.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+
+#include <queue>
+#include <set>
 
 namespace nix {
 
 template<typename T>
-using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
+using GetEdgesAsync = std::function<asio::awaitable<std::set<T>>(const T & elt)>;
 
-template<typename T>
-void computeClosure(const set<T> startElts, set<T> & res, GetEdgesAsync<T> getEdgesAsync)
+template<typename T, typename CompletionToken>
+auto computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges, CompletionToken && token)
 {
-    struct State
-    {
-        size_t pending;
-        set<T> & res;
-        std::exception_ptr exc;
-    };
+    auto initiator = [&res, startElts = std::move(startElts), getEdges = std::move(getEdges)](auto handler) {
+        auto executor = asio::make_strand(asio::get_associated_executor(handler));
 
-    Sync<State> state_(State{0, res, 0});
-
-    std::condition_variable done;
-
-    auto enqueue = [&](this auto & enqueue, const T & current) -> void {
+        /* Hand-rolled dynamic async graph traversal. ASIO/Cobalt and standard
+         * C++ will get channels and task groups at some point, but this will
+         * have to suffice for now. */
+        struct State : std::enable_shared_from_this<State>
         {
-            auto state(state_.lock());
-            if (state->exc)
-                return;
-            if (!state->res.insert(current).second)
-                return;
-            state->pending++;
+            decltype(executor) executor;
+            decltype(getEdges) getEdges;
+            decltype(handler) handler;
+            std::set<T> & res;
+            /**
+             * Needed to keep the ctx.run() alive because actual work might be happening on another thread
+             * (like with FileTransfer case).
+             */
+            asio::executor_work_guard<decltype(State::executor)> workGuard;
+            std::size_t pending = 0;
+            /**
+             * Whether the completion handler has been called.
+             */
+            bool done = false;
+            /**
+             * Amount of coroutines currently in flight.
+             */
+            std::size_t inFlight = 0;
+            /**
+             * Maximum number of concurrent coroutines. Implements primitive rate limiting.
+             */
+            std::size_t maxConcurrent = 64;
+            /**
+             * Nodes to handle next.
+             */
+            std::queue<T> todo;
+
+            State(
+                decltype(executor) executor_, decltype(getEdges) getEdges, decltype(handler) handler, std::set<T> & res)
+                : executor(executor_)
+                , getEdges(std::move(getEdges))
+                , handler(std::move(handler))
+                , res(res)
+                , workGuard(asio::make_work_guard(executor_))
+            {
+            }
+
+            void complete(std::exception_ptr ex)
+            {
+                if (std::exchange(done, true))
+                    return;
+                workGuard.reset(); /* We are done and we can release the lock. */
+                asio::post(executor, [state = this->shared_from_this(), ex] { state->handler(ex); });
+            }
+
+            void enqueue(const std::set<T> & elts)
+            {
+                for (const auto & elt : elts)
+                    enqueue(elt);
+            }
+
+            void spawnWorker(const T & elt)
+            {
+                ++inFlight;
+                auto state = this->shared_from_this();
+                asio::post(executor, [state = this->shared_from_this(), elt = std::move(elt)] {
+                    asio::co_spawn(
+                        state->executor,
+                        [state, elt]() -> asio::awaitable<void> {
+                            try {
+                                state->enqueue(co_await state->getEdges(elt));
+                            } catch (...) {
+                                state->complete(std::current_exception());
+                            }
+                            state->onWorkDone();
+                        },
+                        asio::detached);
+                });
+            }
+
+            void onWorkDone()
+            {
+                --inFlight;
+                --pending;
+
+                if (!todo.empty()) {
+                    auto next = std::move(todo.front());
+                    todo.pop();
+                    asio::post(executor, [state = this->shared_from_this(), next = std::move(next)]() mutable {
+                        state->spawnWorker(std::move(next));
+                    });
+                } else if (pending == 0) {
+                    complete(std::exception_ptr{});
+                }
+            }
+
+            void enqueue(const T & elt)
+            {
+                if (done)
+                    return;
+
+                if (!res.insert(elt).second)
+                    return;
+
+                ++pending;
+
+                if (inFlight < maxConcurrent) {
+                    spawnWorker(elt);
+                } else {
+                    todo.push(elt);
+                }
+            }
+        };
+
+        auto state = make_ref<State>(executor, std::move(getEdges), std::move(handler), res);
+        if (startElts.empty()) {
+            /* No work to do. */
+            state->complete(std::exception_ptr{});
+            return;
         }
 
-        getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
-            try {
-                auto children = prom.get_future().get();
-                for (auto & child : children)
-                    enqueue(child);
-                {
-                    auto state(state_.lock());
-                    assert(state->pending);
-                    if (!--state->pending)
-                        done.notify_one();
-                }
-            } catch (...) {
-                auto state(state_.lock());
-                if (!state->exc)
-                    state->exc = std::current_exception();
-                assert(state->pending);
-                if (!--state->pending)
-                    done.notify_one();
-            };
-        });
+        asio::post(executor, [state, startElts = std::move(startElts)] { state->enqueue(startElts); });
     };
 
-    for (auto & startElt : startElts)
-        enqueue(startElt);
+    return asio::async_initiate<CompletionToken, void(std::exception_ptr)>(std::move(initiator), token);
+}
 
-    {
-        auto state(state_.lock());
-        while (state->pending)
-            state.wait(done);
-        if (state->exc)
-            std::rethrow_exception(state->exc);
-    }
+template<typename T>
+void computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges)
+{
+    asio::io_context ctx;
+    std::exception_ptr ex = nullptr;
+    computeClosure(
+        std::move(startElts),
+        res,
+        std::move(getEdges),
+        asio::bind_executor(ctx.get_executor(), [&](std::exception_ptr ex2) { ex = ex2; }));
+    ctx.run();
+    if (ex)
+        std::rethrow_exception(ex);
 }
 
 } // namespace nix

src/libutil/include/nix/util/compression.hh

@@ -16,6 +16,22 @@ struct CompressionSink : BufferedSink, FinishSink
     using FinishSink::finish;
 };
 
+enum class CompressionAlgo {
+    none,
+    brotli,
+    bzip2,
+    compress,
+    grzip,
+    gzip,
+    lrzip,
+    lz4,
+    lzip,
+    lzma,
+    lzop,
+    xz,
+    zstd,
+};
+
 std::string decompress(const std::string & method, std::string_view in);
 
 std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
@@ -25,6 +41,9 @@ std::string compress(const std::string & method, std::string_view in, const bool
 ref<CompressionSink>
 makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
 
+ref<CompressionSink>
+makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel = false, int level = -1);
+
 MakeError(UnknownCompressionMethod, Error);
 
 MakeError(CompressionError, Error);

src/libutil/include/nix/util/file-descriptor.hh

@@ -2,7 +2,6 @@
 ///@file
 
 #include "nix/util/canon-path.hh"
-#include "nix/util/types.hh"
 #include "nix/util/error.hh"
 
 #ifdef _WIN32
@@ -236,18 +235,6 @@ std::wstring handleToFileName(Descriptor handle);
 #ifndef _WIN32
 namespace unix {
 
-struct SymlinkNotAllowed : public Error
-{
-    CanonPath path;
-
-    SymlinkNotAllowed(CanonPath path)
-        /* Can't provide better error message, since the parent directory is only known to the caller. */
-        : Error("relative path '%s' points to a symlink, which is not allowed", path.rel())
-        , path(std::move(path))
-    {
-    }
-};
-
 /**
  * Safe(r) function to open \param path file relative to \param dirFd, while
  * disallowing escaping from a directory and resolving any symlinks in the
@@ -274,4 +261,14 @@ Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & p
 
 MakeError(EndOfFile, Error);
 
+#ifdef _WIN32
+
+/**
+ * Windows specific replacement for POSIX `lseek` that operates on a `HANDLE` and not
+ * a file descriptor.
+ */
+off_t lseek(Descriptor fd, off_t offset, int whence);
+
+#endif
+
 } // namespace nix

src/libutil/include/nix/util/file-path-impl.hh

@@ -40,6 +40,11 @@ struct UnixPathTrait
     {
         return path.rfind('/', from);
     }
+
+    static size_t rootNameLen(StringView)
+    {
+        return 0;
+    }
 };
 
 /**
@@ -83,6 +88,18 @@ struct WindowsPathTrait
         size_t p2 = path.rfind(preferredSep, from);
         return p1 == String::npos ? p2 : p2 == String::npos ? p1 : std::max(p1, p2);
     }
+
+    static size_t rootNameLen(StringView path)
+    {
+        if (path.size() >= 2 && path[1] == ':') {
+            char driveLetter = path[0];
+            if ((driveLetter >= 'A' && driveLetter <= 'Z') || (driveLetter >= 'a' && driveLetter <= 'z'))
+                return 2;
+        }
+        /* TODO: This needs to also handle UNC paths.
+         * https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats#unc-paths */
+        return 0;
+    }
 };
 
 template<typename CharT>
@@ -116,6 +133,11 @@ typename PathDict::String canonPathInner(typename PathDict::StringView remaining
     typename PathDict::String result;
     result.reserve(256);
 
+    if (auto rootNameLength = PathDict::rootNameLen(remaining)) {
+        result += remaining.substr(0, rootNameLength); /* Copy drive letter verbatim. */
+        remaining.remove_prefix(rootNameLength);
+    }
+
     while (true) {
 
         /* Skip slashes. */