Make the git sink support bare files

It is not actually used, and I rather not keep such "fictitious value"
around.

.github/workflows/backport.yml

@@ -26,7 +26,7 @@ jobs:
           # required to find all branches
           fetch-depth: 0
       - name: Create backport PRs
-        uses: korthout/backport-action@d07416681cab29bf2661702f925f020aaa962997 # v3.4.1
+        uses: korthout/backport-action@c656f5d5851037b2b38fb5db2691a03fa229e3b2 # v4.0.1
         id: backport
         with:
           # Config README: https://github.com/korthout/backport-action#backport-action

.github/workflows/ci.yml

@@ -125,13 +125,13 @@ jobs:
         cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
       if: ${{ matrix.instrumented }}
     - name: Upload coverage reports
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: coverage-reports
         path: coverage-reports/
       if: ${{ matrix.instrumented }}
     - name: Upload installer tarball
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: installer-${{matrix.os}}
         path: out/*
@@ -164,7 +164,7 @@ jobs:
     steps:
     - uses: actions/checkout@v6
     - name: Download installer tarball
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v7
       with:
         name: installer-${{matrix.os}}
         path: out
@@ -174,7 +174,7 @@ jobs:
         echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
         TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
         echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
-    - uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
+    - uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
       if: ${{ !matrix.experimental-installer }}
       with:
         install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}

.version

@@ -1 +1 @@
-2.33.0
+2.34.0

doc/manual/rl-next/channels-subdomain.md

@@ -1,9 +0,0 @@
----
-synopsis: Channel URLs migrated to channels.nixos.org subdomain
-prs: [14518]
-issues: [14517]
----
-
-Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix.
-
-The subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.

doc/manual/rl-next/json-format-changes.md

@@ -1,88 +0,0 @@
----
-synopsis: "JSON format changes for store path info and derivations"
-prs: []
-issues: []
----
-
-JSON formats for store path info and derivations have been updated with new versions and structured fields.
-
-## Store Path Info JSON
-
-`nix path-info --json` now requires a `--json-format` flag to specify the output format version.
-Using `--json` without `--json-format` is deprecated and will become an error in a future release.
-For now, it defaults to version 1 with a warning, for a smoother migration.
-
-### Version 1 (`--json-format 1`)
-
-This is the legacy format, preserved for backwards compatibility:
-
-- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
-- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
-- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
-- Now includes `"storeDir"` field at the top level
-
-### Version 2 (`--json-format 2`)
-
-The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
-
-- **Nested structure with top-level metadata**:
-
-  The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
-
-  ```json
-  {
-    "version": 2,
-    "storeDir": "/nix/store",
-    "info": { ... }
-  }
-  ```
-
-  The map from store bath base names to store object info is nested under the `info` field.
-
-- **Store path base names instead of full paths**:
-
-  Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
-  Combined with `storeDir`, the full path can be reconstructed.
-
-- **Structured `ca` field**:
-
-  Content address is now a structured JSON object instead of a string:
-
-  - Old: `"ca": "fixed:r:sha256:1abc..."`
-  - New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
-  - Still `null` values for input-addressed store objects
-
-  The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
-
-Nix currently only produces, and doesn't consume this format.
-
-Additionally the following field is added to both formats.
-(The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.)
-
-- **`version` field**:
-
-  All store path info JSON now includes `"version": <1|2>`.
-
-- **`storeDir` field**:
-
-  Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
-
-## Derivation JSON (Version 4)
-
-The derivation JSON format has been updated from version 3 to version 4:
-
-- **Restructured inputs**:
-
-  Inputs are now nested under an `inputs` object:
-
-  - Old: `"inputSrcs": [...], "inputDrvs": {...}`
-  - New: `"inputs": {"srcs": [...], "drvs": {...}}`
-
-- **Consistent content addresses**:
-
-  Fixed content-addressed outputs now use structured JSON format.
-  This is the same format as `ca` in store path info (after the new version).
-
-Version 3 and earlier formats are *not* accepted when reading.
-
-**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.

doc/manual/rl-next/libcurl-pausing.md

@@ -1,12 +0,0 @@
----
-synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
-prs: [14614]
-issues: [11728]
----
-
-The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
-[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
-
-This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
-
-Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).

doc/manual/rl-next/repl-interrupt.md

@@ -1,8 +0,0 @@
----
-synopsis: Interrupting REPL commands works more than once
-issues: [13481]
----
-
-Previously, this only worked once per REPL session; further attempts would be ignored.
-This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
-This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).

doc/manual/rl-next/s3-curl-implementation.md

@@ -1,40 +0,0 @@
----
-synopsis: "Improved S3 binary cache support via HTTP"
-prs: [13752, 13823, 14026, 14120, 14131, 14135, 14144, 14170, 14190, 14198, 14206, 14209, 14222, 14223, 14330, 14333, 14335, 14336, 14337, 14350, 14356, 14357, 14374, 14375, 14376, 14377, 14391, 14393, 14420, 14421]
-issues: [13084, 12671, 11748, 12403]
----
-
-S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native
-AWS SigV4 authentication instead of the AWS C++ SDK, providing significant
-improvements:
-
-- **Reduced memory usage**: Eliminates memory buffering issues that caused
-  segfaults with large files
-- **Fixed upload reliability**: Resolves AWS SDK chunking errors
-  (`InvalidChunkSizeError`)
-- **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full
-  `aws-cpp-sdk`, reducing build complexity
-
-The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential
-management.
-
-All existing S3 URL formats and parameters remain supported, however the store
-settings for configuring multipart uploads have changed:
-
-- **`multipart-upload`** (default: `false`): Enable multipart uploads for large
-  files. When enabled, files exceeding the multipart threshold will be uploaded
-  in multiple parts.
-
-- **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using
-  multipart uploads. Files smaller than this will use regular PUT requests.
-  Only takes effect when `multipart-upload` is enabled.
-
-- **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart
-  uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes
-  reduce the number of requests but use more memory.
-
-- **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
-
-Note that this change also means Nix now supports S3 binary cache stores even
-if built without `aws-crt-cpp`, but only for public buckets which do not
-require authentication.

doc/manual/rl-next/s3-object-versioning.md

@@ -1,14 +0,0 @@
----
-synopsis: "S3 URLs now support object versioning via versionId parameter"
-prs: [14274]
-issues: [13955]
----
-
-S3 URLs now support a `versionId` query parameter to fetch specific versions
-of objects from S3 buckets with versioning enabled. This allows pinning to
-exact object versions for reproducibility and protection against unexpected
-changes:
-
-```
-s3://bucket/key?region=us-east-1&versionId=abc123def456
-```

doc/manual/rl-next/s3-storage-class.md

@@ -1,21 +0,0 @@
----
-synopsis: "S3 binary cache stores now support storage class configuration"
-prs: [14464]
-issues: [7015]
----
-
-S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
-
-Example usage:
-
-```bash
-# Use Glacier storage for long-term archival
-nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
-
-# Use Intelligent Tiering for automatic cost optimization
-nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
-```
-
-The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
-
-See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.

doc/manual/source/SUMMARY.md.in

@@ -151,6 +151,7 @@
   - [Contributing](development/contributing.md)
 - [Releases](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
+  - [Release 2.33 (2025-12-09)](release-notes/rl-2.33.md)
   - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md)
   - [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md)
   - [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md)

doc/manual/source/development/building.md

@@ -3,6 +3,10 @@
 This section provides some notes on how to start hacking on Nix.
 To get the latest version of Nix from GitHub:
 
+> **Note**
+>
+> When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo.
+
 ```console
 $ git clone https://github.com/NixOS/nix.git
 $ cd nix

doc/manual/source/development/debugging.md

@@ -6,14 +6,7 @@ Additionally, see [Testing Nix](./testing.md) for further instructions on how to
 
 ## Building Nix with Debug Symbols
 
-In the development shell, set the `mesonBuildType` environment variable to `debug` before configuring the build:
-
-```console
-[nix-shell]$ export mesonBuildType=debugoptimized
-```
-
-Then, proceed to build Nix as described in [Building Nix](./building.md).
-This will build Nix with debug symbols, which are essential for effective debugging.
+In the development shell, `mesonBuildType` is set automatically to `debugoptimized`. This builds Nix with debug symbols, which are essential for effective debugging.
 
 It is also possible to build without optimization for faster build:
 

doc/manual/source/protocols/json/build-trace-entry.md

@@ -1,27 +1,21 @@
-{{#include build-trace-entry-v1-fixed.md}}
+{{#include build-trace-entry-v2-fixed.md}}
 
 ## Examples
 
 ### Simple build trace entry
 
 ```json
-{{#include schema/build-trace-entry-v1/simple.json}}
-```
-
-### Build trace entry with dependencies
-
-```json
-{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}}
+{{#include schema/build-trace-entry-v2/simple.json}}
 ```
 
 ### Build trace entry with signature
 
 ```json
-{{#include schema/build-trace-entry-v1/with-signature.json}}
+{{#include schema/build-trace-entry-v2/with-signature.json}}
 ```
 
 <!--
 ## Raw Schema
 
-[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v1.json)
--->
+[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v2.json)
+-->

doc/manual/source/protocols/json/meson.build

@@ -17,7 +17,7 @@ schemas = [
   'derivation-v4',
   'derivation-options-v1',
   'deriving-path-v1',
-  'build-trace-entry-v1',
+  'build-trace-entry-v2',
   'build-result-v1',
   'store-v1',
 ]

doc/manual/source/protocols/json/schema/build-result-v1.yaml

@@ -83,7 +83,7 @@ properties:
         description: |
           A mapping from output names to their build trace entries.
         additionalProperties:
-          "$ref": "build-trace-entry-v1.yaml"
+          "$ref": "build-trace-entry-v2.yaml"
 
   failure:
     type: object

doc/manual/source/protocols/json/schema/build-trace-entry-v2.yaml

@@ -1,5 +1,5 @@
 "$schema": "http://json-schema.org/draft-04/schema"
-"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json"
 title: Build Trace Entry
 description: |
   A record of a successful build outcome for a specific derivation output.
@@ -11,10 +11,17 @@ description: |
   > This JSON format is currently
   > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
   > and subject to change.
+
+  Verision history:
+
+  - Version 1: Original format
+
+  - Version 2: Remove `dependentRealisations`
+
+type: object
 required:
   - id
   - outPath
-  - dependentRealisations
   - signatures
 allOf:
   - "$ref": "#/$defs/key"
@@ -22,9 +29,11 @@ allOf:
 properties:
   id: {}
   outPath: {}
-  dependentRealisations: {}
   signatures: {}
-additionalProperties: false
+additionalProperties:
+  dependentRealisations:
+    description: deprecated field
+    type: object
 
 "$defs":
   key:
@@ -60,7 +69,6 @@ additionalProperties: false
     type: object
     required:
       - outPath
-      - dependentRealisations
       - signatures
     properties:
       outPath:
@@ -69,19 +77,6 @@ additionalProperties: false
         description: |
           The path to the store object that resulted from building this derivation for the given output name.
 
-      dependentRealisations:
-        type: object
-        title: Underlying Base Build Trace
-        description: |
-          This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
-
-          Keys are derivation output IDs (same format as the main `id` field).
-          Values are the store paths that those dependencies resolved to.
-
-          As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
-          This is the set of base build trace entries that this derived build trace is derived from.
-          (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
-
         patternProperties:
           "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
             "$ref": "store-path-v1.yaml"

doc/manual/source/protocols/json/schema/store-v1.yaml

@@ -70,7 +70,7 @@ properties:
       "^[A-Za-z0-9+/]{43}=$":
         type: object
         additionalProperties:
-          "$ref": "./build-trace-entry-v1.yaml#/$defs/value"
+          "$ref": "./build-trace-entry-v2.yaml#/$defs/value"
     additionalProperties: false
 
 "$defs":

doc/manual/source/release-notes/rl-2.33.md

@@ -0,0 +1,281 @@
+# Release 2.33.0 (2025-12-09)
+
+## New features
+
+- New command `nix registry resolve` [#14595](https://github.com/NixOS/nix/pull/14595)
+
+  This command looks up a flake registry input name and returns the flakeref it resolves to.
+
+  For example, looking up Nixpkgs:
+
+  ```
+  $ nix registry resolve nixpkgs
+  github:NixOS/nixpkgs/nixpkgs-unstable
+  ```
+
+  Upstreamed from [Determinate Nix 3.14.0](https://github.com/DeterminateSystems/nix-src/pull/273).
+
+- `nix flake clone` supports all input types [#14581](https://github.com/NixOS/nix/pull/14581)
+
+  `nix flake clone` now supports arbitrary input types. In particular, this allows you to clone tarball flakes, such as flakes on FlakeHub.
+
+  Upstreamed from [Determinate Nix 3.12.0](https://github.com/DeterminateSystems/nix-src/pull/229).
+
+## Performance improvements
+
+- Git fetcher computes `revCount`s using multiple threads [#14462](https://github.com/NixOS/nix/pull/14462)
+
+  When using Git repositories with a long history, calculating the `revCount` attribute can take a long time. Nix now computes `revCount` using multiple threads, making it much faster (e.g. 9.1s to 3.7s for Nixpkgs).
+
+  Note that if you don't need `revCount`, you can disable it altogether by setting the flake input attribute `shallow = true`.
+
+  Upstreamed from [Determinate Nix 3.12.2](https://github.com/DeterminateSystems/nix-src/pull/245).
+
+- `builtins.stringLength` now runs in constant time [#14442](https://github.com/NixOS/nix/pull/14442)
+
+  The internal representation of strings has been replaced with a size-prefixed Pascal style string. Previously Nix stored strings as a NUL-terminated array of bytes, necessitating a linear scan to calculate the length.
+
+- Uploads to `http://` and `https://` binary cache stores now run in constant memory [#14390](https://github.com/NixOS/nix/pull/14390)
+
+  Nix used to buffer the whole compressed NAR contents in memory. It now reads it in a streaming fashion.
+
+- Channel URLs migrated to channels.nixos.org subdomain [#14517](https://github.com/NixOS/nix/issues/14517) [#14518](https://github.com/NixOS/nix/pull/14518)
+
+  Channel URLs have been updated from `https://nixos.org/channels/` to `https://channels.nixos.org/` throughout Nix. This subdomain provides better reliability with IPv6 support and improved CDN distribution. The old domain apex (`nixos.org/channels/`) currently redirects to the new location but may be deprecated in the future.
+
+- Fix `download buffer is full; consider increasing the 'download-buffer-size' setting` warning [#11728](https://github.com/NixOS/nix/issues/11728) [#14614](https://github.com/NixOS/nix/pull/14614)
+
+  The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
+  [libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
+
+  This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
+
+  Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).
+
+- Significantly improve tarball unpacking performance [#14689](https://github.com/NixOS/nix/pull/14689) [#14696](https://github.com/NixOS/nix/pull/14696) [#10683](https://github.com/NixOS/nix/issues/10683) [#11098](https://github.com/NixOS/nix/issues/11098)
+
+  Nix uses a content-addressed cache backed by libgit2 for deduplicating files fetched via `fetchTarball` and `github`, `tarball` flake inputs. Its usage has been significantly optimised to reduce the amount of I/O operations that are performed. For a typical nixpkgs source tarball this results in 200 times fewer system calls on Linux. In combination with libcurl pausing this alleviates performance regressions stemming from the tarball cache.
+
+- Already valid derivations are no longer copied to the store [#14219](https://github.com/NixOS/nix/pull/14219)
+
+  This results in a modest speedup when using the Nix daemon.
+
+- `nix nar ls` and `nix nar cat` are significantly faster and no longer buffer the whole NAR in memory [#14273](https://github.com/NixOS/nix/pull/14273) [#14732](https://github.com/NixOS/nix/pull/14732)
+
+## S3 improvements
+
+- Improved S3 binary cache support via HTTP [#11748](https://github.com/NixOS/nix/issues/11748) [#12403](https://github.com/NixOS/nix/issues/12403) [#12671](https://github.com/NixOS/nix/issues/12671) [#13084](https://github.com/NixOS/nix/issues/13084) [#13752](https://github.com/NixOS/nix/pull/13752) [#13823](https://github.com/NixOS/nix/pull/13823) [#14026](https://github.com/NixOS/nix/pull/14026) [#14120](https://github.com/NixOS/nix/pull/14120) [#14131](https://github.com/NixOS/nix/pull/14131) [#14135](https://github.com/NixOS/nix/pull/14135) [#14144](https://github.com/NixOS/nix/pull/14144) [#14170](https://github.com/NixOS/nix/pull/14170) [#14190](https://github.com/NixOS/nix/pull/14190) [#14198](https://github.com/NixOS/nix/pull/14198) [#14206](https://github.com/NixOS/nix/pull/14206) [#14209](https://github.com/NixOS/nix/pull/14209) [#14222](https://github.com/NixOS/nix/pull/14222) [#14223](https://github.com/NixOS/nix/pull/14223) [#14330](https://github.com/NixOS/nix/pull/14330) [#14333](https://github.com/NixOS/nix/pull/14333) [#14335](https://github.com/NixOS/nix/pull/14335) [#14336](https://github.com/NixOS/nix/pull/14336) [#14337](https://github.com/NixOS/nix/pull/14337) [#14350](https://github.com/NixOS/nix/pull/14350) [#14356](https://github.com/NixOS/nix/pull/14356) [#14357](https://github.com/NixOS/nix/pull/14357) [#14374](https://github.com/NixOS/nix/pull/14374) [#14375](https://github.com/NixOS/nix/pull/14375) [#14376](https://github.com/NixOS/nix/pull/14376) [#14377](https://github.com/NixOS/nix/pull/14377) [#14391](https://github.com/NixOS/nix/pull/14391) [#14393](https://github.com/NixOS/nix/pull/14393) [#14420](https://github.com/NixOS/nix/pull/14420) [#14421](https://github.com/NixOS/nix/pull/14421)
+
+  S3 binary cache operations now happen via HTTP, leveraging `libcurl`'s native AWS SigV4 authentication instead of the AWS C++ SDK, providing significant improvements:
+
+  - **Reduced memory usage**: Eliminates memory buffering issues that caused segfaults with large files
+  - **Fixed upload reliability**: Resolves AWS SDK chunking errors (`InvalidChunkSizeError`)
+  - **Lighter dependencies**: Uses lightweight `aws-crt-cpp` instead of full `aws-cpp-sdk`, reducing build complexity
+
+  The new implementation requires curl >= 7.75.0 and `aws-crt-cpp` for credential management.
+
+  All existing S3 URL formats and parameters remain supported, however the store settings for configuring multipart uploads have changed:
+
+  - **`multipart-upload`** (default: `false`): Enable multipart uploads for large files. When enabled, files exceeding the multipart threshold will be uploaded in multiple parts.
+
+  - **`multipart-threshold`** (default: `100 MiB`): Minimum file size for using multipart uploads. Files smaller than this will use regular PUT requests. Only takes effect when `multipart-upload` is enabled.
+
+  - **`multipart-chunk-size`** (default: `5 MiB`): Size of each part in multipart uploads. Must be at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the number of requests but use more memory.
+
+  - **`buffer-size`**: Has been replaced by `multipart-chunk-size` and is now an alias to it.
+
+  Note that this change also means Nix now supports S3 binary cache stores even if built without `aws-crt-cpp`, but only for public buckets which do not require authentication.
+
+- S3 URLs now support object versioning via `versionId` parameter [#13955](https://github.com/NixOS/nix/issues/13955) [#14274](https://github.com/NixOS/nix/pull/14274)
+
+  S3 URLs now support a `versionId` query parameter to fetch specific versions
+  of objects from S3 buckets with versioning enabled. This allows pinning to
+  exact object versions for reproducibility and protection against unexpected
+  changes:
+
+  ```
+  s3://bucket/key?region=us-east-1&versionId=abc123def456
+  ```
+
+- S3 binary cache stores now support storage class configuration [#7015](https://github.com/NixOS/nix/issues/7015) [#14464](https://github.com/NixOS/nix/pull/14464)
+
+  S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
+
+  Example usage:
+
+  ```bash
+  # Use Glacier storage for long-term archival
+  nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
+
+  # Use Intelligent Tiering for automatic cost optimization
+  nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
+  ```
+
+  The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
+
+  See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.
+
+
+## Store path info JSON format changes
+
+The JSON format emitted by `nix path-info --json` has been updated to a new version with improved structure.
+
+To maintain compatibility, `nix path-info --json` now requires a `--json-format` flag to specify the output format version.
+Using `--json` without `--json-format` is deprecated and will become an error in a future release.
+For now, it defaults to version 1 with a warning, for a smoother migration.
+
+### Version 1 (`--json-format 1`)
+
+This is the legacy format, preserved for backwards compatibility:
+
+- String-based hash values (e.g., `"narHash": "sha256:FePFYIlM..."`)
+- String-based content addresses (e.g., `"ca": "fixed:r:sha256:1abc..."`)
+- Full store paths for map keys and references (e.g., `"/nix/store/abc...-foo"`)
+- Now includes `"storeDir"` field at the top level
+
+### Version 2 (`--json-format 2`)
+
+The new structured format follows the [JSON guidelines](@docroot@/development/json-guideline.md) with the following changes:
+
+- **Nested structure with top-level metadata**:
+
+  The output is now wrapped in an object with `version`, `storeDir`, and `info` fields:
+
+  ```json
+  {
+    "version": 2,
+    "storeDir": "/nix/store",
+    "info": { ... }
+  }
+  ```
+
+  The map from store path base names to store object info is nested under the `info` field.
+
+- **Store path base names instead of full paths**:
+
+  Map keys and references use store path base names (e.g., `"abc...-foo"`) instead of full absolute store paths.
+  Combined with `storeDir`, the full path can be reconstructed.
+
+- **Structured `ca` field**:
+
+  Content address is now a structured JSON object instead of a string:
+
+  - Old: `"ca": "fixed:r:sha256:1abc..."`
+  - New: `"ca": {"method": "nar", "hash": "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}`
+  - Still `null` values for input-addressed store objects
+
+  The `hash` field uses the [SRI](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) format like other hashes.
+
+Additionally the following fields are added to both formats:
+
+  - **`version` field**:
+
+    All store path info JSON now includes `"version": <1|2>`. The `version` tracks breaking changes, and adding fields to outputted JSON is not a breaking change.
+
+  - **`storeDir` field**:
+
+    Top-level `"storeDir"` field contains the store directory path (e.g., `"/nix/store"`).
+
+## Derivation JSON format changes
+
+The derivation JSON format has been updated from version 3 to version 4:
+
+- **Nested structure with top-level metadata**:
+
+  The output of `nix derivation show` is now wrapped in an object with `version` and `derivations` fields:
+
+  ```json
+  {
+    "version": 4,
+    "derivations": { ... }
+  }
+  ```
+
+  The map from derivation paths to derivation info is nested under the `derivations` field.
+
+  This matches the structure used for `nix path-info --json --json-format 2`, and likewise brings this command into compliance with the JSON guidelines.
+
+- **Restructured inputs**:
+
+  Inputs are now nested under an `inputs` object:
+
+  - Old: `"inputSrcs": [...], "inputDrvs": {...}`
+  - New: `"inputs": {"srcs": [...], "drvs": {...}}`
+
+- **Consistent content addresses**:
+
+  Fixed content-addressed outputs now use structured JSON format.
+  This is the same format as `ca` in store path info (after the new version).
+
+Version 3 and earlier formats are *not* accepted when reading.
+
+**Affected command**: `nix derivation`, namely its `show` and `add` sub-commands.
+
+## Miscellaneous changes
+
+- Git fetcher: Restore progress indication [#14487](https://github.com/NixOS/nix/pull/14487)
+
+  Nix used to feel "stuck" while it was cloning large repositories. Nix now shows Git's native progress indicator while fetching.
+
+  Upstreamed from [Determinate Nix 3.13.0](https://github.com/DeterminateSystems/nix-src/pull/250).
+
+- Interrupting REPL commands works more than once [#13481](https://github.com/NixOS/nix/issues/13481)
+
+  Previously, this only worked once per REPL session; further attempts would be ignored.
+  This issue is now fixed, so REPL commands such as `:b` or `:p` can be canceled consistently.
+  This is a cherry-pick of the change from the [Lix project](https://gerrit.lix.systems/c/lix/+/1097).
+
+- NAR unpacking code has been rewritten to make use of dirfd-based `openat` and `openat2` system calls when available [#14597](https://github.com/NixOS/nix/pull/14597)
+
+- Dynamic size unit rendering [#14423](https://github.com/NixOS/nix/pull/14423) [#14364](https://github.com/NixOS/nix/pull/14364)
+
+  Various commands and the progress bar now use dynamically determined size units instead
+  of always using `MiB`. For example, the progress bar now reports download status like:
+
+  ```
+  [1/196/197 copied (773.7 MiB/2.1 GiB), 172.4/421.5 MiB DL]
+  ```
+
+  Instead of:
+
+  ```
+  [1/196/197 copied (773.7/2147.3 MiB), 172.4/421.5 MiB DL]
+  ```
+
+## Contributors
+
+This release was made possible by the following 33 contributors:
+
+- Adam Dinwoodie [**(@me-and)**](https://github.com/me-and)
+- jonhermansen [**(@jonhermansen)**](https://github.com/jonhermansen)
+- Arnout Engelen [**(@raboof)**](https://github.com/raboof)
+- Jean-François Roche [**(@jfroche)**](https://github.com/jfroche)
+- tomberek [**(@tomberek)**](https://github.com/tomberek)
+- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra)
+- Marcel [**(@MarcelCoding)**](https://github.com/MarcelCoding)
+- David McFarland [**(@corngood)**](https://github.com/corngood)
+- Soumyadip Sarkar [**(@neuralsorcerer)**](https://github.com/neuralsorcerer)
+- Cole Helbling [**(@cole-h)**](https://github.com/cole-h)
+- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314)
+- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy)
+- Alex Auvolat [**(@Alexis211)**](https://github.com/Alexis211)
+- edef [**(@edef1c)**](https://github.com/edef1c)
+- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium)
+- Vinayak Goyal [**(@vinayakankugoyal)**](https://github.com/vinayakankugoyal)
+- Graham Dennis [**(@GrahamDennis)**](https://github.com/GrahamDennis)
+- Aspen Smith [**(@glittershark)**](https://github.com/glittershark)
+- Jens Petersen [**(@juhp)**](https://github.com/juhp)
+- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault)
+- Peter Bynum [**(@pkpbynum)**](https://github.com/pkpbynum)
+- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92)
+- Alex Decious [**(@adeci)**](https://github.com/adeci)
+- Matthieu Coudron [**(@teto)**](https://github.com/teto)
+- Domen Kožar [**(@domenkozar)**](https://github.com/domenkozar)
+- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii)
+- Seth Flynn [**(@getchoo)**](https://github.com/getchoo)
+- Robert Hensing [**(@roberth)**](https://github.com/roberth)
+- Vladimir Panteleev [**(@CyberShadow)**](https://github.com/CyberShadow)
+- bryango [**(@bryango)**](https://github.com/bryango)
+- Henry [**(@cootshk)**](https://github.com/cootshk)
+- Martin Joerg [**(@mjoerg)**](https://github.com/mjoerg)
+- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria)

maintainers/data/release-credits-email-to-handle.json

@@ -224,5 +224,25 @@
     "42688647+netadr@users.noreply.github.com": "netadr",
     "matej.urbas@gmail.com": "urbas",
     "ethanalexevans@gmail.com": "ethanavatar",
-    "greg.marti@gmail.com": "gmarti"
+    "greg.marti@gmail.com": "gmarti",
+    "arnout@bzzt.net": "raboof",
+    "vinayakankugoyal@gmail.com": "vinayakankugoyal",
+    "Radvendii@users.noreply.github.com": "Radvendii",
+    "jon@jh86.org": "jonhermansen",
+    "edef@edef.eu": "edef1c",
+    "pkpbynum@gmail.com": "pkpbynum",
+    "886074+teto@users.noreply.github.com": "teto",
+    "alex@adnab.me": "Alexis211",
+    "root@gws.fyi": "glittershark",
+    "me@m4rc3l.de": "MarcelCoding",
+    "taeer.bar-yam@bevuta.com": "Radvendii",
+    "martin.joerg@gmail.com": "mjoerg",
+    "git@cy.md": "CyberShadow",
+    "cootshk@duck.com": "cootshk",
+    "adam@dinwoodie.org": "me-and",
+    "domen@cachix.org": "domenkozar",
+    "alex.decious@gmail.com": "adeci",
+    "soumya.papanvk18@gmail.com": "neuralsorcerer",
+    "gdennis@anduril.com": null,
+    "graham.dennis@gmail.com": "GrahamDennis"
 }

maintainers/data/release-credits-handle-to-name.json

@@ -196,5 +196,21 @@
     "gmarti": "Gr\u00e9gory Marti",
     "lovesegfault": "Bernardo Meurer",
     "EphraimSiegfried": "Ephraim Siegfried",
-    "hgl": "Glen Huang"
+    "hgl": "Glen Huang",
+    "mjoerg": "Martin Joerg",
+    "Alexis211": "Alex Auvolat",
+    "domenkozar": "Domen Ko\u017ear",
+    "edef1c": "edef",
+    "cootshk": "Henry",
+    "raboof": "Arnout Engelen",
+    "pkpbynum": "Peter Bynum",
+    "glittershark": "Aspen Smith",
+    "MarcelCoding": "Marcel",
+    "teto": "Matthieu Coudron",
+    "jonhermansen": null,
+    "neuralsorcerer": "Soumyadip Sarkar",
+    "adeci": "Alex Decious",
+    "vinayakankugoyal": "Vinayak Goyal",
+    "me-and": "Adam Dinwoodie",
+    "GrahamDennis": "Graham Dennis"
 }

packaging/dev-shell.nix

@@ -258,10 +258,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
 
     # We use this shell with the local checkout, not unpackPhase.
     src = null;
+
     # Workaround https://sourceware.org/pipermail/gdb-patches/2025-October/221398.html
     # Remove when gdb fix is rolled out everywhere.
     separateDebugInfo = false;
 
+    mesonBuildType = "debugoptimized";
+
     env = {
       # For `make format`, to work without installing pre-commit
       _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml"

src/json-schema-checks/meson.build

@@ -62,9 +62,11 @@ schemas = [
   },
   {
     'stem' : 'build-trace-entry',
-    'schema' : schema_dir / 'build-trace-entry-v1.yaml',
+    'schema' : schema_dir / 'build-trace-entry-v2.yaml',
     'files' : [
       'simple.json',
+      # The field is no longer supported, but we want to show that we
+      # ignore it during parsing.
       'with-dependent-realisations.json',
       'with-signature.json',
     ],

src/libexpr/eval.cc

@@ -741,6 +741,11 @@ public:
         inDebugger = true;
     }
 
+    DebuggerGuard(DebuggerGuard &&) = delete;
+    DebuggerGuard(const DebuggerGuard &) = delete;
+    DebuggerGuard & operator=(DebuggerGuard &&) = delete;
+    DebuggerGuard & operator=(const DebuggerGuard &) = delete;
+
     ~DebuggerGuard()
     {
         inDebugger = false;

src/libexpr/include/nix/expr/attr-set.hh

@@ -107,6 +107,8 @@ private:
     Bindings & operator=(const Bindings &) = delete;
     Bindings & operator=(Bindings &&) = delete;
 
+    ~Bindings() = default;
+
     friend class BindingsBuilder;
 
     /**

src/libexpr/include/nix/expr/value.hh

@@ -164,8 +164,6 @@ public:
     Value ** elems;
     ListBuilder(EvalMemory & mem, size_t size);
 
-    // NOTE: Can be noexcept because we are just copying integral values and
-    // raw pointers.
     ListBuilder(ListBuilder && x) noexcept
         : size(x.size)
         , inlineElems{x.inlineElems[0], x.inlineElems[1]}
@@ -173,6 +171,11 @@ public:
     {
     }
 
+    ListBuilder(const ListBuilder &) = delete;
+    ListBuilder & operator=(ListBuilder &&) = delete;
+    ListBuilder & operator=(const ListBuilder &) = delete;
+    ~ListBuilder() = default;
+
     Value *& operator[](size_t n)
     {
         return elems[n];

src/libexpr/primops.cc

@@ -4061,6 +4061,8 @@ static RegisterPrimOp primop_sort({
 
         1. Transitivity
 
+        If a is less than b and b is less than c, then it follows that a is less than c.
+
         ```nix
         comparator a b && comparator b c -> comparator a c
         ```
@@ -4073,9 +4075,23 @@ static RegisterPrimOp primop_sort({
 
         1. Transitivity of equivalence
 
+        First, two values a and b are considered equivalent with respect to the comparator if:
+
+        ```
+        !comparator a b && !comparator b a
+        ```
+
+        In other words, neither is considered "less than" the other.
+
+        Transitivity of equivalence means:
+
+        If a is equivalent to b, and b is equivalent to c, then a must also be equivalent to c.
+
         ```nix
-        let equiv = a: b: (!comparator a b && !comparator b a); in
-        equiv a b && equiv b c -> equiv a c
+        let
+          equiv = x: y: (!comparator x y && !comparator y x);
+        in
+          equiv a b && equiv b c -> equiv a c
         ```
 
       If the *comparator* violates any of these properties, then `builtins.sort`

src/libfetchers-tests/git-utils.cc

@@ -48,7 +48,7 @@ public:
 
     ref<GitRepo> openRepo()
     {
-        return GitRepo::openRepo(tmpDir, true, false);
+        return GitRepo::openRepo(tmpDir, {.create = true});
     }
 
     std::string getRepoName() const
@@ -70,12 +70,6 @@ TEST_F(GitUtilsTest, sink_basic)
     auto repo = openRepo();
     auto sink = repo->getFileSystemObjectSink();
 
-    // TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory
-    // here
-    //                The sync method does not document this behavior, should probably renamed because it's not very
-    //                general, and I can't imagine that "non-conventional" archives or any other source to be handled by
-    //                this sink.
-
     sink->createDirectory(CanonPath("foo-1.1"));
 
     sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) {
@@ -91,7 +85,7 @@ TEST_F(GitUtilsTest, sink_basic)
 
     // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
 
-    auto result = repo->dereferenceSingletonDirectory(sink->flush());
+    auto result = repo->dereferenceSingletonDirectory(sink->flush().hash);
     auto accessor = repo->getAccessor(result, {}, getRepoName());
     auto entries = accessor->readDirectory(CanonPath::root);
     ASSERT_EQ(entries.size(), 5u);
@@ -123,6 +117,51 @@ TEST_F(GitUtilsTest, sink_hardlink)
     }
 };
 
+TEST_F(GitUtilsTest, sink_root_file)
+{
+    auto repo = openRepo();
+    auto sink = repo->getFileSystemObjectSink();
+
+    sink->createRegularFile(
+        CanonPath::root, [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
+
+    auto result = sink->flush();
+    ASSERT_EQ(result.mode, nix::git::Mode::Regular);
+    auto accessor = repo->getAccessor(result.hash, false, getRepoName());
+    ASSERT_EQ(accessor->readFile(CanonPath::root), "hello world");
+};
+
+TEST_F(GitUtilsTest, sink_root_symlink)
+{
+    auto repo = openRepo();
+    auto sink = repo->getFileSystemObjectSink();
+
+    sink->createSymlink(CanonPath::root, "target");
+
+    auto result = sink->flush();
+    ASSERT_EQ(result.mode, nix::git::Mode::Symlink);
+    auto accessor = repo->getAccessor(result.hash, false, getRepoName());
+    ASSERT_EQ(accessor->readLink(CanonPath::root), "target");
+};
+
+TEST_F(GitUtilsTest, sink_hardlink_to_root)
+{
+    auto repo = openRepo();
+    auto sink = repo->getFileSystemObjectSink();
+
+    sink->createRegularFile(
+        CanonPath("hello"), [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
+
+    sink->createDirectory(CanonPath("subdir"));
+    sink->createHardlink(CanonPath("subdir/link"), CanonPath("hello"));
+
+    auto result = sink->flush();
+    ASSERT_EQ(result.mode, nix::git::Mode::Directory);
+    auto accessor = repo->getAccessor(result.hash, false, getRepoName());
+    ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
+    ASSERT_EQ(accessor->readFile(CanonPath("subdir/link")), "hello world");
+};
+
 TEST_F(GitUtilsTest, peel_reference)
 {
     // Create a commit in the repo

src/libfetchers/git-utils.cc

@@ -203,16 +203,19 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
 
 } // extern "C"
 
-static void initRepoAtomically(std::filesystem::path & path, bool bare)
+static void initRepoAtomically(std::filesystem::path & path, GitRepo::Options options)
 {
     if (pathExists(path.string()))
         return;
 
+    if (!options.create)
+        throw Error("Git repository %s does not exist.", path);
+
     std::filesystem::path tmpDir = createTempDir(path.parent_path());
     AutoDelete delTmpDir(tmpDir, true);
     Repository tmpRepo;
 
-    if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), bare))
+    if (git_repository_init(Setter(tmpRepo), tmpDir.string().c_str(), options.bare))
         throw Error("creating Git repository %s: %s", path, git_error_last()->message);
     try {
         std::filesystem::rename(tmpDir, path);
@@ -234,7 +237,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     /** Location of the repository on disk. */
     std::filesystem::path path;
 
-    bool bare;
+    Options options;
 
     /**
      * libgit2 repository. Note that new objects are not written to disk,
@@ -255,18 +258,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
      */
     git_odb_backend * packBackend = nullptr;
 
-    GitRepoImpl(std::filesystem::path _path, bool create, bool bare, bool packfilesOnly = false)
+    GitRepoImpl(std::filesystem::path _path, Options _options)
         : path(std::move(_path))
-        , bare(bare)
+        , options(_options)
     {
         initLibGit2();
 
-        initRepoAtomically(path, bare);
+        initRepoAtomically(path, options);
         if (git_repository_open(Setter(repo), path.string().c_str()))
             throw Error("opening Git repository %s: %s", path, git_error_last()->message);
 
         ObjectDb odb;
-        if (packfilesOnly) {
+        if (options.packfilesOnly) {
             /* Create a fresh object database because by default the repo also
                loose object backends. We are not using any of those for the
                tarball cache, but libgit2 still does a bunch of unnecessary
@@ -295,7 +298,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         if (git_odb_add_backend(odb.get(), mempackBackend, 999))
             throw Error("adding mempack backend to Git object database: %s", git_error_last()->message);
 
-        if (packfilesOnly) {
+        if (options.packfilesOnly) {
             if (git_repository_set_odb(repo.get(), odb.get()))
                 throw Error("setting Git object database: %s", git_error_last()->message);
         }
@@ -366,7 +369,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     {
         // TODO: as an optimization, it would be nice to include `this` in the pool.
         return Pool<GitRepoImpl>(std::numeric_limits<size_t>::max(), [this]() -> ref<GitRepoImpl> {
-            return make_ref<GitRepoImpl>(path, false, bare);
+            return make_ref<GitRepoImpl>(path, options);
         });
     }
 
@@ -712,9 +715,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 };
 
-ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare, bool packfilesOnly)
+ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, GitRepo::Options options)
 {
-    return make_ref<GitRepoImpl>(path, create, bare, packfilesOnly);
+    return make_ref<GitRepoImpl>(path, options);
 }
 
 /**
@@ -959,15 +962,15 @@ struct GitSourceAccessor : SourceAccessor
 
     Blob getBlob(State & state, const CanonPath & path, bool expectSymlink)
     {
-        if (!expectSymlink && git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
-            return dupObject<Blob>((git_blob *) &*state.root);
-
         auto notExpected = [&]() {
             throw Error(expectSymlink ? "'%s' is not a symlink" : "'%s' is not a regular file", showPath(path));
         };
 
-        if (path.isRoot())
+        if (path.isRoot()) {
+            if (git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
+                return dupObject<Blob>((git_blob *) &*state.root);
             notExpected();
+        }
 
         auto entry = need(state, path);
 
@@ -1051,16 +1054,24 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
     }
 };
 
-struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
-{
-    ref<GitRepoImpl> repo;
+namespace {
 
+struct DirEntry
+{
+    git_oid oid;
+    git_filemode_t mode;
+};
+
+struct DirectoryState
+{
     struct PendingDir
     {
         std::string name;
         TreeBuilder builder;
     };
 
+    GitRepoImpl & repo;
+    TreeBuilder rootBuilder;
     std::vector<PendingDir> pendingDirs;
 
     /**
@@ -1075,30 +1086,9 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
      */
     decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr;
 
-    void pushBuilder(std::string name)
-    {
-        const git_tree_entry * entry;
-        Tree prevTree = nullptr;
-
-        if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) {
-            /* Clone a tree that we've already finished. This happens
-               if a tarball has directory entries that are not
-               contiguous. */
-            if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
-                throw Error("parent of '%s' is not a directory", name);
-
-            if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry))
-                throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
-        }
-
-        git_treebuilder * b;
-        if (git_treebuilder_new(&b, *repo, prevTree.get()))
-            throw Error("creating a tree builder: %s", git_error_last()->message);
-        pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)});
-    };
-
-    GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
+    DirectoryState(GitRepoImpl & repo)
         : repo(repo)
+        , rootBuilder(makeBuilder())
     {
         /* Monkey-patching the pack backend to only read the pack directory
            once. Otherwise it will do a readdir for each added oid when it's
@@ -1123,62 +1113,145 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
              */
             packfileOdbRefresh = std::exchange(backend->refresh, nullptr);
         }
-        pushBuilder("");
     }
 
-    std::pair<git_oid, std::string> popBuilder()
-    {
-        assert(!pendingDirs.empty());
-        auto pending = std::move(pendingDirs.back());
-        git_oid oid;
-        if (git_treebuilder_write(&oid, pending.builder.get()))
-            throw Error("creating a tree object: %s", git_error_last()->message);
-        pendingDirs.pop_back();
-        return {oid, pending.name};
-    };
+    TreeBuilder makeBuilder(git_tree * prevTree = nullptr);
 
-    void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
-    {
-        assert(!pendingDirs.empty());
-        auto & pending = pendingDirs.back();
-        if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
-            throw Error("adding a file to a tree builder: %s", git_error_last()->message);
-    };
+    std::vector<std::string> prepareDirs(const CanonPath & path);
+    void updateBuilders(std::span<const std::string> names);
+    std::pair<git_oid, std::string> popBuilder();
+    void pushBuilder(std::string name);
+    void addToTree(const std::string & name, const DirEntry & entry);
+};
 
-    void updateBuilders(std::span<const std::string> names)
-    {
-        // Find the common prefix of pendingDirs and names.
-        size_t prefixLen = 0;
-        for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
-            if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
-                break;
+TreeBuilder DirectoryState::makeBuilder(git_tree * prevTree)
+{
+    TreeBuilder builder;
+    if (git_treebuilder_new(Setter(builder), repo, prevTree))
+        throw Error("creating a tree builder: %s", git_error_last()->message);
+    return builder;
+}
 
-        // Finish the builders that are not part of the common prefix.
-        for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
-            auto [oid, name] = popBuilder();
-            addToTree(name, oid, GIT_FILEMODE_TREE);
+git_oid finishBuilder(TreeBuilder & builder)
+{
+    git_oid oid;
+    if (git_treebuilder_write(&oid, builder.get()))
+        throw Error("creating a tree object: %s", git_error_last()->message);
+    return oid;
+}
+
+void DirectoryState::pushBuilder(std::string name)
+{
+    auto & parentBuilder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
+    const git_tree_entry * entry;
+    Tree prevTree = nullptr;
+
+    if ((entry = git_treebuilder_get(parentBuilder.get(), name.c_str()))) {
+        /* Clone a tree that we've already finished. This happens
+           if a tarball has directory entries that are not
+           contiguous. */
+        if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+            throw Error("parent of '%s' is not a directory", name);
+
+        if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), repo, entry))
+            throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
+    }
+
+    pendingDirs.push_back({
+        .name = std::move(name),
+        .builder = makeBuilder(prevTree.get()),
+    });
+}
+
+std::pair<git_oid, std::string> DirectoryState::popBuilder()
+{
+    assert(!pendingDirs.empty());
+    auto pending = std::move(pendingDirs.back());
+    pendingDirs.pop_back();
+    return {finishBuilder(pending.builder), pending.name};
+}
+
+void DirectoryState::addToTree(const std::string & name, const DirEntry & entry)
+{
+    auto & builder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
+    if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &entry.oid, entry.mode))
+        throw Error("adding a file to a tree builder: %s", git_error_last()->message);
+}
+
+void DirectoryState::updateBuilders(std::span<const std::string> names)
+{
+    // Find the common prefix of pendingDirs and names.
+    size_t prefixLen = 0;
+    for (; prefixLen < names.size() && prefixLen < pendingDirs.size(); ++prefixLen)
+        if (names[prefixLen] != pendingDirs[prefixLen].name)
+            break;
+
+    // Finish the builders that are not part of the common prefix.
+    for (auto n = pendingDirs.size(); n > prefixLen; --n) {
+        auto [oid, name] = popBuilder();
+        addToTree(name, DirEntry{.oid = oid, .mode = GIT_FILEMODE_TREE});
+    }
+
+    // Create builders for the new directories.
+    for (auto n = prefixLen; n < names.size(); ++n)
+        pushBuilder(names[n]);
+}
+
+std::vector<std::string> DirectoryState::prepareDirs(const CanonPath & path)
+{
+    std::vector<std::string> pathComponents;
+    for (auto & c : path)
+        pathComponents.emplace_back(c);
+
+    updateBuilders(pathComponents);
+
+    return pathComponents;
+}
+
+struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
+{
+    ref<GitRepoImpl> repo;
+
+    /**
+     * Root state:
+     * - nullopt: nothing created yet
+     * - DirEntry: root is a single non-directory object (file/symlink)
+     * - DirectoryState: root is a directory with builders
+     */
+    std::optional<std::variant<DirEntry, DirectoryState>> root;
+
+    DirectoryState & ensureDirectoryState()
+    {
+        if (!root) {
+            root.emplace<DirectoryState>(*repo);
         }
+        if (auto * dirState = std::get_if<DirectoryState>(&*root)) {
+            return *dirState;
+        }
+        throw Error("cannot create directory entry: root is not a directory");
+    }
 
-        // Create builders for the new directories.
-        for (auto n = prefixLen; n < names.size(); ++n)
-            pushBuilder(names[n]);
-    };
-
-    bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
+    GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
+        : repo(repo)
     {
-        std::span<const std::string> pathComponents2{pathComponents};
+    }
 
-        updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1));
-
-        return true;
+    void addEntry(const CanonPath & path, const DirEntry & entry)
+    {
+        if (auto res = path.parentAndChild()) {
+            auto & [parent, child] = *res;
+            auto & dirState = ensureDirectoryState();
+            dirState.prepareDirs(parent);
+            dirState.addToTree(std::string(child), entry);
+        } else {
+            if (root)
+                throw Error("root already exists");
+            root = entry;
+        }
     }
 
     void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
     {
-        auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
-        if (!prepareDirs(pathComponents, false))
-            return;
-
         using WriteStream = std::unique_ptr<::git_writestream, decltype([](::git_writestream * stream) {
                                                 if (stream)
                                                     stream->free(stream);
@@ -1257,60 +1330,68 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
                     "creating a blob object for '%s' from in-memory buffer: %s", path, git_error_last()->message);
         }
 
-        addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB);
+        addEntry(
+            path,
+            DirEntry{
+                .oid = oid,
+                .mode = crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB,
+            });
     }
 
-    void createDirectory(const CanonPath & path) override
+    void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
     {
-        auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
-        (void) prepareDirs(pathComponents, true);
+        ensureDirectoryState().prepareDirs(path);
+        if (callback)
+            (*callback)(*this, path);
     }
 
     void createSymlink(const CanonPath & path, const std::string & target) override
     {
-        auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
-        if (!prepareDirs(pathComponents, false))
-            return;
-
         git_oid oid;
         if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
             throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
 
-        addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
+        addEntry(
+            path,
+            DirEntry{
+                .oid = oid,
+                .mode = GIT_FILEMODE_LINK,
+            });
     }
 
     void createHardlink(const CanonPath & path, const CanonPath & target) override
     {
-        std::vector<std::string> pathComponents;
-        for (auto & c : path)
-            pathComponents.emplace_back(c);
+        auto & dirState = ensureDirectoryState();
 
-        if (!prepareDirs(pathComponents, false))
-            return;
+        auto res = path.parentAndChild();
+        if (!res)
+            throw Error("hard link cannot be root file system object");
+        auto & [parent, name] = *res;
+
+        dirState.prepareDirs(parent);
 
         // We can't just look up the path from the start of the root, since
         // some parent directories may not have finished yet, so we compute
         // a relative path that helps us find the right git_tree_builder or object.
-        auto relTarget = CanonPath(path).parent()->makeRelative(target);
+        auto relTarget = parent.makeRelative(target);
 
-        auto dir = pendingDirs.rbegin();
+        auto dir = dirState.pendingDirs.rbegin();
 
         // For each ../ component at the start, go up one directory.
         // CanonPath::makeRelative() always puts all .. elements at the start,
         // so they're all handled by this loop:
         std::string_view relTargetLeft(relTarget);
         while (hasPrefix(relTargetLeft, "../")) {
-            if (dir == pendingDirs.rend())
+            if (dir == dirState.pendingDirs.rend())
                 throw Error("invalid hard link target '%s' for path '%s'", target, path);
             ++dir;
             relTargetLeft = relTargetLeft.substr(3);
         }
-        if (dir == pendingDirs.rend())
-            throw Error("invalid hard link target '%s' for path '%s'", target, path);
 
         // Look up the remainder of the target, starting at the
         // top-most `git_treebuilder`.
-        std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
+        std::variant<git_treebuilder *, git_oid> curDir{
+            dir == dirState.pendingDirs.rend() ? dirState.rootBuilder.get() : dir->builder.get()};
         Object tree; // needed to keep `entry` alive
         const git_tree_entry * entry = nullptr;
 
@@ -1330,26 +1411,41 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
         assert(entry);
 
-        addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry));
+        dirState.addToTree(
+            std::string(name),
+            DirEntry{.oid = *git_tree_entry_id(entry), .mode = git_tree_entry_filemode(entry)});
     }
 
-    Hash flush() override
+    git::TreeEntry flush() override
     {
-        updateBuilders({});
+        if (!root)
+            throw Error("nothing to flush");
 
-        auto [oid, _name] = popBuilder();
+        auto [oid, mode] = std::visit(
+            overloaded{
+                [](DirEntry & entry) -> std::pair<git_oid, git_filemode_t> {
+                    return {entry.oid, entry.mode};
+                },
+                [](DirectoryState & dirState) -> std::pair<git_oid, git_filemode_t> {
+                    dirState.updateBuilders({});
+                    return {finishBuilder(dirState.rootBuilder), GIT_FILEMODE_TREE};
+                },
+            },
+            *root);
 
         if (auto * backend = repo->packBackend) {
             /* We are done writing blobs, can restore refresh functionality. */
-            backend->refresh = packfileOdbRefresh;
+            backend->refresh = dirState.packfileOdbRefresh;
         }
 
         repo->flush();
 
-        return toHash(oid);
+        return git::TreeEntry{.mode = static_cast<git::Mode>(mode), .hash = toHash(oid)};
     }
 };
 
+}
+
 ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
@@ -1427,8 +1523,12 @@ namespace fetchers {
 
 ref<GitRepo> Settings::getTarballCache() const
 {
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
-    return GitRepo::openRepo(repoDir, /*create=*/true, /*bare=*/true, /*packfilesOnly=*/true);
+    /* v1: Had either only loose objects or thin packfiles referring to loose objects
+     * v2: Must have only packfiles with no loose objects. Should get repacked periodically
+     * for optimal packfiles.
+     */
+    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache-v2";
+    return GitRepo::openRepo(repoDir, {.create = true, .bare = true, .packfilesOnly = true});
 }
 
 } // namespace fetchers
@@ -1442,7 +1542,7 @@ GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path &
         if (i != cache->end())
             return i->second;
     }
-    auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo();
+    auto workdirInfo = GitRepo::openRepo(path, {})->getWorkdirInfo();
     _cache.lock()->emplace(path, workdirInfo);
     return workdirInfo;
 }

src/libfetchers/git.cc

@@ -637,11 +637,6 @@ struct GitInputScheme : InputScheme
                     url);
             }
 
-            // If we don't check here for the path existence, then we can give libgit2 any directory
-            // and it will initialize them as git directories.
-            if (!pathExists(path)) {
-                throw Error("The path '%s' does not exist.", path);
-            }
             repoInfo.location = std::filesystem::absolute(path);
         } else {
             if (url.scheme == "file")
@@ -703,7 +698,7 @@ struct GitInputScheme : InputScheme
         if (auto res = cache->lookup(key))
             return getIntAttr(*res, "lastModified");
 
-        auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev);
+        auto lastModified = GitRepo::openRepo(repoDir, {})->getLastModified(rev);
 
         cache->upsert(key, {{"lastModified", lastModified}});
 
@@ -726,7 +721,7 @@ struct GitInputScheme : InputScheme
         Activity act(
             *logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.locationToArg()));
 
-        auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev);
+        auto revCount = GitRepo::openRepo(repoDir, {})->getRevCount(rev);
 
         cache->upsert(key, Attrs{{"revCount", revCount}});
 
@@ -737,7 +732,7 @@ struct GitInputScheme : InputScheme
     {
         auto head = std::visit(
             overloaded{
-                [&](const std::filesystem::path & path) { return GitRepo::openRepo(path)->getWorkdirRef(); },
+                [&](const std::filesystem::path & path) { return GitRepo::openRepo(path, {})->getWorkdirRef(); },
                 [&](const ParsedURL & url) { return readHeadCached(url.to_string(), shallow); }},
             repoInfo.location);
         if (!head) {
@@ -795,7 +790,7 @@ struct GitInputScheme : InputScheme
         if (auto repoPath = repoInfo.getPath()) {
             repoDir = *repoPath;
             if (!input.getRev())
-                input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
+                input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir, {})->resolveRef(ref).gitRev());
         } else {
             auto repoUrl = std::get<ParsedURL>(repoInfo.location);
             std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
@@ -805,7 +800,7 @@ struct GitInputScheme : InputScheme
             std::filesystem::create_directories(cacheDir.parent_path());
             PathLocks cacheDirLock({cacheDir.string()});
 
-            auto repo = GitRepo::openRepo(cacheDir, true, true);
+            auto repo = GitRepo::openRepo(cacheDir, {.create = true, .bare = true});
 
             // We need to set the origin so resolving submodule URLs works
             repo->setRemote("origin", repoUrl.to_string());
@@ -876,7 +871,7 @@ struct GitInputScheme : InputScheme
             // the remainder
         }
 
-        auto repo = GitRepo::openRepo(repoDir);
+        auto repo = GitRepo::openRepo(repoDir, {});
 
         auto isShallow = repo->isShallow();
 
@@ -963,7 +958,7 @@ struct GitInputScheme : InputScheme
             for (auto & submodule : repoInfo.workdirInfo.submodules)
                 repoInfo.workdirInfo.files.insert(submodule.path);
 
-        auto repo = GitRepo::openRepo(repoPath, false, false);
+        auto repo = GitRepo::openRepo(repoPath, {});
 
         auto exportIgnore = getExportIgnoreAttr(input);
 
@@ -1003,7 +998,7 @@ struct GitInputScheme : InputScheme
         }
 
         if (!repoInfo.workdirInfo.isDirty) {
-            auto repo = GitRepo::openRepo(repoPath);
+            auto repo = GitRepo::openRepo(repoPath, {});
 
             if (auto ref = repo->getWorkdirRef())
                 input.attrs.insert_or_assign("ref", *ref);

src/libfetchers/github.cc

@@ -321,12 +321,12 @@ struct GitArchiveInputScheme : InputScheme
         auto tarballCache = settings.getTarballCache();
         auto parseSink = tarballCache->getFileSystemObjectSink();
         auto lastModified = unpackTarfileToSink(archive, *parseSink);
-        auto tree = parseSink->flush();
+        auto treeEntry = parseSink->flush();
 
         act.reset();
 
         TarballInfo tarballInfo{
-            .treeHash = tarballCache->dereferenceSingletonDirectory(tree), .lastModified = lastModified};
+            .treeHash = tarballCache->dereferenceSingletonDirectory(treeEntry.hash), .lastModified = lastModified};
 
         cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}});
         cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}});

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -2,6 +2,7 @@
 
 #include "nix/fetchers/filtering-source-accessor.hh"
 #include "nix/util/fs-sink.hh"
+#include "nix/util/git.hh"
 
 namespace nix {
 
@@ -17,9 +18,9 @@ struct Settings;
 struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
 {
     /**
-     * Flush builder and return a final Git hash.
+     * Flush builder and return the final Git tree entry (hash and mode).
      */
-    virtual Hash flush() = 0;
+    virtual git::TreeEntry flush() = 0;
 };
 
 struct GitAccessorOptions
@@ -32,8 +33,14 @@ struct GitRepo
 {
     virtual ~GitRepo() {}
 
-    static ref<GitRepo>
-    openRepo(const std::filesystem::path & path, bool create = false, bool bare = false, bool packfilesOnly = false);
+    struct Options
+    {
+        bool create = false;
+        bool bare = false;
+        bool packfilesOnly = false;
+    };
+
+    static ref<GitRepo> openRepo(const std::filesystem::path & path, Options options);
 
     virtual uint64_t getRevCount(const Hash & rev) = 0;
 

src/libfetchers/tarball.cc

@@ -182,7 +182,7 @@ static DownloadTarballResult downloadTarball_(
     auto tarballCache = settings.getTarballCache();
     auto parseSink = tarballCache->getFileSystemObjectSink();
     auto lastModified = unpackTarfileToSink(archive, *parseSink);
-    auto tree = parseSink->flush();
+    auto treeEntry = parseSink->flush();
 
     act.reset();
 
@@ -196,7 +196,7 @@ static DownloadTarballResult downloadTarball_(
         infoAttrs = cached->value;
     } else {
         infoAttrs.insert_or_assign("etag", res->etag);
-        infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(tree).gitRev());
+        infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(treeEntry.hash).gitRev());
         infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified));
         if (res->immutableUrl)
             infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl);

src/libstore-test-support/include/nix/store/tests/protocol.hh

@@ -88,25 +88,38 @@ public:
     }
 };
 
-#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    TEST_F(FIXTURE, NAME##_read)                                                     \
-    {                                                                                \
-        readProtoTest(STEM, VERSION, VALUE);                                         \
-    }                                                                                \
-    TEST_F(FIXTURE, NAME##_write)                                                    \
-    {                                                                                \
-        writeProtoTest(STEM, VERSION, VALUE);                                        \
+#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_read)                                                          \
+    {                                                                                     \
+        readProtoTest(STEM, VERSION, VALUE);                                              \
     }
 
-#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
-    VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    TEST_F(FIXTURE, NAME##_json_read)                                            \
-    {                                                                            \
-        readJsonTest(STEM, VALUE);                                               \
-    }                                                                            \
-    TEST_F(FIXTURE, NAME##_json_write)                                           \
-    {                                                                            \
-        writeJsonTest(STEM, VALUE);                                              \
+#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_write)                                                          \
+    {                                                                                      \
+        writeProtoTest(STEM, VERSION, VALUE);                                              \
     }
 
+#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)  \
+    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
+
+#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
+    VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_json_read)                                                 \
+    {                                                                                 \
+        readJsonTest(STEM, VALUE);                                                    \
+    }
+
+#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_json_write)                                                 \
+    {                                                                                  \
+        writeJsonTest(STEM, VALUE);                                                    \
+    }
+
+#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)  \
+    VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
+
 } // namespace nix

src/libstore-tests/common-protocol.cc

@@ -47,24 +47,30 @@ public:
     }
 };
 
-#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
-    TEST_F(CommonProtoTest, NAME##_read)         \
-    {                                            \
-        readProtoTest(STEM, VALUE);              \
-    }                                            \
-    TEST_F(CommonProtoTest, NAME##_write)        \
-    {                                            \
-        writeProtoTest(STEM, VALUE);             \
-    }                                            \
-    TEST_F(CommonProtoTest, NAME##_json_read)    \
-    {                                            \
-        readJsonTest(STEM, VALUE);               \
-    }                                            \
-    TEST_F(CommonProtoTest, NAME##_json_write)   \
-    {                                            \
-        writeJsonTest(STEM, VALUE);              \
+#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
+    TEST_F(CommonProtoTest, NAME##_read)              \
+    {                                                 \
+        readProtoTest(STEM, VALUE);                   \
+    }                                                 \
+    TEST_F(CommonProtoTest, NAME##_json_read)         \
+    {                                                 \
+        readJsonTest(STEM, VALUE);                    \
     }
 
+#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
+    TEST_F(CommonProtoTest, NAME##_write)              \
+    {                                                  \
+        writeProtoTest(STEM, VALUE);                   \
+    }                                                  \
+    TEST_F(CommonProtoTest, NAME##_json_write)         \
+    {                                                  \
+        writeJsonTest(STEM, VALUE);                    \
+    }
+
+#define CHARACTERIZATION_TEST(NAME, STEM, VALUE)  \
+    READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
+    WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE)
+
 CHARACTERIZATION_TEST(
     string,
     "string",
@@ -141,7 +147,7 @@ CHARACTERIZATION_TEST(
         },
     }))
 
-CHARACTERIZATION_TEST(
+READ_CHARACTERIZATION_TEST(
     realisation_with_deps,
     "realisation-with-deps",
     (std::tuple<Realisation>{
@@ -149,16 +155,6 @@ CHARACTERIZATION_TEST(
             {
                 .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
                 .signatures = {"asdf", "qwer"},
-                .dependentRealisations =
-                    {
-                        {
-                            DrvOutput{
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "quux",
-                            },
-                            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
-                        },
-                    },
             },
             {
                 .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),

src/libstore-tests/data/worker-substitution/ca-drv/store-after.json

@@ -0,0 +1,58 @@
+{
+  "buildTrace": {
+    "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
+      "contents": {
+        "contents": "I am the output of a CA derivation",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+        "narSize": 152,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {
+    "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "test-ca-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/ca-drv/store-before.json

@@ -0,0 +1,27 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {
+    "vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "test-ca-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/ca-drv/substituter.json

@@ -0,0 +1,39 @@
+{
+  "buildTrace": {
+    "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
+      "contents": {
+        "contents": "I am the output of a CA derivation",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
+        "narSize": 152,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/issue-11928/store-after.json

@@ -0,0 +1,83 @@
+{
+  "buildTrace": {
+    "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
+      "contents": {
+        "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+        "narSize": 232,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {
+    "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {
+          "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+            "dynamicOutputs": {},
+            "outputs": [
+              "out"
+            ]
+          }
+        },
+        "srcs": []
+      },
+      "name": "root-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    },
+    "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "dep-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/issue-11928/store-before.json

@@ -0,0 +1,52 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {
+    "11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {
+          "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+            "dynamicOutputs": {},
+            "outputs": [
+              "out"
+            ]
+          }
+        },
+        "srcs": []
+      },
+      "name": "root-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    },
+    "vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "dep-drv",
+      "outputs": {
+        "out": {
+          "hashAlgo": "sha256",
+          "method": "nar"
+        }
+      },
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/worker-substitution/issue-11928/substituter.json

@@ -0,0 +1,68 @@
+{
+  "buildTrace": {
+    "8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
+        "signatures": []
+      }
+    },
+    "gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
+      "contents": {
+        "contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
+        "narSize": 232,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    },
+    "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": {
+      "contents": {
+        "contents": "I am the dependency output",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
+        "narSize": 144,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/single/substituter.json

@@ -0,0 +1,31 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": {
+      "contents": {
+        "contents": "Hello, world!",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
+        "narSize": 128,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/with-dep/store.json

@@ -0,0 +1,55 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
+      "contents": {
+        "contents": "I am a dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+        "narSize": 136,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    },
+    "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
+      "contents": {
+        "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+        "narSize": 184,
+        "references": [
+          "4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
+        ],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/worker-substitution/with-dep/substituter.json

@@ -0,0 +1,55 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
+      "contents": {
+        "contents": "I am a dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
+        "narSize": 136,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    },
+    "k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
+      "contents": {
+        "contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
+        "narSize": 184,
+        "references": [
+          "4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
+        ],
+        "registrationTime": null,
+        "signatures": [],
+        "storeDir": "/nix/store",
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/derivation-parser-bench.cc

@@ -2,7 +2,7 @@
 #include "nix/store/derivations.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/experimental-features.hh"
-#include "nix/util/environment-variables.hh"
+#include "nix/util/tests/test-data.hh"
 #include "nix/store/store-open.hh"
 #include <fstream>
 #include <sstream>
@@ -50,11 +50,7 @@ static void BM_UnparseRealDerivationFile(benchmark::State & state, const std::st
 }
 
 // Register benchmarks for actual test derivation files if they exist
-BENCHMARK_CAPTURE(
-    BM_ParseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
-BENCHMARK_CAPTURE(
-    BM_ParseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
-BENCHMARK_CAPTURE(
-    BM_UnparseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
-BENCHMARK_CAPTURE(
-    BM_UnparseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
+BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
+BENCHMARK_CAPTURE(BM_ParseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());
+BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, hello, (getUnitTestData() / "derivation/hello.drv").string());
+BENCHMARK_CAPTURE(BM_UnparseRealDerivationFile, firefox, (getUnitTestData() / "derivation/firefox.drv").string());

src/libstore-tests/meson.build

@@ -85,6 +85,7 @@ sources = files(
   'store-reference.cc',
   'uds-remote-store.cc',
   'worker-protocol.cc',
+  'worker-substitution.cc',
   'write-derivation.cc',
 )
 

src/libstore-tests/nix_api_store.cc

@@ -8,6 +8,7 @@
 #include "nix/store/tests/nix_api_store.hh"
 #include "nix/store/globals.hh"
 #include "nix/util/tests/string_callback.hh"
+#include "nix/util/tests/test-data.hh"
 #include "nix/util/url.hh"
 
 #include "store-tests-config.hh"
@@ -302,7 +303,7 @@ public:
 
         store = open_local_store();
 
-        std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+        std::filesystem::path unitTestData = nix::getUnitTestData();
         std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
         std::stringstream buffer;
         buffer << t.rdbuf();
@@ -357,7 +358,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
 
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
@@ -404,7 +405,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
     buffer << t.rdbuf();
@@ -449,7 +450,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
     buffer << t.rdbuf();
@@ -494,7 +495,7 @@ TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
 
     auto * store = open_local_store();
 
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
     std::stringstream buffer;
     buffer << t.rdbuf();
@@ -870,7 +871,7 @@ TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagati
  */
 static std::string load_json_from_test_data(const char * filename)
 {
-    std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
+    std::filesystem::path unitTestData = nix::getUnitTestData();
     std::ifstream t{unitTestData / filename};
     std::stringstream buffer;
     buffer << t.rdbuf();

src/libstore-tests/realisation.cc

@@ -44,54 +44,45 @@ TEST_P(RealisationJsonTest, to_json)
     writeJsonTest(name, value);
 }
 
+Realisation simple{
+    {
+        .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
+    },
+    {
+        .drvHash = Hash::parseExplicitFormatUnprefixed(
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+            HashAlgorithm::SHA256,
+            HashFormat::Base16),
+        .outputName = "foo",
+    },
+};
+
 INSTANTIATE_TEST_SUITE_P(
     RealisationJSON,
     RealisationJsonTest,
-    ([] {
-        Realisation simple{
-            {
-                .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
-            },
-            {
-                .drvHash = Hash::parseExplicitFormatUnprefixed(
-                    "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
-                    HashAlgorithm::SHA256,
-                    HashFormat::Base16),
-                .outputName = "foo",
-            },
-        };
-        return ::testing::Values(
-            std::pair{
-                "simple",
-                simple,
-            },
-            std::pair{
-                "with-signature",
-                [&] {
-                    auto r = simple;
-                    // FIXME actually sign properly
-                    r.signatures = {"asdfasdfasdf"};
-                    return r;
-                }()},
-            std::pair{
-                "with-dependent-realisations",
-                [&] {
-                    auto r = simple;
-                    r.dependentRealisations = {{
-                        {
-                            .drvHash = Hash::parseExplicitFormatUnprefixed(
-                                "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
-                                HashAlgorithm::SHA256,
-                                HashFormat::Base16),
-                            .outputName = "foo",
-                        },
-                        StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
-                    }};
-                    return r;
-                }(),
-            });
-    }
+    ::testing::Values(
+        std::pair{
+            "simple",
+            simple,
+        },
+        std::pair{
+            "with-signature",
+            [&] {
+                auto r = simple;
+                // FIXME actually sign properly
+                r.signatures = {"asdfasdfasdf"};
+                return r;
+            }(),
+        }));
 
-     ()));
+/**
+ * We no longer have a notion of "dependent realisations", but we still
+ * want to parse old realisation files. So make this just be a read test
+ * (no write direction), accordingly.
+ */
+TEST_F(RealisationTest, dependent_realisations_from_json)
+{
+    readJsonTest("with-dependent-realisations", simple);
+}
 
 } // namespace nix

src/libstore-tests/serve-protocol.cc

@@ -118,7 +118,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         },
     }))
 
-VERSIONED_CHARACTERIZATION_TEST(
+VERSIONED_READ_CHARACTERIZATION_TEST(
     ServeProtoTest,
     realisation_with_deps,
     "realisation-with-deps",
@@ -128,16 +128,6 @@ VERSIONED_CHARACTERIZATION_TEST(
             {
                 .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
                 .signatures = {"asdf", "qwer"},
-                .dependentRealisations =
-                    {
-                        {
-                            DrvOutput{
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "quux",
-                            },
-                            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
-                        },
-                    },
             },
             {
                 .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),

src/libstore-tests/worker-protocol.cc

@@ -171,7 +171,7 @@ VERSIONED_CHARACTERIZATION_TEST(
         },
     }))
 
-VERSIONED_CHARACTERIZATION_TEST(
+VERSIONED_READ_CHARACTERIZATION_TEST(
     WorkerProtoTest,
     realisation_with_deps,
     "realisation-with-deps",
@@ -181,16 +181,6 @@ VERSIONED_CHARACTERIZATION_TEST(
             {
                 .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
                 .signatures = {"asdf", "qwer"},
-                .dependentRealisations =
-                    {
-                        {
-                            DrvOutput{
-                                .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
-                                .outputName = "quux",
-                            },
-                            StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
-                        },
-                    },
             },
             {
                 .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),

src/libstore-tests/worker-substitution.cc

@@ -0,0 +1,450 @@
+#include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
+
+#include "nix/store/build/worker.hh"
+#include "nix/store/derivations.hh"
+#include "nix/store/dummy-store-impl.hh"
+#include "nix/store/globals.hh"
+#include "nix/util/memory-source-accessor.hh"
+
+#include "nix/store/tests/libstore.hh"
+#include "nix/util/tests/json-characterization.hh"
+
+namespace nix {
+
+class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest<ref<DummyStore>>
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution";
+
+protected:
+    ref<DummyStore> dummyStore;
+    ref<DummyStore> substituter;
+
+    WorkerSubstitutionTest()
+        : LibStoreTest([] {
+            auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
+            config->readOnly = false;
+            return config->openDummyStore();
+        }())
+        , dummyStore(store.dynamic_pointer_cast<DummyStore>())
+        , substituter([] {
+            auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
+            config->readOnly = false;
+            config->isTrusted = true;
+            return config->openDummyStore();
+        }())
+    {
+    }
+
+public:
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+
+    static void SetUpTestSuite()
+    {
+        initLibStore(false);
+    }
+};
+
+TEST_F(WorkerSubstitutionTest, singleStoreObject)
+{
+    // Add a store path to the substituter
+    auto pathInSubstituter = substituter->addToStore(
+        "hello",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "Hello, world!",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Snapshot the substituter (has one store object)
+    checkpointJson("single/substituter", substituter);
+
+    // Snapshot the destination store before (should be empty)
+    checkpointJson("../dummy-store/empty", dummyStore);
+
+    // The path should not exist in the destination store yet
+    ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituerAsStore = substituter;
+    worker.getSubstituters = [substituerAsStore]() -> std::list<ref<Store>> { return {substituerAsStore}; };
+
+    // Create a substitution goal for the path
+    auto goal = worker.makePathSubstitutionGoal(pathInSubstituter);
+
+    // Run the worker with -j0 semantics (no local builds, only substitution)
+    // The worker.run() takes a set of goals
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after (should match the substituter)
+    checkpointJson("single/substituter", dummyStore);
+
+    // The path should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter));
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+}
+
+TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject)
+{
+    // First, add a dependency store path to the substituter
+    auto dependencyPath = substituter->addToStore(
+        "dependency",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I am a dependency",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Now add a store path that references the dependency
+    auto mainPath = substituter->addToStore(
+        "main",
+        SourcePath{
+            [&] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                // Include a reference to the dependency path in the contents
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I depend on " + substituter->printStorePath(dependencyPath),
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256,
+        StorePathSet{dependencyPath});
+
+    // Snapshot the substituter (has two store objects)
+    checkpointJson("with-dep/substituter", substituter);
+
+    // Snapshot the destination store before (should be empty)
+    checkpointJson("../dummy-store/empty", dummyStore);
+
+    // Neither path should exist in the destination store yet
+    ASSERT_FALSE(dummyStore->isValidPath(dependencyPath));
+    ASSERT_FALSE(dummyStore->isValidPath(mainPath));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituterAsStore = substituter;
+    worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
+
+    // Create a substitution goal for the main path only
+    // The worker should automatically substitute the dependency as well
+    auto goal = worker.makePathSubstitutionGoal(mainPath);
+
+    // Run the worker
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after (should match the substituter)
+    checkpointJson("with-dep/substituter", dummyStore);
+
+    // Both paths should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(dependencyPath));
+    ASSERT_TRUE(dummyStore->isValidPath(mainPath));
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+}
+
+TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
+{
+    // Enable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
+
+    // Create a CA floating output derivation
+    Derivation drv;
+    drv.name = "test-ca-drv";
+    drv.outputs = {
+        {
+            "out",
+            DerivationOutput{DerivationOutput::CAFloating{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hashAlgo = HashAlgorithm::SHA256,
+            }},
+        },
+    };
+
+    // Write the derivation to the destination store
+    auto drvPath = writeDerivation(*dummyStore, drv);
+
+    // Snapshot the destination store before
+    checkpointJson("ca-drv/store-before", dummyStore);
+
+    // Compute the hash modulo of the derivation
+    // For CA floating derivations, the kind is Deferred since outputs aren't known until build
+    auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
+    ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
+    auto drvHash = hashModulo.hashes.at("out");
+
+    // Create the output store object
+    auto outputPath = substituter->addToStore(
+        "test-ca-drv-out",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I am the output of a CA derivation",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Add the realisation (build trace) to the substituter
+    substituter->buildTrace.insert_or_assign(
+        drvHash,
+        std::map<std::string, UnkeyedRealisation>{
+            {
+                "out",
+                UnkeyedRealisation{
+                    .outPath = outputPath,
+                },
+            },
+        });
+
+    // Snapshot the substituter
+    checkpointJson("ca-drv/substituter", substituter);
+
+    // The realisation should not exist in the destination store yet
+    DrvOutput drvOutput{drvHash, "out"};
+    ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituterAsStore = substituter;
+    worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
+
+    // Create a derivation goal for the CA derivation output
+    // The worker should substitute the output rather than building
+    auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true);
+
+    // Run the worker
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after
+    checkpointJson("ca-drv/store-after", dummyStore);
+
+    // The output path should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(outputPath));
+
+    // The realisation should now exist in the destination store
+    auto realisation = dummyStore->queryRealisation(drvOutput);
+    ASSERT_TRUE(realisation);
+    ASSERT_EQ(realisation->outPath, outputPath);
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+
+    // Disable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "");
+}
+
+/**
+ * Test for issue #11928: substituting a CA derivation output should not
+ * require fetching the output of an input derivation when that output
+ * is not referenced.
+ */
+TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
+{
+    // Enable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
+
+    // Create the dependency CA floating derivation
+    Derivation depDrv;
+    depDrv.name = "dep-drv";
+    depDrv.outputs = {
+        {
+            "out",
+            DerivationOutput{DerivationOutput::CAFloating{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hashAlgo = HashAlgorithm::SHA256,
+            }},
+        },
+    };
+
+    // Write the dependency derivation to the destination store
+    auto depDrvPath = writeDerivation(*dummyStore, depDrv);
+
+    // Compute the hash modulo for the dependency derivation
+    auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
+    ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
+    auto depDrvHash = depHashModulo.hashes.at("out");
+
+    // Create the output store object for the dependency in the substituter
+    auto depOutputPath = substituter->addToStore(
+        "dep-drv-out",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents = "I am the dependency output",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // Add the realisation for the dependency to the substituter
+    substituter->buildTrace.insert_or_assign(
+        depDrvHash,
+        std::map<std::string, UnkeyedRealisation>{
+            {
+                "out",
+                UnkeyedRealisation{
+                    .outPath = depOutputPath,
+                },
+            },
+        });
+
+    // Create the root CA floating derivation that depends on depDrv
+    Derivation rootDrv;
+    rootDrv.name = "root-drv";
+    rootDrv.outputs = {
+        {
+            "out",
+            DerivationOutput{DerivationOutput::CAFloating{
+                .method = ContentAddressMethod::Raw::NixArchive,
+                .hashAlgo = HashAlgorithm::SHA256,
+            }},
+        },
+    };
+    // Add the dependency derivation as an input
+    rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
+
+    // Write the root derivation to the destination store
+    auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
+
+    // Snapshot the destination store before
+    checkpointJson("issue-11928/store-before", dummyStore);
+
+    // Compute the hash modulo for the root derivation
+    auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
+    ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
+    auto rootDrvHash = rootHashModulo.hashes.at("out");
+
+    // Create the output store object for the root derivation
+    // Note: it does NOT reference the dependency's output
+    auto rootOutputPath = substituter->addToStore(
+        "root-drv-out",
+        SourcePath{
+            [] {
+                auto sc = make_ref<MemorySourceAccessor>();
+                sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
+                    .executable = false,
+                    .contents =
+                        "I am the root output. "
+                        "I don't reference anything because the other derivation's output is just needed at build time.",
+                }};
+                return sc;
+            }(),
+        },
+        ContentAddressMethod::Raw::NixArchive,
+        HashAlgorithm::SHA256);
+
+    // The DrvOutputs for both derivations
+    DrvOutput depDrvOutput{depDrvHash, "out"};
+    DrvOutput rootDrvOutput{rootDrvHash, "out"};
+
+    // Add the realisation for the root derivation to the substituter
+    substituter->buildTrace.insert_or_assign(
+        rootDrvHash,
+        std::map<std::string, UnkeyedRealisation>{
+            {
+                "out",
+                UnkeyedRealisation{
+                    .outPath = rootOutputPath,
+                },
+            },
+        });
+
+    // Snapshot the substituter
+    // Note: it has realisations for both drvs, but only the root's output store object
+    checkpointJson("issue-11928/substituter", substituter);
+
+    // The realisations should not exist in the destination store yet
+    ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput));
+    ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput));
+
+    // Create a worker with our custom substituter
+    Worker worker{*dummyStore, *dummyStore};
+
+    // Override the substituters to use our dummy store substituter
+    ref<Store> substituterAsStore = substituter;
+    worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
+
+    // Create a derivation goal for the root derivation output
+    // The worker should substitute the output rather than building
+    auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false);
+
+    // Run the worker
+    Goals goals;
+    goals.insert(upcast_goal(goal));
+    worker.run(goals);
+
+    // Snapshot the destination store after
+    checkpointJson("issue-11928/store-after", dummyStore);
+
+    // The root output path should now exist in the destination store
+    ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath));
+
+    // The root realisation should now exist in the destination store
+    auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput);
+    ASSERT_TRUE(rootRealisation);
+    ASSERT_EQ(rootRealisation->outPath, rootOutputPath);
+
+    // #11928: The dependency's REALISATION should be fetched, because
+    // it is needed to resolve the underlying derivation. Currently the
+    // realisation is not fetched (bug). Once fixed: Change
+    // depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the
+    // ASSERT_EQ
+    auto depRealisation = dummyStore->queryRealisation(depDrvOutput);
+    ASSERT_FALSE(depRealisation);
+    // ASSERT_EQ(depRealisation->outPath, depOutputPath);
+
+    // The dependency's OUTPUT is correctly not fetched (not referenced by root output)
+    ASSERT_FALSE(dummyStore->isValidPath(depOutputPath));
+
+    // Verify the goal succeeded
+    ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
+
+    // Disable CA derivations experimental feature
+    experimentalFeatureSettings.set("extra-experimental-features", "");
+}
+
+} // namespace nix

src/libstore/aws-creds.cc

@@ -4,15 +4,15 @@
 
 #  include <aws/crt/Types.h>
 #  include "nix/store/s3-url.hh"
-#  include "nix/util/finally.hh"
 #  include "nix/util/logging.hh"
-#  include "nix/util/url.hh"
-#  include "nix/util/util.hh"
 
 #  include <aws/crt/Api.h>
 #  include <aws/crt/auth/Credentials.h>
 #  include <aws/crt/io/Bootstrap.h>
 
+// C library headers for SSO provider support
+#  include <aws/auth/credentials.h>
+
 #  include <boost/unordered/concurrent_flat_map.hpp>
 
 #  include <chrono>
@@ -30,6 +30,46 @@ AwsAuthError::AwsAuthError(int errorCode)
 
 namespace {
 
+/**
+ * Helper function to wrap a C credentials provider in the C++ interface.
+ * This replicates the static s_CreateWrappedProvider from aws-crt-cpp.
+ */
+static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createWrappedProvider(
+    aws_credentials_provider * rawProvider, Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
+{
+    if (rawProvider == nullptr) {
+        return nullptr;
+    }
+
+    auto provider = Aws::Crt::MakeShared<Aws::Crt::Auth::CredentialsProvider>(allocator, rawProvider, allocator);
+    return std::static_pointer_cast<Aws::Crt::Auth::ICredentialsProvider>(provider);
+}
+
+/**
+ * Create an SSO credentials provider using the C library directly.
+ * The C++ wrapper doesn't expose SSO, so we call the C library and wrap the result.
+ * Returns nullptr if SSO provider creation fails (e.g., profile doesn't have SSO config).
+ */
+static std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> createSSOProvider(
+    const std::string & profileName,
+    Aws::Crt::Io::ClientBootstrap * bootstrap,
+    Aws::Crt::Io::TlsContext * tlsContext,
+    Aws::Crt::Allocator * allocator = Aws::Crt::ApiAllocator())
+{
+    aws_credentials_provider_sso_options options;
+    AWS_ZERO_STRUCT(options);
+
+    options.bootstrap = bootstrap->GetUnderlyingHandle();
+    options.tls_ctx = tlsContext ? tlsContext->GetUnderlyingHandle() : nullptr;
+    if (!profileName.empty()) {
+        options.profile_name_override = aws_byte_cursor_from_c_str(profileName.c_str());
+    }
+
+    // Create the SSO provider - will return nullptr if SSO isn't configured for this profile
+    // createWrappedProvider handles nullptr gracefully
+    return createWrappedProvider(aws_credentials_provider_new_sso(allocator, &options), allocator);
+}
+
 static AwsCredentials getCredentialsFromProvider(std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider> provider)
 {
     if (!provider || !provider->IsValid()) {
@@ -91,6 +131,22 @@ public:
             logLevel = Aws::Crt::LogLevel::Warn;
         }
         apiHandle.InitializeLogging(logLevel, stderr);
+
+        // Create a shared TLS context for SSO (required for HTTPS connections)
+        auto allocator = Aws::Crt::ApiAllocator();
+        auto tlsCtxOptions = Aws::Crt::Io::TlsContextOptions::InitDefaultClient(allocator);
+        tlsContext =
+            std::make_shared<Aws::Crt::Io::TlsContext>(tlsCtxOptions, Aws::Crt::Io::TlsMode::CLIENT, allocator);
+        if (!tlsContext || !*tlsContext) {
+            warn("failed to create TLS context for AWS SSO; SSO authentication will be unavailable");
+            tlsContext = nullptr;
+        }
+
+        // Get bootstrap (lives as long as apiHandle)
+        bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
+        if (!bootstrap) {
+            throw AwsAuthError("failed to create AWS client bootstrap");
+        }
     }
 
     AwsCredentials getCredentialsRaw(const std::string & profile);
@@ -111,6 +167,8 @@ public:
 
 private:
     Aws::Crt::ApiHandle apiHandle;
+    std::shared_ptr<Aws::Crt::Io::TlsContext> tlsContext;
+    Aws::Crt::Io::ClientBootstrap * bootstrap;
     boost::concurrent_flat_map<std::string, std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>>
         credentialProviderCache;
 };
@@ -118,23 +176,58 @@ private:
 std::shared_ptr<Aws::Crt::Auth::ICredentialsProvider>
 AwsCredentialProviderImpl::createProviderForProfile(const std::string & profile)
 {
-    debug(
-        "[pid=%d] creating new AWS credential provider for profile '%s'",
-        getpid(),
-        profile.empty() ? "(default)" : profile.c_str());
+    // profileDisplayName is only used for debug logging - SDK uses its default profile
+    // when ProfileNameOverride is not set
+    const char * profileDisplayName = profile.empty() ? "(default)" : profile.c_str();
 
-    if (profile.empty()) {
-        Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config;
-        config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
-        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config);
+    debug("[pid=%d] creating new AWS credential provider for profile '%s'", getpid(), profileDisplayName);
+
+    // Build a custom credential chain: Environment → SSO → Profile → IMDS
+    // This works for both default and named profiles, ensuring consistent behavior
+    // including SSO support and proper TLS context for STS-based role assumption.
+    Aws::Crt::Auth::CredentialsProviderChainConfig chainConfig;
+    auto allocator = Aws::Crt::ApiAllocator();
+
+    auto addProviderToChain = [&](std::string_view name, auto createProvider) {
+        if (auto provider = createProvider()) {
+            chainConfig.Providers.push_back(provider);
+            debug("Added AWS %s Credential Provider to chain for profile '%s'", name, profileDisplayName);
+        } else {
+            debug("Skipped AWS %s Credential Provider for profile '%s'", name, profileDisplayName);
+        }
+    };
+
+    // 1. Environment variables (highest priority)
+    addProviderToChain("Environment", [&]() {
+        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderEnvironment(allocator);
+    });
+
+    // 2. SSO provider (try it, will fail gracefully if not configured)
+    if (tlsContext) {
+        addProviderToChain("SSO", [&]() { return createSSOProvider(profile, bootstrap, tlsContext.get(), allocator); });
+    } else {
+        debug("Skipped AWS SSO Credential Provider for profile '%s': TLS context unavailable", profileDisplayName);
     }
 
-    Aws::Crt::Auth::CredentialsProviderProfileConfig config;
-    config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap();
-    // This is safe because the underlying C library will copy this string
-    // c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220
-    config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
-    return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config);
+    // 3. Profile provider (for static credentials and role_arn/source_profile with STS)
+    addProviderToChain("Profile", [&]() {
+        Aws::Crt::Auth::CredentialsProviderProfileConfig profileConfig;
+        profileConfig.Bootstrap = bootstrap;
+        profileConfig.TlsContext = tlsContext.get();
+        if (!profile.empty()) {
+            profileConfig.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str());
+        }
+        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(profileConfig, allocator);
+    });
+
+    // 4. IMDS provider (for EC2 instances, lowest priority)
+    addProviderToChain("IMDS", [&]() {
+        Aws::Crt::Auth::CredentialsProviderImdsConfig imdsConfig;
+        imdsConfig.Bootstrap = bootstrap;
+        return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderImds(imdsConfig, allocator);
+    });
+
+    return Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChain(chainConfig, allocator);
 }
 
 AwsCredentials AwsCredentialProviderImpl::getCredentialsRaw(const std::string & profile)

src/libstore/build/derivation-building-goal.cc

@@ -71,14 +71,6 @@ void DerivationBuildingGoal::killChild()
 #endif
 }
 
-void DerivationBuildingGoal::timedOut(Error && ex)
-{
-    killChild();
-    // We're not inside a coroutine, hence we can't use co_return here.
-    // Thus we ignore the return value.
-    [[maybe_unused]] Done _ = doneFailure({BuildResult::Failure::TimedOut, std::move(ex)});
-}
-
 std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & drv)
 {
     std::string msg;
@@ -443,7 +435,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
     if (useHook) {
         buildResult.startTime = time(0); // inexact
         started();
-        co_await Suspend{};
+
+        while (true) {
+            auto event = co_await WaitForChildEvent{};
+            if (auto * output = std::get_if<ChildOutput>(&event)) {
+                co_await processChildOutput(output->fd, output->data);
+            } else if (std::get_if<ChildEOF>(&event)) {
+                if (!currentLogLine.empty())
+                    flushLine();
+                break;
+            } else if (auto * timeout = std::get_if<TimedOut>(&event)) {
+                killChild();
+                co_return doneFailure(std::move(*timeout));
+            }
+        }
 
 #ifndef _WIN32
         assert(hook);
@@ -664,7 +669,20 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
     worker.childStarted(shared_from_this(), {builderOut}, true, true);
 
     started();
-    co_await Suspend{};
+
+    while (true) {
+        auto event = co_await WaitForChildEvent{};
+        if (auto * output = std::get_if<ChildOutput>(&event)) {
+            co_await processChildOutput(output->fd, output->data);
+        } else if (std::get_if<ChildEOF>(&event)) {
+            if (!currentLogLine.empty())
+                flushLine();
+            break;
+        } else if (auto * timeout = std::get_if<TimedOut>(&event)) {
+            killChild();
+            co_return doneFailure(std::move(*timeout));
+        }
+    }
 
     trace("build done");
 
@@ -970,7 +988,7 @@ Path DerivationBuildingGoal::openLogFile()
     logFileSink = std::make_shared<FdSink>(fdLogFile.get());
 
     if (settings.compressLog)
-        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
+        logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
     else
         logSink = logFileSink;
 
@@ -997,7 +1015,7 @@ bool DerivationBuildingGoal::isReadDesc(Descriptor fd)
 #endif
 }
 
-void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view data)
+Goal::Co DerivationBuildingGoal::processChildOutput(Descriptor fd, std::string_view data)
 {
     // local & `ssh://`-builds are dealt with here.
     auto isWrittenToLog = isReadDesc(fd);
@@ -1005,14 +1023,11 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
         logSize += data.size();
         if (settings.maxLogSize && logSize > settings.maxLogSize) {
             killChild();
-            // We're not inside a coroutine, hence we can't use co_return here.
-            // Thus we ignore the return value.
-            [[maybe_unused]] Done _ = doneFailure(BuildError(
+            co_return doneFailure(BuildError(
                 BuildResult::Failure::LogLimitExceeded,
                 "%s killed after writing more than %d bytes of log output",
                 getName(),
                 settings.maxLogSize));
-            return;
         }
 
         for (auto c : data)
@@ -1065,13 +1080,7 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d
                 currentHookLine += c;
     }
 #endif
-}
-
-void DerivationBuildingGoal::handleEOF(Descriptor fd)
-{
-    if (!currentLogLine.empty())
-        flushLine();
-    worker.wakeUp(shared_from_this());
+    co_return Return{};
 }
 
 void DerivationBuildingGoal::flushLine()

src/libstore/build/derivation-goal.cc

@@ -1,4 +1,5 @@
 #include "nix/store/build/derivation-goal.hh"
+#include "nix/store/build/drv-output-substitution-goal.hh"
 #include "nix/store/build/derivation-building-goal.hh"
 #include "nix/store/build/derivation-resolution-goal.hh"
 #ifndef _WIN32 // TODO enable build hook on Windows
@@ -100,9 +101,24 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
            through substitutes.  If that doesn't work, we'll build
            them. */
         if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
-            if (!checkResult)
-                waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput})));
-            else {
+            if (!checkResult) {
+                DrvOutput id{outputHash, wantedOutput};
+                auto g = worker.makeDrvOutputSubstitutionGoal(id);
+                waitees.insert(g);
+                co_await await(std::move(waitees));
+
+                if (nrFailed == 0) {
+                    waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath)));
+                    co_await await(std::move(waitees));
+
+                    trace("output path substituted");
+
+                    if (nrFailed == 0)
+                        worker.store.registerDrvOutput({*g->outputInfo, id});
+                    else
+                        debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
+                }
+            } else {
                 auto * cap = getDerivationCA(*drv);
                 waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
                     checkResult->first.outPath,
@@ -210,11 +226,6 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
                         .outputName = wantedOutput,
                     }};
                 newRealisation.signatures.clear();
-                if (!drv->type().isFixed()) {
-                    auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
-                    newRealisation.dependentRealisations =
-                        drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
-                }
                 worker.store.signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
             }

src/libstore/build/drv-output-substitution-goal.cc

@@ -3,8 +3,6 @@
 #include "nix/store/build/worker.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/util/callback.hh"
-#include "nix/store/store-open.hh"
-#include "nix/store/globals.hh"
 
 namespace nix {
 
@@ -21,11 +19,11 @@ Goal::Co DrvOutputSubstitutionGoal::init()
     trace("init");
 
     /* If the derivation already exists, we’re done */
-    if (worker.store.queryRealisation(id)) {
+    if ((outputInfo = worker.store.queryRealisation(id))) {
         co_return amDone(ecSuccess);
     }
 
-    auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+    auto subs = worker.getSubstituters();
 
     bool substituterFailed = false;
 
@@ -66,16 +64,19 @@ Goal::Co DrvOutputSubstitutionGoal::init()
             true,
             false);
 
-        co_await Suspend{};
+        while (true) {
+            auto event = co_await WaitForChildEvent{};
+            if (std::get_if<ChildOutput>(&event)) {
+                // Doesn't process child output
+            } else if (std::get_if<ChildEOF>(&event)) {
+                break;
+            } else if (std::get_if<TimedOut>(&event)) {
+                unreachable();
+            }
+        }
 
         worker.childTerminated(this);
 
-        /*
-         * The realisation corresponding to the given output id.
-         * Will be filled once we can get it.
-         */
-        std::shared_ptr<const UnkeyedRealisation> outputInfo;
-
         try {
             outputInfo = promise->get_future().get();
         } catch (std::exception & e) {
@@ -86,45 +87,6 @@ Goal::Co DrvOutputSubstitutionGoal::init()
         if (!outputInfo)
             continue;
 
-        bool failed = false;
-
-        Goals waitees;
-
-        for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
-            if (depId != id) {
-                if (auto localOutputInfo = worker.store.queryRealisation(depId);
-                    localOutputInfo && localOutputInfo->outPath != depPath) {
-                    warn(
-                        "substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
-                        "Local:  %s\n"
-                        "Remote: %s",
-                        sub->config.getHumanReadableURI(),
-                        depId.to_string(),
-                        worker.store.printStorePath(localOutputInfo->outPath),
-                        worker.store.printStorePath(depPath));
-                    failed = true;
-                    break;
-                }
-                waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
-            }
-        }
-
-        if (failed)
-            continue;
-
-        waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
-
-        co_await await(std::move(waitees));
-
-        trace("output path substituted");
-
-        if (nrFailed > 0) {
-            debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
-            co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
-        }
-
-        worker.store.registerDrvOutput({*outputInfo, id});
-
         trace("finished");
         co_return amDone(ecSuccess);
     }
@@ -149,9 +111,4 @@ std::string DrvOutputSubstitutionGoal::key()
     return "a$" + std::string(id.to_string());
 }
 
-void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd)
-{
-    worker.wakeUp(shared_from_this());
-}
-
 } // namespace nix

src/libstore/build/goal.cc

@@ -4,8 +4,58 @@
 
 namespace nix {
 
+TimedOut::TimedOut(time_t maxDuration)
+    : BuildError(BuildResult::Failure::TimedOut, "timed out after %1% seconds", maxDuration)
+    , maxDuration(maxDuration)
+{
+}
+
 using Co = nix::Goal::Co;
 using promise_type = nix::Goal::promise_type;
+using ChildEvents = decltype(promise_type::childEvents);
+
+void ChildEvents::pushChildEvent(ChildOutput event)
+{
+    if (childTimeout)
+        return; // Already timed out, ignore
+    childOutputs.push(std::move(event));
+}
+
+void ChildEvents::pushChildEvent(ChildEOF event)
+{
+    if (childTimeout)
+        return; // Already timed out, ignore
+    assert(!childEOF);
+    childEOF = std::move(event);
+}
+
+void ChildEvents::pushChildEvent(TimedOut event)
+{
+    // Timeout is immediate - flush pending events
+    childOutputs = {};
+    childEOF.reset();
+    childTimeout = std::move(event);
+}
+
+bool ChildEvents::hasChildEvent() const
+{
+    return !childOutputs.empty() || childEOF || childTimeout;
+}
+
+Goal::ChildEvent ChildEvents::popChildEvent()
+{
+    if (!childOutputs.empty()) {
+        auto event = std::move(childOutputs.front());
+        childOutputs.pop();
+        return event;
+    }
+    if (childEOF)
+        return *std::exchange(childEOF, std::nullopt);
+    if (childTimeout)
+        return *std::exchange(childTimeout, std::nullopt);
+    unreachable();
+}
+
 using handle_type = nix::Goal::handle_type;
 using Suspend = nix::Goal::Suspend;
 
@@ -206,6 +256,27 @@ void Goal::work()
     assert(top_co || exitCode != ecBusy);
 }
 
+void Goal::handleChildOutput(Descriptor fd, std::string_view data)
+{
+    assert(top_co);
+    top_co->handle.promise().childEvents.pushChildEvent(ChildOutput{fd, std::string{data}});
+    worker.wakeUp(shared_from_this());
+}
+
+void Goal::handleEOF(Descriptor fd)
+{
+    assert(top_co);
+    top_co->handle.promise().childEvents.pushChildEvent(ChildEOF{fd});
+    worker.wakeUp(shared_from_this());
+}
+
+void Goal::timedOut(TimedOut && ex)
+{
+    assert(top_co);
+    top_co->handle.promise().childEvents.pushChildEvent(std::move(ex));
+    worker.wakeUp(shared_from_this());
+}
+
 Goal::Co Goal::yield()
 {
     worker.wakeUp(shared_from_this());

src/libstore/build/substitution-goal.cc

@@ -1,5 +1,4 @@
 #include "nix/store/build/worker.hh"
-#include "nix/store/store-open.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/store/nar-info.hh"
 #include "nix/util/finally.hh"
@@ -60,7 +59,7 @@ Goal::Co PathSubstitutionGoal::init()
         throw Error(
             "cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
 
-    auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+    auto subs = worker.getSubstituters();
 
     bool substituterFailed = false;
     std::optional<Error> lastStoresException = std::nullopt;
@@ -258,7 +257,16 @@ Goal::Co PathSubstitutionGoal::tryToRun(
         true,
         false);
 
-    co_await Suspend{};
+    while (true) {
+        auto event = co_await WaitForChildEvent{};
+        if (std::get_if<ChildOutput>(&event)) {
+            // Substitution doesn't process child output
+        } else if (std::get_if<ChildEOF>(&event)) {
+            break;
+        } else if (std::get_if<TimedOut>(&event)) {
+            unreachable(); // Substitution doesn't use timeouts
+        }
+    }
 
     trace("substitute finished");
 
@@ -310,11 +318,6 @@ Goal::Co PathSubstitutionGoal::tryToRun(
     co_return doneSuccess(BuildResult::Success::Substituted);
 }
 
-void PathSubstitutionGoal::handleEOF(Descriptor fd)
-{
-    worker.wakeUp(shared_from_this());
-}
-
 void PathSubstitutionGoal::cleanup()
 {
     try {

src/libstore/build/worker.cc

@@ -1,5 +1,6 @@
 #include "nix/store/local-store.hh"
 #include "nix/store/machines.hh"
+#include "nix/store/store-open.hh"
 #include "nix/store/build/worker.hh"
 #include "nix/store/build/substitution-goal.hh"
 #include "nix/store/build/drv-output-substitution-goal.hh"
@@ -21,6 +22,7 @@ Worker::Worker(Store & store, Store & evalStore)
     , actSubstitutions(*logger, actCopyPaths)
     , store(store)
     , evalStore(evalStore)
+    , getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
 {
     nrLocalBuilds = 0;
     nrSubstitutions = 0;
@@ -479,14 +481,13 @@ void Worker::waitForInput()
 
         if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts
             && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) {
-            goal->timedOut(
-                Error("%1% timed out after %2% seconds of silence", goal->getName(), settings.maxSilentTime));
+            goal->timedOut(TimedOut(settings.maxSilentTime));
         }
 
         else if (
             goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts
             && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) {
-            goal->timedOut(Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout));
+            goal->timedOut(TimedOut(settings.buildTimeout));
         }
     }
 

src/libstore/derivation-options.cc

@@ -341,8 +341,12 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
 
                 if (parsed) {
                     auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
-                    if (!e || !e->is_object())
+                    if (!e)
                         return ret;
+                    if (!e->is_object()) {
+                        warn("'exportReferencesGraph' in structured attrs is not a JSON object, ignoring");
+                        return ret;
+                    }
                     for (auto & [key, storePathsJson] : getObject(*e)) {
                         StringSet ss;
                         flatten(storePathsJson, ss);

src/libstore/derivations.cc

@@ -1489,8 +1489,6 @@ adl_serializer<DerivationOutput>::from_json(const json & _json, const Experiment
     }
 }
 
-static unsigned constexpr expectedJsonVersionDerivation = 4;
-
 void adl_serializer<Derivation>::to_json(json & res, const Derivation & d)
 {
     res = nlohmann::json::object();

src/libstore/filetransfer.cc

@@ -41,9 +41,27 @@ FileTransferSettings fileTransferSettings;
 
 static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings);
 
+namespace {
+
+using curlSList = std::unique_ptr<::curl_slist, decltype([](::curl_slist * list) { ::curl_slist_free_all(list); })>;
+using curlMulti = std::unique_ptr<::CURLM, decltype([](::CURLM * multi) { ::curl_multi_cleanup(multi); })>;
+
+struct curlMultiError : Error
+{
+    ::CURLMcode code;
+
+    curlMultiError(::CURLMcode code)
+        : Error{"unexpected curl multi error: %s", ::curl_multi_strerror(code)}
+    {
+        assert(code != CURLM_OK);
+    }
+};
+
+} // namespace
+
 struct curlFileTransfer : public FileTransfer
 {
-    CURLM * curlm = 0;
+    curlMulti curlm;
 
     std::random_device rd;
     std::mt19937 mt19937;
@@ -69,7 +87,7 @@ struct curlFileTransfer : public FileTransfer
            has been reached. */
         std::chrono::steady_clock::time_point embargo;
 
-        struct curl_slist * requestHeaders = 0;
+        curlSList requestHeaders;
 
         std::string encoding;
 
@@ -92,6 +110,15 @@ struct curlFileTransfer : public FileTransfer
             return httpStatus;
         }
 
+        void appendHeaders(const std::string & header)
+        {
+            curlSList tmpSList = curlSList(::curl_slist_append(requestHeaders.get(), requireCString(header)));
+            if (!tmpSList)
+                throw std::bad_alloc();
+            requestHeaders.release();
+            requestHeaders = std::move(tmpSList);
+        }
+
         TransferItem(
             curlFileTransfer & fileTransfer,
             const FileTransferRequest & request,
@@ -131,13 +158,13 @@ struct curlFileTransfer : public FileTransfer
         {
             result.urls.push_back(request.uri.to_string());
 
-            requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
+            appendHeaders("Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
             if (!request.expectedETag.empty())
-                requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
+                appendHeaders("If-None-Match: " + request.expectedETag);
             if (!request.mimeType.empty())
-                requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
+                appendHeaders("Content-Type: " + request.mimeType);
             for (auto it = request.headers.begin(); it != request.headers.end(); ++it) {
-                requestHeaders = curl_slist_append(requestHeaders, fmt("%s: %s", it->first, it->second).c_str());
+                appendHeaders(fmt("%s: %s", it->first, it->second));
             }
         }
 
@@ -145,11 +172,9 @@ struct curlFileTransfer : public FileTransfer
         {
             if (req) {
                 if (active)
-                    curl_multi_remove_handle(fileTransfer.curlm, req);
+                    curl_multi_remove_handle(fileTransfer.curlm.get(), req);
                 curl_easy_cleanup(req);
             }
-            if (requestHeaders)
-                curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
                     fail(FileTransferError(
@@ -348,7 +373,7 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
         }
 
-#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+#if !defined(_WIN32)
         static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose)
         {
             unix::closeOnExec(curlfd);
@@ -411,15 +436,11 @@ struct curlFileTransfer : public FileTransfer
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion
                  + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : ""))
                     .c_str());
-#if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
-#endif
-#if LIBCURL_VERSION_NUM >= 0x072f00
             if (fileTransferSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             else
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
-#endif
             curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
             curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
@@ -429,7 +450,7 @@ struct curlFileTransfer : public FileTransfer
             curl_easy_setopt(req, CURLOPT_XFERINFODATA, this);
             curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0);
 
-            curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
+            curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders.get());
 
             if (settings.downloadSpeed.get() > 0)
                 curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
@@ -459,10 +480,9 @@ struct curlFileTransfer : public FileTransfer
             if (settings.caFile != "")
                 curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
 
-#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+#if !defined(_WIN32)
             curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
 #endif
-
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
@@ -694,13 +714,6 @@ struct curlFileTransfer : public FileTransfer
 
     Sync<State> state_;
 
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-    /* We can't use a std::condition_variable to wake up the curl
-       thread, because it only monitors file descriptors. So use a
-       pipe instead. */
-    Pipe wakeupPipe;
-#endif
-
     std::thread workerThread;
 
     curlFileTransfer()
@@ -709,43 +722,35 @@ struct curlFileTransfer : public FileTransfer
         static std::once_flag globalInit;
         std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL);
 
-        curlm = curl_multi_init();
+        curlm = curlMulti(curl_multi_init());
 
-#if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0
-        curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
-#endif
-#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
-        curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
-#endif
-
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        wakeupPipe.create();
-        fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK);
-#endif
+        curl_multi_setopt(curlm.get(), CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
+        curl_multi_setopt(curlm.get(), CURLMOPT_MAX_TOTAL_CONNECTIONS, fileTransferSettings.httpConnections.get());
 
         workerThread = std::thread([&]() { workerThreadEntry(); });
     }
 
     ~curlFileTransfer()
     {
-        stopWorkerThread();
-
+        try {
+            stopWorkerThread();
+        } catch (...) {
+            ignoreExceptionInDestructor();
+        }
         workerThread.join();
-
-        if (curlm)
-            curl_multi_cleanup(curlm);
     }
 
     void stopWorkerThread()
     {
         /* Signal the worker thread to exit. */
-        {
-            auto state(state_.lock());
-            state->quit();
-        }
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ", false);
-#endif
+        state_.lock()->quit();
+        wakeupMulti();
+    }
+
+    void wakeupMulti()
+    {
+        if (auto ec = ::curl_multi_wakeup(curlm.get()))
+            throw curlMultiError(ec);
     }
 
     void workerThreadMain()
@@ -775,32 +780,25 @@ struct curlFileTransfer : public FileTransfer
 
             /* Let curl do its thing. */
             int running;
-            CURLMcode mc = curl_multi_perform(curlm, &running);
+            CURLMcode mc = curl_multi_perform(curlm.get(), &running);
             if (mc != CURLM_OK)
                 throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc));
 
             /* Set the promises of any finished requests. */
             CURLMsg * msg;
             int left;
-            while ((msg = curl_multi_info_read(curlm, &left))) {
+            while ((msg = curl_multi_info_read(curlm.get(), &left))) {
                 if (msg->msg == CURLMSG_DONE) {
                     auto i = items.find(msg->easy_handle);
                     assert(i != items.end());
                     i->second->finish(msg->data.result);
-                    curl_multi_remove_handle(curlm, i->second->req);
+                    curl_multi_remove_handle(curlm.get(), i->second->req);
                     i->second->active = false;
                     items.erase(i);
                 }
             }
 
             /* Wait for activity, including wakeup events. */
-            int numfds = 0;
-            struct curl_waitfd extraFDs[1];
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-            extraFDs[0].fd = wakeupPipe.readSide.get();
-            extraFDs[0].events = CURL_WAIT_POLLIN;
-            extraFDs[0].revents = 0;
-#endif
             long maxSleepTimeMs = items.empty() ? 10000 : 100;
             auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point()
                                    ? std::max(
@@ -809,23 +807,14 @@ struct curlFileTransfer : public FileTransfer
                                              nextWakeup - std::chrono::steady_clock::now())
                                              .count())
                                    : maxSleepTimeMs;
-            vomit("download thread waiting for %d ms", sleepTimeMs);
-            mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
+
+            int numfds = 0;
+            mc = curl_multi_poll(curlm.get(), nullptr, 0, sleepTimeMs, &numfds);
             if (mc != CURLM_OK)
-                throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc));
+                throw curlMultiError(mc);
 
             nextWakeup = std::chrono::steady_clock::time_point();
 
-            /* Add new curl requests from the incoming requests queue,
-               except for requests that are embargoed (waiting for a
-               retry timeout to expire). */
-            if (extraFDs[0].revents & CURL_WAIT_POLLIN) {
-                char buf[1024];
-                auto res = read(extraFDs[0].fd, buf, sizeof(buf));
-                if (res == -1 && errno != EINTR)
-                    throw SysError("reading curl wakeup socket");
-            }
-
             std::vector<std::shared_ptr<TransferItem>> incoming;
             auto now = std::chrono::steady_clock::now();
 
@@ -848,7 +837,7 @@ struct curlFileTransfer : public FileTransfer
             for (auto & item : incoming) {
                 debug("starting %s of %s", item->request.noun(), item->request.uri);
                 item->init();
-                curl_multi_add_handle(curlm, item->req);
+                curl_multi_add_handle(curlm.get(), item->req);
                 item->active = true;
                 items[item->req] = item;
             }
@@ -899,10 +888,8 @@ struct curlFileTransfer : public FileTransfer
                 throw nix::Error("cannot enqueue download request because the download thread is shutting down");
             state->incoming.push(item);
         }
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ");
-#endif
 
+        wakeupMulti();
         return ItemHandle(static_cast<Item &>(*item));
     }
 
@@ -922,9 +909,7 @@ struct curlFileTransfer : public FileTransfer
     {
         auto state(state_.lock());
         state->unpause.push_back(std::move(item));
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ");
-#endif
+        wakeupMulti();
     }
 
     void unpauseTransfer(ItemHandle handle) override

src/libstore/include/nix/store/build/derivation-builder.hh

@@ -79,7 +79,7 @@ struct DerivationBuilderParams
      */
     const StorePathSet & inputPaths;
 
-    const std::map<std::string, InitialOutput> & initialOutputs;
+    const std::map<std::string, InitialOutput> initialOutputs;
 
     const BuildMode & buildMode;
 

src/libstore/include/nix/store/build/derivation-building-goal.hh

@@ -100,8 +100,6 @@ private:
 
     std::map<ActivityId, Activity> builderActivities;
 
-    void timedOut(Error && ex) override;
-
     std::string key() override;
 
     /**
@@ -129,10 +127,10 @@ private:
     bool isReadDesc(Descriptor fd);
 
     /**
-     * Callback used by the worker to write to the log.
+     * Process output from a child process.
      */
-    void handleChildOutput(Descriptor fd, std::string_view data) override;
-    void handleEOF(Descriptor fd) override;
+    Co processChildOutput(Descriptor fd, std::string_view data);
+
     void flushLine();
 
     /**

src/libstore/include/nix/store/build/derivation-goal.hh

@@ -52,11 +52,6 @@ struct DerivationGoal : public Goal
         bool storeDerivation);
     ~DerivationGoal() = default;
 
-    void timedOut(Error && ex) override
-    {
-        unreachable();
-    };
-
     std::string key() override;
 
     JobCategory jobCategory() const override

src/libstore/include/nix/store/build/derivation-resolution-goal.hh

@@ -43,8 +43,6 @@ struct DerivationResolutionGoal : public Goal
      */
     std::unique_ptr<std::pair<StorePath, BasicDerivation>> resolvedDrv;
 
-    void timedOut(Error && ex) override {}
-
 private:
 
     /**

src/libstore/include/nix/store/build/derivation-trampoline-goal.hh

@@ -109,8 +109,6 @@ struct DerivationTrampolineGoal : public Goal
 
     virtual ~DerivationTrampolineGoal();
 
-    void timedOut(Error && ex) override {}
-
     std::string key() override;
 
     JobCategory jobCategory() const override

src/libstore/include/nix/store/build/drv-output-substitution-goal.hh

@@ -14,11 +14,14 @@ namespace nix {
 class Worker;
 
 /**
- * Substitution of a derivation output.
- * This is done in three steps:
- * 1. Fetch the output info from a substituter
- * 2. Substitute the corresponding output path
- * 3. Register the output info
+ * Fetch a `Realisation` (drv ⨯ output name -> output path) from a
+ * substituter.
+ *
+ * If the output store object itself should also be substituted, that is
+ * the responsibility of the caller to do so.
+ *
+ * @todo rename this `BuidlTraceEntryGoal`, which will make sense
+ * especially once `Realisation` is renamed to `BuildTraceEntry`.
  */
 class DrvOutputSubstitutionGoal : public Goal
 {
@@ -31,17 +34,16 @@ class DrvOutputSubstitutionGoal : public Goal
 public:
     DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
 
+    /**
+     * The realisation corresponding to the given output id.
+     * Will be filled once we can get it.
+     */
+    std::shared_ptr<const UnkeyedRealisation> outputInfo;
+
     Co init();
 
-    void timedOut(Error && ex) override
-    {
-        unreachable();
-    };
-
     std::string key() override;
 
-    void handleEOF(Descriptor fd) override;
-
     JobCategory jobCategory() const override
     {
         return JobCategory::Substitution;

src/libstore/include/nix/store/build/goal.hh

@@ -5,9 +5,18 @@
 #include "nix/store/build-result.hh"
 
 #include <coroutine>
+#include <queue>
+#include <variant>
 
 namespace nix {
 
+struct TimedOut : BuildError
+{
+    time_t maxDuration;
+
+    TimedOut(time_t maxDuration);
+};
+
 /**
  * Forward definition.
  */
@@ -138,6 +147,29 @@ public:
         friend Goal;
     };
 
+    /**
+     * Event types for child process communication, delivered via coroutines.
+     */
+    struct ChildOutput
+    {
+        Descriptor fd;
+        std::string data;
+    };
+
+    struct ChildEOF
+    {
+        Descriptor fd;
+    };
+
+    using ChildEvent = std::variant<ChildOutput, ChildEOF, TimedOut>;
+
+    /**
+     * Tag type for `co_await`-ing child events.
+     * Returns a `ChildEvent` when resumed.
+     */
+    struct WaitForChildEvent
+    {};
+
     // forward declaration of promise_type, see below
     struct promise_type;
 
@@ -276,6 +308,28 @@ public:
          */
         bool alive = true;
 
+        class
+        {
+            /**
+             * Structured queue of child events:
+             * - outputs: stream of data from child
+             * - eof: optional end-of-stream marker
+             * - timeout: optional timeout that flushes/overrides other events
+             */
+            std::queue<ChildOutput> childOutputs;
+            std::optional<ChildEOF> childEOF;
+            std::optional<TimedOut> childTimeout;
+
+        public:
+
+            void pushChildEvent(ChildOutput event);
+            void pushChildEvent(ChildEOF event);
+            void pushChildEvent(TimedOut event);
+            bool hasChildEvent() const;
+            ChildEvent popChildEvent();
+
+        } childEvents;
+
         /**
          * The awaiter used by @ref final_suspend.
          */
@@ -369,13 +423,66 @@ public:
             return static_cast<Co &&>(co);
         }
 
+        /**
+         * Awaiter for @ref Suspend. Always suspends, but asserts
+         * there are no pending child events (those should be
+         * consumed first via @ref WaitForChildEvent).
+         */
+        struct SuspendAwaiter
+        {
+            promise_type & promise;
+
+            bool await_ready()
+            {
+                assert(!promise.childEvents.hasChildEvent());
+                return false;
+            }
+
+            void await_suspend(handle_type) {}
+
+            void await_resume() {}
+        };
+
         /**
          * Allows awaiting a @ref Suspend.
          * Always suspends.
          */
-        std::suspend_always await_transform(Suspend)
+        SuspendAwaiter await_transform(Suspend)
         {
-            return {};
+            return SuspendAwaiter{*this};
+        };
+
+        /**
+         * Awaiter for child events. Suspends and returns the
+         * pending child event when resumed.
+         */
+        struct ChildEventAwaiter
+        {
+            handle_type handle;
+
+            bool await_ready()
+            {
+                return handle && handle.promise().childEvents.hasChildEvent();
+            }
+
+            void await_suspend(handle_type h)
+            {
+                handle = h;
+            }
+
+            ChildEvent await_resume()
+            {
+                assert(handle);
+                return handle.promise().childEvents.popChildEvent();
+            }
+        };
+
+        /**
+         * Allows awaiting child events (output, EOF, timeout).
+         */
+        ChildEventAwaiter await_transform(WaitForChildEvent)
+        {
+            return ChildEventAwaiter{handle_type::from_promise(*this)};
         };
     };
 
@@ -432,15 +539,23 @@ public:
 
     void work();
 
-    virtual void handleChildOutput(Descriptor fd, std::string_view data)
-    {
-        unreachable();
-    }
+    /**
+     * Called by the worker when data is received from a child process.
+     * Stores the event and resumes the coroutine.
+     */
+    void handleChildOutput(Descriptor fd, std::string_view data);
 
-    virtual void handleEOF(Descriptor fd)
-    {
-        unreachable();
-    }
+    /**
+     * Called by the worker when EOF is received from a child process.
+     * Stores the event and resumes the coroutine.
+     */
+    void handleEOF(Descriptor fd);
+
+    /**
+     * Called by the worker when a build times out.
+     * Stores the event and resumes the coroutine.
+     */
+    void timedOut(TimedOut && ex);
 
     void trace(std::string_view s);
 
@@ -449,13 +564,6 @@ public:
         return name;
     }
 
-    /**
-     * Callback in case of a timeout.  It should wake up its waiters,
-     * get rid of any running child processes that are being monitored
-     * by the worker (important!), etc.
-     */
-    virtual void timedOut(Error && ex) = 0;
-
     /**
      * Used for comparisons. The order matters a bit for scheduling. We
      * want:

src/libstore/include/nix/store/build/substitution-goal.hh

@@ -53,11 +53,6 @@ public:
         std::optional<ContentAddress> ca = std::nullopt);
     ~PathSubstitutionGoal();
 
-    void timedOut(Error && ex) override
-    {
-        unreachable();
-    };
-
     std::string key() override
     {
         return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
@@ -72,12 +67,6 @@ public:
         StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed);
     Co finished();
 
-    /**
-     * Callback used by the worker to write to the log.
-     */
-    void handleChildOutput(Descriptor fd, std::string_view data) override {};
-    void handleEOF(Descriptor fd) override;
-
     /* Called by destructor, can't be overridden */
     void cleanup() override final;
 

src/libstore/include/nix/store/build/worker.hh

@@ -8,6 +8,7 @@
 #include "nix/store/realisation.hh"
 #include "nix/util/muxable-pipe.hh"
 
+#include <functional>
 #include <future>
 #include <thread>
 
@@ -171,6 +172,14 @@ public:
     Store & store;
     Store & evalStore;
 
+    /**
+     * Function to get the substituters to use for path substitution.
+     *
+     * Defaults to `getDefaultSubstituters`. This allows tests to
+     * inject custom substituters.
+     */
+    std::function<std::list<ref<Store>>()> getSubstituters;
+
 #ifndef _WIN32 // TODO Enable building on Windows
     std::unique_ptr<HookInstance> hook;
 #endif

src/libstore/include/nix/store/derivations.hh

@@ -586,6 +586,12 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva
  */
 std::string hashPlaceholder(const OutputNameView outputName);
 
+/**
+ * The expected JSON version for derivation serialization.
+ * Used by `nix derivation show` and `nix derivation add`.
+ */
+constexpr unsigned expectedJsonVersionDerivation = 4;
+
 } // namespace nix
 
 JSON_IMPL_WITH_XP_FEATURES(nix::DerivationOutput)

src/libstore/include/nix/store/globals.hh

@@ -1143,7 +1143,7 @@ public:
 
     Setting<std::string> netrcFile{
         this,
-        fmt("%s/%s", nixConfDir, "netrc"),
+        (nixConfDir / "netrc").string(),
         "netrc-file",
         R"(
           If set to an absolute path to a `netrc` file, Nix uses the HTTP

src/libstore/include/nix/store/pathlocks.hh

@@ -33,6 +33,22 @@ private:
 public:
     PathLocks();
     PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
+
+    PathLocks(PathLocks && other) noexcept
+        : fds(std::exchange(other.fds, {}))
+        , deletePaths(other.deletePaths)
+    {
+    }
+
+    PathLocks & operator=(PathLocks && other) noexcept
+    {
+        fds = std::exchange(other.fds, {});
+        deletePaths = other.deletePaths;
+        return *this;
+    }
+
+    PathLocks(const PathLocks &) = delete;
+    PathLocks & operator=(const PathLocks &) = delete;
     bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
     ~PathLocks();
     void unlock();
@@ -45,6 +61,10 @@ struct FdLock
     bool acquired = false;
 
     FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
+    FdLock(const FdLock &) = delete;
+    FdLock & operator=(const FdLock &) = delete;
+    FdLock(FdLock &&) = delete;
+    FdLock & operator=(FdLock &&) = delete;
 
     ~FdLock()
     {

src/libstore/include/nix/store/realisation.hh

@@ -56,14 +56,6 @@ struct UnkeyedRealisation
 
     StringSet signatures;
 
-    /**
-     * The realisations that are required for the current one to be valid.
-     *
-     * When importing this realisation, the store will first check that all its
-     * dependencies exist, and map to the correct output path
-     */
-    std::map<DrvOutput, StorePath> dependentRealisations;
-
     std::string fingerprint(const DrvOutput & key) const;
 
     void sign(const DrvOutput & key, const Signer &);
@@ -87,10 +79,6 @@ struct Realisation : UnkeyedRealisation
 
     bool isCompatibleWith(const UnkeyedRealisation & other) const;
 
-    static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
-
-    static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
-
     bool operator==(const Realisation &) const = default;
     auto operator<=>(const Realisation &) const = default;
 };
@@ -154,10 +142,6 @@ struct RealisedPath
      */
     const StorePath & path() const &;
 
-    void closure(Store & store, Set & ret) const;
-    static void closure(Store & store, const Set & startPaths, Set & ret);
-    Set closure(Store & store) const;
-
     bool operator==(const RealisedPath &) const = default;
     auto operator<=>(const RealisedPath &) const = default;
 };

src/libstore/include/nix/store/store-api.hh

@@ -1007,9 +1007,6 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashR
 
 const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 
-std::map<DrvOutput, StorePath>
-drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr);
-
 template<>
 struct json_avoids_null<TrustedFlag> : std::true_type
 {};

src/libstore/local-store.cc

@@ -110,8 +110,6 @@ struct LocalStore::State::Stmts
     SQLiteStmt QueryAllRealisedOutputs;
     SQLiteStmt QueryPathFromHashPart;
     SQLiteStmt QueryValidPaths;
-    SQLiteStmt QueryRealisationReferences;
-    SQLiteStmt AddRealisationReference;
 };
 
 LocalStore::LocalStore(ref<const Config> config)
@@ -390,21 +388,6 @@ LocalStore::LocalStore(ref<const Config> config)
                     where drvPath = ?
                     ;
             )");
-        state->stmts->QueryRealisationReferences.create(
-            state->db,
-            R"(
-                select drvPath, outputName from Realisations
-                    join RealisationsRefs on realisationReference = Realisations.id
-                    where referrer = ?;
-            )");
-        state->stmts->AddRealisationReference.create(
-            state->db,
-            R"(
-                insert or replace into RealisationsRefs (referrer, realisationReference)
-                values (
-                    (select id from Realisations where drvPath = ? and outputName = ?),
-                    (select id from Realisations where drvPath = ? and outputName = ?));
-            )");
     }
 }
 
@@ -654,25 +637,6 @@ void LocalStore::registerDrvOutput(const Realisation & info)
                     concatStringsSep(" ", info.signatures))
                 .exec();
         }
-        for (auto & [outputId, depPath] : info.dependentRealisations) {
-            auto localRealisation = queryRealisationCore_(*state, outputId);
-            if (!localRealisation)
-                throw Error(
-                    "unable to register the derivation '%s' as it "
-                    "depends on the non existent '%s'",
-                    info.id.to_string(),
-                    outputId.to_string());
-            if (localRealisation->second.outPath != depPath)
-                throw Error(
-                    "unable to register the derivation '%s' as it "
-                    "depends on a realisation of '%s' that doesn’t"
-                    "match what we have locally",
-                    info.id.to_string(),
-                    outputId.to_string());
-            state->stmts->AddRealisationReference
-                .use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName)
-                .exec();
-        }
     });
 }
 
@@ -1606,21 +1570,6 @@ std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(LocalStore
         return std::nullopt;
     auto [realisationDbId, res] = *maybeCore;
 
-    std::map<DrvOutput, StorePath> dependentRealisations;
-    auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId));
-    while (useRealisationRefs.next()) {
-        auto depId = DrvOutput{
-            Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
-            useRealisationRefs.getStr(1),
-        };
-        auto dependentRealisation = queryRealisationCore_(state, depId);
-        assert(dependentRealisation); // Enforced by the db schema
-        auto outputPath = dependentRealisation->second.outPath;
-        dependentRealisations.insert({depId, outputPath});
-    }
-
-    res.dependentRealisations = dependentRealisations;
-
     return {res};
 }
 

src/libstore/meson.build

@@ -39,28 +39,32 @@ deps_public_maybe_subproject = [
 ]
 subdir('nix-meson-build-support/subprojects')
 
-run_command(
-  'ln',
-  '-s',
-  meson.project_build_root() / '__nothing_link_target',
-  meson.project_build_root() / '__nothing_symlink',
-  # native doesn't allow dangling symlinks, which the tests require
-  env : {'MSYS' : 'winsymlinks:lnk'},
-  check : true,
-)
-can_link_symlink = run_command(
-  'ln',
-  meson.project_build_root() / '__nothing_symlink',
-  meson.project_build_root() / '__nothing_hardlink',
-  check : false,
-).returncode() == 0
-run_command(
-  'rm',
-  '-f',
-  meson.project_build_root() / '__nothing_symlink',
-  meson.project_build_root() / '__nothing_hardlink',
-  check : true,
-)
+can_link_symlink = false
+native_ln = find_program('ln', required : false, native : true)
+if native_ln.found()
+  run_command(
+    native_ln,
+    '-s',
+    meson.project_build_root() / '__nothing_link_target',
+    meson.project_build_root() / '__nothing_symlink',
+    # native doesn't allow dangling symlinks, which the tests require
+    env : {'MSYS' : 'winsymlinks:lnk'},
+    check : true,
+  )
+  can_link_symlink = run_command(
+    native_ln,
+    meson.project_build_root() / '__nothing_symlink',
+    meson.project_build_root() / '__nothing_hardlink',
+    check : false,
+  ).returncode() == 0
+  run_command(
+    'rm',
+    '-f',
+    meson.project_build_root() / '__nothing_symlink',
+    meson.project_build_root() / '__nothing_hardlink',
+    check : true,
+  )
+endif
 summary('can hardlink to symlink', can_link_symlink, bool_yn : true)
 configdata_priv.set('CAN_LINK_SYMLINK', can_link_symlink.to_int())
 
@@ -160,6 +164,8 @@ if s3_aws_auth.enabled()
   deps_other += aws_crt_cpp
   aws_c_common = cxx.find_library('aws-c-common', required : true)
   deps_other += aws_c_common
+  aws_c_auth = cxx.find_library('aws-c-auth', required : true)
+  deps_other += aws_c_auth
 endif
 
 configdata_pub.set('NIX_WITH_AWS_AUTH', s3_aws_auth.enabled().to_int())

src/libstore/misc.cc

@@ -334,65 +334,6 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
         result);
 }
 
-std::map<DrvOutput, StorePath>
-drvOutputReferences(const std::set<Realisation> & inputRealisations, const StorePathSet & pathReferences)
-{
-    std::map<DrvOutput, StorePath> res;
-
-    for (const auto & input : inputRealisations) {
-        if (pathReferences.count(input.outPath)) {
-            res.insert({input.id, input.outPath});
-        }
-    }
-
-    return res;
-}
-
-std::map<DrvOutput, StorePath>
-drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_)
-{
-    auto & evalStore = evalStore_ ? *evalStore_ : store;
-
-    std::set<Realisation> inputRealisations;
-
-    auto accumRealisations = [&](this auto & self,
-                                 const StorePath & inputDrv,
-                                 const DerivedPathMap<StringSet>::ChildNode & inputNode) -> void {
-        if (!inputNode.value.empty()) {
-            auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
-            for (const auto & outputName : inputNode.value) {
-                auto outputHash = get(outputHashes, outputName);
-                if (!outputHash)
-                    throw Error(
-                        "output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv));
-                DrvOutput key{*outputHash, outputName};
-                auto thisRealisation = store.queryRealisation(key);
-                if (!thisRealisation)
-                    throw Error(
-                        "output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv));
-                inputRealisations.insert({*thisRealisation, std::move(key)});
-            }
-        }
-        if (!inputNode.value.empty()) {
-            auto d = makeConstantStorePathRef(inputDrv);
-            for (const auto & [outputName, childNode] : inputNode.childMap) {
-                SingleDerivedPath next = SingleDerivedPath::Built{d, outputName};
-                self(
-                    // TODO deep resolutions for dynamic derivations, issue #8947, would go here.
-                    resolveDerivedPath(store, next, evalStore_),
-                    childNode);
-            }
-        }
-    };
-
-    for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
-        accumRealisations(inputDrv, inputNode);
-
-    auto info = store.queryPathInfo(outputPath);
-
-    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
-}
-
 OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
 {
     auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);

src/libstore/realisation.cc

@@ -1,6 +1,5 @@
 #include "nix/store/realisation.hh"
 #include "nix/store/store-api.hh"
-#include "nix/util/closure.hh"
 #include "nix/util/signature/local-keys.hh"
 #include "nix/util/json-utils.hh"
 #include <nlohmann/json.hpp>
@@ -26,41 +25,6 @@ std::string DrvOutput::to_string() const
     return strHash() + "!" + outputName;
 }
 
-std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
-{
-    std::set<Realisation> res;
-    Realisation::closure(store, startOutputs, res);
-    return res;
-}
-
-void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
-{
-    auto getDeps = [&](const Realisation & current) -> std::set<Realisation> {
-        std::set<Realisation> res;
-        for (auto & [currentDep, _] : current.dependentRealisations) {
-            if (auto currentRealisation = store.queryRealisation(currentDep))
-                res.insert({*currentRealisation, currentDep});
-            else
-                throw Error("Unrealised derivation '%s'", currentDep.to_string());
-        }
-        return res;
-    };
-
-    computeClosure<Realisation>(
-        startOutputs,
-        res,
-        [&](const Realisation & current, std::function<void(std::promise<std::set<Realisation>> &)> processEdges) {
-            std::promise<std::set<Realisation>> promise;
-            try {
-                auto res = getDeps(current);
-                promise.set_value(res);
-            } catch (...) {
-                promise.set_exception(std::current_exception());
-            }
-            return processEdges(promise);
-        });
-}
-
 std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
 {
     nlohmann::json serialized = Realisation{*this, key};
@@ -99,43 +63,7 @@ const StorePath & RealisedPath::path() const &
 
 bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
 {
-    if (outPath == other.outPath) {
-        if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
-            warn(
-                "Encountered a realisation for '%s' with an empty set of "
-                "dependencies. This is likely an artifact from an older Nix. "
-                "I’ll try to fix the realisation if I can",
-                id.to_string());
-            return true;
-        } else if (dependentRealisations == other.dependentRealisations) {
-            return true;
-        }
-    }
-    return false;
-}
-
-void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret)
-{
-    // FIXME: This only builds the store-path closure, not the real realisation
-    // closure
-    StorePathSet initialStorePaths, pathsClosure;
-    for (auto & path : startPaths)
-        initialStorePaths.insert(path.path());
-    store.computeFSClosure(initialStorePaths, pathsClosure);
-    ret.insert(startPaths.begin(), startPaths.end());
-    ret.insert(pathsClosure.begin(), pathsClosure.end());
-}
-
-void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const
-{
-    RealisedPath::closure(store, {*this}, ret);
-}
-
-RealisedPath::Set RealisedPath::closure(Store & store) const
-{
-    RealisedPath::Set ret;
-    closure(store, ret);
-    return ret;
+    return outPath == other.outPath;
 }
 
 } // namespace nix
@@ -162,27 +90,19 @@ UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & js
     if (auto signaturesOpt = optionalValueAt(json, "signatures"))
         signatures = *signaturesOpt;
 
-    std::map<DrvOutput, StorePath> dependentRealisations;
-    if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations"))
-        for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies))
-            dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath});
-
     return UnkeyedRealisation{
         .outPath = valueAt(json, "outPath"),
         .signatures = signatures,
-        .dependentRealisations = dependentRealisations,
     };
 }
 
 void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
 {
-    auto jsonDependentRealisations = nlohmann::json::object();
-    for (auto & [depId, depOutPath] : r.dependentRealisations)
-        jsonDependentRealisations.emplace(depId.to_string(), depOutPath);
     json = {
         {"outPath", r.outPath},
         {"signatures", r.signatures},
-        {"dependentRealisations", jsonDependentRealisations},
+        // back-compat
+        {"dependentRealisations", json::object()},
     };
 }
 

src/libstore/restricted-store.cc

@@ -292,7 +292,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
     next->computeFSClosure(newPaths, closure);
     for (auto & path : closure)
         goal.addDependency(path);
-    for (auto & real : Realisation::closure(*next, newRealisations))
+    for (auto & real : newRealisations)
         goal.addedDrvOutputs.insert(real.id);
 
     return results;

src/libstore/store-api.cc

@@ -915,36 +915,21 @@ std::map<StorePath, StorePath> copyPaths(
     SubstituteFlag substitute)
 {
     StorePathSet storePaths;
-    std::set<Realisation> toplevelRealisations;
+    std::vector<const Realisation *> realisations;
     for (auto & path : paths) {
         storePaths.insert(path.path());
         if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
             experimentalFeatureSettings.require(Xp::CaDerivations);
-            toplevelRealisations.insert(*realisation);
+            realisations.push_back(realisation);
         }
     }
 
     auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
 
     try {
-        // Copy the realisation closure
-        processGraph<Realisation>(
-            Realisation::closure(srcStore, toplevelRealisations),
-            [&](const Realisation & current) -> std::set<Realisation> {
-                std::set<Realisation> children;
-                for (const auto & [drvOutput, _] : current.dependentRealisations) {
-                    auto currentChild = srcStore.queryRealisation(drvOutput);
-                    if (!currentChild)
-                        throw Error(
-                            "incomplete realisation closure: '%s' is a "
-                            "dependency of '%s' but isn't registered",
-                            drvOutput.to_string(),
-                            current.id.to_string());
-                    children.insert({*currentChild, drvOutput});
-                }
-                return children;
-            },
-            [&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); });
+        // Copy the realisations. TODO batch this
+        for (const auto * realisation : realisations)
+            dstStore.registerDrvOutput(*realisation, checkSigs);
     } catch (MissingExperimentalFeature & e) {
         // Don't fail if the remote doesn't support CA derivations is it might
         // not be within our control to change that, and we might still want
@@ -1055,8 +1040,19 @@ void copyClosure(
     if (&srcStore == &dstStore)
         return;
 
-    RealisedPath::Set closure;
-    RealisedPath::closure(srcStore, paths, closure);
+    StorePathSet closure0;
+    for (auto & path : paths) {
+        if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
+            closure0.insert(opaquePath->path);
+        }
+    }
+
+    StorePathSet closure1;
+    srcStore.computeFSClosure(closure0, closure1);
+
+    RealisedPath::Set closure = paths;
+    for (auto && path : closure1)
+        closure.insert({std::move(path)});
 
     copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
 }

src/libutil-test-support/include/nix/util/tests/characterization.hh

@@ -4,20 +4,11 @@
 #include <gtest/gtest.h>
 
 #include "nix/util/types.hh"
-#include "nix/util/environment-variables.hh"
 #include "nix/util/file-system.hh"
+#include "nix/util/tests/test-data.hh"
 
 namespace nix {
 
-/**
- * The path to the unit test data directory. See the contributing guide
- * in the manual for further details.
- */
-static inline std::filesystem::path getUnitTestData()
-{
-    return getEnv("_NIX_TEST_UNIT_DATA").value();
-}
-
 /**
  * Whether we should update "golden masters" instead of running tests
  * against them. See the contributing guide in the manual for further

src/libutil-test-support/include/nix/util/tests/json-characterization.hh

@@ -83,6 +83,31 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go
     }
 }
 
+/**
+ * Specialization for when we need to do "JSON -> `ref<T>`" in one
+ * direction, but "`const T &` -> JSON" in the other direction.
+ */
+template<typename T>
+void checkpointJson(CharacterizationTest & test, PathView testStem, const ref<T> & got)
+{
+    using namespace nlohmann;
+
+    auto file = test.goldenMaster(Path{testStem} + ".json");
+
+    json gotJson = static_cast<json>(*got);
+
+    if (testAccept()) {
+        std::filesystem::create_directories(file.parent_path());
+        writeFile(file, gotJson.dump(2) + "\n");
+        ADD_FAILURE() << "Updating golden master " << file;
+    } else {
+        json expectedJson = json::parse(readFile(file));
+        ASSERT_EQ(gotJson, expectedJson);
+        ref<T> expected = adl_serializer<ref<T>>::from_json(expectedJson);
+        ASSERT_EQ(*got, *expected);
+    }
+}
+
 /**
  * Mixin class for writing characterization tests for `nlohmann::json`
  * conversions for a given type.

src/libutil-test-support/include/nix/util/tests/meson.build

@@ -10,4 +10,5 @@ headers = files(
   'json-characterization.hh',
   'nix_api_util.hh',
   'string_callback.hh',
+  'test-data.hh',
 )

src/libutil-test-support/include/nix/util/tests/test-data.hh

@@ -0,0 +1,24 @@
+#pragma once
+///@file
+
+#include <filesystem>
+#include "nix/util/environment-variables.hh"
+#include "nix/util/error.hh"
+
+namespace nix {
+
+/**
+ * The path to the unit test data directory. See the contributing guide
+ * in the manual for further details.
+ */
+static inline std::filesystem::path getUnitTestData()
+{
+    auto data = getEnv("_NIX_TEST_UNIT_DATA");
+    if (!data)
+        throw Error(
+            "_NIX_TEST_UNIT_DATA environment variable is not set. "
+            "Recommendation: use meson, example: 'meson test -C build --gdb'");
+    return std::filesystem::path(*data);
+}
+
+} // namespace nix

src/libutil-tests/meson.build

@@ -33,6 +33,9 @@ deps_private += rapidcheck
 gtest = dependency('gtest', main : true)
 deps_private += gtest
 
+gmock = dependency('gmock')
+deps_private += gmock
+
 configdata = configuration_data()
 configdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 
@@ -72,6 +75,7 @@ sources = files(
   'position.cc',
   'processes.cc',
   'sort.cc',
+  'source-accessor.cc',
   'spawn.cc',
   'strings.cc',
   'suggestions.cc',

src/libutil-tests/source-accessor.cc

@@ -0,0 +1,138 @@
+#include "nix/util/fs-sink.hh"
+#include "nix/util/file-system.hh"
+#include "nix/util/processes.hh"
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+#include <rapidcheck/gtest.h>
+
+namespace nix {
+
+MATCHER_P2(HasContents, path, expected, "")
+{
+    auto stat = arg->maybeLstat(path);
+    if (!stat) {
+        *result_listener << arg->showPath(path) << " does not exist";
+        return false;
+    }
+    if (stat->type != SourceAccessor::tRegular) {
+        *result_listener << arg->showPath(path) << " is not a regular file";
+        return false;
+    }
+    auto actual = arg->readFile(path);
+    if (actual != expected) {
+        *result_listener << arg->showPath(path) << " has contents " << ::testing::PrintToString(actual);
+        return false;
+    }
+    return true;
+}
+
+MATCHER_P2(HasSymlink, path, target, "")
+{
+    auto stat = arg->maybeLstat(path);
+    if (!stat) {
+        *result_listener << arg->showPath(path) << " does not exist";
+        return false;
+    }
+    if (stat->type != SourceAccessor::tSymlink) {
+        *result_listener << arg->showPath(path) << " is not a symlink";
+        return false;
+    }
+    auto actual = arg->readLink(path);
+    if (actual != target) {
+        *result_listener << arg->showPath(path) << " points to " << ::testing::PrintToString(actual);
+        return false;
+    }
+    return true;
+}
+
+MATCHER_P2(HasDirectory, path, dirents, "")
+{
+    auto stat = arg->maybeLstat(path);
+    if (!stat) {
+        *result_listener << arg->showPath(path) << " does not exist";
+        return false;
+    }
+    if (stat->type != SourceAccessor::tDirectory) {
+        *result_listener << arg->showPath(path) << " is not a directory";
+        return false;
+    }
+    auto actual = arg->readDirectory(path);
+    std::set<std::string> actualKeys, expectedKeys(dirents.begin(), dirents.end());
+    for (auto & [k, _] : actual)
+        actualKeys.insert(k);
+    if (actualKeys != expectedKeys) {
+        *result_listener << arg->showPath(path) << " has entries " << ::testing::PrintToString(actualKeys);
+        return false;
+    }
+    return true;
+}
+
+class FSSourceAccessorTest : public ::testing::Test
+{
+protected:
+    std::filesystem::path tmpDir;
+    std::unique_ptr<nix::AutoDelete> delTmpDir;
+
+    void SetUp() override
+    {
+        tmpDir = nix::createTempDir();
+        delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
+    }
+
+    void TearDown() override
+    {
+        delTmpDir.reset();
+    }
+};
+
+TEST_F(FSSourceAccessorTest, works)
+{
+    {
+        RestoreSink sink(false);
+        sink.dstPath = tmpDir;
+        sink.dirFd = openDirectory(tmpDir);
+        sink.createDirectory(CanonPath("subdir"));
+        sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); });
+        sink.createRegularFile(CanonPath("subdir/file2"), [](CreateRegularFileSink & crf) { crf("content2"); });
+        sink.createSymlink(CanonPath("rootlink"), "target");
+        sink.createDirectory(CanonPath("a"));
+        sink.createSymlink(CanonPath("a/dirlink"), "../subdir");
+    }
+
+    EXPECT_THAT(makeFSSourceAccessor(tmpDir / "file1"), HasContents(CanonPath::root, "content1"));
+    EXPECT_THAT(makeFSSourceAccessor(tmpDir / "rootlink"), HasSymlink(CanonPath::root, "target"));
+    EXPECT_THAT(
+        makeFSSourceAccessor(tmpDir),
+        HasDirectory(CanonPath::root, std::set<std::string>{"file1", "subdir", "rootlink", "a"}));
+    EXPECT_THAT(makeFSSourceAccessor(tmpDir / "subdir"), HasDirectory(CanonPath::root, std::set<std::string>{"file2"}));
+
+    {
+        auto accessor = makeFSSourceAccessor(tmpDir);
+        EXPECT_THAT(accessor, HasContents(CanonPath("file1"), "content1"));
+        EXPECT_THAT(accessor, HasContents(CanonPath("subdir/file2"), "content2"));
+
+        EXPECT_TRUE(accessor->pathExists(CanonPath("file1")));
+        EXPECT_FALSE(accessor->pathExists(CanonPath("nonexistent")));
+
+        EXPECT_THROW(accessor->readFile(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
+        EXPECT_THROW(accessor->maybeLstat(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
+        EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), SymlinkNotAllowed);
+        EXPECT_THROW(accessor->pathExists(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
+    }
+
+    {
+        auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent");
+        EXPECT_FALSE(accessor->maybeLstat(CanonPath::root));
+        EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError);
+    }
+
+    {
+        auto accessor = makeFSSourceAccessor(tmpDir, true);
+        EXPECT_EQ(accessor->getLastModified(), 0);
+        accessor->maybeLstat(CanonPath("file1"));
+        EXPECT_GT(accessor->getLastModified(), 0);
+    }
+}
+
+} // namespace nix

src/libutil/canon-path.cc

@@ -54,6 +54,17 @@ std::optional<CanonPath> CanonPath::parent() const
     return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/'))));
 }
 
+std::optional<std::pair<CanonPath, std::string_view>> CanonPath::parentAndChild() const
+{
+    if (isRoot())
+        return std::nullopt;
+    auto slash = path.rfind('/');
+    return {{
+        CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, slash))),
+        std::string_view{path}.substr(slash + 1)
+    }};
+}
+
 void CanonPath::pop()
 {
     assert(!isRoot());

src/libutil/compression.cc

@@ -69,24 +69,54 @@ struct ArchiveDecompressionSource : Source
     }
 };
 
+/* These strings are a part of the public API in store parameters and such. Do not change!
+   Happens to match enum names. */
+#define NIX_FOR_EACH_LA_ALGO(MACRO) \
+    MACRO(bzip2)                    \
+    MACRO(compress)                 \
+    MACRO(grzip)                    \
+    MACRO(gzip)                     \
+    MACRO(lrzip)                    \
+    MACRO(lz4)                      \
+    MACRO(lzip)                     \
+    MACRO(lzma)                     \
+    MACRO(lzop)                     \
+    MACRO(xz)                       \
+    MACRO(zstd)
+
 struct ArchiveCompressionSink : CompressionSink
 {
     Sink & nextSink;
     struct archive * archive;
 
-    ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
+    ArchiveCompressionSink(
+        Sink & nextSink, CompressionAlgo method, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
         : nextSink(nextSink)
     {
         archive = archive_write_new();
         if (!archive)
             throw Error("failed to initialize libarchive");
-        check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
+
+        auto [addFilter, format] = [method]() -> std::pair<int (*)(struct archive *), const char *> {
+            switch (method) {
+            case CompressionAlgo::none:
+            case CompressionAlgo::brotli:
+                unreachable();
+#define NIX_DEF_LA_ALGO_CASE(algo) \
+    case CompressionAlgo::algo:    \
+        return {archive_write_add_filter_##algo, #algo};
+                NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
+#undef NIX_DEF_LA_ALGO_CASE
+            }
+            unreachable();
+        }();
+
+        check(addFilter(archive), "couldn't initialize compression (%s)");
         check(archive_write_set_format_raw(archive));
         if (parallel)
-            check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
+            check(archive_write_set_filter_option(archive, format, "threads", "0"));
         if (level != COMPRESSION_LEVEL_DEFAULT)
-            check(archive_write_set_filter_option(
-                archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
+            check(archive_write_set_filter_option(archive, format, "compression-level", std::to_string(level).c_str()));
         // disable internal buffering
         check(archive_write_set_bytes_per_block(archive, 0));
         // disable output padding
@@ -289,19 +319,52 @@ struct BrotliCompressionSink : ChunkedCompressionSink
     }
 };
 
+/* Parses a *compression* method into the corresponding enum. This is only used
+   in the *compression* case and user interface. Content-Encoding should not use
+   these. */
+static CompressionAlgo parseNixCompressionAlgoString(std::string_view method)
+{
+    static const std::unordered_map<std::string_view, CompressionAlgo> lookupTable = {
+        {"none", CompressionAlgo::none},
+        {"br", CompressionAlgo::brotli},
+#define NIX_DEF_LA_ALGO_NAME(algo) {#algo, CompressionAlgo::algo},
+        NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_NAME)
+#undef NIX_DEF_LA_ALGO_NAME
+    };
+
+    if (auto it = lookupTable.find(method); it != lookupTable.end())
+        return it->second;
+
+    static const StringSet allNames = [&]() {
+        StringSet res;
+        for (auto & [name, _] : lookupTable)
+            res.emplace(name);
+        return res;
+    }();
+
+    throw UnknownCompressionMethod(
+        Suggestions::bestMatches(allNames, method), "unknown compression method '%s'", method);
+}
+
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
 {
-    std::vector<std::string> la_supports = {
-        "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
-    if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
-        return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
-    }
-    if (method == "none")
+    return makeCompressionSink(parseNixCompressionAlgoString(method), nextSink, parallel, level);
+}
+
+ref<CompressionSink> makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level)
+{
+    switch (method) {
+    case CompressionAlgo::none:
         return make_ref<NoneSink>(nextSink);
-    else if (method == "br")
+    case CompressionAlgo::brotli:
         return make_ref<BrotliCompressionSink>(nextSink);
-    else
-        throw UnknownCompressionMethod("unknown compression method '%s'", method);
+        /* Everything else is supported via libarchive. */
+#define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo:
+        NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
+        return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
+#undef NIX_DEF_LA_ALGO_CASE
+    }
+    unreachable();
 }
 
 std::string compress(const std::string & method, std::string_view in, const bool parallel, int level)

src/libutil/freebsd/include/nix/util/freebsd-jail.hh

@@ -11,6 +11,24 @@ class AutoRemoveJail
     bool del;
 public:
     AutoRemoveJail(int jid);
+    AutoRemoveJail(const AutoRemoveJail &) = delete;
+    AutoRemoveJail & operator=(const AutoRemoveJail &) = delete;
+
+    AutoRemoveJail(AutoRemoveJail && other) noexcept
+        : jid(other.jid)
+        , del(other.del)
+    {
+        other.cancel();
+    }
+
+    AutoRemoveJail & operator=(AutoRemoveJail && other) noexcept
+    {
+        jid = other.jid;
+        del = other.del;
+        other.cancel();
+        return *this;
+    }
+
     AutoRemoveJail();
     ~AutoRemoveJail();
     void cancel();

src/libutil/fs-sink.cc

@@ -70,31 +70,7 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can
     return dst;
 }
 
-#ifndef _WIN32
-void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
-{
-    if (path.isRoot()) {
-        createDirectory(path);
-        callback(*this, path);
-        return;
-    }
-
-    createDirectory(path);
-    assert(dirFd); // If that's not true the above call must have thrown an exception.
-
-    RestoreSink dirSink{startFsync};
-    dirSink.dstPath = append(dstPath, path);
-    dirSink.dirFd =
-        unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
-
-    if (!dirSink.dirFd)
-        throw SysError("opening directory '%s'", dirSink.dstPath.string());
-
-    callback(dirSink, CanonPath::root);
-}
-#endif
-
-void RestoreSink::createDirectory(const CanonPath & path)
+void RestoreSink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
 {
     auto p = append(dstPath, path);
 
@@ -107,6 +83,16 @@ void RestoreSink::createDirectory(const CanonPath & path)
         if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1)
             throw SysError("creating directory '%s'", p.string());
 
+        if (callback) {
+            RestoreSink dirSink{startFsync};
+            dirSink.dstPath =
+                unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
+
+            if (!dirSink.dirFd)
+                throw SysError("opening directory '%s'", dirSink.dstPath.string());
+
+            (*callback)(dirSink, CanonPath::root);
+        }
         return;
     }
 #endif
@@ -125,7 +111,10 @@ void RestoreSink::createDirectory(const CanonPath & path)
             throw SysError("creating directory '%1%'", p.string());
     }
 #endif
-};
+
+    if (callback)
+        (*callback)(*this, path);
+}
 
 struct RestoreRegularFile : CreateRegularFileSink
 {

src/libutil/include/nix/util/canon-path.hh

@@ -238,6 +238,13 @@ public:
         return ((std::string_view) path).substr(path.rfind('/') + 1);
     }
 
+    /**
+     * Combination of `parent` and `baseName`.
+     *
+     * @note that because of the `std::string_view`, we are borrowing for the base name.
+     */
+    std::optional<std::pair<CanonPath, std::string_view>> parentAndChild() const;
+
     bool operator==(const CanonPath & x) const
     {
         return path == x.path;

src/libutil/include/nix/util/compression.hh

@@ -16,6 +16,22 @@ struct CompressionSink : BufferedSink, FinishSink
     using FinishSink::finish;
 };
 
+enum class CompressionAlgo {
+    none,
+    brotli,
+    bzip2,
+    compress,
+    grzip,
+    gzip,
+    lrzip,
+    lz4,
+    lzip,
+    lzma,
+    lzop,
+    xz,
+    zstd,
+};
+
 std::string decompress(const std::string & method, std::string_view in);
 
 std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
@@ -25,6 +41,9 @@ std::string compress(const std::string & method, std::string_view in, const bool
 ref<CompressionSink>
 makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
 
+ref<CompressionSink>
+makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel = false, int level = -1);
+
 MakeError(UnknownCompressionMethod, Error);
 
 MakeError(CompressionError, Error);

src/libutil/include/nix/util/file-descriptor.hh

@@ -2,7 +2,6 @@
 ///@file
 
 #include "nix/util/canon-path.hh"
-#include "nix/util/types.hh"
 #include "nix/util/error.hh"
 
 #ifdef _WIN32
@@ -236,18 +235,6 @@ std::wstring handleToFileName(Descriptor handle);
 #ifndef _WIN32
 namespace unix {
 
-struct SymlinkNotAllowed : public Error
-{
-    CanonPath path;
-
-    SymlinkNotAllowed(CanonPath path)
-        /* Can't provide better error message, since the parent directory is only known to the caller. */
-        : Error("relative path '%s' points to a symlink, which is not allowed", path.rel())
-        , path(std::move(path))
-    {
-    }
-};
-
 /**
  * Safe(r) function to open \param path file relative to \param dirFd, while
  * disallowing escaping from a directory and resolving any symlinks in the

src/libutil/include/nix/util/file-system.hh

@@ -480,8 +480,23 @@ class AutoUnmount
     Path path;
     bool del;
 public:
-    AutoUnmount(Path &);
     AutoUnmount();
+    AutoUnmount(Path &);
+    AutoUnmount(const AutoUnmount &) = delete;
+
+    AutoUnmount(AutoUnmount && other) noexcept
+        : path(std::move(other.path))
+        , del(std::exchange(other.del, false))
+    {
+    }
+
+    AutoUnmount & operator=(AutoUnmount && other) noexcept
+    {
+        path = std::move(other.path);
+        del = std::exchange(other.del, false);
+        return *this;
+    }
+
     ~AutoUnmount();
     void cancel();
 };

src/libutil/include/nix/util/fs-sink.hh

@@ -34,8 +34,6 @@ struct FileSystemObjectSink
 {
     virtual ~FileSystemObjectSink() = default;
 
-    virtual void createDirectory(const CanonPath & path) = 0;
-
     using DirectoryCreatedCallback = std::function<void(FileSystemObjectSink & dirSink, const CanonPath & dirRelPath)>;
 
     /**
@@ -47,11 +45,7 @@ struct FileSystemObjectSink
      * freshly created directory. Use this when it's important to disallow any
      * intermediate path components from being symlinks.
      */
-    virtual void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
-    {
-        createDirectory(path);
-        callback(*this, path);
-    }
+    virtual void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) = 0;
 
     /**
      * This function in general is no re-entrant. Only one file can be
@@ -86,7 +80,11 @@ void copyRecursive(
  */
 struct NullFileSystemObjectSink : FileSystemObjectSink
 {
-    void createDirectory(const CanonPath & path) override {}
+    void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
+    {
+        if (callback)
+            (*callback)(*this, CanonPath::root);
+    }
 
     void createSymlink(const CanonPath & path, const std::string & target) override {}
 
@@ -118,11 +116,7 @@ struct RestoreSink : FileSystemObjectSink
     {
     }
 
-    void createDirectory(const CanonPath & path) override;
-
-#ifndef _WIN32
-    void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override;
-#endif
+    void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
 
     void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
 
@@ -144,9 +138,15 @@ struct RegularFileSink : FileSystemObjectSink
     {
     }
 
-    void createDirectory(const CanonPath & path) override
+    void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
     {
         regular = false;
+        if (callback) {
+            NullFileSystemObjectSink s;
+            /* The children of the
+               directory that cannot exist also cannot exist */
+            (*callback)(s, CanonPath::root);
+        }
     }
 
     void createSymlink(const CanonPath & path, const std::string & target) override

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -153,7 +153,7 @@ struct MemorySink : FileSystemObjectSink
     {
     }
 
-    void createDirectory(const CanonPath & path) override;
+    void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
 
     void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
 

src/libutil/include/nix/util/serialise.hh

@@ -159,6 +159,8 @@ struct FdSink : BufferedSink
     }
 
     FdSink(FdSink &&) = default;
+    FdSink(const FdSink &) = delete;
+    FdSink & operator=(const FdSink &) = delete;
 
     FdSink & operator=(FdSink && s)
     {
@@ -200,8 +202,10 @@ struct FdSource : BufferedSource, RestartableSource
     }
 
     FdSource(FdSource &&) = default;
-
     FdSource & operator=(FdSource && s) = default;
+    FdSource(const FdSource &) = delete;
+    FdSource & operator=(const FdSource & s) = delete;
+    ~FdSource() = default;
 
     bool good() override;
     void restart() override;
@@ -452,6 +456,11 @@ struct LambdaSink : Sink
     {
     }
 
+    LambdaSink(LambdaSink &&) = delete;
+    LambdaSink(const LambdaSink &) = delete;
+    LambdaSink & operator=(LambdaSink &&) = delete;
+    LambdaSink & operator=(const LambdaSink &) = delete;
+
     ~LambdaSink()
     {
         cleanupFun();
@@ -628,6 +637,11 @@ struct FramedSource : Source
     {
     }
 
+    FramedSource(FramedSource &&) = delete;
+    FramedSource(const FramedSource &) = delete;
+    FramedSource & operator=(FramedSource &&) = delete;
+    FramedSource & operator=(const FramedSource &) = delete;
+
     ~FramedSource()
     {
         try {
@@ -685,6 +699,11 @@ struct FramedSink : nix::BufferedSink
     {
     }
 
+    FramedSink(FramedSink &&) = delete;
+    FramedSink(const FramedSink &) = delete;
+    FramedSink & operator=(FramedSink &&) = delete;
+    FramedSink & operator=(const FramedSink &) = delete;
+
     ~FramedSink()
     {
         try {

src/libutil/include/nix/util/source-accessor.hh

@@ -222,6 +222,24 @@ ref<SourceAccessor> makeEmptySourceAccessor();
  */
 MakeError(RestrictedPathError, Error);
 
+struct SymlinkNotAllowed : public Error
+{
+    CanonPath path;
+
+    SymlinkNotAllowed(CanonPath path)
+        : Error("relative path '%s' points to a symlink, which is not allowed", path.rel())
+        , path(std::move(path))
+    {
+    }
+
+    template<typename... Args>
+    SymlinkNotAllowed(CanonPath path, const std::string & fs, Args &&... args)
+        : Error(fs, std::forward<Args>(args)...)
+        , path(std::move(path))
+    {
+    }
+};
+
 /**
  * Return an accessor for the root filesystem.
  */
@@ -233,7 +251,7 @@ ref<SourceAccessor> getFSSourceAccessor();
  * elements, and that absolute symlinks are resolved relative to
  * `root`.
  */
-ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root);
+ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false);
 
 /**
  * Construct an accessor that presents a "union" view of a vector of