libstore: Fix double quotes in debug logs for pathlocks

libmain: Fix download progress rendering

.github/workflows/backport.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           app-id: ${{ vars.CI_APP_ID }}
           private-key: ${{ secrets.CI_APP_PRIVATE_KEY }}
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           # required to find all branches

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
   eval:
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
     - uses: ./.github/actions/install-nix-action
@@ -40,7 +40,7 @@ jobs:
     name: pre-commit checks
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: ./.github/actions/install-nix-action
         with:
           dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
@@ -87,7 +87,7 @@ jobs:
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
     - uses: ./.github/actions/install-nix-action
@@ -162,7 +162,7 @@ jobs:
     name: installer test ${{ matrix.scenario }}
     runs-on: ${{ matrix.runs-on }}
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
     - name: Download installer tarball
       uses: actions/download-artifact@v6
       with:
@@ -174,7 +174,7 @@ jobs:
         echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
         TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
         echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
-    - uses: cachix/install-nix-action@7ec16f2c061ab07b235a7245e06ed46fe9a1cab6 # v31.8.3
+    - uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
       if: ${{ !matrix.experimental-installer }}
       with:
         install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
@@ -227,7 +227,7 @@ jobs:
       github.ref_name == 'master'
     runs-on: ubuntu-24.04
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
     - uses: ./.github/actions/install-nix-action
@@ -276,14 +276,14 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout nix
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Checkout flake-regressions
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           repository: NixOS/flake-regressions
           path: flake-regressions
       - name: Checkout flake-regressions-data
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           repository: NixOS/flake-regressions-data
           path: flake-regressions/tests
@@ -303,7 +303,7 @@ jobs:
       github.event_name == 'push' &&
       github.ref_name == 'master'
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
     - uses: ./.github/actions/install-nix-action

doc/manual/meson.build

@@ -88,7 +88,7 @@ manual = custom_target(
         @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
         @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
         sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
-        @4@ -r -L --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
+        @4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
         (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
         rm -rf @2@/manual
         mv @2@/html @2@/manual

doc/manual/package.nix

@@ -37,14 +37,17 @@ mkMesonDerivation (finalAttrs: {
       (fileset.unions [
         ../../.version
         # For example JSON
+        ../../src/libutil-tests/data/memory-source-accessor
         ../../src/libutil-tests/data/hash
         ../../src/libstore-tests/data/content-address
         ../../src/libstore-tests/data/store-path
         ../../src/libstore-tests/data/realisation
+        ../../src/libstore-tests/data/derivation
         ../../src/libstore-tests/data/derived-path
         ../../src/libstore-tests/data/path-info
         ../../src/libstore-tests/data/nar-info
         ../../src/libstore-tests/data/build-result
+        ../../src/libstore-tests/data/dummy-store
         # Too many different types of files to filter for now
         ../../doc/manual
         ./.

doc/manual/rl-next/libcurl-pausing.md

@@ -0,0 +1,12 @@
+---
+synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
+prs: [14614]
+issues: [11728]
+---
+
+The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
+[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
+
+This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
+
+Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).

doc/manual/source/SUMMARY.md.in

@@ -121,14 +121,17 @@
 - [Architecture and Design](architecture/architecture.md)
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
+    - [File System Object](protocols/json/file-system-object.md)
     - [Hash](protocols/json/hash.md)
     - [Content Address](protocols/json/content-address.md)
     - [Store Path](protocols/json/store-path.md)
     - [Store Object Info](protocols/json/store-object-info.md)
-    - [Derivation](protocols/json/derivation.md)
+    - [Derivation](protocols/json/derivation/index.md)
+      - [Derivation Options](protocols/json/derivation/options.md)
     - [Deriving Path](protocols/json/deriving-path.md)
     - [Build Trace Entry](protocols/json/build-trace-entry.md)
     - [Build Result](protocols/json/build-result.md)
+    - [Store](protocols/json/store.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Store Path Specification](protocols/store-path.md)
   - [Nix Archive (NAR) Format](protocols/nix-archive/index.md)

doc/manual/source/command-ref/nix-build.md

@@ -36,7 +36,7 @@ to a temporary location. The tarball must include a single top-level
 directory containing at least a file named `default.nix`.
 
 `nix-build` is essentially a wrapper around
-[`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix
+[`nix-instantiate`](./nix-instantiate.md) (to translate a high-level Nix
 expression to a low-level [store derivation]) and [`nix-store
 --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
 derivation).
@@ -52,8 +52,8 @@ derivation).
 # Options
 
 All options not listed here are passed to
-[`nix-store --realise`](nix-store/realise.md),
-except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md).
+[`nix-store --realise`](./nix-store/realise.md),
+except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](./nix-instantiate.md).
 
 - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
 

doc/manual/source/command-ref/nix-instantiate.md

@@ -32,7 +32,7 @@ standard input.
 
 - `--add-root` *path*
 
-  See the [corresponding option](nix-store.md) in `nix-store`.
+  See the [corresponding option](./nix-store.md) in `nix-store`.
 
 - `--parse`
 

doc/manual/source/development/debugging.md

@@ -15,7 +15,7 @@ In the development shell, set the `mesonBuildType` environment variable to `debu
 Then, proceed to build Nix as described in [Building Nix](./building.md).
 This will build Nix with debug symbols, which are essential for effective debugging.
 
-It is also possible to build without debugging for faster build:
+It is also possible to build without optimization for faster build:
 
 ```console
 [nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify)

doc/manual/source/development/testing.md

@@ -137,6 +137,12 @@ $ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v
 will regenerate the "golden master" expected result for the `libnixstore` characterisation tests.
 The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything.
 
+### JSON Schema testing
+
+In `doc/manual/source/protocols/json/` we have a number of manual pages generated from [JSON Schema](https://json-schema.org/).
+That JSON schema is tested against the JSON file test data used in [characterisation tests](#characterisation-testing-unit ) for JSON (de)serialization, in `src/json-schema-checks`.
+Between the JSON (de)serialization testing, and this testing of the same data against the schema, we make sure that the manual, the implementation, and a machine-readable schema are are all in sync.
+
 ### Unit test support libraries
 
 There are headers and code which are not just used to test the library in question, but also downstream libraries.

doc/manual/source/protocols/json/derivation.md

@@ -1,7 +0,0 @@
-{{#include derivation-v4-fixed.md}}
-
-<!-- need to convert YAML to JSON first
-## Raw Schema
-
-[JSON Schema for Derivation v3](schema/derivation-v4.json)
--->

doc/manual/source/protocols/json/derivation/index.md

@@ -0,0 +1,7 @@
+{{#include ../derivation-v4-fixed.md}}
+
+<!-- need to convert YAML to JSON first
+## Raw Schema
+
+[JSON Schema for Derivation v4](schema/derivation-v4.json)
+-->

doc/manual/source/protocols/json/derivation/options.md

@@ -0,0 +1,49 @@
+{{#include ../derivation-options-v1-fixed.md}}
+
+## Examples
+
+### Input-addressed derivations
+
+#### Default options
+
+```json
+{{#include ../schema/derivation-options-v1/ia/defaults.json}}
+```
+
+#### All options set
+
+```json
+{{#include ../schema/derivation-options-v1/ia/all_set.json}}
+```
+
+#### Default options (structured attributes)
+
+```json
+{{#include ../schema/derivation-options-v1/ia/structuredAttrs_defaults.json}}
+```
+
+#### All options set (structured attributes)
+
+```json
+{{#include ../schema/derivation-options-v1/ia/structuredAttrs_all_set.json}}
+```
+
+### Content-addressed derivations
+
+#### All options set
+
+```json
+{{#include ../schema/derivation-options-v1/ca/all_set.json}}
+```
+
+#### All options set (structured attributes)
+
+```json
+{{#include ../schema/derivation-options-v1/ca/structuredAttrs_all_set.json}}
+```
+
+<!-- need to convert YAML to JSON first
+## Raw Schema
+
+[JSON Schema for Derivation Options v1](schema/derivation-options-v1.json)
+-->

doc/manual/source/protocols/json/file-system-object.md

@@ -0,0 +1,21 @@
+{{#include file-system-object-v1-fixed.md}}
+
+## Examples
+
+### Simple
+
+```json
+{{#include schema/file-system-object-v1/simple.json}}
+```
+
+### Complex
+
+```json
+{{#include schema/file-system-object-v1/complex.json}}
+```
+
+<!-- need to convert YAML to JSON first
+## Raw Schema
+
+[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
+-->

doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed

@@ -11,7 +11,8 @@ s/\\`/`/g
 #
 # As we have more such relative links, more replacements of this nature
 # should appear below.
-s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
-s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
-s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
-s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g
+s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
+s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](@docroot@/protocols/json/hash.html#algorithm)^g
+s^\(./hash-v1.yaml\)^[JSON format for `Hash`](@docroot@/protocols/json/hash.html)^g
+s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html#method)^g
+s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](@docroot@/protocols/json/content-address.html)^g

doc/manual/source/protocols/json/meson.build

@@ -9,14 +9,17 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
 json_schema_config = files('json-schema-for-humans-config.yaml')
 
 schemas = [
+  'file-system-object-v1',
   'hash-v1',
   'content-address-v1',
   'store-path-v1',
   'store-object-info-v2',
   'derivation-v4',
+  'derivation-options-v1',
   'deriving-path-v1',
   'build-trace-entry-v1',
   'build-result-v1',
+  'store-v1',
 ]
 
 schema_files = files()

doc/manual/source/protocols/json/schema/build-trace-entry-v1.yaml

@@ -4,71 +4,97 @@ title: Build Trace Entry
 description: |
   A record of a successful build outcome for a specific derivation output.
 
-  This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md) entry.
+  This schema describes the JSON representation of a [build trace entry](@docroot@/store/build-trace.md).
 
   > **Warning**
   >
   > This JSON format is currently
   > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
   > and subject to change.
-
-type: object
 required:
   - id
   - outPath
   - dependentRealisations
   - signatures
+allOf:
+  - "$ref": "#/$defs/key"
+  - "$ref": "#/$defs/value"
 properties:
-  id:
-    type: string
-    title: Derivation Output ID
-    pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$"
-    description: |
-      Unique identifier for the derivation output that was built.
-
-      Format: `{hash-quotient-drv}!{output-name}`
-
-      - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv).
-        Begins with `sha256:`.
-
-      - **output-name**: Name of the specific output (e.g., "out", "dev", "doc")
-
-      Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"`
-
-  outPath:
-    "$ref": "store-path-v1.yaml"
-    title: Output Store Path
-    description: |
-      The path to the store object that resulted from building this derivation for the given output name.
-
-  dependentRealisations:
-    type: object
-    title: Underlying Base Build Trace
-    description: |
-      This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
-
-      Keys are derivation output IDs (same format as the main `id` field).
-      Values are the store paths that those dependencies resolved to.
-
-      As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
-      This is the set of base build trace entries that this derived build trace is derived from.
-      (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
-
-    patternProperties:
-      "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
-        $ref: "store-path-v1.yaml"
-        title: Dependent Store Path
-        description: Store path that this dependency resolved to during the build
-    additionalProperties: false
-
-  signatures:
-    type: array
-    title: Build Signatures
-    description: |
-      A set of cryptographic signatures attesting to the authenticity of this build trace entry.
-    items:
-      type: string
-      title: Signature
-      description: A single cryptographic signature
-
+  id: {}
+  outPath: {}
+  dependentRealisations: {}
+  signatures: {}
 additionalProperties: false
+
+"$defs":
+  key:
+    title: Build Trace Key
+    description: |
+      A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair.
+      This is the "key" part, refering to a derivation and output.
+    type: object
+    required:
+      - id
+    properties:
+      id:
+        type: string
+        title: Derivation Output ID
+        pattern: "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$"
+        description: |
+          Unique identifier for the derivation output that was built.
+
+          Format: `{hash-quotient-drv}!{output-name}`
+
+          - **hash-quotient-drv**: SHA-256 [hash of the quotient derivation](@docroot@/store/derivation/outputs/input-address.md#hash-quotient-drv).
+            Begins with `sha256:`.
+
+          - **output-name**: Name of the specific output (e.g., "out", "dev", "doc")
+
+          Example: `"sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo"`
+
+  value:
+    title: Build Trace Value
+    description: |
+      A [build trace entry](@docroot@/store/build-trace.md) is a key-value pair.
+      This is the "value" part, describing an output.
+    type: object
+    required:
+      - outPath
+      - dependentRealisations
+      - signatures
+    properties:
+      outPath:
+        "$ref": "store-path-v1.yaml"
+        title: Output Store Path
+        description: |
+          The path to the store object that resulted from building this derivation for the given output name.
+
+      dependentRealisations:
+        type: object
+        title: Underlying Base Build Trace
+        description: |
+          This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
+
+          Keys are derivation output IDs (same format as the main `id` field).
+          Values are the store paths that those dependencies resolved to.
+
+          As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
+          This is the set of base build trace entries that this derived build trace is derived from.
+          (The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
+
+        patternProperties:
+          "^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
+            "$ref": "store-path-v1.yaml"
+            title: Dependent Store Path
+            description: Store path that this dependency resolved to during the build
+        additionalProperties: false
+
+      signatures:
+        type: array
+        title: Build Signatures
+        description: |
+          A set of cryptographic signatures attesting to the authenticity of this build trace entry.
+        items:
+          type: string
+          title: Signature
+          description: A single cryptographic signature

doc/manual/source/protocols/json/schema/derivation-options-v1

@@ -0,0 +1 @@
+../../../../../../src/libstore-tests/data/derivation

doc/manual/source/protocols/json/schema/derivation-options-v1.yaml

@@ -0,0 +1,242 @@
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-options-v1.json"
+title: Derivation Options
+description: |
+  JSON representation of Nix's `DerivationOptions` type.
+
+  This schema describes various build-time options and constraints that can be specified for a derivation.
+
+  > **Warning**
+  >
+  > This JSON format is currently
+  > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
+  > and subject to change.
+
+type: object
+required:
+  - outputChecks
+  - unsafeDiscardReferences
+  - passAsFile
+  - exportReferencesGraph
+  - additionalSandboxProfile
+  - noChroot
+  - impureHostDeps
+  - impureEnvVars
+  - allowLocalNetworking
+  - requiredSystemFeatures
+  - preferLocalBuild
+  - allowSubstitutes
+properties:
+  outputChecks:
+    type: object
+    title: Output Check
+    description: |
+      Constraints on what the derivation's outputs can and cannot reference.
+      Can either apply to all outputs or be specified per output.
+    oneOf:
+      - title: Output Checks For All Outputs
+        description: |
+          Output checks that apply to all outputs of the derivation.
+        required:
+          - forAllOutputs
+        properties:
+          forAllOutputs:
+            "$ref": "#/$defs/outputCheckSpec"
+        additionalProperties: false
+
+      - title: Output Checks Per Output
+        description: |
+          Output checks specified individually for each output.
+        required:
+          - perOutput
+        properties:
+          perOutput:
+            type: object
+            additionalProperties:
+              "$ref": "#/$defs/outputCheckSpec"
+        additionalProperties: false
+
+  unsafeDiscardReferences:
+    type: object
+    title: Unsafe Discard References
+    description: |
+      A map specifying which references should be unsafely discarded from each output.
+      This is generally not recommended and requires special permissions.
+    additionalProperties:
+      type: array
+      items:
+        type: string
+
+  passAsFile:
+    type: array
+    title: Pass As File
+    description: |
+      List of environment variable names whose values should be passed as files rather than directly.
+    items:
+      type: string
+
+  exportReferencesGraph:
+    type: object
+    title: Export References Graph
+    description: |
+      Specify paths whose references graph should be exported to files.
+    additionalProperties:
+      type: array
+      items:
+        "$ref": "deriving-path-v1.yaml"
+
+  additionalSandboxProfile:
+    type: string
+    title: Additional Sandbox Profile
+    description: |
+      Additional sandbox profile directives (macOS specific).
+
+  noChroot:
+    type: boolean
+    title: No Chroot
+    description: |
+      Whether to disable the build sandbox, if allowed.
+
+  impureHostDeps:
+    type: array
+    title: Impure Host Dependencies
+    description: |
+      List of host paths that the build can access.
+    items:
+      type: string
+
+  impureEnvVars:
+    type: array
+    title: Impure Environment Variables
+    description: |
+      List of environment variable names that should be passed through to the build from the calling environment.
+    items:
+      type: string
+
+  allowLocalNetworking:
+    type: boolean
+    title: Allow Local Networking
+    description: |
+      Whether the build should have access to local network (macOS specific).
+
+  requiredSystemFeatures:
+    type: array
+    title: Required System Features
+    description: |
+      List of system features required to build this derivation (e.g., "kvm", "nixos-test").
+    items:
+      type: string
+
+  preferLocalBuild:
+    type: boolean
+    title: Prefer Local Build
+    description: |
+      Whether this derivation should preferably be built locally rather than its outputs substituted.
+
+  allowSubstitutes:
+    type: boolean
+    title: Allow Substitutes
+    description: |
+      Whether substituting from other stores should be allowed for this derivation's outputs.
+
+additionalProperties: false
+
+$defs:
+
+  outputCheckSpec:
+    type: object
+    title: Output Check Specification
+    description: |
+      Constraints on what a specific output can reference.
+    required:
+      - ignoreSelfRefs
+      - maxSize
+      - maxClosureSize
+      - allowedReferences
+      - allowedRequisites
+      - disallowedReferences
+      - disallowedRequisites
+    properties:
+      ignoreSelfRefs:
+        type: boolean
+        title: Ignore Self References
+        description: |
+          Whether references from this output to itself should be ignored when checking references.
+
+      maxSize:
+        type: ["integer", "null"]
+        title: Maximum Size
+        description: |
+          Maximum allowed size of this output in bytes, or null for no limit.
+        minimum: 0
+
+      maxClosureSize:
+        type: ["integer", "null"]
+        title: Maximum Closure Size
+        description: |
+          Maximum allowed size of this output's closure in bytes, or null for no limit.
+        minimum: 0
+
+      allowedReferences:
+        oneOf:
+          - type: array
+            items:
+              "$ref": "#/$defs/drvRef"
+          - type: "null"
+        title: Allowed References
+        description: |
+          If set, the output can only reference paths in this list.
+          If null, no restrictions apply.
+
+      allowedRequisites:
+        oneOf:
+          - type: array
+            items:
+              "$ref": "#/$defs/drvRef"
+          - type: "null"
+        title: Allowed Requisites
+        description: |
+          If set, the output's closure can only contain paths in this list.
+          If null, no restrictions apply.
+
+      disallowedReferences:
+        type: array
+        title: Disallowed References
+        description: |
+          The output must not reference any paths in this list.
+        items:
+          "$ref": "#/$defs/drvRef"
+
+      disallowedRequisites:
+        type: array
+        title: Disallowed Requisites
+        description: |
+          The output's closure must not contain any paths in this list.
+        items:
+          "$ref": "#/$defs/drvRef"
+    additionalProperties: false
+
+  drvRef:
+    # TODO fix bug in checker, should be `oneOf`
+    anyOf:
+      - type: object
+        title: Current derivation Output Reference
+        description: |
+          A reference to a specific output of the current derivation.
+        required:
+          - drvPath
+          - output
+        properties:
+          drvPath:
+            type: string
+            const: "self"
+            title: This derivation
+            description: |
+              Won't be confused for a deriving path
+          output:
+            type: string
+            title: Output Name
+            description: |
+              The name of the output being referenced.
+        additionalProperties: false
+      - "$ref": "deriving-path-v1.yaml"

doc/manual/source/protocols/json/schema/file-system-object-v1

@@ -0,0 +1 @@
+../../../../../../src/libutil-tests/data/memory-source-accessor

doc/manual/source/protocols/json/schema/file-system-object-v1.yaml

@@ -0,0 +1,71 @@
+"$schema": http://json-schema.org/draft-04/schema#
+"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
+title: File System Object
+description: |
+  This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
+
+  The schema is recursive because file system objects contain other file system objects.
+type: object
+required: ["type"]
+properties:
+  type:
+    type: string
+    enum: ["regular", "symlink", "directory"]
+
+# Enforce conditional structure based on `type`
+anyOf:
+  - $ref: "#/$defs/regular"
+    required: ["type", "contents"]
+
+  - $ref: "#/$defs/directory"
+    required: ["type", "entries"]
+
+  - $ref: "#/$defs/symlink"
+    required: ["type", "target"]
+
+"$defs":
+  regular:
+    title: Regular File
+    description: |
+      See [Regular File](@docroot@/store/file-system-object.md#regular) in the manual for details.
+    required: ["contents"]
+    properties:
+      type:
+        const: "regular"
+      contents:
+        type: string
+        description: File contents
+      executable:
+        type: boolean
+        description: Whether the file is executable.
+        default: false
+    additionalProperties: false
+
+  directory:
+    title: Directory
+    description: |
+      See [Directory](@docroot@/store/file-system-object.md#directory) in the manual for details.
+    required: ["entries"]
+    properties:
+      type:
+        const: "directory"
+      entries:
+        type: object
+        description: |
+          Map of names to nested file system objects (for type=directory)
+        additionalProperties:
+          $ref: "#"
+    additionalProperties: false
+
+  symlink:
+    title: Symbolic Link
+    description: |
+      See [Symbolic Link](@docroot@/store/file-system-object.md#symlink) in the manual for details.
+    required: ["target"]
+    properties:
+      type:
+        const: "symlink"
+      target:
+        type: string
+        description: Target path of the symlink.
+    additionalProperties: false

doc/manual/source/protocols/json/schema/store-v1

@@ -0,0 +1 @@
+../../../../../../src/libstore-tests/data/dummy-store

doc/manual/source/protocols/json/schema/store-v1.yaml

@@ -0,0 +1,90 @@
+"$schema": "http://json-schema.org/draft-04/schema"
+"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-v1.json"
+title: Store
+description: |
+  Experimental JSON representation of a Nix [Store](@docroot@/store/index.md).
+
+  This schema describes the JSON serialization of a Nix store.
+  We use it for (de)serializing in-memory "dummy stores" used for testing, but in principle the data represented in this schema could live in any type of store.
+
+  > **Warning**
+  >
+  > This JSON format is currently
+  > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
+  > and subject to change.
+
+type: object
+required:
+  - config
+  - contents
+  - derivations
+  - buildTrace
+properties:
+  config:
+    "$ref": "#/$defs/storeConfig"
+
+  contents:
+    type: object
+    title: Store Objects
+    description: |
+      Map of [store path](@docroot@/store/store-path.md) base names to [store objects](@docroot@/store/store-object.md).
+    patternProperties:
+      "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$":
+        type: object
+        title: Store Object
+        required:
+          - info
+          - contents
+        properties:
+          info:
+            "$ref": "./store-object-info-v2.yaml#/$defs/impure"
+            title: Store Object Info
+            description: |
+              Metadata about the [store object](@docroot@/store/store-object.md) including hash, size, references, etc.
+          contents:
+            "$ref": "./file-system-object-v1.yaml"
+            title: File System Object Contents
+            description: |
+              The actual [file system object](@docroot@/store/file-system-object.md) contents of this store path.
+        additionalProperties: false
+    additionalProperties: false
+
+  derivations:
+    type: object
+    title: Derivations
+    description: |
+      Map of [store path](@docroot@/store/store-path.md) base names (always ending in `.drv`) to [derivations](@docroot@/store/derivation/index.md).
+    patternProperties:
+      "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$":
+        "$ref": "./derivation-v4.yaml"
+    additionalProperties: false
+
+  buildTrace:
+    type: object
+    title: Build Trace
+    description: |
+      Map of output hashes (base64 SHA256) to maps of output names to realisations.
+      Records which outputs have been built and their realisations.
+      See [Build Trace](@docroot@/store/build-trace.md) for more details.
+    patternProperties:
+      "^[A-Za-z0-9+/]{43}=$":
+        type: object
+        additionalProperties:
+          "$ref": "./build-trace-entry-v1.yaml#/$defs/value"
+    additionalProperties: false
+
+"$defs":
+  storeConfig:
+    title: Store Configuration
+    description: |
+      Configuration for the store, including the store directory path.
+    type: object
+    required:
+      - store
+    properties:
+      store:
+        type: string
+        title: Store Directory
+        description: |
+          The store directory path (e.g., `/nix/store`).
+    additionalProperties: false

doc/manual/source/protocols/json/store.md

@@ -0,0 +1,21 @@
+{{#include store-v1-fixed.md}}
+
+## Examples
+
+### Empty store
+
+```json
+{{#include schema/store-v1/empty.json}}
+```
+
+### Store with one file
+
+```json
+{{#include schema/store-v1/one-flat-file.json}}
+```
+
+### Store with one derivation
+
+```json
+{{#include schema/store-v1/one-derivation.json}}
+```

doc/manual/source/release-notes/rl-2.32.md

@@ -12,7 +12,7 @@
 
   We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands.
 
-  See the documentation on the [JSON format for derivations](@docroot@/protocols/json/derivation.md) for further details.
+  See the documentation on the [JSON format for derivations](@docroot@/protocols/json/derivation/index.md) for further details.
 
 - C API: `nix_get_attr_name_byidx`, `nix_get_attr_byidx` take a `nix_value *` instead of `const nix_value *` [#13987](https://github.com/NixOS/nix/pull/13987)
 

doc/manual/source/store/derivation/index.md

@@ -192,7 +192,7 @@ There are two formats, documented separately:
 
 - The legacy ["ATerm" format](@docroot@/protocols/derivation-aterm.md)
 
-- The experimental, currently under development and changing [JSON format](@docroot@/protocols/json/derivation.md)
+- The experimental, currently under development and changing [JSON format](@docroot@/protocols/json/derivation/index.md)
 
 Every derivation has a canonical choice of encoding used to serialize it to a store object.
 This ensures that there is a canonical [store path] used to refer to the derivation, as described in [Referencing derivations](#derivation-path).

doc/manual/source/store/file-system-object.md

@@ -3,19 +3,23 @@
 Nix uses a simplified model of the file system, which consists of file system objects.
 Every file system object is one of the following:
 
- - File
+ - [**Regular File**]{#regular}
 
    - A possibly empty sequence of bytes for contents
    - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission
 
- - Directory
+ - [**Directory**]{#directory}
 
    Mapping of names to child file system objects
 
- - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link)
+ - [**Symbolic link**]{#symlink}
 
-   An arbitrary string.
-   Nix does not assign any semantics to symbolic links.
+   An arbitrary string, known as the *target* of the symlink.
+
+   In general, Nix does not assign any semantics to symbolic links.
+   Certain operations however, may make additional assumptions and attempt to use the target to find another file system object.
+
+   > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept.
 
 File system objects and their children form a tree.
 A bare file or symlink can be a root file system object.

maintainers/flake-module.nix

@@ -79,6 +79,8 @@
               # Not supported by nixfmt
               ''^tests/functional/lang/eval-okay-deprecate-cursed-or\.nix$''
               ''^tests/functional/lang/eval-okay-attrs5\.nix$''
+              ''^tests/functional/lang/eval-fail-dynamic-attrs-inherit\.nix$''
+              ''^tests/functional/lang/eval-fail-dynamic-attrs-inherit-2\.nix$''
 
               # More syntax tests
               # These tests, or parts of them, should have been parse-* test cases.

packaging/dev-shell.nix

@@ -130,15 +130,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
     havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
     ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
 
+    availableComponents = lib.filterAttrs (
+      k: v: lib.meta.availableOn pkgs.hostPlatform v
+    ) allComponents;
+
     activeComponents = buildInputsClosureCond isInternal (
-      lib.attrValues (finalAttrs.passthru.config.getComponents allComponents)
+      lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents)
     );
 
     allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2;
     internalDrvs = byDrvPath (
       # Drop the attr names (not present in buildInputs anyway)
-      lib.attrValues allComponents
-      ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues allComponents)
+      lib.attrValues availableComponents
+      ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents)
     );
 
     isInternal =
@@ -187,19 +191,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
         );
 
       small =
-        (finalAttrs.finalPackage.withActiveComponents (c: {
-          inherit (c)
-            nix-cli
-            nix-util-tests
-            nix-store-tests
-            nix-expr-tests
-            nix-fetchers-tests
-            nix-flake-tests
-            nix-functional-tests
-            # Currently required
-            nix-perl-bindings
-            ;
-        })).overrideAttrs
+        (finalAttrs.finalPackage.withActiveComponents (
+          c:
+          lib.intersectAttrs (lib.genAttrs [
+            "nix-cli"
+            "nix-util-tests"
+            "nix-store-tests"
+            "nix-expr-tests"
+            "nix-fetchers-tests"
+            "nix-flake-tests"
+            "nix-functional-tests"
+            "nix-perl-bindings"
+          ] (_: null)) c
+        )).overrideAttrs
           (o: {
             mesonFlags = o.mesonFlags ++ [
               # TODO: infer from activeComponents or vice versa

src/json-schema-checks/derivation-options

@@ -0,0 +1 @@
+../libstore-tests/data/derivation

src/json-schema-checks/file-system-object

@@ -0,0 +1 @@
+../../src/libutil-tests/data/memory-source-accessor

src/json-schema-checks/meson.build

@@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema'
 
 # Get all example files
 schemas = [
+  {
+    'stem' : 'file-system-object',
+    'schema' : schema_dir / 'file-system-object-v1.yaml',
+    'files' : [
+      'simple.json',
+      'complex.json',
+    ],
+  },
   {
     'stem' : 'hash',
     'schema' : schema_dir / 'hash-v1.yaml',
@@ -63,6 +71,18 @@ schemas = [
       'with-signature.json',
     ],
   },
+  {
+    'stem' : 'derivation-options',
+    'schema' : schema_dir / 'derivation-options-v1.yaml',
+    'files' : [
+      'ia' / 'defaults.json',
+      'ia' / 'all_set.json',
+      'ia' / 'structuredAttrs_defaults.json',
+      'ia' / 'structuredAttrs_all_set.json',
+      'ca' / 'all_set.json',
+      'ca' / 'structuredAttrs_all_set.json',
+    ],
+  },
 ]
 
 # Derivation and Derivation output
@@ -192,6 +212,19 @@ schemas += [
   },
 ]
 
+# Dummy store
+schemas += [
+  {
+    'stem' : 'store',
+    'schema' : schema_dir / 'store-v1.yaml',
+    'files' : [
+      'empty.json',
+      'one-flat-file.json',
+      'one-derivation.json',
+    ],
+  },
+]
+
 # Validate each example against the schema
 foreach schema : schemas
   stem = schema['stem']

src/json-schema-checks/package.nix

@@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: {
   fileset = lib.fileset.unions [
     ../../.version
     ../../doc/manual/source/protocols/json/schema
+    ../../src/libutil-tests/data/memory-source-accessor
     ../../src/libutil-tests/data/hash
     ../../src/libstore-tests/data/content-address
     ../../src/libstore-tests/data/store-path
@@ -29,6 +30,7 @@ mkMesonDerivation (finalAttrs: {
     ../../src/libstore-tests/data/path-info
     ../../src/libstore-tests/data/nar-info
     ../../src/libstore-tests/data/build-result
+    ../../src/libstore-tests/data/dummy-store
     ./.
   ];
 

src/json-schema-checks/store

@@ -0,0 +1 @@
+../../src/libstore-tests/data/dummy-store

src/kaitai-struct-checks/package.nix

@@ -65,6 +65,6 @@ mkMesonDerivation (finalAttrs: {
   '';
 
   meta = {
-    platforms = lib.platforms.all;
+    platforms = lib.platforms.unix;
   };
 })

src/libcmd/command.cc

@@ -297,7 +297,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables)
 
 MixDefaultProfile::MixDefaultProfile()
 {
-    profile = getDefaultProfile();
+    profile = getDefaultProfile().string();
 }
 
 MixEnvironment::MixEnvironment()
@@ -391,7 +391,7 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu
                     auto symlink = outLink;
                     if (i)
                         symlink += fmt("-%d", i);
-                    store.addPermRoot(bo.path, absPath(symlink.string()));
+                    store.addPermRoot(bo.path, absPath(symlink).string());
                 },
                 [&](const BuiltPath::Built & bfd) {
                     for (auto & output : bfd.outputs) {
@@ -400,7 +400,7 @@ void createOutLinks(const std::filesystem::path & outLink, const BuiltPaths & bu
                             symlink += fmt("-%d", i);
                         if (output.first != "out")
                             symlink += fmt("-%s", output.first);
-                        store.addPermRoot(output.second, absPath(symlink.string()));
+                        store.addPermRoot(output.second, absPath(symlink).string());
                     }
                 },
             },

src/libcmd/common-eval-args.cc

@@ -132,7 +132,7 @@ MixEvalArgs::MixEvalArgs()
             fetchers::Attrs extraAttrs;
             if (to.subdir != "")
                 extraAttrs["dir"] = to.subdir;
-            fetchers::overrideRegistry(fetchSettings, from.input, to.input, extraAttrs);
+            fetchers::overrideRegistry(from.input, to.input, extraAttrs);
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             completeFlakeRef(completions, openStore(), prefix);
@@ -165,19 +165,19 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
                         state.parseExprFromString(
                             arg.expr,
                             compatibilitySettings.nixShellShebangArgumentsRelativeToScript
-                                ? state.rootPath(absPath(getCommandBaseDir()))
+                                ? state.rootPath(absPath(getCommandBaseDir()).string())
                                 : state.rootPath(".")));
                 },
-                [&](const AutoArgString & arg) { v->mkString(arg.s); },
-                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string())); },
-                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO)); }},
+                [&](const AutoArgString & arg) { v->mkString(arg.s, state.mem); },
+                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string()), state.mem); },
+                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO), state.mem); }},
             arg);
         res.insert(state.symbols.create(name), v);
     }
     return res.finish();
 }
 
-SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
+SourcePath lookupFileArg(EvalState & state, std::string_view s, const std::filesystem::path * baseDir)
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto accessor = fetchers::downloadTarball(*state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
@@ -197,12 +197,13 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
     }
 
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
-        Path p(s.substr(1, s.size() - 2));
+        // Should perhaps be a `CanonPath`?
+        std::string p(s.substr(1, s.size() - 2));
         return state.findFile(p);
     }
 
     else
-        return state.rootPath(baseDir ? absPath(s, *baseDir) : absPath(s));
+        return state.rootPath(absPath(std::filesystem::path{s}, baseDir).string());
 }
 
 } // namespace nix

src/libcmd/include/nix/cmd/command.hh

@@ -134,7 +134,7 @@ struct MixFlakeOptions : virtual Args, EvalCommand
 
 struct SourceExprCommand : virtual Args, MixFlakeOptions
 {
-    std::optional<Path> file;
+    std::optional<std::filesystem::path> file;
     std::optional<std::string> expr;
 
     SourceExprCommand();
@@ -310,7 +310,7 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
 
 struct MixProfile : virtual StoreCommand
 {
-    std::optional<Path> profile;
+    std::optional<std::filesystem::path> profile;
 
     MixProfile();
 

src/libcmd/include/nix/cmd/common-eval-args.hh

@@ -84,6 +84,6 @@ private:
 /**
  * @param baseDir Optional [base directory](https://nix.dev/manual/nix/development/glossary#gloss-base-directory)
  */
-SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr);
+SourcePath lookupFileArg(EvalState & state, std::string_view s, const std::filesystem::path * baseDir = nullptr);
 
 } // namespace nix

src/libcmd/include/nix/cmd/installable-value.hh

@@ -17,7 +17,7 @@ class AttrCursor;
 struct App
 {
     std::vector<DerivedPath> context;
-    Path program;
+    std::filesystem::path program;
     // FIXME: add args, sandbox settings, metadata, ...
 };
 

src/libcmd/include/nix/cmd/repl.hh

@@ -19,7 +19,16 @@ struct AbstractNixRepl
 
     typedef std::vector<std::pair<Value *, std::string>> AnnotatedValues;
 
-    using RunNix = void(Path program, const Strings & args, const std::optional<std::string> & input);
+    /**
+     * Run a nix executable
+     *
+     * @todo this is a layer violation
+     *
+     * @param programName Name of the command, e.g. `nix` or `nix-env`.
+     * @param args aguments to the command.
+     */
+    using RunNix =
+        void(const std::string & programName, const Strings & args, const std::optional<std::string> & input);
 
     /**
      * @param runNix Function to run the nix CLI to support various

src/libcmd/installables.cc

@@ -132,7 +132,7 @@ MixFlakeOptions::MixFlakeOptions()
             lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputAttrPath(inputAttrPath),
-                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true));
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string(), true));
         }},
         .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
             if (n == 0) {
@@ -173,7 +173,7 @@ MixFlakeOptions::MixFlakeOptions()
             auto flake = flake::lockFlake(
                 flakeSettings,
                 *evalState,
-                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())),
+                parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()).string()),
                 {.writeLockFile = false});
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
@@ -185,7 +185,6 @@ MixFlakeOptions::MixFlakeOptions()
                     }
 
                     overrideRegistry(
-                        fetchSettings,
                         fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
                         extraAttrs);
@@ -264,7 +263,7 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s
 
             evalSettings.pureEval = false;
             auto state = getEvalState();
-            auto e = state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, *file)));
+            auto e = state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, file->string())));
 
             Value root;
             state->eval(e, root);
@@ -466,10 +465,10 @@ Installables SourceExprCommand::parseInstallables(ref<Store> store, std::vector<
             state->eval(e, *vFile);
         } else if (file) {
             auto dir = absPath(getCommandBaseDir());
-            state->evalFile(lookupFileArg(*state, *file, &dir), *vFile);
+            state->evalFile(lookupFileArg(*state, file->string(), &dir), *vFile);
         } else {
-            Path dir = absPath(getCommandBaseDir());
-            auto e = state->parseExprFromString(*expr, state->rootPath(dir));
+            auto dir = absPath(getCommandBaseDir());
+            auto e = state->parseExprFromString(*expr, state->rootPath(dir.string()));
             state->eval(e, *vFile);
         }
 
@@ -802,7 +801,8 @@ std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
     std::vector<FlakeRef> res;
     res.reserve(rawInstallables.size());
     for (const auto & i : rawInstallables)
-        res.push_back(parseFlakeRefWithFragment(fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first);
+        res.push_back(
+            parseFlakeRefWithFragment(fetchSettings, expandTilde(i), absPath(getCommandBaseDir()).string()).first);
     return res;
 }
 
@@ -821,7 +821,8 @@ void RawInstallablesCommand::run(ref<Store> store)
 
 std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
 {
-    return {parseFlakeRefWithFragment(fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first};
+    return {parseFlakeRefWithFragment(fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir()).string())
+                .first};
 }
 
 void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables)

src/libcmd/repl.cc

@@ -58,8 +58,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
 {
     size_t debugTraceIndex;
 
-    // Arguments passed to :load, saved so they can be reloaded with :reload
-    Strings loadedFiles;
+    std::list<std::filesystem::path> loadedFiles;
     // Arguments passed to :load-flake, saved so they can be reloaded with :reload
     Strings loadedFlakes;
     std::function<AnnotatedValues()> getValues;
@@ -73,7 +72,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
 
     RunNix * runNixPtr;
 
-    void runNix(Path program, const Strings & args, const std::optional<std::string> & input = {});
+    void runNix(const std::string & program, const Strings & args, const std::optional<std::string> & input = {});
 
     std::unique_ptr<ReplInteracter> interacter;
 
@@ -92,7 +91,7 @@ struct NixRepl : AbstractNixRepl, detail::ReplCompleterMixin, gc
     StorePath getDerivationPath(Value & v);
     ProcessLineResult processLine(std::string line);
 
-    void loadFile(const Path & path);
+    void loadFile(const std::filesystem::path & path);
     void loadFlake(const std::string & flakeRef);
     void loadFiles();
     void loadFlakes();
@@ -539,7 +538,9 @@ ProcessLineResult NixRepl::processLine(std::string line)
         Value v;
         evalString(arg, v);
         StorePath drvPath = getDerivationPath(v);
-        Path drvPathRaw = state->store->printStorePath(drvPath);
+        // N.B. This need not be a local / native file path. For
+        // example, we might be using an SSH store to a different OS.
+        std::string drvPathRaw = state->store->printStorePath(drvPath);
 
         if (command == ":b" || command == ":bl") {
             state->store->buildPaths({
@@ -712,12 +713,12 @@ ProcessLineResult NixRepl::processLine(std::string line)
     return ProcessLineResult::PromptAgain;
 }
 
-void NixRepl::loadFile(const Path & path)
+void NixRepl::loadFile(const std::filesystem::path & path)
 {
     loadedFiles.remove(path);
     loadedFiles.push_back(path);
     Value v, v2;
-    state->evalFile(lookupFileArg(*state, path), v);
+    state->evalFile(lookupFileArg(*state, path.string()), v);
     state->autoCallFunction(*autoArgs, v, v2);
     addAttrsToScope(v2);
 }
@@ -790,7 +791,7 @@ void NixRepl::reloadFilesAndFlakes()
 
 void NixRepl::loadFiles()
 {
-    Strings old = loadedFiles;
+    decltype(loadedFiles) old = loadedFiles;
     loadedFiles.clear();
 
     for (auto & i : old) {
@@ -888,7 +889,7 @@ void NixRepl::evalString(std::string s, Value & v)
     state->forceValue(v, v.determinePos(noPos));
 }
 
-void NixRepl::runNix(Path program, const Strings & args, const std::optional<std::string> & input)
+void NixRepl::runNix(const std::string & program, const Strings & args, const std::optional<std::string> & input)
 {
     if (runNixPtr)
         (*runNixPtr)(program, args, input);

src/libexpr-c/nix_api_expr.cc

@@ -69,8 +69,8 @@ nix_err nix_expr_eval_from_string(
         context->last_err_code = NIX_OK;
     try {
         nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
-        state->state.eval(parsedExpr, value->value);
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.eval(parsedExpr, *value->value);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -80,8 +80,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.callFunction(*fn->value, *arg->value, *value->value, nix::noPos);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -91,9 +91,15 @@ nix_err nix_value_call_multi(
 {
     if (context)
         context->last_err_code = NIX_OK;
+
+    std::vector<nix::Value *> internal_args;
+    internal_args.reserve(nargs);
+    for (size_t i = 0; i < nargs; i++)
+        internal_args.push_back(args[i]->value);
+
     try {
-        state->state.callFunction(fn->value, {(nix::Value **) args, nargs}, value->value, nix::noPos);
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.callFunction(*fn->value, {internal_args.data(), nargs}, *value->value, nix::noPos);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -103,7 +109,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -113,7 +119,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValueDeep(value->value);
+        state->state.forceValueDeep(*value->value);
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-c/nix_api_expr_internal.h

@@ -39,7 +39,13 @@ struct ListBuilder
 
 struct nix_value
 {
-    nix::Value value;
+    nix::Value * value;
+    /**
+     * As we move to a managed heap, we need EvalMemory in more places. Ideally, we would take in EvalState or
+     * EvalMemory as an argument when we need it, but we don't want to make changes to the stable C api, so we stuff it
+     * into the nix_value that will get passed in to the relevant functions.
+     */
+    nix::EvalMemory * mem;
 };
 
 struct nix_string_return

src/libexpr-c/nix_api_value.cc

@@ -20,7 +20,7 @@ static const nix::Value & check_value_not_null(const nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return *((const nix::Value *) value);
+    return *value->value;
 }
 
 static nix::Value & check_value_not_null(nix_value * value)
@@ -28,7 +28,7 @@ static nix::Value & check_value_not_null(nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return value->value;
+    return *value->value;
 }
 
 static const nix::Value & check_value_in(const nix_value * value)
@@ -58,9 +58,14 @@ static nix::Value & check_value_out(nix_value * value)
     return v;
 }
 
-static inline nix_value * as_nix_value_ptr(nix::Value * v)
+static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
 {
-    return reinterpret_cast<nix_value *>(v);
+    nix_value * ret = new (mem.allocBytes(sizeof(nix_value))) nix_value{
+        .value = v,
+        .mem = &mem,
+    };
+    nix_gc_incref(nullptr, ret);
+    return ret;
 }
 
 /**
@@ -69,7 +74,13 @@ static inline nix_value * as_nix_value_ptr(nix::Value * v)
  * Deals with errors and converts arguments from C++ into C types.
  */
 static void nix_c_primop_wrapper(
-    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+    PrimOpFun f,
+    void * userdata,
+    int arity,
+    nix::EvalState & state,
+    const nix::PosIdx pos,
+    nix::Value ** args,
+    nix::Value & v)
 {
     nix_c_context ctx;
 
@@ -85,8 +96,15 @@ static void nix_c_primop_wrapper(
     // ok because we don't see a need for this yet (e.g. inspecting thunks,
     // or maybe something to make blackholes work better; we don't know).
     nix::Value vTmp;
+    nix_value * vTmpPtr = new_nix_value(&vTmp, state.mem);
 
-    f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
+    std::vector<nix_value *> external_args;
+    external_args.reserve(arity);
+    for (int i = 0; i < arity; i++) {
+        nix_value * external_arg = new_nix_value(args[i], state.mem);
+        external_args.push_back(external_arg);
+    }
+    f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr);
 
     if (ctx.last_err_code != NIX_OK) {
         /* TODO: Throw different errors depending on the error code */
@@ -135,7 +153,7 @@ PrimOp * nix_alloc_primop(
                     .args = {},
                     .arity = (size_t) arity,
                     .doc = doc,
-                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)};
         if (args)
             for (size_t i = 0; args[i]; i++)
                 p->args.emplace_back(*args);
@@ -160,8 +178,7 @@ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        nix_value * res = as_nix_value_ptr(state->state.allocValue());
-        nix_gc_incref(nullptr, res);
+        nix_value * res = new_nix_value(state->state.allocValue(), state->state.mem);
         return res;
     }
     NIXC_CATCH_ERRS_NULL
@@ -331,10 +348,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
             return nullptr;
         }
         auto * p = v.listView()[ix];
-        nix_gc_incref(nullptr, p);
-        if (p != nullptr)
-            state->state.forceValue(*p, nix::noPos);
-        return as_nix_value_ptr(p);
+        if (p == nullptr)
+            return nullptr;
+        state->state.forceValue(*p, nix::noPos);
+        return new_nix_value(p, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -352,9 +369,8 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt
             return nullptr;
         }
         auto * p = v.listView()[ix];
-        nix_gc_incref(nullptr, p);
         // Note: intentionally NOT calling forceValue() to keep the element lazy
-        return as_nix_value_ptr(p);
+        return new_nix_value(p, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -369,9 +385,8 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
-            nix_gc_incref(nullptr, attr->value);
             state->state.forceValue(*attr->value, nix::noPos);
-            return as_nix_value_ptr(attr->value);
+            return new_nix_value(attr->value, state->state.mem);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -390,9 +405,8 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
-            nix_gc_incref(nullptr, attr->value);
             // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-            return as_nix_value_ptr(attr->value);
+            return new_nix_value(attr->value, state->state.mem);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -440,9 +454,8 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
-        nix_gc_incref(nullptr, a.value);
         state->state.forceValue(*a.value, nix::noPos);
-        return as_nix_value_ptr(a.value);
+        return new_nix_value(a.value, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -461,9 +474,8 @@ nix_value * nix_get_attr_byidx_lazy(
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
-        nix_gc_incref(nullptr, a.value);
         // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-        return as_nix_value_ptr(a.value);
+        return new_nix_value(a.value, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -503,7 +515,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkString(std::string_view(str));
+        v.mkString(std::string_view(str), *value->mem);
     }
     NIXC_CATCH_ERRS
 }
@@ -514,7 +526,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)), s->state.mem);
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-tests/json.cc

@@ -73,7 +73,7 @@ TEST_F(JSONValueTest, StringQuotes)
 TEST_F(JSONValueTest, DISABLED_Path)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("/test")));
+    v.mkPath(state.rootPath(CanonPath("/test")), state.mem);
     ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
 }
 } /* namespace nix */

src/libexpr-tests/value/print.cc

@@ -268,7 +268,7 @@ struct StringPrintingTests : LibExprTest
     void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args)
     {
         Value v;
-        v.mkString(literal);
+        v.mkString(literal, state.mem);
 
         std::stringstream out;
         printValue(state, out, v, PrintOptions{.maxStringLength = maxLength});
@@ -353,7 +353,7 @@ TEST_F(ValuePrintingTests, ansiColorsStringElided)
 TEST_F(ValuePrintingTests, ansiColorsPath)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("puppy")));
+    v.mkPath(state.rootPath(CanonPath("puppy")), state.mem);
 
     test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true});
 }

src/libexpr/eval-settings.cc

@@ -60,18 +60,18 @@ EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lo
 Strings EvalSettings::getDefaultNixPath()
 {
     Strings res;
-    auto add = [&](const Path & p, const std::string & s = std::string()) {
+    auto add = [&](const std::filesystem::path & p, const std::string & s = std::string()) {
         if (std::filesystem::exists(p)) {
             if (s.empty()) {
-                res.push_back(p);
+                res.push_back(p.string());
             } else {
-                res.push_back(s + "=" + p);
+                res.push_back(s + "=" + p.string());
             }
         }
     };
 
-    add(getNixDefExpr() + "/channels");
-    add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
+    add(std::filesystem::path{getNixDefExpr()} / "channels");
+    add(rootChannelsDir() / "nixpkgs", "nixpkgs");
     add(rootChannelsDir());
 
     return res;
@@ -108,4 +108,4 @@ Path getNixDefExpr()
     return settings.useXDGBaseDirectories ? getStateDir() + "/defexpr" : getHome() + "/.nix-defexpr";
 }
 
-} // namespace nix
+} // namespace nix

src/libexpr/eval.cc

@@ -81,20 +81,20 @@ static const char * makeImmutableString(std::string_view s)
     return t;
 }
 
-StringData & StringData::alloc(size_t size)
+StringData & StringData::alloc(EvalMemory & mem, size_t size)
 {
-    void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
+    void * t = mem.allocBytes(sizeof(StringData) + size + 1);
     if (!t)
         throw std::bad_alloc();
     auto res = new (t) StringData(size);
     return *res;
 }
 
-const StringData & StringData::make(std::string_view s)
+const StringData & StringData::make(EvalMemory & mem, std::string_view s)
 {
     if (s.empty())
         return ""_sds;
-    auto & res = alloc(s.size());
+    auto & res = alloc(mem, s.size());
     std::memcpy(&res.data_, s.data(), s.size());
     res.data_[s.size()] = '\0';
     return res;
@@ -849,9 +849,9 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
         evalState.runDebugRepl(nullptr, trace.env, trace.expr);
 }
 
-void Value::mkString(std::string_view s)
+void Value::mkString(std::string_view s, EvalMemory & mem)
 {
-    mkStringNoCopy(StringData::make(s));
+    mkStringNoCopy(StringData::make(mem, s));
 }
 
 Value::StringWithContext::Context *
@@ -862,13 +862,13 @@ Value::StringWithContext::Context::fromBuilder(const NixStringContext & context,
 
     auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
     std::ranges::transform(
-        context, ctx->elems, [](const NixStringContextElem & elt) { return &StringData::make(elt.to_string()); });
+        context, ctx->elems, [&](const NixStringContextElem & elt) { return &StringData::make(mem, elt.to_string()); });
     return ctx;
 }
 
 void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem)
 {
-    mkStringNoCopy(StringData::make(s), Value::StringWithContext::Context::fromBuilder(context, mem));
+    mkStringNoCopy(StringData::make(mem, s), Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
 void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem)
@@ -876,9 +876,9 @@ void Value::mkStringMove(const StringData & s, const NixStringContext & context,
     mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
-void Value::mkPath(const SourcePath & path)
+void Value::mkPath(const SourcePath & path, EvalMemory & mem)
 {
-    mkPath(&*path.accessor, StringData::make(path.path.abs()));
+    mkPath(&*path.accessor, StringData::make(mem, path.path.abs()));
 }
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@@ -943,7 +943,7 @@ void EvalState::mkPos(Value & v, PosIdx p)
     auto origin = positions.originOf(p);
     if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(s.file).mkString(path->path.abs());
+        attrs.alloc(s.file).mkString(path->path.abs(), mem);
         makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column));
         v.mkAttrs(attrs);
     } else
@@ -1751,9 +1751,9 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    SmallValueVector<4> vArgs(args.size());
-    for (size_t i = 0; i < args.size(); ++i)
-        vArgs[i] = args[i]->maybeThunk(state, env);
+    SmallValueVector<4> vArgs(args->size());
+    for (size_t i = 0; i < args->size(); ++i)
+        vArgs[i] = (*args)[i]->maybeThunk(state, env);
 
     state.callFunction(vFun, vArgs, v, pos);
 }
@@ -2139,9 +2139,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         for (const auto & part : strings) {
             resultStr += *part;
         }
-        v.mkPath(state.rootPath(CanonPath(resultStr)));
+        v.mkPath(state.rootPath(CanonPath(resultStr)), state.mem);
     } else {
-        auto & resultStr = StringData::alloc(sSize);
+        auto & resultStr = StringData::alloc(state.mem, sSize);
         auto * tmp = resultStr.data();
         for (const auto & part : strings) {
             std::memcpy(tmp, part->data(), part->size());
@@ -2188,6 +2188,8 @@ void EvalState::forceValueDeep(Value & v)
     std::set<const Value *> seen;
 
     [&, &state(*this)](this const auto & recurse, Value & v) {
+        auto _level = state.addCallDepth(v.determinePos(noPos));
+
         if (!seen.insert(&v).second)
             return;
 
@@ -2214,8 +2216,15 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
+            size_t index = 0;
             for (auto v2 : v.listView())
-                recurse(*v2);
+                try {
+                    recurse(*v2);
+                    index++;
+                } catch (Error & e) {
+                    state.addErrorTrace(e, "while evaluating list element at index %1%", index);
+                    throw;
+                }
         }
     }(v);
 }

src/libexpr/include/nix/expr/nixexpr.hh

@@ -438,6 +438,7 @@ struct ExprAttrs : Expr
     std::shared_ptr<const StaticEnv> bindInheritSources(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
     Env * buildInheritFromEnv(EvalState & state, Env & up);
     void showBindings(const SymbolTable & symbols, std::ostream & str) const;
+    void moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc);
 };
 
 struct ExprList : Expr
@@ -592,11 +593,14 @@ public:
 struct ExprCall : Expr
 {
     Expr * fun;
-    std::vector<Expr *> args;
+    /**
+     * args will never be null. See comment on ExprAttrs::AttrDefs below.
+     */
+    std::optional<std::pmr::vector<Expr *>> args;
     PosIdx pos;
     std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
+    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -604,7 +608,7 @@ struct ExprCall : Expr
     {
     }
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -619,6 +623,7 @@ struct ExprCall : Expr
 
     virtual void resetCursedOr() override;
     virtual void warnIfCursedOr(const SymbolTable & symbols, const PosTable & positions) override;
+    void moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc);
     COMMON_METHODS
 };
 
@@ -836,7 +841,7 @@ public:
     // we define some calls to add explicitly so that the argument can be passed in as initializer lists
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
+    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args));
@@ -844,7 +849,7 @@ public:
 
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos));

src/libexpr/include/nix/expr/parser-state.hh

@@ -47,6 +47,79 @@ struct ParserLocation
     }
 };
 
+/**
+ * This represents a string-like parse that possibly has yet to be constructed.
+ *
+ * Examples:
+ * "foo"
+ * ${"foo" + "bar"}
+ * "foo.bar"
+ * "foo-${a}"
+ *
+ * Using this type allows us to avoid construction altogether in cases where what we actually need is the string
+ * contents. For example in foo."bar.baz", there is no need to construct an AST node for "bar.baz", but we don't know
+ * that until we bubble the value up during parsing and see that it's a node in an AttrPath.
+ */
+class ToBeStringyExpr
+{
+private:
+    using Raw = std::variant<std::monostate, std::string_view, Expr *>;
+    Raw raw;
+
+public:
+    ToBeStringyExpr() = default;
+
+    ToBeStringyExpr(std::string_view v)
+        : raw(v)
+    {
+    }
+
+    ToBeStringyExpr(Expr * expr)
+        : raw(expr)
+    {
+        assert(expr);
+    }
+
+    /**
+     * Visits the expression and invokes an overloaded functor object \ref f.
+     * If the underlying Expr has a dynamic type of ExprString the overload taking std::string_view
+     * is invoked.
+     *
+     * Used to consistently handle simple StringExpr ${"string"} as non-dynamic attributes.
+     * @see https://github.com/NixOS/nix/issues/14642
+     */
+    template<class F>
+    void visit(F && f)
+    {
+        std::visit(
+            overloaded{
+                [&](std::string_view str) { f(str); },
+                [&](Expr * expr) {
+                    ExprString * str = dynamic_cast<ExprString *>(expr);
+                    if (str)
+                        f(str->v.string_view());
+                    else
+                        f(expr);
+                },
+                [](std::monostate) { unreachable(); }},
+            raw);
+    }
+
+    /**
+     * Get or create an Expr from either an existing Expr or from a string.
+     * Delays the allocation or an AST node in case the parser only cares about string contents.
+     */
+    Expr * toExpr(Exprs & exprs)
+    {
+        return std::visit(
+            overloaded{
+                [&](std::string_view str) -> Expr * { return exprs.add<ExprString>(exprs.alloc, str); },
+                [&](Expr * expr) { return expr; },
+                [](std::monostate) -> Expr * { unreachable(); }},
+            raw);
+    }
+};
+
 struct LexerState
 {
     /**

src/libexpr/include/nix/expr/value.hh

@@ -232,13 +232,13 @@ public:
      * Allocate StringData on the (possibly) GC-managed heap and copy
      * the contents of s to it.
      */
-    static const StringData & make(std::string_view s);
+    static const StringData & make(EvalMemory & mem, std::string_view s);
 
     /**
      * Allocate StringData on the (possibly) GC-managed heap.
      * @param size Length of the string (without the NUL terminator).
      */
-    static StringData & alloc(size_t size);
+    static StringData & alloc(EvalMemory & mem, size_t size);
 
     size_t size() const
     {
@@ -1147,13 +1147,13 @@ public:
         setStorage(StringWithContext{.str = &s, .context = context});
     }
 
-    void mkString(std::string_view s);
+    void mkString(std::string_view s, EvalMemory & mem);
 
     void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem);
 
     void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem);
 
-    void mkPath(const SourcePath & path);
+    void mkPath(const SourcePath & path, EvalMemory & mem);
 
     inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept
     {

src/libexpr/json-to-value.cc

@@ -151,7 +151,7 @@ public:
     bool string(string_t & val) override
     {
         forceNoNullByte(val);
-        rs->value(state).mkString(val);
+        rs->value(state).mkString(val, state.mem);
         rs->add();
         return true;
     }

src/libexpr/nixexpr.cc

@@ -191,7 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << '(';
     fun->show(symbols, str);
-    for (auto e : args) {
+    for (auto e : *args) {
         str << ' ';
         e->show(symbols, str);
     }
@@ -399,18 +399,19 @@ ExprAttrs::bindInheritSources(EvalState & es, const std::shared_ptr<const Static
     return inner;
 }
 
+void ExprAttrs::moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc)
+{
+    AttrDefs newAttrs{std::move(*attrs), alloc};
+    attrs.emplace(std::move(newAttrs), alloc);
+    DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), alloc};
+    dynamicAttrs.emplace(std::move(newDynamicAttrs), alloc);
+    if (inheritFromExprs)
+        inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), alloc);
+}
+
 void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    // Move storage into the Exprs arena
-    {
-        auto arena = es.mem.exprs.alloc;
-        AttrDefs newAttrs{std::move(*attrs), arena};
-        attrs.emplace(std::move(newAttrs), arena);
-        DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), arena};
-        dynamicAttrs.emplace(std::move(newDynamicAttrs), arena);
-        if (inheritFromExprs)
-            inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), arena);
-    }
+    moveDataToAllocator(es.mem.exprs.alloc);
 
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
@@ -484,18 +485,26 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
     body->bindVars(es, newEnv);
 }
 
+void ExprCall::moveDataToAllocator(std::pmr::polymorphic_allocator<char> & alloc)
+{
+    std::pmr::vector<Expr *> newArgs{std::move(*args), alloc};
+    args.emplace(std::move(newArgs), alloc);
+}
+
 void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
+    moveDataToAllocator(es.mem.exprs.alloc);
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     fun->bindVars(es, env);
-    for (auto e : args)
+    for (auto e : *args)
         e->bindVars(es, env);
 }
 
 void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
+    attrs->moveDataToAllocator(es.mem.exprs.alloc);
     auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
         auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs->size());
 

src/libexpr/parser.y

@@ -115,7 +115,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P
 
 static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
     if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
-        e2->args.push_back(arg);
+        e2->args->push_back(arg);
         return fn;
     }
     return exprs.add<ExprCall>(pos, fn, {arg});
@@ -129,7 +129,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
 %type <Expr *> start expr expr_function expr_if expr_op
 %type <Expr *> expr_select expr_simple expr_app
 %type <Expr *> expr_pipe_from expr_pipe_into
-%type <std::vector<Expr *>> list
+%type <std::pmr::vector<Expr *>> list
 %type <ExprAttrs *> binds binds1
 %type <FormalsBuilder> formals formal_set
 %type <Formal> formal
@@ -138,7 +138,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
 %type <std::vector<std::pair<PosIdx, Expr *>>> string_parts_interpolated
 %type <std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>> ind_string_parts
 %type <Expr *> path_start
-%type <std::variant<Expr *, std::string_view>> string_parts string_attr
+%type <ToBeStringyExpr> string_parts string_attr
 %type <StringToken> attr
 %token <StringToken> ID
 %token <StringToken> STR IND_STR
@@ -297,12 +297,7 @@ expr_simple
   }
   | INT_LIT { $$ = state->exprs.add<ExprInt>($1); }
   | FLOAT_LIT { $$ = state->exprs.add<ExprFloat>($1); }
-  | '"' string_parts '"' {
-      std::visit(overloaded{
-          [&](std::string_view str) { $$ = state->exprs.add<ExprString>(state->exprs.alloc, str); },
-          [&](Expr * expr) { $$ = expr; }},
-      $2);
-  }
+  | '"' string_parts '"' { $$ = $2.toExpr(state->exprs); }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
       $$ = state->stripIndentation(CUR_POS, $2);
   }
@@ -342,9 +337,9 @@ expr_simple
   ;
 
 string_parts
-  : STR { $$ = $1; }
-  | string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1); }
-  | { $$ = std::string_view(); }
+  : STR { $$ = {$1}; }
+  | string_parts_interpolated { $$ = {state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1)}; }
+  | { $$ = {std::string_view()}; }
   ;
 
 string_parts_interpolated
@@ -447,15 +442,15 @@ attrs
   : attrs attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($2), state->at(@2)); }
   | attrs string_attr
     { $$ = std::move($1);
-      std::visit(overloaded {
+      $2.visit(overloaded{
           [&](std::string_view str) { $$.emplace_back(state->symbols.create(str), state->at(@2)); },
           [&](Expr * expr) {
-              throw ParseError({
-                  .msg = HintFmt("dynamic attributes not allowed in inherit"),
-                  .pos = state->positions[state->at(@2)]
-              });
-          }
-      }, $2);
+                throw ParseError({
+                    .msg = HintFmt("dynamic attributes not allowed in inherit"),
+                    .pos = state->positions[state->at(@2)]
+                });
+          }}
+      );
     }
   | { }
   ;
@@ -464,17 +459,17 @@ attrpath
   : attrpath '.' attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($3)); }
   | attrpath '.' string_attr
     { $$ = std::move($1);
-      std::visit(overloaded {
+      $3.visit(overloaded{
           [&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
-          [&](Expr * expr) { $$.emplace_back(expr); }
-      }, std::move($3));
+          [&](Expr * expr) { $$.emplace_back(expr); }}
+      );
     }
   | attr { $$.emplace_back(state->symbols.create($1)); }
   | string_attr
-    { std::visit(overloaded {
+    { $1.visit(overloaded{
           [&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
-          [&](Expr * expr) { $$.emplace_back(expr); }
-      }, std::move($1));
+          [&](Expr * expr) { $$.emplace_back(expr); }}
+      );
     }
   ;
 
@@ -485,7 +480,7 @@ attr
 
 string_attr
   : '"' string_parts '"' { $$ = std::move($2); }
-  | DOLLAR_CURLY expr '}' { $$ = $2; }
+  | DOLLAR_CURLY expr '}' { $$ = {$2}; }
   ;
 
 list

src/libexpr/primops.cc

@@ -53,7 +53,7 @@ RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
 static inline Value * mkString(EvalState & state, const std::csub_match & match)
 {
     Value * v = state.allocValue();
-    v->mkString({match.first, match.second});
+    v->mkString({match.first, match.second}, state.mem);
     return v;
 }
 
@@ -230,12 +230,12 @@ void derivationToValue(
                 NixStringContextElem::DrvDeep{.drvPath = storePath},
             },
             state.mem);
-    attrs.alloc(state.s.name).mkString(drv.env["name"]);
+    attrs.alloc(state.s.name).mkString(drv.env["name"], state.mem);
 
     auto list = state.buildList(drv.outputs.size());
     for (const auto & [i, o] : enumerate(drv.outputs)) {
         mkOutputString(state, attrs, storePath, o);
-        (list[i] = state.allocValue())->mkString(o.first);
+        (list[i] = state.allocValue())->mkString(o.first, state.mem);
     }
     attrs.alloc(state.s.outputs).mkList(list);
 
@@ -519,7 +519,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
         v.mkStringNoCopy("lambda"_sds);
         break;
     case nExternal:
-        v.mkString(args[0]->external()->typeOf());
+        v.mkString(args[0]->external()->typeOf(), state.mem);
         break;
     case nFloat:
         v.mkStringNoCopy("float"_sds);
@@ -1176,7 +1176,7 @@ static void prim_getEnv(EvalState & state, const PosIdx pos, Value ** args, Valu
 {
     std::string name(
         state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv"));
-    v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""));
+    v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""), state.mem);
 }
 
 static RegisterPrimOp primop_getEnv({
@@ -1774,28 +1774,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
             drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{});
         }
 
-        auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
-        switch (hashModulo.kind) {
-        case DrvHash::Kind::Regular:
-            for (auto & i : outputs) {
-                auto h = get(hashModulo.hashes, i);
-                if (!h)
-                    state.error<AssertionError>("derivation produced no hash for output '%s'", i).atPos(v).debugThrow();
-                auto outPath = state.store->makeOutputPath(i, *h, drvName);
-                drv.env[i] = state.store->printStorePath(outPath);
-                drv.outputs.insert_or_assign(
-                    i,
-                    DerivationOutput::InputAddressed{
-                        .path = std::move(outPath),
-                    });
-            }
-            break;
-            ;
-        case DrvHash::Kind::Deferred:
-            for (auto & i : outputs) {
-                drv.outputs.insert_or_assign(i, DerivationOutput::Deferred{});
-            }
-        }
+        drv.fillInOutputPaths(*state.store);
     }
 
     /* Write the resulting term into the Nix store directory. */
@@ -1842,8 +1821,10 @@ static RegisterPrimOp primop_derivationStrict(
    ‘out’. */
 static void prim_placeholder(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
-    v.mkString(hashPlaceholder(
-        state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder")));
+    v.mkString(
+        hashPlaceholder(state.forceStringNoCtx(
+            *args[0], pos, "while evaluating the first argument passed to builtins.placeholder")),
+        state.mem);
 }
 
 static RegisterPrimOp primop_placeholder({
@@ -2027,7 +2008,7 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
     state.forceValue(*args[0], pos);
     if (args[0]->type() == nPath) {
         auto path = args[0]->path();
-        v.mkPath(path.path.isRoot() ? path : path.parent());
+        v.mkPath(path.path.isRoot() ? path : path.parent(), state.mem);
     } else {
         NixStringContext context;
         auto path = state.coerceToString(
@@ -2144,7 +2125,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
     auto path =
         state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
 
-    v.mkPath(state.findFile(lookupPath, path, pos));
+    v.mkPath(state.findFile(lookupPath, path, pos), state.mem);
 }
 
 static RegisterPrimOp primop_findFile(
@@ -2293,7 +2274,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
 
     auto path = realisePath(state, pos, *args[1]);
 
-    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem);
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -2382,7 +2363,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Val
             // detailed node info quickly in this case we produce a thunk to
             // query the file type lazily.
             auto epath = state.allocValue();
-            epath->mkPath(path / name);
+            epath->mkPath(path / name, state.mem);
             if (!readFileType)
                 readFileType = &state.getBuiltin("readFileType");
             attr.mkApp(readFileType, epath);
@@ -2763,7 +2744,7 @@ bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosId
     /* Call the filter function.  The first argument is the path, the
        second is a string indicating the type of the file. */
     Value arg1;
-    arg1.mkString(path.path.abs());
+    arg1.mkString(path.path.abs(), mem);
 
     // assert that type is not "unknown"
     Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))};
@@ -4541,7 +4522,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value ** args,
     auto s =
         state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
-    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false), state.mem);
 }
 
 static RegisterPrimOp primop_hashString({
@@ -4574,7 +4555,7 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value ** args,
     HashFormat hf = parseHashFormat(
         state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
 
-    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
+    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI), state.mem);
 }
 
 static RegisterPrimOp primop_convertHash({
@@ -4992,8 +4973,8 @@ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value ** args
         state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName");
     DrvName parsed(name);
     auto attrs = state.buildBindings(2);
-    attrs.alloc(state.s.name).mkString(parsed.name);
-    attrs.alloc("version").mkString(parsed.version);
+    attrs.alloc(state.s.name).mkString(parsed.name, state.mem);
+    attrs.alloc("version").mkString(parsed.version, state.mem);
     v.mkAttrs(attrs);
 }
 
@@ -5048,7 +5029,7 @@ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value ** args
     }
     auto list = state.buildList(components.size());
     for (const auto & [n, component] : enumerate(components))
-        (list[n] = state.allocValue())->mkString(std::move(component));
+        (list[n] = state.allocValue())->mkString(std::move(component), state.mem);
     v.mkList(list);
 }
 
@@ -5192,7 +5173,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
         });
 
     if (!settings.pureEval)
-        v.mkString(settings.getCurrentSystem());
+        v.mkString(settings.getCurrentSystem(), mem);
     addConstant(
         "__currentSystem",
         v,
@@ -5224,7 +5205,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
             .impureOnly = true,
         });
 
-    v.mkString(nixVersion);
+    v.mkString(nixVersion, mem);
     addConstant(
         "__nixVersion",
         v,
@@ -5249,7 +5230,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
         )",
         });
 
-    v.mkString(store->storeDir);
+    v.mkString(store->storeDir, mem);
     addConstant(
         "__storeDir",
         v,
@@ -5314,8 +5295,8 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
     auto list = buildList(lookupPath.elements.size());
     for (const auto & [n, i] : enumerate(lookupPath.elements)) {
         auto attrs = buildBindings(2);
-        attrs.alloc("path").mkString(i.path.s);
-        attrs.alloc("prefix").mkString(i.prefix.s);
+        attrs.alloc("path").mkString(i.path.s, mem);
+        attrs.alloc("prefix").mkString(i.prefix.s, mem);
         (list[n] = allocValue())->mkAttrs(attrs);
     }
     v.mkList(list);

src/libexpr/primops/context.cc

@@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
     NixStringContext context;
     auto s = state.coerceToString(
         pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
-    v.mkString(*s);
+    v.mkString(*s, state.mem);
 }
 
 static RegisterPrimOp primop_unsafeDiscardStringContext({
@@ -218,7 +218,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
         if (!info.second.outputs.empty()) {
             auto list = state.buildList(info.second.outputs.size());
             for (const auto & [i, output] : enumerate(info.second.outputs))
-                (list[i] = state.allocValue())->mkString(output);
+                (list[i] = state.allocValue())->mkString(output, state.mem);
             infoAttrs.alloc(state.s.outputs).mkList(list);
         }
         attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);

src/libexpr/primops/fetchMercurial.cc

@@ -86,12 +86,12 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
     auto attrs2 = state.buildBindings(8);
     state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));
     if (input2.getRef())
-        attrs2.alloc("branch").mkString(*input2.getRef());
+        attrs2.alloc("branch").mkString(*input2.getRef(), state.mem);
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
     auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
-    attrs2.alloc("rev").mkString(rev2.gitRev());
-    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
+    attrs2.alloc("rev").mkString(rev2.gitRev(), state.mem);
+    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12), state.mem);
     if (auto revCount = input2.getRevCount())
         attrs2.alloc("revCount").mkInt(*revCount);
     v.mkAttrs(attrs2);

src/libexpr/primops/fetchTree.cc

@@ -35,7 +35,7 @@ void emitTreeAttrs(
     // FIXME: support arbitrary input attributes.
 
     if (auto narHash = input.getNarHash())
-        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true), state.mem);
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
@@ -43,13 +43,13 @@ void emitTreeAttrs(
     if (!forceDirty) {
 
         if (auto rev = input.getRev()) {
-            attrs.alloc("rev").mkString(rev->gitRev());
-            attrs.alloc("shortRev").mkString(rev->gitShortRev());
+            attrs.alloc("rev").mkString(rev->gitRev(), state.mem);
+            attrs.alloc("shortRev").mkString(rev->gitShortRev(), state.mem);
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
             auto emptyHash = Hash(HashAlgorithm::SHA1);
-            attrs.alloc("rev").mkString(emptyHash.gitRev());
-            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
+            attrs.alloc("rev").mkString(emptyHash.gitRev(), state.mem);
+            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem);
         }
 
         if (auto revCount = input.getRevCount())
@@ -59,13 +59,14 @@ void emitTreeAttrs(
     }
 
     if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) {
-        attrs.alloc("dirtyRev").mkString(*dirtyRev);
-        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"));
+        attrs.alloc("dirtyRev").mkString(*dirtyRev, state.mem);
+        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"), state.mem);
     }
 
     if (auto lastModified = input.getLastModified()) {
         attrs.alloc("lastModified").mkInt(*lastModified);
-        attrs.alloc("lastModifiedDate").mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
+        attrs.alloc("lastModifiedDate")
+            .mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")), state.mem);
     }
 
     v.mkAttrs(attrs);

src/libexpr/primops/fromTOML.cc

@@ -126,7 +126,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
         case toml::value_t::string: {
             auto s = toml::get<std::string_view>(t);
             forceNoNullByte(s);
-            v.mkString(s);
+            v.mkString(s, state.mem);
         } break;
         case toml::value_t::local_datetime:
         case toml::value_t::offset_datetime:
@@ -142,7 +142,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 s << t;
                 auto str = s.view();
                 forceNoNullByte(str);
-                attrs.alloc("value").mkString(str);
+                attrs.alloc("value").mkString(str, state.mem);
                 v.mkAttrs(attrs);
             } else {
                 throw std::runtime_error("Dates and times are not supported");

src/libexpr/value-to-json.cc

@@ -16,6 +16,8 @@ json printValueAsJSON(
 {
     checkInterrupt();
 
+    auto _level = state.addCallDepth(pos);
+
     if (strict)
         state.forceValue(v, pos);
 

src/libfetchers-tests/git-utils.cc

@@ -92,7 +92,7 @@ TEST_F(GitUtilsTest, sink_basic)
     // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
 
     auto result = repo->dereferenceSingletonDirectory(sink->flush());
-    auto accessor = repo->getAccessor(result, false, getRepoName());
+    auto accessor = repo->getAccessor(result, {}, getRepoName());
     auto entries = accessor->readDirectory(CanonPath::root);
     ASSERT_EQ(entries.size(), 5u);
     ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");

src/libfetchers/fetchers.cc

@@ -495,9 +495,9 @@ void InputScheme::clone(
 
     Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
 
-    auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); });
-
-    restorePath(destDir, *source);
+    RestoreSink sink(/*startFsync=*/false);
+    sink.dstPath = destDir;
+    copyRecursive(*accessor, CanonPath::root, sink, CanonPath::root);
 }
 
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const

src/libfetchers/git-utils.cc

@@ -541,14 +541,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 
     /**
-     * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
+     * A 'GitSourceAccessor' with no regard for export-ignore.
      */
-    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, bool smudgeLfs = false);
+    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, const GitAccessorOptions & options);
 
     ref<SourceAccessor>
-    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) override;
+    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) override;
 
-    ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
+    ref<SourceAccessor>
+    getAccessor(const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError e) override;
 
     ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
 
@@ -668,7 +669,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
     {
-        auto accessor = getAccessor(treeHash, false, "");
+        auto accessor = getAccessor(treeHash, {}, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
@@ -716,15 +717,17 @@ struct GitSourceAccessor : SourceAccessor
         ref<GitRepoImpl> repo;
         Object root;
         std::optional<lfs::Fetch> lfsFetch = std::nullopt;
+        GitAccessorOptions options;
     };
 
     Sync<State> state_;
 
-    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
+    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, const GitAccessorOptions & options)
         : state_{State{
               .repo = repo_,
               .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
-              .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+              .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+              .options = options,
           }}
     {
     }
@@ -1254,26 +1257,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, bool smudgeLfs)
+ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
+    return make_ref<GitSourceAccessor>(self, rev, options);
 }
 
 ref<SourceAccessor>
-GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs)
+GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
+    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
-    if (exportIgnore)
+    if (options.exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
     else
         return rawGitAccessor;
 }
 
-ref<SourceAccessor>
-GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
+ref<SourceAccessor> GitRepoImpl::getAccessor(
+    const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create(
@@ -1283,10 +1286,9 @@ GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllow
                                            boost::unordered_flat_set<CanonPath>{CanonPath::root},
                                            std::move(makeNotAllowedError))
                                            .cast<SourceAccessor>();
-    if (exportIgnore)
-        return make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
-    else
-        return fileAccessor;
+    if (options.exportIgnore)
+        fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
+    return fileAccessor;
 }
 
 ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()
@@ -1299,7 +1301,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
 
-    auto accessor = getAccessor(rev, exportIgnore, "");
+    auto accessor = getAccessor(rev, {.exportIgnore = exportIgnore}, "");
     if (!accessor->pathExists(modulesFile))
         return {};
 
@@ -1316,7 +1318,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
 
     std::vector<std::tuple<Submodule, Hash>> result;
 
-    auto rawAccessor = getRawAccessor(rev);
+    auto rawAccessor = getRawAccessor(rev, {});
 
     for (auto & submodule : parseSubmodules(pathTemp)) {
         /* Filter out .gitmodules entries that don't exist or are not
@@ -1332,10 +1334,8 @@ namespace fetchers {
 
 ref<GitRepo> Settings::getTarballCache() const
 {
-    auto tarballCache(_tarballCache.lock());
-    if (!*tarballCache)
-        *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
-    return ref<GitRepo>(*tarballCache);
+    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
+    return GitRepo::openRepo(repoDir, true, true);
 }
 
 } // namespace fetchers

src/libfetchers/git.cc

@@ -900,7 +900,8 @@ struct GitInputScheme : InputScheme
 
         bool exportIgnore = getExportIgnoreAttr(input);
         bool smudgeLfs = getLfsAttr(input);
-        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
+        auto accessor = repo->getAccessor(
+            rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»");
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -967,7 +968,7 @@ struct GitInputScheme : InputScheme
         auto exportIgnore = getExportIgnoreAttr(input);
 
         ref<SourceAccessor> accessor =
-            repo->getAccessor(repoInfo.workdirInfo, exportIgnore, makeNotAllowedError(repoPath));
+            repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath));
 
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the

src/libfetchers/github.cc

@@ -351,7 +351,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
         auto accessor =
-            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
+            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string() + "»");
 
         return {accessor, input};
     }

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -135,8 +135,6 @@ struct Settings : public Config
 
 private:
     mutable Sync<std::shared_ptr<Cache>> _cache;
-
-    mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
 };
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -22,6 +22,12 @@ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
     virtual Hash flush() = 0;
 };
 
+struct GitAccessorOptions
+{
+    bool exportIgnore = false;
+    bool smudgeLfs = false;
+};
+
 struct GitRepo
 {
     virtual ~GitRepo() {}
@@ -89,10 +95,10 @@ struct GitRepo
     virtual bool hasObject(const Hash & oid) = 0;
 
     virtual ref<SourceAccessor>
-    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) = 0;
+    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0;
 
-    virtual ref<SourceAccessor>
-    getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
+    virtual ref<SourceAccessor> getAccessor(
+        const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) = 0;
 
     virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
 

src/libfetchers/include/nix/fetchers/registry.hh

@@ -13,8 +13,6 @@ namespace nix::fetchers {
 
 struct Registry
 {
-    const Settings & settings;
-
     enum RegistryType {
         Flag = 0,
         User = 1,
@@ -34,9 +32,8 @@ struct Registry
 
     std::vector<Entry> entries;
 
-    Registry(const Settings & settings, RegistryType type)
-        : settings{settings}
-        , type{type}
+    Registry(RegistryType type)
+        : type{type}
     {
     }
 
@@ -59,7 +56,7 @@ Path getUserRegistryPath();
 
 Registries getRegistries(const Settings & settings, Store & store);
 
-void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs);
+void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs);
 
 enum class UseRegistries : int {
     No,

src/libfetchers/registry.cc

@@ -14,10 +14,10 @@ std::shared_ptr<Registry> Registry::read(const Settings & settings, const Source
 {
     debug("reading registry '%s'", path);
 
-    auto registry = std::make_shared<Registry>(settings, type);
+    auto registry = std::make_shared<Registry>(type);
 
     if (!path.pathExists())
-        return std::make_shared<Registry>(settings, type);
+        return std::make_shared<Registry>(type);
 
     try {
 
@@ -125,15 +125,15 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
     return customRegistry;
 }
 
-std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
+std::shared_ptr<Registry> getFlagRegistry()
 {
-    static auto flagRegistry = std::make_shared<Registry>(settings, Registry::Flag);
+    static auto flagRegistry = std::make_shared<Registry>(Registry::Flag);
     return flagRegistry;
 }
 
-void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs)
+void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs)
 {
-    getFlagRegistry(settings)->add(from, to, extraAttrs);
+    getFlagRegistry()->add(from, to, extraAttrs);
 }
 
 static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, Store & store)
@@ -141,7 +141,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
     static auto reg = [&]() {
         auto path = settings.flakeRegistry.get();
         if (path == "") {
-            return std::make_shared<Registry>(settings, Registry::Global); // empty registry
+            return std::make_shared<Registry>(Registry::Global); // empty registry
         }
 
         return Registry::read(
@@ -165,7 +165,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
 Registries getRegistries(const Settings & settings, Store & store)
 {
     Registries registries;
-    registries.push_back(getFlagRegistry(settings));
+    registries.push_back(getFlagRegistry());
     registries.push_back(getUserRegistry(settings));
     registries.push_back(getSystemRegistry(settings));
     registries.push_back(getGlobalRegistry(settings, store));

src/libfetchers/tarball.cc

@@ -136,7 +136,7 @@ static DownloadTarballResult downloadTarball_(
             .treeHash = treeHash,
             .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
             .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
-            .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix),
+            .accessor = settings.getTarballCache()->getAccessor(treeHash, {}, displayPrefix),
         };
     };
 

src/libflake-c/nix_api_flake.cc

@@ -200,7 +200,7 @@ nix_value * nix_locked_flake_get_output_attrs(
     nix_clear_err(context);
     try {
         auto v = nix_alloc_value(context, evalState);
-        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value);
+        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value);
         return v;
     }
     NIXC_CATCH_ERRS_NULL

src/libflake/flake-primops.cc

@@ -93,7 +93,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
         auto & vv = binds.alloc(s);
         std::visit(
             overloaded{
-                [&vv](const std::string & value) { vv.mkString(value); },
+                [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); },
                 [&vv](const uint64_t & value) { vv.mkInt(value); },
                 [&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }},
             value);
@@ -156,7 +156,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
         }
     }
     auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs);
-    v.mkString(flakeRef.to_string());
+    v.mkString(flakeRef.to_string(), state.mem);
 }
 
 nix::PrimOp flakeRefToString({

src/libflake/flake.cc

@@ -956,7 +956,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
         auto key = keyMap.find(node);
         assert(key != keyMap.end());
 
-        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel());
+        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem);
 
         overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
     }
@@ -966,7 +966,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
     Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix"));
 
     auto vLocks = state.allocValue();
-    vLocks->mkString(lockFileStr);
+    vLocks->mkString(lockFileStr, state.mem);
 
     auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree");
     assert(vFetchFinalTree);

src/libmain/progress-bar.cc

@@ -470,7 +470,8 @@ public:
         std::string res;
 
         auto renderActivity =
-            [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
+            [&] [[nodiscard]] (
+                ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
                 auto & act = state.activitiesByType[type];
                 uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
                 for (auto & j : act.its) {
@@ -514,7 +515,7 @@ public:
                 return s;
             };
 
-        auto renderSizeActivity = [&](ActivityType type, const std::string & itemFmt = "%s") {
+        auto renderSizeActivity = [&] [[nodiscard]] (ActivityType type, const std::string & itemFmt = "%s") {
             auto & act = state.activitiesByType[type];
             uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
             for (auto & j : act.its) {
@@ -573,14 +574,17 @@ public:
             return s;
         };
 
+        auto maybeAppendToResult = [&](std::string_view s) {
+            if (s.empty())
+                return;
+            if (!res.empty())
+                res += ", ";
+            res += s;
+        };
+
         auto showActivity =
             [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
-                auto s = renderActivity(type, itemFmt, numberFmt, unit);
-                if (s.empty())
-                    return;
-                if (!res.empty())
-                    res += ", ";
-                res += s;
+                maybeAppendToResult(renderActivity(type, itemFmt, numberFmt, unit));
             };
 
         showActivity(actBuilds, "%s built");
@@ -602,7 +606,7 @@ public:
             }
         }
 
-        renderSizeActivity(actFileTransfer, "%s DL");
+        maybeAppendToResult(renderSizeActivity(actFileTransfer, "%s DL"));
 
         {
             auto s = renderActivity(actOptimiseStore, "%s paths optimised");

src/libstore-c/nix_api_store.cc

@@ -1,3 +1,6 @@
+#include <cstring>
+#include <span>
+
 #include "nix_api_store.h"
 #include "nix_api_store_internal.h"
 #include "nix_api_util.h"
@@ -8,6 +11,7 @@
 #include "nix/store/store-open.hh"
 #include "nix/store/build-result.hh"
 #include "nix/store/local-fs-store.hh"
+#include "nix/util/base-nix-32.hh"
 
 #include "nix/store/globals.hh"
 
@@ -215,7 +219,65 @@ void nix_derivation_free(nix_derivation * drv)
 
 StorePath * nix_store_path_clone(const StorePath * p)
 {
-    return new StorePath{p->path};
+    try {
+        return new StorePath{p->path};
+    } catch (...) {
+        return nullptr;
+    }
+}
+
+} // extern "C"
+
+template<size_t S>
+static auto to_cpp_array(const uint8_t (&r)[S])
+{
+    return reinterpret_cast<const std::array<std::byte, S> &>(r);
+}
+
+extern "C" {
+
+nix_err
+nix_store_path_hash(nix_c_context * context, const StorePath * store_path, nix_store_path_hash_part * hash_part_out)
+{
+    try {
+        auto hashPart = store_path->path.hashPart();
+        // Decode from Nix32 (base32) encoding to raw bytes
+        auto decoded = nix::BaseNix32::decode(hashPart);
+
+        assert(decoded.size() == sizeof(hash_part_out->bytes));
+        std::memcpy(hash_part_out->bytes, decoded.data(), sizeof(hash_part_out->bytes));
+        return NIX_OK;
+    }
+    NIXC_CATCH_ERRS
+}
+
+StorePath * nix_store_create_from_parts(
+    nix_c_context * context, const nix_store_path_hash_part * hash, const char * name, size_t name_len)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        // Encode the 20 raw bytes to Nix32 (base32) format
+        auto hashStr = nix::BaseNix32::encode(std::span<const std::byte>{to_cpp_array(hash->bytes)});
+
+        // Construct the store path basename: <hash>-<name>
+        std::string baseName;
+        baseName += hashStr;
+        baseName += "-";
+        baseName += std::string_view{name, name_len};
+
+        return new StorePath{nix::StorePath(std::move(baseName))};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_derivation * nix_derivation_clone(const nix_derivation * d)
+{
+    try {
+        return new nix_derivation{d->drv};
+    } catch (...) {
+        return nullptr;
+    }
 }
 
 nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json)
@@ -223,17 +285,25 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        auto drv = static_cast<nix::Derivation>(nlohmann::json::parse(json));
-
-        auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true);
-
-        drv.checkInvariants(*store->ptr, drvPath);
-
-        return new nix_derivation{drv};
+        return new nix_derivation{nix::Derivation::parseJsonAndValidate(*store->ptr, nlohmann::json::parse(json))};
     }
     NIXC_CATCH_ERRS_NULL
 }
 
+nix_err nix_derivation_to_json(
+    nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto result = static_cast<nlohmann::json>(drv->drv).dump();
+        if (callback) {
+            callback(result.data(), result.size(), userdata);
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
 StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation)
 {
     if (context)
@@ -258,4 +328,14 @@ nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store
     NIXC_CATCH_ERRS
 }
 
+nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        return new nix_derivation{store->ptr->derivationFromPath(path->path)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
 } // extern "C"

src/libstore-c/nix_api_store.h

@@ -106,7 +106,7 @@ nix_err
 nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data);
 
 /**
- * @brief Parse a Nix store path into a StorePath
+ * @brief Parse a Nix store path that includes the store dir into a StorePath
  *
  * @note Don't forget to free this path using nix_store_path_free()!
  * @param[out] context Optional, stores error information
@@ -188,9 +188,16 @@ nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_cal
 /**
  * @brief Create a `nix_derivation` from a JSON representation of that derivation.
  *
+ * @note Unlike `nix_derivation_to_json`, this needs a `Store`. This is because
+ * over time we expect the internal representation of derivations in Nix to
+ * differ from accepted derivation formats. The store argument is here to help
+ * any logic needed to convert from JSON to the internal representation, in
+ * excess of just parsing.
+ *
  * @param[out] context Optional, stores error information.
  * @param[in] store nix store reference.
  * @param[in] json JSON of the derivation as a string.
+ * @return A new derivation, or NULL on error. Free with `nix_derivation_free` when done using the `nix_derivation`.
  */
 nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json);
 
@@ -242,6 +249,16 @@ nix_err nix_store_get_fs_closure(
     void * userdata,
     void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path));
 
+/**
+ * @brief Returns the derivation associated with the store path
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] store The nix store
+ * @param[in] path The nix store path
+ * @return A new derivation, or NULL on error. Free with `nix_derivation_free` when done using the `nix_derivation`.
+ */
+nix_derivation * nix_store_drv_from_store_path(nix_c_context * context, Store * store, const StorePath * path);
+
 // cffi end
 #ifdef __cplusplus
 }

src/libstore-c/nix_api_store/derivation.h

@@ -20,6 +20,14 @@ extern "C" {
 /** @brief Nix Derivation */
 typedef struct nix_derivation nix_derivation;
 
+/**
+ * @brief Copy a `nix_derivation`
+ *
+ * @param[in] d the derivation to copy
+ * @return a new `nix_derivation`
+ */
+nix_derivation * nix_derivation_clone(const nix_derivation * d);
+
 /**
  * @brief Deallocate a `nix_derivation`
  *
@@ -28,6 +36,17 @@ typedef struct nix_derivation nix_derivation;
  */
 void nix_derivation_free(nix_derivation * drv);
 
+/**
+ * @brief Gets the derivation as a JSON string
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] drv The derivation
+ * @param[in] callback Called with the JSON string
+ * @param[in] userdata Arbitrary data passed to the callback
+ */
+nix_err nix_derivation_to_json(
+    nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata);
+
 // cffi end
 #ifdef __cplusplus
 }

src/libstore-c/nix_api_store/store_path.h

@@ -10,6 +10,9 @@
  * @brief Store path operations
  */
 
+#include <stddef.h>
+#include <stdint.h>
+
 #include "nix_api_util.h"
 
 #ifdef __cplusplus
@@ -44,6 +47,45 @@ void nix_store_path_free(StorePath * p);
  */
 void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
 
+/**
+ * @brief A store path hash
+ *
+ * Once decoded from "nix32" encoding, a store path hash is 20 raw bytes.
+ */
+typedef struct nix_store_path_hash_part
+{
+    uint8_t bytes[20];
+} nix_store_path_hash_part;
+
+/**
+ * @brief Get the path hash (e.g. "<hash>" in /nix/store/<hash>-<name>)
+ *
+ * The hash is returned as raw bytes, decoded from "nix32" encoding.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] store_path the path to get the hash from
+ * @param[out] hash_part_out the decoded hash as 20 raw bytes
+ * @return NIX_OK on success, error code on failure
+ */
+nix_err
+nix_store_path_hash(nix_c_context * context, const StorePath * store_path, nix_store_path_hash_part * hash_part_out);
+
+/**
+ * @brief Create a StorePath from its constituent parts (hash and name)
+ *
+ * This function constructs a store path from a hash and name, without needing
+ * a Store reference or the store directory prefix.
+ *
+ * @note Don't forget to free this path using nix_store_path_free()!
+ * @param[out] context Optional, stores error information
+ * @param[in] hash The store path hash (20 raw bytes)
+ * @param[in] name The store path name (the part after the hash)
+ * @param[in] name_len Length of the name string
+ * @return owned store path, NULL on error
+ */
+StorePath * nix_store_create_from_parts(
+    nix_c_context * context, const nix_store_path_hash_part * hash, const char name[/*name_len*/], size_t name_len);
+
 // cffi end
 #ifdef __cplusplus
 }

src/libstore-test-support/include/nix/store/tests/nix_api_store.hh

@@ -50,7 +50,8 @@ protected:
 #else
         // resolve any symlinks in i.e. on macOS /tmp -> /private/tmp
         // because this is not allowed for a nix store.
-        auto tmpl = nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", true);
+        auto tmpl =
+            nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", std::nullopt, true);
         nixDir = mkdtemp((char *) tmpl.c_str());
 #endif
 

src/libstore-tests/data/derivation/ca/all_set.json

@@ -4,10 +4,13 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+      {
+        "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+        "output": "out"
+      }
     ],
     "refs2": [
-      "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+      "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -20,18 +23,36 @@
   "outputChecks": {
     "forAllOutputs": {
       "allowedReferences": [
-        "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+        {
+          "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+          "output": "out"
+        }
       ],
       "allowedRequisites": [
-        "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
-        "bin"
+        {
+          "drvPath": "self",
+          "output": "bin"
+        },
+        {
+          "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+          "output": "dev"
+        }
       ],
       "disallowedReferences": [
-        "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
-        "dev"
+        {
+          "drvPath": "self",
+          "output": "dev"
+        },
+        {
+          "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+          "output": "out"
+        }
       ],
       "disallowedRequisites": [
-        "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
+        {
+          "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+          "output": "dev"
+        }
       ],
       "ignoreSelfRefs": true,
       "maxClosureSize": null,

src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json

@@ -4,10 +4,13 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+      {
+        "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+        "output": "out"
+      }
     ],
     "refs2": [
-      "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+      "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -23,11 +26,20 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
-          "dev"
+          {
+            "drvPath": "self",
+            "output": "dev"
+          },
+          {
+            "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+            "output": "out"
+          }
         ],
         "disallowedRequisites": [
-          "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
+          {
+            "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+            "output": "dev"
+          }
         ],
         "ignoreSelfRefs": false,
         "maxClosureSize": null,
@@ -44,11 +56,20 @@
       },
       "out": {
         "allowedReferences": [
-          "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+          {
+            "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+            "output": "out"
+          }
         ],
         "allowedRequisites": [
-          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
-          "bin"
+          {
+            "drvPath": "self",
+            "output": "bin"
+          },
+          {
+            "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+            "output": "dev"
+          }
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/data/derivation/ia/all_set.json

@@ -4,10 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+      "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
     ],
     "refs2": [
-      "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+      "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -20,18 +20,24 @@
   "outputChecks": {
     "forAllOutputs": {
       "allowedReferences": [
-        "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+        "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
       ],
       "allowedRequisites": [
-        "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
-        "bin"
+        {
+          "drvPath": "self",
+          "output": "bin"
+        },
+        "z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
       ],
       "disallowedReferences": [
-        "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
-        "dev"
+        {
+          "drvPath": "self",
+          "output": "dev"
+        },
+        "r5cff30838majxk5mp3ip2diffi8vpaj-bar"
       ],
       "disallowedRequisites": [
-        "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
+        "9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
       ],
       "ignoreSelfRefs": true,
       "maxClosureSize": null,

src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json

@@ -4,10 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+      "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
     ],
     "refs2": [
-      "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+      "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -23,11 +23,14 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
-          "dev"
+          {
+            "drvPath": "self",
+            "output": "dev"
+          },
+          "r5cff30838majxk5mp3ip2diffi8vpaj-bar"
         ],
         "disallowedRequisites": [
-          "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
+          "9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
         ],
         "ignoreSelfRefs": false,
         "maxClosureSize": null,
@@ -44,11 +47,14 @@
       },
       "out": {
         "allowedReferences": [
-          "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+          "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
         ],
         "allowedRequisites": [
-          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
-          "bin"
+          {
+            "drvPath": "self",
+            "output": "bin"
+          },
+          "z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/data/derivation/invariants/bad-depends-on-drv-pre.json

@@ -0,0 +1,27 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "InputAddressed throws when should be deferred",
+    "out": ""
+  },
+  "inputs": {
+    "drvs": {
+      "lg4c4b8r9hlczwprl6kgnzfd9mc1xmkk-dependency.drv": {
+        "dynamicOutputs": {},
+        "outputs": [
+          "out"
+        ]
+      }
+    },
+    "srcs": []
+  },
+  "name": "depends-on-drv",
+  "outputs": {
+    "out": {
+      "path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
+    }
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/bad-env-var.json

@@ -0,0 +1,18 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Wrong env var value throws error",
+    "out": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
+  },
+  "inputs": {
+    "drvs": {},
+    "srcs": []
+  },
+  "name": "bad-env-var",
+  "outputs": {
+    "out": {}
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/bad-path.json

@@ -0,0 +1,20 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Wrong InputAddressed path throws error",
+    "out": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
+  },
+  "inputs": {
+    "drvs": {},
+    "srcs": []
+  },
+  "name": "bad-path",
+  "outputs": {
+    "out": {
+      "path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-wrong-name"
+    }
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/depends-on-drv-pre.json

@@ -0,0 +1,25 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Deferred stays deferred with CA dependencies",
+    "out": ""
+  },
+  "inputs": {
+    "drvs": {
+      "lg4c4b8r9hlczwprl6kgnzfd9mc1xmkk-dependency.drv": {
+        "dynamicOutputs": {},
+        "outputs": [
+          "out"
+        ]
+      }
+    },
+    "srcs": []
+  },
+  "name": "depends-on-drv",
+  "outputs": {
+    "out": {}
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/filled-in-deferred-empty-env-var-post.json

@@ -0,0 +1,20 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Fill in deferred output with empty env var",
+    "out": "/nix/store/bilpz1nq8qi9r3bzsp72n34yjgqg43ws-filled-in-deferred-empty-env-var"
+  },
+  "inputs": {
+    "drvs": {},
+    "srcs": []
+  },
+  "name": "filled-in-deferred-empty-env-var",
+  "outputs": {
+    "out": {
+      "path": "bilpz1nq8qi9r3bzsp72n34yjgqg43ws-filled-in-deferred-empty-env-var"
+    }
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/filled-in-deferred-empty-env-var-pre.json

@@ -0,0 +1,18 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Fill in deferred output with empty env var",
+    "out": ""
+  },
+  "inputs": {
+    "drvs": {},
+    "srcs": []
+  },
+  "name": "filled-in-deferred-empty-env-var",
+  "outputs": {
+    "out": {}
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/filled-in-deferred-no-env-var-post.json

@@ -0,0 +1,20 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Fill in deferred with missing env var",
+    "out": "/nix/store/wpk9qrgg77fyswhailap0gicgw98izx9-filled-in-deferred-no-env-var"
+  },
+  "inputs": {
+    "drvs": {},
+    "srcs": []
+  },
+  "name": "filled-in-deferred-no-env-var",
+  "outputs": {
+    "out": {
+      "path": "wpk9qrgg77fyswhailap0gicgw98izx9-filled-in-deferred-no-env-var"
+    }
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/filled-in-deferred-no-env-var-pre.json

@@ -0,0 +1,17 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Fill in deferred with missing env var"
+  },
+  "inputs": {
+    "drvs": {},
+    "srcs": []
+  },
+  "name": "filled-in-deferred-no-env-var",
+  "outputs": {
+    "out": {}
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/derivation/invariants/filled-in-idempotent.json

@@ -0,0 +1,20 @@
+{
+  "args": [],
+  "builder": "/bin/sh",
+  "env": {
+    "__doc": "Correct path stays unchanged",
+    "out": "/nix/store/w4bk7hpyxzgy2gx8fsa8f952435pll3i-filled-in-already"
+  },
+  "inputs": {
+    "drvs": {},
+    "srcs": []
+  },
+  "name": "filled-in-already",
+  "outputs": {
+    "out": {
+      "path": "w4bk7hpyxzgy2gx8fsa8f952435pll3i-filled-in-already"
+    }
+  },
+  "system": "x86_64-linux",
+  "version": 4
+}

src/libstore-tests/data/dummy-store/empty.json

@@ -0,0 +1,8 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {}
+}

src/libstore-tests/data/dummy-store/one-derivation.json

@@ -0,0 +1,22 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {
+    "rlqjbbb65ggcx9hy577hvnn929wz1aj0-foo.drv": {
+      "args": [],
+      "builder": "",
+      "env": {},
+      "inputs": {
+        "drvs": {},
+        "srcs": []
+      },
+      "name": "foo",
+      "outputs": {},
+      "system": "",
+      "version": 4
+    }
+  }
+}

src/libstore-tests/data/dummy-store/one-flat-file.json

@@ -0,0 +1,38 @@
+{
+  "buildTrace": {},
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {
+    "5hizn7xyyrhxr0k2magvxl5ccvk0ci9n-my-file": {
+      "contents": {
+        "contents": "asdf",
+        "executable": false,
+        "type": "regular"
+      },
+      "info": {
+        "ca": {
+          "hash": {
+            "algorithm": "sha256",
+            "format": "base64",
+            "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
+          },
+          "method": "nar"
+        },
+        "deriver": null,
+        "narHash": {
+          "algorithm": "sha256",
+          "format": "base64",
+          "hash": "f1eduuSIYC1BofXA1tycF79Ai2NSMJQtUErx5DxLYSU="
+        },
+        "narSize": 120,
+        "references": [],
+        "registrationTime": null,
+        "signatures": [],
+        "ultimate": false,
+        "version": 2
+      }
+    }
+  },
+  "derivations": {}
+}

src/libstore-tests/data/dummy-store/one-realisation.json

@@ -0,0 +1,16 @@
+{
+  "buildTrace": {
+    "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=": {
+      "out": {
+        "dependentRealisations": {},
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+        "signatures": []
+      }
+    }
+  },
+  "config": {
+    "store": "/nix/store"
+  },
+  "contents": {},
+  "derivations": {}
+}

src/libstore-tests/derivation-advanced-attrs.cc

@@ -3,7 +3,7 @@
 
 #include "nix/util/experimental-features.hh"
 #include "nix/store/derivations.hh"
-#include "nix/store/derivations.hh"
+#include "nix/store/derived-path.hh"
 #include "nix/store/derivation-options.hh"
 #include "nix/store/parsed-derivations.hh"
 #include "nix/util/types.hh"
@@ -17,7 +17,7 @@ namespace nix {
 using namespace nlohmann;
 
 class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>,
-                                    public JsonCharacterizationTest<DerivationOptions>,
+                                    public JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>,
                                     public LibStoreTest
 {
 protected:
@@ -42,7 +42,8 @@ public:
     {
         this->readTest(fileName, [&](auto encoded) {
             auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
-            DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+            auto options = derivationOptionsFromStructuredAttrs(
+                *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
             EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures);
         });
     }
@@ -51,11 +52,14 @@ public:
      * Helper function to test DerivationOptions parsing and comparison
      */
     void testDerivationOptions(
-        const std::string & fileName, const DerivationOptions & expected, const StringSet & expectedSystemFeatures)
+        const std::string & fileName,
+        const DerivationOptions<SingleDerivedPath> & expected,
+        const StringSet & expectedSystemFeatures)
     {
         this->readTest(fileName, [&](auto encoded) {
             auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
-            DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+            auto options = derivationOptionsFromStructuredAttrs(
+                *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
             EXPECT_EQ(options, expected);
             EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures);
@@ -131,22 +135,38 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute
  * Since these are both repeated and sensative opaque values, it makes
  * sense to give them names in this file.
  */
-static std::string pathFoo = "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
-                   pathFooDev = "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
-                   pathBar = "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
-                   pathBarDev = "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
-                   pathBarDrvIA = "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
-                   pathBarDrvCA = "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
-                   placeholderFoo = "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
-                   placeholderFooDev = "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
-                   placeholderBar = "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
-                   placeholderBarDev = "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8";
+static SingleDerivedPath
+    pathFoo = SingleDerivedPath::Opaque{StorePath{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}},
+    pathFooDev = SingleDerivedPath::Opaque{StorePath{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}},
+    pathBar = SingleDerivedPath::Opaque{StorePath{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}},
+    pathBarDev = SingleDerivedPath::Opaque{StorePath{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}},
+    pathBarDrvIA = SingleDerivedPath::Opaque{StorePath{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}},
+    pathBarDrvCA = SingleDerivedPath::Opaque{StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}},
+    placeholderFoo =
+        SingleDerivedPath::Built{
+            .drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
+            .output = "out",
+        },
+    placeholderFooDev =
+        SingleDerivedPath::Built{
+            .drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
+            .output = "dev",
+        },
+    placeholderBar =
+        SingleDerivedPath::Built{
+            .drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
+            .output = "out",
+        },
+    placeholderBarDev = SingleDerivedPath::Built{
+        .drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
+        .output = "dev",
+    };
 
-using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph);
+using ExportReferencesMap = decltype(DerivationOptions<SingleDerivedPath>::exportReferencesGraph);
 
-static const DerivationOptions advancedAttributes_defaults = {
+static const DerivationOptions<SingleDerivedPath> advancedAttributes_defaults = {
     .outputChecks =
-        DerivationOptions::OutputChecks{
+        DerivationOptions<SingleDerivedPath>::OutputChecks{
             .ignoreSelfRefs = true,
         },
     .unsafeDiscardReferences = {},
@@ -167,7 +187,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
     this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(!got.structuredAttrs);
 
@@ -192,9 +213,9 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
 
 TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
 {
-    DerivationOptions expected = {
+    DerivationOptions<SingleDerivedPath> expected = {
         .outputChecks =
-            DerivationOptions::OutputChecks{
+            DerivationOptions<SingleDerivedPath>::OutputChecks{
                 .ignoreSelfRefs = true,
             },
         .unsafeDiscardReferences = {},
@@ -212,12 +233,13 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
     this->readTest("advanced-attributes.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(!got.structuredAttrs);
 
         // Reset fields that vary between test cases to enable whole-object comparison
-        options.outputChecks = DerivationOptions::OutputChecks{.ignoreSelfRefs = true};
+        options.outputChecks = DerivationOptions<SingleDerivedPath>::OutputChecks{.ignoreSelfRefs = true};
         options.exportReferencesGraph = {};
 
         EXPECT_EQ(options, expected);
@@ -227,14 +249,14 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
     });
 };
 
-DerivationOptions advancedAttributes_ia = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_ia = {
     .outputChecks =
-        DerivationOptions::OutputChecks{
+        DerivationOptions<SingleDerivedPath>::OutputChecks{
             .ignoreSelfRefs = true,
-            .allowedReferences = StringSet{pathFoo},
-            .disallowedReferences = StringSet{pathBar, "dev"},
-            .allowedRequisites = StringSet{pathFooDev, "bin"},
-            .disallowedRequisites = StringSet{pathBarDev},
+            .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo},
+            .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}},
+            .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}},
+            .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev},
         },
     .unsafeDiscardReferences = {},
     .passAsFile = {},
@@ -257,14 +279,14 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_ia)
     testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"});
 };
 
-DerivationOptions advancedAttributes_ca = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_ca = {
     .outputChecks =
-        DerivationOptions::OutputChecks{
+        DerivationOptions<SingleDerivedPath>::OutputChecks{
             .ignoreSelfRefs = true,
-            .allowedReferences = StringSet{placeholderFoo},
-            .disallowedReferences = StringSet{placeholderBar, "dev"},
-            .allowedRequisites = StringSet{placeholderFooDev, "bin"},
-            .disallowedRequisites = StringSet{placeholderBarDev},
+            .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo},
+            .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}},
+            .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}},
+            .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev},
         },
     .unsafeDiscardReferences = {},
     .passAsFile = {},
@@ -287,8 +309,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
     testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"});
 };
 
-DerivationOptions advancedAttributes_structuredAttrs_defaults = {
-    .outputChecks = std::map<std::string, DerivationOptions::OutputChecks>{},
+DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_defaults = {
+    .outputChecks = std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{},
     .unsafeDiscardReferences = {},
     .passAsFile = {},
     .exportReferencesGraph = {},
@@ -307,7 +329,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
     this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(got.structuredAttrs);
 
@@ -332,11 +355,11 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
 
 TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
 {
-    DerivationOptions expected = {
+    DerivationOptions<SingleDerivedPath> expected = {
         .outputChecks =
-            std::map<std::string, DerivationOptions::OutputChecks>{
+            std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
                 {"dev",
-                 DerivationOptions::OutputChecks{
+                 DerivationOptions<SingleDerivedPath>::OutputChecks{
                      .maxSize = 789,
                      .maxClosureSize = 5909,
                  }},
@@ -357,7 +380,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
     this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(got.structuredAttrs);
 
@@ -365,7 +389,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
         {
             // Delete all keys but "dev" in options.outputChecks
             auto * outputChecksMapP =
-                std::get_if<std::map<std::string, DerivationOptions::OutputChecks>>(&options.outputChecks);
+                std::get_if<std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>>(
+                    &options.outputChecks);
             ASSERT_TRUE(outputChecksMapP);
             auto & outputChecksMap = *outputChecksMapP;
             auto devEntry = outputChecksMap.find("dev");
@@ -385,21 +410,21 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
     });
 };
 
-DerivationOptions advancedAttributes_structuredAttrs_ia = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ia = {
     .outputChecks =
-        std::map<std::string, DerivationOptions::OutputChecks>{
+        std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
             {"out",
-             DerivationOptions::OutputChecks{
-                 .allowedReferences = StringSet{pathFoo},
-                 .allowedRequisites = StringSet{pathFooDev, "bin"},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo},
+                 .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}},
              }},
             {"bin",
-             DerivationOptions::OutputChecks{
-                 .disallowedReferences = StringSet{pathBar, "dev"},
-                 .disallowedRequisites = StringSet{pathBarDev},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}},
+                 .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev},
              }},
             {"dev",
-             DerivationOptions::OutputChecks{
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
                  .maxSize = 789,
                  .maxClosureSize = 5909,
              }},
@@ -427,21 +452,21 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
         "advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"});
 };
 
-DerivationOptions advancedAttributes_structuredAttrs_ca = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ca = {
     .outputChecks =
-        std::map<std::string, DerivationOptions::OutputChecks>{
+        std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
             {"out",
-             DerivationOptions::OutputChecks{
-                 .allowedReferences = StringSet{placeholderFoo},
-                 .allowedRequisites = StringSet{placeholderFooDev, "bin"},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo},
+                 .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}},
              }},
             {"bin",
-             DerivationOptions::OutputChecks{
-                 .disallowedReferences = StringSet{placeholderBar, "dev"},
-                 .disallowedRequisites = StringSet{placeholderBarDev},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}},
+                 .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev},
              }},
             {"dev",
-             DerivationOptions::OutputChecks{
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
                  .maxSize = 789,
                  .maxClosureSize = 5909,
              }},
@@ -471,14 +496,16 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
         {"rainbow", "uid-range", "ca-derivations"});
 };
 
-#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2)                                                             \
-    TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json)                                                  \
-    {                                                                                                      \
-        this->JsonCharacterizationTest<DerivationOptions>::readJsonTest(#VAR, advancedAttributes_##VAR2);  \
-    }                                                                                                      \
-    TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json)                                                    \
-    {                                                                                                      \
-        this->JsonCharacterizationTest<DerivationOptions>::writeJsonTest(#VAR, advancedAttributes_##VAR2); \
+#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2)                                               \
+    TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json)                                    \
+    {                                                                                        \
+        this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::readJsonTest(  \
+            #VAR, advancedAttributes_##VAR2);                                                \
+    }                                                                                        \
+    TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json)                                      \
+    {                                                                                        \
+        this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::writeJsonTest( \
+            #VAR, advancedAttributes_##VAR2);                                                \
     }
 
 TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults)

src/libstore-tests/derivation/external-formats.cc

@@ -1,57 +1,14 @@
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 
-#include "nix/util/experimental-features.hh"
 #include "nix/store/derivations.hh"
-
-#include "nix/store/tests/libstore.hh"
+#include "derivation/test-support.hh"
 #include "nix/util/tests/json-characterization.hh"
 
 namespace nix {
 
 using nlohmann::json;
 
-class DerivationTest : public virtual CharacterizationTest, public LibStoreTest
-{
-    std::filesystem::path unitTestData = getUnitTestData() / "derivation";
-
-public:
-    std::filesystem::path goldenMaster(std::string_view testStem) const override
-    {
-        return unitTestData / testStem;
-    }
-
-    /**
-     * We set these in tests rather than the regular globals so we don't have
-     * to worry about race conditions if the tests run concurrently.
-     */
-    ExperimentalFeatureSettings mockXpSettings;
-};
-
-class CaDerivationTest : public DerivationTest
-{
-    void SetUp() override
-    {
-        mockXpSettings.set("experimental-features", "ca-derivations");
-    }
-};
-
-class DynDerivationTest : public DerivationTest
-{
-    void SetUp() override
-    {
-        mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
-    }
-};
-
-class ImpureDerivationTest : public DerivationTest
-{
-    void SetUp() override
-    {
-        mockXpSettings.set("experimental-features", "impure-derivations");
-    }
-};
-
 TEST_F(DerivationTest, BadATerm_version)
 {
     ASSERT_THROW(