Expose builtins.fetchFinalTree to the user

This may be more efficient than fetchTree since it allows substitution
from binary caches if `narHash` is specified in the input
attributes. This is what call-flake.nix already used internally.

.github/workflows/ci.yml

@@ -174,7 +174,7 @@ jobs:
         echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
         TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
         echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
-    - uses: cachix/install-nix-action@7ec16f2c061ab07b235a7245e06ed46fe9a1cab6 # v31.8.3
+    - uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
       if: ${{ !matrix.experimental-installer }}
       with:
         install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}

doc/manual/package.nix

@@ -37,6 +37,7 @@ mkMesonDerivation (finalAttrs: {
       (fileset.unions [
         ../../.version
         # For example JSON
+        ../../src/libutil-tests/data/memory-source-accessor
         ../../src/libutil-tests/data/hash
         ../../src/libstore-tests/data/content-address
         ../../src/libstore-tests/data/store-path

doc/manual/rl-next/libcurl-pausing.md

@@ -0,0 +1,12 @@
+---
+synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
+prs: [14614]
+issues: [11728]
+---
+
+The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
+[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
+
+This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
+
+Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).

doc/manual/source/SUMMARY.md.in

@@ -121,6 +121,7 @@
 - [Architecture and Design](architecture/architecture.md)
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
+    - [File System Object](protocols/json/file-system-object.md)
     - [Hash](protocols/json/hash.md)
     - [Content Address](protocols/json/content-address.md)
     - [Store Path](protocols/json/store-path.md)

doc/manual/source/command-ref/nix-build.md

@@ -36,7 +36,7 @@ to a temporary location. The tarball must include a single top-level
 directory containing at least a file named `default.nix`.
 
 `nix-build` is essentially a wrapper around
-[`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix
+[`nix-instantiate`](./nix-instantiate.md) (to translate a high-level Nix
 expression to a low-level [store derivation]) and [`nix-store
 --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
 derivation).
@@ -52,8 +52,8 @@ derivation).
 # Options
 
 All options not listed here are passed to
-[`nix-store --realise`](nix-store/realise.md),
-except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md).
+[`nix-store --realise`](./nix-store/realise.md),
+except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](./nix-instantiate.md).
 
 - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
 

doc/manual/source/command-ref/nix-instantiate.md

@@ -32,7 +32,7 @@ standard input.
 
 - `--add-root` *path*
 
-  See the [corresponding option](nix-store.md) in `nix-store`.
+  See the [corresponding option](./nix-store.md) in `nix-store`.
 
 - `--parse`
 

doc/manual/source/development/debugging.md

@@ -15,7 +15,7 @@ In the development shell, set the `mesonBuildType` environment variable to `debu
 Then, proceed to build Nix as described in [Building Nix](./building.md).
 This will build Nix with debug symbols, which are essential for effective debugging.
 
-It is also possible to build without debugging for faster build:
+It is also possible to build without optimization for faster build:
 
 ```console
 [nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify)

doc/manual/source/protocols/json/file-system-object.md

@@ -0,0 +1,21 @@
+{{#include file-system-object-v1-fixed.md}}
+
+## Examples
+
+### Simple
+
+```json
+{{#include schema/file-system-object-v1/simple.json}}
+```
+
+### Complex
+
+```json
+{{#include schema/file-system-object-v1/complex.json}}
+```
+
+<!-- need to convert YAML to JSON first
+## Raw Schema
+
+[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
+-->

doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed

@@ -11,6 +11,7 @@ s/\\`/`/g
 #
 # As we have more such relative links, more replacements of this nature
 # should appear below.
+s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
 s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
 s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
 s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g

doc/manual/source/protocols/json/meson.build

@@ -9,6 +9,7 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
 json_schema_config = files('json-schema-for-humans-config.yaml')
 
 schemas = [
+  'file-system-object-v1',
   'hash-v1',
   'content-address-v1',
   'store-path-v1',

doc/manual/source/protocols/json/schema/file-system-object-v1

@@ -0,0 +1 @@
+../../../../../../src/libutil-tests/data/memory-source-accessor

doc/manual/source/protocols/json/schema/file-system-object-v1.yaml

@@ -0,0 +1,71 @@
+"$schema": http://json-schema.org/draft-04/schema#
+"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
+title: File System Object
+description: |
+  This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
+
+  The schema is recursive because file system objects contain other file system objects.
+type: object
+required: ["type"]
+properties:
+  type:
+    type: string
+    enum: ["regular", "symlink", "directory"]
+
+# Enforce conditional structure based on `type`
+anyOf:
+  - $ref: "#/$defs/regular"
+    required: ["type", "contents"]
+
+  - $ref: "#/$defs/directory"
+    required: ["type", "entries"]
+
+  - $ref: "#/$defs/symlink"
+    required: ["type", "target"]
+
+"$defs":
+  regular:
+    title: Regular File
+    description: |
+      See [Regular File](@docroot@/store/file-system-object.md#regular) in the manual for details.
+    required: ["contents"]
+    properties:
+      type:
+        const: "regular"
+      contents:
+        type: string
+        description: File contents
+      executable:
+        type: boolean
+        description: Whether the file is executable.
+        default: false
+    additionalProperties: false
+
+  directory:
+    title: Directory
+    description: |
+      See [Directory](@docroot@/store/file-system-object.md#directory) in the manual for details.
+    required: ["entries"]
+    properties:
+      type:
+        const: "directory"
+      entries:
+        type: object
+        description: |
+          Map of names to nested file system objects (for type=directory)
+        additionalProperties:
+          $ref: "#"
+    additionalProperties: false
+
+  symlink:
+    title: Symbolic Link
+    description: |
+      See [Symbolic Link](@docroot@/store/file-system-object.md#symlink) in the manual for details.
+    required: ["target"]
+    properties:
+      type:
+        const: "symlink"
+      target:
+        type: string
+        description: Target path of the symlink.
+    additionalProperties: false

doc/manual/source/store/file-system-object.md

@@ -3,19 +3,23 @@
 Nix uses a simplified model of the file system, which consists of file system objects.
 Every file system object is one of the following:
 
- - File
+ - [**Regular File**]{#regular}
 
    - A possibly empty sequence of bytes for contents
    - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission
 
- - Directory
+ - [**Directory**]{#directory}
 
    Mapping of names to child file system objects
 
- - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link)
+ - [**Symbolic link**]{#symlink}
 
-   An arbitrary string.
-   Nix does not assign any semantics to symbolic links.
+   An arbitrary string, known as the *target* of the symlink.
+
+   In general, Nix does not assign any semantics to symbolic links.
+   Certain operations however, may make additional assumptions and attempt to use the target to find another file system object.
+
+   > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept.
 
 File system objects and their children form a tree.
 A bare file or symlink can be a root file system object.

packaging/dev-shell.nix

@@ -130,15 +130,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
     havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
     ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
 
+    availableComponents = lib.filterAttrs (
+      k: v: lib.meta.availableOn pkgs.hostPlatform v
+    ) allComponents;
+
     activeComponents = buildInputsClosureCond isInternal (
-      lib.attrValues (finalAttrs.passthru.config.getComponents allComponents)
+      lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents)
     );
 
     allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2;
     internalDrvs = byDrvPath (
       # Drop the attr names (not present in buildInputs anyway)
-      lib.attrValues allComponents
-      ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues allComponents)
+      lib.attrValues availableComponents
+      ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents)
     );
 
     isInternal =
@@ -187,19 +191,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
         );
 
       small =
-        (finalAttrs.finalPackage.withActiveComponents (c: {
-          inherit (c)
-            nix-cli
-            nix-util-tests
-            nix-store-tests
-            nix-expr-tests
-            nix-fetchers-tests
-            nix-flake-tests
-            nix-functional-tests
-            # Currently required
-            nix-perl-bindings
-            ;
-        })).overrideAttrs
+        (finalAttrs.finalPackage.withActiveComponents (
+          c:
+          lib.intersectAttrs (lib.genAttrs [
+            "nix-cli"
+            "nix-util-tests"
+            "nix-store-tests"
+            "nix-expr-tests"
+            "nix-fetchers-tests"
+            "nix-flake-tests"
+            "nix-functional-tests"
+            "nix-perl-bindings"
+          ] (_: null)) c
+        )).overrideAttrs
           (o: {
             mesonFlags = o.mesonFlags ++ [
               # TODO: infer from activeComponents or vice versa

src/json-schema-checks/file-system-object

@@ -0,0 +1 @@
+../../src/libutil-tests/data/memory-source-accessor

src/json-schema-checks/meson.build

@@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema'
 
 # Get all example files
 schemas = [
+  {
+    'stem' : 'file-system-object',
+    'schema' : schema_dir / 'file-system-object-v1.yaml',
+    'files' : [
+      'simple.json',
+      'complex.json',
+    ],
+  },
   {
     'stem' : 'hash',
     'schema' : schema_dir / 'hash-v1.yaml',

src/json-schema-checks/package.nix

@@ -20,6 +20,7 @@ mkMesonDerivation (finalAttrs: {
   fileset = lib.fileset.unions [
     ../../.version
     ../../doc/manual/source/protocols/json/schema
+    ../../src/libutil-tests/data/memory-source-accessor
     ../../src/libutil-tests/data/hash
     ../../src/libstore-tests/data/content-address
     ../../src/libstore-tests/data/store-path

src/kaitai-struct-checks/package.nix

@@ -65,6 +65,6 @@ mkMesonDerivation (finalAttrs: {
   '';
 
   meta = {
-    platforms = lib.platforms.all;
+    platforms = lib.platforms.unix;
   };
 })

src/libcmd/common-eval-args.cc

@@ -132,7 +132,7 @@ MixEvalArgs::MixEvalArgs()
             fetchers::Attrs extraAttrs;
             if (to.subdir != "")
                 extraAttrs["dir"] = to.subdir;
-            fetchers::overrideRegistry(fetchSettings, from.input, to.input, extraAttrs);
+            fetchers::overrideRegistry(from.input, to.input, extraAttrs);
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             completeFlakeRef(completions, openStore(), prefix);
@@ -168,9 +168,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
                                 ? state.rootPath(absPath(getCommandBaseDir()))
                                 : state.rootPath(".")));
                 },
-                [&](const AutoArgString & arg) { v->mkString(arg.s); },
-                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string())); },
-                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO)); }},
+                [&](const AutoArgString & arg) { v->mkString(arg.s, state.mem); },
+                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string()), state.mem); },
+                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO), state.mem); }},
             arg);
         res.insert(state.symbols.create(name), v);
     }

src/libcmd/installables.cc

@@ -185,7 +185,6 @@ MixFlakeOptions::MixFlakeOptions()
                     }
 
                     overrideRegistry(
-                        fetchSettings,
                         fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
                         extraAttrs);

src/libexpr-c/nix_api_expr.cc

@@ -69,8 +69,8 @@ nix_err nix_expr_eval_from_string(
         context->last_err_code = NIX_OK;
     try {
         nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
-        state->state.eval(parsedExpr, value->value);
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.eval(parsedExpr, *value->value);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -80,8 +80,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.callFunction(*fn->value, *arg->value, *value->value, nix::noPos);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -91,9 +91,15 @@ nix_err nix_value_call_multi(
 {
     if (context)
         context->last_err_code = NIX_OK;
+
+    std::vector<nix::Value *> internal_args;
+    internal_args.reserve(nargs);
+    for (size_t i = 0; i < nargs; i++)
+        internal_args.push_back(args[i]->value);
+
     try {
-        state->state.callFunction(fn->value, {(nix::Value **) args, nargs}, value->value, nix::noPos);
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.callFunction(*fn->value, {internal_args.data(), nargs}, *value->value, nix::noPos);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -103,7 +109,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValue(value->value, nix::noPos);
+        state->state.forceValue(*value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -113,7 +119,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValueDeep(value->value);
+        state->state.forceValueDeep(*value->value);
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-c/nix_api_expr_internal.h

@@ -39,7 +39,13 @@ struct ListBuilder
 
 struct nix_value
 {
-    nix::Value value;
+    nix::Value * value;
+    /**
+     * As we move to a managed heap, we need EvalMemory in more places. Ideally, we would take in EvalState or
+     * EvalMemory as an argument when we need it, but we don't want to make changes to the stable C api, so we stuff it
+     * into the nix_value that will get passed in to the relevant functions.
+     */
+    nix::EvalMemory * mem;
 };
 
 struct nix_string_return

src/libexpr-c/nix_api_value.cc

@@ -20,7 +20,7 @@ static const nix::Value & check_value_not_null(const nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return *((const nix::Value *) value);
+    return *value->value;
 }
 
 static nix::Value & check_value_not_null(nix_value * value)
@@ -28,7 +28,7 @@ static nix::Value & check_value_not_null(nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return value->value;
+    return *value->value;
 }
 
 static const nix::Value & check_value_in(const nix_value * value)
@@ -58,9 +58,14 @@ static nix::Value & check_value_out(nix_value * value)
     return v;
 }
 
-static inline nix_value * as_nix_value_ptr(nix::Value * v)
+static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
 {
-    return reinterpret_cast<nix_value *>(v);
+    nix_value * ret = new (mem.allocBytes(sizeof(nix_value))) nix_value{
+        .value = v,
+        .mem = &mem,
+    };
+    nix_gc_incref(nullptr, ret);
+    return ret;
 }
 
 /**
@@ -69,7 +74,13 @@ static inline nix_value * as_nix_value_ptr(nix::Value * v)
  * Deals with errors and converts arguments from C++ into C types.
  */
 static void nix_c_primop_wrapper(
-    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+    PrimOpFun f,
+    void * userdata,
+    int arity,
+    nix::EvalState & state,
+    const nix::PosIdx pos,
+    nix::Value ** args,
+    nix::Value & v)
 {
     nix_c_context ctx;
 
@@ -85,8 +96,15 @@ static void nix_c_primop_wrapper(
     // ok because we don't see a need for this yet (e.g. inspecting thunks,
     // or maybe something to make blackholes work better; we don't know).
     nix::Value vTmp;
+    nix_value * vTmpPtr = new_nix_value(&vTmp, state.mem);
 
-    f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
+    std::vector<nix_value *> external_args;
+    external_args.reserve(arity);
+    for (int i = 0; i < arity; i++) {
+        nix_value * external_arg = new_nix_value(args[i], state.mem);
+        external_args.push_back(external_arg);
+    }
+    f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr);
 
     if (ctx.last_err_code != NIX_OK) {
         /* TODO: Throw different errors depending on the error code */
@@ -135,7 +153,7 @@ PrimOp * nix_alloc_primop(
                     .args = {},
                     .arity = (size_t) arity,
                     .doc = doc,
-                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)};
         if (args)
             for (size_t i = 0; args[i]; i++)
                 p->args.emplace_back(*args);
@@ -160,8 +178,7 @@ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        nix_value * res = as_nix_value_ptr(state->state.allocValue());
-        nix_gc_incref(nullptr, res);
+        nix_value * res = new_nix_value(state->state.allocValue(), state->state.mem);
         return res;
     }
     NIXC_CATCH_ERRS_NULL
@@ -331,10 +348,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
             return nullptr;
         }
         auto * p = v.listView()[ix];
-        nix_gc_incref(nullptr, p);
-        if (p != nullptr)
-            state->state.forceValue(*p, nix::noPos);
-        return as_nix_value_ptr(p);
+        if (p == nullptr)
+            return nullptr;
+        state->state.forceValue(*p, nix::noPos);
+        return new_nix_value(p, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -352,9 +369,8 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt
             return nullptr;
         }
         auto * p = v.listView()[ix];
-        nix_gc_incref(nullptr, p);
         // Note: intentionally NOT calling forceValue() to keep the element lazy
-        return as_nix_value_ptr(p);
+        return new_nix_value(p, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -369,9 +385,8 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
-            nix_gc_incref(nullptr, attr->value);
             state->state.forceValue(*attr->value, nix::noPos);
-            return as_nix_value_ptr(attr->value);
+            return new_nix_value(attr->value, state->state.mem);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -390,9 +405,8 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
-            nix_gc_incref(nullptr, attr->value);
             // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-            return as_nix_value_ptr(attr->value);
+            return new_nix_value(attr->value, state->state.mem);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -440,9 +454,8 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
-        nix_gc_incref(nullptr, a.value);
         state->state.forceValue(*a.value, nix::noPos);
-        return as_nix_value_ptr(a.value);
+        return new_nix_value(a.value, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -461,9 +474,8 @@ nix_value * nix_get_attr_byidx_lazy(
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
-        nix_gc_incref(nullptr, a.value);
         // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-        return as_nix_value_ptr(a.value);
+        return new_nix_value(a.value, state->state.mem);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -503,7 +515,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkString(std::string_view(str));
+        v.mkString(std::string_view(str), *value->mem);
     }
     NIXC_CATCH_ERRS
 }
@@ -514,7 +526,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)), s->state.mem);
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-tests/json.cc

@@ -73,7 +73,7 @@ TEST_F(JSONValueTest, StringQuotes)
 TEST_F(JSONValueTest, DISABLED_Path)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("/test")));
+    v.mkPath(state.rootPath(CanonPath("/test")), state.mem);
     ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
 }
 } /* namespace nix */

src/libexpr-tests/value/print.cc

@@ -268,7 +268,7 @@ struct StringPrintingTests : LibExprTest
     void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args)
     {
         Value v;
-        v.mkString(literal);
+        v.mkString(literal, state.mem);
 
         std::stringstream out;
         printValue(state, out, v, PrintOptions{.maxStringLength = maxLength});
@@ -353,7 +353,7 @@ TEST_F(ValuePrintingTests, ansiColorsStringElided)
 TEST_F(ValuePrintingTests, ansiColorsPath)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("puppy")));
+    v.mkPath(state.rootPath(CanonPath("puppy")), state.mem);
 
     test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true});
 }

src/libexpr/eval.cc

@@ -81,20 +81,20 @@ static const char * makeImmutableString(std::string_view s)
     return t;
 }
 
-StringData & StringData::alloc(size_t size)
+StringData & StringData::alloc(EvalMemory & mem, size_t size)
 {
-    void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
+    void * t = mem.allocBytes(sizeof(StringData) + size + 1);
     if (!t)
         throw std::bad_alloc();
     auto res = new (t) StringData(size);
     return *res;
 }
 
-const StringData & StringData::make(std::string_view s)
+const StringData & StringData::make(EvalMemory & mem, std::string_view s)
 {
     if (s.empty())
         return ""_sds;
-    auto & res = alloc(s.size());
+    auto & res = alloc(mem, s.size());
     std::memcpy(&res.data_, s.data(), s.size());
     res.data_[s.size()] = '\0';
     return res;
@@ -849,9 +849,9 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
         evalState.runDebugRepl(nullptr, trace.env, trace.expr);
 }
 
-void Value::mkString(std::string_view s)
+void Value::mkString(std::string_view s, EvalMemory & mem)
 {
-    mkStringNoCopy(StringData::make(s));
+    mkStringNoCopy(StringData::make(mem, s));
 }
 
 Value::StringWithContext::Context *
@@ -862,13 +862,13 @@ Value::StringWithContext::Context::fromBuilder(const NixStringContext & context,
 
     auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
     std::ranges::transform(
-        context, ctx->elems, [](const NixStringContextElem & elt) { return &StringData::make(elt.to_string()); });
+        context, ctx->elems, [&](const NixStringContextElem & elt) { return &StringData::make(mem, elt.to_string()); });
     return ctx;
 }
 
 void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem)
 {
-    mkStringNoCopy(StringData::make(s), Value::StringWithContext::Context::fromBuilder(context, mem));
+    mkStringNoCopy(StringData::make(mem, s), Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
 void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem)
@@ -876,9 +876,9 @@ void Value::mkStringMove(const StringData & s, const NixStringContext & context,
     mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
-void Value::mkPath(const SourcePath & path)
+void Value::mkPath(const SourcePath & path, EvalMemory & mem)
 {
-    mkPath(&*path.accessor, StringData::make(path.path.abs()));
+    mkPath(&*path.accessor, StringData::make(mem, path.path.abs()));
 }
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@@ -943,7 +943,7 @@ void EvalState::mkPos(Value & v, PosIdx p)
     auto origin = positions.originOf(p);
     if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(s.file).mkString(path->path.abs());
+        attrs.alloc(s.file).mkString(path->path.abs(), mem);
         makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column));
         v.mkAttrs(attrs);
     } else
@@ -1751,9 +1751,9 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    SmallValueVector<4> vArgs(args.size());
-    for (size_t i = 0; i < args.size(); ++i)
-        vArgs[i] = args[i]->maybeThunk(state, env);
+    SmallValueVector<4> vArgs(args->size());
+    for (size_t i = 0; i < args->size(); ++i)
+        vArgs[i] = (*args)[i]->maybeThunk(state, env);
 
     state.callFunction(vFun, vArgs, v, pos);
 }
@@ -2139,9 +2139,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         for (const auto & part : strings) {
             resultStr += *part;
         }
-        v.mkPath(state.rootPath(CanonPath(resultStr)));
+        v.mkPath(state.rootPath(CanonPath(resultStr)), state.mem);
     } else {
-        auto & resultStr = StringData::alloc(sSize);
+        auto & resultStr = StringData::alloc(state.mem, sSize);
         auto * tmp = resultStr.data();
         for (const auto & part : strings) {
             std::memcpy(tmp, part->data(), part->size());
@@ -2188,6 +2188,8 @@ void EvalState::forceValueDeep(Value & v)
     std::set<const Value *> seen;
 
     [&, &state(*this)](this const auto & recurse, Value & v) {
+        auto _level = state.addCallDepth(v.determinePos(noPos));
+
         if (!seen.insert(&v).second)
             return;
 
@@ -2214,8 +2216,15 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
+            size_t index = 0;
             for (auto v2 : v.listView())
-                recurse(*v2);
+                try {
+                    recurse(*v2);
+                    index++;
+                } catch (Error & e) {
+                    state.addErrorTrace(e, "while evaluating list element at index %1%", index);
+                    throw;
+                }
         }
     }(v);
 }

src/libexpr/include/nix/expr/nixexpr.hh

@@ -592,11 +592,14 @@ public:
 struct ExprCall : Expr
 {
     Expr * fun;
-    std::vector<Expr *> args;
+    /**
+     * args will never be null. See comment on ExprAttrs::AttrDefs below.
+     */
+    std::optional<std::pmr::vector<Expr *>> args;
     PosIdx pos;
     std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
+    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -604,7 +607,7 @@ struct ExprCall : Expr
     {
     }
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -836,7 +839,7 @@ public:
     // we define some calls to add explicitly so that the argument can be passed in as initializer lists
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
+    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args));
@@ -844,7 +847,7 @@ public:
 
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos));

src/libexpr/include/nix/expr/value.hh

@@ -232,13 +232,13 @@ public:
      * Allocate StringData on the (possibly) GC-managed heap and copy
      * the contents of s to it.
      */
-    static const StringData & make(std::string_view s);
+    static const StringData & make(EvalMemory & mem, std::string_view s);
 
     /**
      * Allocate StringData on the (possibly) GC-managed heap.
      * @param size Length of the string (without the NUL terminator).
      */
-    static StringData & alloc(size_t size);
+    static StringData & alloc(EvalMemory & mem, size_t size);
 
     size_t size() const
     {
@@ -1147,13 +1147,13 @@ public:
         setStorage(StringWithContext{.str = &s, .context = context});
     }
 
-    void mkString(std::string_view s);
+    void mkString(std::string_view s, EvalMemory & mem);
 
     void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem);
 
     void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem);
 
-    void mkPath(const SourcePath & path);
+    void mkPath(const SourcePath & path, EvalMemory & mem);
 
     inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept
     {

src/libexpr/json-to-value.cc

@@ -151,7 +151,7 @@ public:
     bool string(string_t & val) override
     {
         forceNoNullByte(val);
-        rs->value(state).mkString(val);
+        rs->value(state).mkString(val, state.mem);
         rs->add();
         return true;
     }

src/libexpr/nixexpr.cc

@@ -191,7 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << '(';
     fun->show(symbols, str);
-    for (auto e : args) {
+    for (auto e : *args) {
         str << ' ';
         e->show(symbols, str);
     }
@@ -486,11 +486,17 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
 
 void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
+    // Move storage into the Exprs arena
+    {
+        auto arena = es.mem.exprs.alloc;
+        std::pmr::vector<Expr *> newArgs{std::move(*args), arena};
+        args.emplace(std::move(newArgs), arena);
+    }
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     fun->bindVars(es, env);
-    for (auto e : args)
+    for (auto e : *args)
         e->bindVars(es, env);
 }
 

src/libexpr/parser.y

@@ -115,7 +115,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P
 
 static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
     if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
-        e2->args.push_back(arg);
+        e2->args->push_back(arg);
         return fn;
     }
     return exprs.add<ExprCall>(pos, fn, {arg});
@@ -129,7 +129,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
 %type <Expr *> start expr expr_function expr_if expr_op
 %type <Expr *> expr_select expr_simple expr_app
 %type <Expr *> expr_pipe_from expr_pipe_into
-%type <std::vector<Expr *>> list
+%type <std::pmr::vector<Expr *>> list
 %type <ExprAttrs *> binds binds1
 %type <FormalsBuilder> formals formal_set
 %type <Formal> formal

src/libexpr/primops.cc

@@ -53,7 +53,7 @@ RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
 static inline Value * mkString(EvalState & state, const std::csub_match & match)
 {
     Value * v = state.allocValue();
-    v->mkString({match.first, match.second});
+    v->mkString({match.first, match.second}, state.mem);
     return v;
 }
 
@@ -230,12 +230,12 @@ void derivationToValue(
                 NixStringContextElem::DrvDeep{.drvPath = storePath},
             },
             state.mem);
-    attrs.alloc(state.s.name).mkString(drv.env["name"]);
+    attrs.alloc(state.s.name).mkString(drv.env["name"], state.mem);
 
     auto list = state.buildList(drv.outputs.size());
     for (const auto & [i, o] : enumerate(drv.outputs)) {
         mkOutputString(state, attrs, storePath, o);
-        (list[i] = state.allocValue())->mkString(o.first);
+        (list[i] = state.allocValue())->mkString(o.first, state.mem);
     }
     attrs.alloc(state.s.outputs).mkList(list);
 
@@ -519,7 +519,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
         v.mkStringNoCopy("lambda"_sds);
         break;
     case nExternal:
-        v.mkString(args[0]->external()->typeOf());
+        v.mkString(args[0]->external()->typeOf(), state.mem);
         break;
     case nFloat:
         v.mkStringNoCopy("float"_sds);
@@ -1176,7 +1176,7 @@ static void prim_getEnv(EvalState & state, const PosIdx pos, Value ** args, Valu
 {
     std::string name(
         state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv"));
-    v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""));
+    v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""), state.mem);
 }
 
 static RegisterPrimOp primop_getEnv({
@@ -1842,8 +1842,10 @@ static RegisterPrimOp primop_derivationStrict(
    ‘out’. */
 static void prim_placeholder(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
-    v.mkString(hashPlaceholder(
-        state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder")));
+    v.mkString(
+        hashPlaceholder(state.forceStringNoCtx(
+            *args[0], pos, "while evaluating the first argument passed to builtins.placeholder")),
+        state.mem);
 }
 
 static RegisterPrimOp primop_placeholder({
@@ -2027,7 +2029,7 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
     state.forceValue(*args[0], pos);
     if (args[0]->type() == nPath) {
         auto path = args[0]->path();
-        v.mkPath(path.path.isRoot() ? path : path.parent());
+        v.mkPath(path.path.isRoot() ? path : path.parent(), state.mem);
     } else {
         NixStringContext context;
         auto path = state.coerceToString(
@@ -2144,7 +2146,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
     auto path =
         state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
 
-    v.mkPath(state.findFile(lookupPath, path, pos));
+    v.mkPath(state.findFile(lookupPath, path, pos), state.mem);
 }
 
 static RegisterPrimOp primop_findFile(
@@ -2293,7 +2295,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
 
     auto path = realisePath(state, pos, *args[1]);
 
-    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem);
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -2382,7 +2384,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Val
             // detailed node info quickly in this case we produce a thunk to
             // query the file type lazily.
             auto epath = state.allocValue();
-            epath->mkPath(path / name);
+            epath->mkPath(path / name, state.mem);
             if (!readFileType)
                 readFileType = &state.getBuiltin("readFileType");
             attr.mkApp(readFileType, epath);
@@ -2763,7 +2765,7 @@ bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosId
     /* Call the filter function.  The first argument is the path, the
        second is a string indicating the type of the file. */
     Value arg1;
-    arg1.mkString(path.path.abs());
+    arg1.mkString(path.path.abs(), mem);
 
     // assert that type is not "unknown"
     Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))};
@@ -4541,7 +4543,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value ** args,
     auto s =
         state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
-    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
+    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false), state.mem);
 }
 
 static RegisterPrimOp primop_hashString({
@@ -4574,7 +4576,7 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value ** args,
     HashFormat hf = parseHashFormat(
         state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
 
-    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
+    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI), state.mem);
 }
 
 static RegisterPrimOp primop_convertHash({
@@ -4992,8 +4994,8 @@ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value ** args
         state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName");
     DrvName parsed(name);
     auto attrs = state.buildBindings(2);
-    attrs.alloc(state.s.name).mkString(parsed.name);
-    attrs.alloc("version").mkString(parsed.version);
+    attrs.alloc(state.s.name).mkString(parsed.name, state.mem);
+    attrs.alloc("version").mkString(parsed.version, state.mem);
     v.mkAttrs(attrs);
 }
 
@@ -5048,7 +5050,7 @@ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value ** args
     }
     auto list = state.buildList(components.size());
     for (const auto & [n, component] : enumerate(components))
-        (list[n] = state.allocValue())->mkString(std::move(component));
+        (list[n] = state.allocValue())->mkString(std::move(component), state.mem);
     v.mkList(list);
 }
 
@@ -5192,7 +5194,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
         });
 
     if (!settings.pureEval)
-        v.mkString(settings.getCurrentSystem());
+        v.mkString(settings.getCurrentSystem(), mem);
     addConstant(
         "__currentSystem",
         v,
@@ -5224,7 +5226,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
             .impureOnly = true,
         });
 
-    v.mkString(nixVersion);
+    v.mkString(nixVersion, mem);
     addConstant(
         "__nixVersion",
         v,
@@ -5249,7 +5251,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
         )",
         });
 
-    v.mkString(store->storeDir);
+    v.mkString(store->storeDir, mem);
     addConstant(
         "__storeDir",
         v,
@@ -5314,8 +5316,8 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
     auto list = buildList(lookupPath.elements.size());
     for (const auto & [n, i] : enumerate(lookupPath.elements)) {
         auto attrs = buildBindings(2);
-        attrs.alloc("path").mkString(i.path.s);
-        attrs.alloc("prefix").mkString(i.prefix.s);
+        attrs.alloc("path").mkString(i.path.s, mem);
+        attrs.alloc("prefix").mkString(i.prefix.s, mem);
         (list[n] = allocValue())->mkAttrs(attrs);
     }
     v.mkList(list);

src/libexpr/primops/context.cc

@@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
     NixStringContext context;
     auto s = state.coerceToString(
         pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
-    v.mkString(*s);
+    v.mkString(*s, state.mem);
 }
 
 static RegisterPrimOp primop_unsafeDiscardStringContext({
@@ -218,7 +218,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
         if (!info.second.outputs.empty()) {
             auto list = state.buildList(info.second.outputs.size());
             for (const auto & [i, output] : enumerate(info.second.outputs))
-                (list[i] = state.allocValue())->mkString(output);
+                (list[i] = state.allocValue())->mkString(output, state.mem);
             infoAttrs.alloc(state.s.outputs).mkList(list);
         }
         attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);

src/libexpr/primops/fetchMercurial.cc

@@ -86,12 +86,12 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
     auto attrs2 = state.buildBindings(8);
     state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));
     if (input2.getRef())
-        attrs2.alloc("branch").mkString(*input2.getRef());
+        attrs2.alloc("branch").mkString(*input2.getRef(), state.mem);
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
     auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
-    attrs2.alloc("rev").mkString(rev2.gitRev());
-    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
+    attrs2.alloc("rev").mkString(rev2.gitRev(), state.mem);
+    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12), state.mem);
     if (auto revCount = input2.getRevCount())
         attrs2.alloc("revCount").mkInt(*revCount);
     v.mkAttrs(attrs2);

src/libexpr/primops/fetchTree.cc

@@ -35,7 +35,7 @@ void emitTreeAttrs(
     // FIXME: support arbitrary input attributes.
 
     if (auto narHash = input.getNarHash())
-        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true), state.mem);
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
@@ -43,13 +43,13 @@ void emitTreeAttrs(
     if (!forceDirty) {
 
         if (auto rev = input.getRev()) {
-            attrs.alloc("rev").mkString(rev->gitRev());
-            attrs.alloc("shortRev").mkString(rev->gitShortRev());
+            attrs.alloc("rev").mkString(rev->gitRev(), state.mem);
+            attrs.alloc("shortRev").mkString(rev->gitShortRev(), state.mem);
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
             auto emptyHash = Hash(HashAlgorithm::SHA1);
-            attrs.alloc("rev").mkString(emptyHash.gitRev());
-            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
+            attrs.alloc("rev").mkString(emptyHash.gitRev(), state.mem);
+            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem);
         }
 
         if (auto revCount = input.getRevCount())
@@ -59,13 +59,14 @@ void emitTreeAttrs(
     }
 
     if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) {
-        attrs.alloc("dirtyRev").mkString(*dirtyRev);
-        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"));
+        attrs.alloc("dirtyRev").mkString(*dirtyRev, state.mem);
+        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"), state.mem);
     }
 
     if (auto lastModified = input.getLastModified()) {
         attrs.alloc("lastModified").mkInt(*lastModified);
-        attrs.alloc("lastModifiedDate").mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
+        attrs.alloc("lastModifiedDate")
+            .mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")), state.mem);
     }
 
     v.mkAttrs(attrs);
@@ -150,11 +151,6 @@ static void fetchTree(
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
-        // fetchTree should fetch git repos with shallow = true by default
-        if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
-            attrs.emplace("shallow", Explicit<bool>{true});
-        }
-
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>("argument 'name' isn’t supported in call to '%s'", fetcher)
@@ -366,10 +362,13 @@ void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value ** args, Val
 }
 
 static RegisterPrimOp primop_fetchFinalTree({
-    .name = "fetchFinalTree",
+    .name = "__fetchFinalTree",
     .args = {"input"},
+    .doc = R"(
+      Like `fetchTree`, but does not return any additional fetcher attributes (like `revCount`).
+      This allows inputs to be substituted if `narHash` is specified.
+    )",
     .fun = prim_fetchFinalTree,
-    .internal = true,
 });
 
 static void fetch(

src/libexpr/primops/fromTOML.cc

@@ -126,7 +126,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
         case toml::value_t::string: {
             auto s = toml::get<std::string_view>(t);
             forceNoNullByte(s);
-            v.mkString(s);
+            v.mkString(s, state.mem);
         } break;
         case toml::value_t::local_datetime:
         case toml::value_t::offset_datetime:
@@ -142,7 +142,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 s << t;
                 auto str = s.view();
                 forceNoNullByte(str);
-                attrs.alloc("value").mkString(str);
+                attrs.alloc("value").mkString(str, state.mem);
                 v.mkAttrs(attrs);
             } else {
                 throw std::runtime_error("Dates and times are not supported");

src/libexpr/value-to-json.cc

@@ -16,6 +16,8 @@ json printValueAsJSON(
 {
     checkInterrupt();
 
+    auto _level = state.addCallDepth(pos);
+
     if (strict)
         state.forceValue(v, pos);
 

src/libfetchers-tests/git-utils.cc

@@ -92,7 +92,7 @@ TEST_F(GitUtilsTest, sink_basic)
     // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
 
     auto result = repo->dereferenceSingletonDirectory(sink->flush());
-    auto accessor = repo->getAccessor(result, false, getRepoName());
+    auto accessor = repo->getAccessor(result, {}, getRepoName());
     auto entries = accessor->readDirectory(CanonPath::root);
     ASSERT_EQ(entries.size(), 5u);
     ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");

src/libfetchers/fetchers.cc

@@ -269,24 +269,10 @@ void Input::checkLocks(Input specified, Input & result)
         }
     }
 
-    if (auto prevLastModified = specified.getLastModified()) {
-        if (result.getLastModified() != prevLastModified)
-            throw Error(
-                "'lastModified' attribute mismatch in input '%s', expected %d, got %d",
-                result.to_string(),
-                *prevLastModified,
-                result.getLastModified().value_or(-1));
-    }
-
     if (auto prevRev = specified.getRev()) {
         if (result.getRev() != prevRev)
             throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev());
     }
-
-    if (auto prevRevCount = specified.getRevCount()) {
-        if (result.getRevCount() != prevRevCount)
-            throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount);
-    }
 }
 
 std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, Store & store) const
@@ -495,9 +481,9 @@ void InputScheme::clone(
 
     Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
 
-    auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); });
-
-    restorePath(destDir, *source);
+    RestoreSink sink(/*startFsync=*/false);
+    sink.dstPath = destDir;
+    copyRecursive(*accessor, CanonPath::root, sink, CanonPath::root);
 }
 
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const

src/libfetchers/git-utils.cc

@@ -541,14 +541,15 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 
     /**
-     * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
+     * A 'GitSourceAccessor' with no regard for export-ignore.
      */
-    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, bool smudgeLfs = false);
+    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, const GitAccessorOptions & options);
 
     ref<SourceAccessor>
-    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) override;
+    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) override;
 
-    ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
+    ref<SourceAccessor>
+    getAccessor(const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError e) override;
 
     ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
 
@@ -668,7 +669,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
     {
-        auto accessor = getAccessor(treeHash, false, "");
+        auto accessor = getAccessor(treeHash, {}, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
@@ -716,15 +717,17 @@ struct GitSourceAccessor : SourceAccessor
         ref<GitRepoImpl> repo;
         Object root;
         std::optional<lfs::Fetch> lfsFetch = std::nullopt;
+        GitAccessorOptions options;
     };
 
     Sync<State> state_;
 
-    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
+    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, const GitAccessorOptions & options)
         : state_{State{
               .repo = repo_,
               .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
-              .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+              .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+              .options = options,
           }}
     {
     }
@@ -1254,26 +1257,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, bool smudgeLfs)
+ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
+    return make_ref<GitSourceAccessor>(self, rev, options);
 }
 
 ref<SourceAccessor>
-GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs)
+GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
+    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
-    if (exportIgnore)
+    if (options.exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
     else
         return rawGitAccessor;
 }
 
-ref<SourceAccessor>
-GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
+ref<SourceAccessor> GitRepoImpl::getAccessor(
+    const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create(
@@ -1283,10 +1286,9 @@ GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllow
                                            boost::unordered_flat_set<CanonPath>{CanonPath::root},
                                            std::move(makeNotAllowedError))
                                            .cast<SourceAccessor>();
-    if (exportIgnore)
-        return make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
-    else
-        return fileAccessor;
+    if (options.exportIgnore)
+        fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
+    return fileAccessor;
 }
 
 ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()
@@ -1299,7 +1301,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
 
-    auto accessor = getAccessor(rev, exportIgnore, "");
+    auto accessor = getAccessor(rev, {.exportIgnore = exportIgnore}, "");
     if (!accessor->pathExists(modulesFile))
         return {};
 
@@ -1316,7 +1318,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
 
     std::vector<std::tuple<Submodule, Hash>> result;
 
-    auto rawAccessor = getRawAccessor(rev);
+    auto rawAccessor = getRawAccessor(rev, {});
 
     for (auto & submodule : parseSubmodules(pathTemp)) {
         /* Filter out .gitmodules entries that don't exist or are not
@@ -1332,10 +1334,8 @@ namespace fetchers {
 
 ref<GitRepo> Settings::getTarballCache() const
 {
-    auto tarballCache(_tarballCache.lock());
-    if (!*tarballCache)
-        *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
-    return ref<GitRepo>(*tarballCache);
+    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
+    return GitRepo::openRepo(repoDir, true, true);
 }
 
 } // namespace fetchers

src/libfetchers/git.cc

@@ -778,6 +778,16 @@ struct GitInputScheme : InputScheme
         }
     }
 
+    /**
+     * Decide whether we can do a shallow clone, which is faster. This is possible if the user explicitly specified
+     * `shallow = true`, or if we already have a `revCount`.
+     */
+    bool canDoShallow(const Input & input) const
+    {
+        bool shallow = getShallowAttr(input);
+        return shallow || input.getRevCount().has_value();
+    }
+
     std::pair<ref<SourceAccessor>, Input>
     getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const
     {
@@ -786,7 +796,7 @@ struct GitInputScheme : InputScheme
         auto origRev = input.getRev();
 
         auto originalRef = input.getRef();
-        bool shallow = getShallowAttr(input);
+        bool shallow = canDoShallow(input);
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
         input.attrs.insert_or_assign("ref", ref);
 
@@ -797,11 +807,27 @@ struct GitInputScheme : InputScheme
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
         } else {
+            auto rev = input.getRev();
             auto repoUrl = std::get<ParsedURL>(repoInfo.location);
             std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
             repoDir = cacheDir;
             repoInfo.gitDir = ".";
 
+            /* If shallow = false, but we have a non-shallow repo that already contains the desired rev, then use that
+             * repo instead. */
+            std::filesystem::path cacheDirNonShallow = getCachePath(repoUrl.to_string(), false);
+            if (rev && shallow && pathExists(cacheDirNonShallow)) {
+                auto nonShallowRepo = GitRepo::openRepo(cacheDirNonShallow, true, true);
+                if (nonShallowRepo->hasObject(*rev)) {
+                    debug(
+                        "using non-shallow cached repo for '%s' since it contains rev '%s'",
+                        repoUrl.to_string(),
+                        rev->gitRev());
+                    repoDir = cacheDirNonShallow;
+                    goto have_rev;
+                }
+            }
+
             std::filesystem::create_directories(cacheDir.parent_path());
             PathLocks cacheDirLock({cacheDir.string()});
 
@@ -817,7 +843,7 @@ struct GitInputScheme : InputScheme
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (auto rev = input.getRev()) {
+            if (rev) {
                 doFetch = !repo->hasObject(*rev);
             } else {
                 if (getAllRefsAttr(input)) {
@@ -831,7 +857,6 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
-                bool shallow = getShallowAttr(input);
                 try {
                     auto fetchRef = getAllRefsAttr(input)             ? "refs/*:refs/*"
                                     : input.getRev()                  ? input.getRev()->gitRev()
@@ -859,7 +884,7 @@ struct GitInputScheme : InputScheme
                     warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
             }
 
-            if (auto rev = input.getRev()) {
+            if (rev) {
                 if (!repo->hasObject(*rev))
                     throw Error(
                         "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
@@ -876,23 +901,30 @@ struct GitInputScheme : InputScheme
             // the remainder
         }
 
+    have_rev:
         auto repo = GitRepo::openRepo(repoDir);
 
-        auto isShallow = repo->isShallow();
-
-        if (isShallow && !getShallowAttr(input))
-            throw Error(
-                "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
-                repoInfo.locationToArg());
-
         // FIXME: check whether rev is an ancestor of ref?
 
         auto rev = *input.getRev();
 
-        input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
+        /* Skip lastModified computation if it's already supplied by the caller.
+           We don't care if they specify an incorrect value; it doesn't
+           matter for security, unlike narHash. */
+        if (!input.attrs.contains("lastModified"))
+            input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
+
+        /* Like lastModified, skip revCount if supplied by the caller. */
+        if (!shallow && !input.attrs.contains("revCount")) {
+            auto isShallow = repo->isShallow();
+
+            if (isShallow && !shallow)
+                throw Error(
+                    "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
+                    repoInfo.locationToArg());
 
-        if (!getShallowAttr(input))
             input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev));
+        }
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
 
@@ -900,7 +932,8 @@ struct GitInputScheme : InputScheme
 
         bool exportIgnore = getExportIgnoreAttr(input);
         bool smudgeLfs = getLfsAttr(input);
-        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
+        auto accessor = repo->getAccessor(
+            rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»");
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -967,7 +1000,7 @@ struct GitInputScheme : InputScheme
         auto exportIgnore = getExportIgnoreAttr(input);
 
         ref<SourceAccessor> accessor =
-            repo->getAccessor(repoInfo.workdirInfo, exportIgnore, makeNotAllowedError(repoPath));
+            repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath));
 
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the

src/libfetchers/github.cc

@@ -351,7 +351,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
         auto accessor =
-            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
+            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string() + "»");
 
         return {accessor, input};
     }

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -135,8 +135,6 @@ struct Settings : public Config
 
 private:
     mutable Sync<std::shared_ptr<Cache>> _cache;
-
-    mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
 };
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -22,6 +22,12 @@ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
     virtual Hash flush() = 0;
 };
 
+struct GitAccessorOptions
+{
+    bool exportIgnore = false;
+    bool smudgeLfs = false;
+};
+
 struct GitRepo
 {
     virtual ~GitRepo() {}
@@ -89,10 +95,10 @@ struct GitRepo
     virtual bool hasObject(const Hash & oid) = 0;
 
     virtual ref<SourceAccessor>
-    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) = 0;
+    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0;
 
-    virtual ref<SourceAccessor>
-    getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
+    virtual ref<SourceAccessor> getAccessor(
+        const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) = 0;
 
     virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
 

src/libfetchers/include/nix/fetchers/registry.hh

@@ -13,8 +13,6 @@ namespace nix::fetchers {
 
 struct Registry
 {
-    const Settings & settings;
-
     enum RegistryType {
         Flag = 0,
         User = 1,
@@ -34,9 +32,8 @@ struct Registry
 
     std::vector<Entry> entries;
 
-    Registry(const Settings & settings, RegistryType type)
-        : settings{settings}
-        , type{type}
+    Registry(RegistryType type)
+        : type{type}
     {
     }
 
@@ -59,7 +56,7 @@ Path getUserRegistryPath();
 
 Registries getRegistries(const Settings & settings, Store & store);
 
-void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs);
+void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs);
 
 enum class UseRegistries : int {
     No,

src/libfetchers/registry.cc

@@ -14,10 +14,10 @@ std::shared_ptr<Registry> Registry::read(const Settings & settings, const Source
 {
     debug("reading registry '%s'", path);
 
-    auto registry = std::make_shared<Registry>(settings, type);
+    auto registry = std::make_shared<Registry>(type);
 
     if (!path.pathExists())
-        return std::make_shared<Registry>(settings, type);
+        return std::make_shared<Registry>(type);
 
     try {
 
@@ -125,15 +125,15 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
     return customRegistry;
 }
 
-std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
+std::shared_ptr<Registry> getFlagRegistry()
 {
-    static auto flagRegistry = std::make_shared<Registry>(settings, Registry::Flag);
+    static auto flagRegistry = std::make_shared<Registry>(Registry::Flag);
     return flagRegistry;
 }
 
-void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs)
+void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs)
 {
-    getFlagRegistry(settings)->add(from, to, extraAttrs);
+    getFlagRegistry()->add(from, to, extraAttrs);
 }
 
 static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, Store & store)
@@ -141,7 +141,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
     static auto reg = [&]() {
         auto path = settings.flakeRegistry.get();
         if (path == "") {
-            return std::make_shared<Registry>(settings, Registry::Global); // empty registry
+            return std::make_shared<Registry>(Registry::Global); // empty registry
         }
 
         return Registry::read(
@@ -165,7 +165,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
 Registries getRegistries(const Settings & settings, Store & store)
 {
     Registries registries;
-    registries.push_back(getFlagRegistry(settings));
+    registries.push_back(getFlagRegistry());
     registries.push_back(getUserRegistry(settings));
     registries.push_back(getSystemRegistry(settings));
     registries.push_back(getGlobalRegistry(settings, store));

src/libfetchers/tarball.cc

@@ -136,7 +136,7 @@ static DownloadTarballResult downloadTarball_(
             .treeHash = treeHash,
             .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
             .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
-            .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix),
+            .accessor = settings.getTarballCache()->getAccessor(treeHash, {}, displayPrefix),
         };
     };
 

src/libflake-c/nix_api_flake.cc

@@ -200,7 +200,7 @@ nix_value * nix_locked_flake_get_output_attrs(
     nix_clear_err(context);
     try {
         auto v = nix_alloc_value(context, evalState);
-        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value);
+        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value);
         return v;
     }
     NIXC_CATCH_ERRS_NULL

src/libflake/call-flake.nix

@@ -10,9 +10,6 @@ lockFileStr:
 # unlocked trees.
 overrides:
 
-# This is `prim_fetchFinalTree`.
-fetchTreeFinal:
-
 let
   inherit (builtins) mapAttrs;
 
@@ -52,7 +49,7 @@ let
         else
           # FIXME: remove obsolete node.info.
           # Note: lock file entries are always final.
-          fetchTreeFinal (node.info or { } // removeAttrs node.locked [ "dir" ]);
+          builtins.fetchFinalTree (node.info or { } // removeAttrs node.locked [ "dir" ]);
 
       subdir = overrides.${key}.dir or node.locked.dir or "";
 

src/libflake/flake-primops.cc

@@ -93,7 +93,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
         auto & vv = binds.alloc(s);
         std::visit(
             overloaded{
-                [&vv](const std::string & value) { vv.mkString(value); },
+                [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); },
                 [&vv](const uint64_t & value) { vv.mkInt(value); },
                 [&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }},
             value);
@@ -156,7 +156,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
         }
     }
     auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs);
-    v.mkString(flakeRef.to_string());
+    v.mkString(flakeRef.to_string(), state.mem);
 }
 
 nix::PrimOp flakeRefToString({

src/libflake/flake.cc

@@ -956,7 +956,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
         auto key = keyMap.find(node);
         assert(key != keyMap.end());
 
-        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel());
+        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem);
 
         overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
     }
@@ -966,12 +966,9 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
     Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix"));
 
     auto vLocks = state.allocValue();
-    vLocks->mkString(lockFileStr);
+    vLocks->mkString(lockFileStr, state.mem);
 
-    auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree");
-    assert(vFetchFinalTree);
-
-    Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree};
+    Value * args[] = {vLocks, &vOverrides};
     state.callFunction(*vCallFlake, args, vRes, noPos);
 }
 

src/libstore-tests/data/derivation/ca/all_set.json

@@ -4,10 +4,13 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+      {
+        "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+        "output": "out"
+      }
     ],
     "refs2": [
-      "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+      "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -20,18 +23,36 @@
   "outputChecks": {
     "forAllOutputs": {
       "allowedReferences": [
-        "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+        {
+          "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+          "output": "out"
+        }
       ],
       "allowedRequisites": [
-        "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
-        "bin"
+        {
+          "drvPath": "self",
+          "output": "bin"
+        },
+        {
+          "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+          "output": "dev"
+        }
       ],
       "disallowedReferences": [
-        "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
-        "dev"
+        {
+          "drvPath": "self",
+          "output": "dev"
+        },
+        {
+          "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+          "output": "out"
+        }
       ],
       "disallowedRequisites": [
-        "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
+        {
+          "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+          "output": "dev"
+        }
       ],
       "ignoreSelfRefs": true,
       "maxClosureSize": null,

src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json

@@ -4,10 +4,13 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+      {
+        "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+        "output": "out"
+      }
     ],
     "refs2": [
-      "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+      "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -23,11 +26,20 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
-          "dev"
+          {
+            "drvPath": "self",
+            "output": "dev"
+          },
+          {
+            "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+            "output": "out"
+          }
         ],
         "disallowedRequisites": [
-          "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
+          {
+            "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+            "output": "dev"
+          }
         ],
         "ignoreSelfRefs": false,
         "maxClosureSize": null,
@@ -44,11 +56,20 @@
       },
       "out": {
         "allowedReferences": [
-          "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+          {
+            "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+            "output": "out"
+          }
         ],
         "allowedRequisites": [
-          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
-          "bin"
+          {
+            "drvPath": "self",
+            "output": "bin"
+          },
+          {
+            "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
+            "output": "dev"
+          }
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/data/derivation/ia/all_set.json

@@ -4,10 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+      "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
     ],
     "refs2": [
-      "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+      "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -20,18 +20,24 @@
   "outputChecks": {
     "forAllOutputs": {
       "allowedReferences": [
-        "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+        "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
       ],
       "allowedRequisites": [
-        "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
-        "bin"
+        {
+          "drvPath": "self",
+          "output": "bin"
+        },
+        "z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
       ],
       "disallowedReferences": [
-        "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
-        "dev"
+        {
+          "drvPath": "self",
+          "output": "dev"
+        },
+        "r5cff30838majxk5mp3ip2diffi8vpaj-bar"
       ],
       "disallowedRequisites": [
-        "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
+        "9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
       ],
       "ignoreSelfRefs": true,
       "maxClosureSize": null,

src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json

@@ -4,10 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+      "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
     ],
     "refs2": [
-      "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+      "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -23,11 +23,14 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
-          "dev"
+          {
+            "drvPath": "self",
+            "output": "dev"
+          },
+          "r5cff30838majxk5mp3ip2diffi8vpaj-bar"
         ],
         "disallowedRequisites": [
-          "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
+          "9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
         ],
         "ignoreSelfRefs": false,
         "maxClosureSize": null,
@@ -44,11 +47,14 @@
       },
       "out": {
         "allowedReferences": [
-          "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+          "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
         ],
         "allowedRequisites": [
-          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
-          "bin"
+          {
+            "drvPath": "self",
+            "output": "bin"
+          },
+          "z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/derivation-advanced-attrs.cc

@@ -3,7 +3,7 @@
 
 #include "nix/util/experimental-features.hh"
 #include "nix/store/derivations.hh"
-#include "nix/store/derivations.hh"
+#include "nix/store/derived-path.hh"
 #include "nix/store/derivation-options.hh"
 #include "nix/store/parsed-derivations.hh"
 #include "nix/util/types.hh"
@@ -17,7 +17,7 @@ namespace nix {
 using namespace nlohmann;
 
 class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>,
-                                    public JsonCharacterizationTest<DerivationOptions>,
+                                    public JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>,
                                     public LibStoreTest
 {
 protected:
@@ -42,7 +42,8 @@ public:
     {
         this->readTest(fileName, [&](auto encoded) {
             auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
-            DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+            auto options = derivationOptionsFromStructuredAttrs(
+                *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
             EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures);
         });
     }
@@ -51,11 +52,14 @@ public:
      * Helper function to test DerivationOptions parsing and comparison
      */
     void testDerivationOptions(
-        const std::string & fileName, const DerivationOptions & expected, const StringSet & expectedSystemFeatures)
+        const std::string & fileName,
+        const DerivationOptions<SingleDerivedPath> & expected,
+        const StringSet & expectedSystemFeatures)
     {
         this->readTest(fileName, [&](auto encoded) {
             auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
-            DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+            auto options = derivationOptionsFromStructuredAttrs(
+                *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
             EXPECT_EQ(options, expected);
             EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures);
@@ -131,22 +135,38 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute
  * Since these are both repeated and sensative opaque values, it makes
  * sense to give them names in this file.
  */
-static std::string pathFoo = "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
-                   pathFooDev = "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
-                   pathBar = "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
-                   pathBarDev = "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
-                   pathBarDrvIA = "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
-                   pathBarDrvCA = "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
-                   placeholderFoo = "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
-                   placeholderFooDev = "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
-                   placeholderBar = "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
-                   placeholderBarDev = "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8";
+static SingleDerivedPath
+    pathFoo = SingleDerivedPath::Opaque{StorePath{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}},
+    pathFooDev = SingleDerivedPath::Opaque{StorePath{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}},
+    pathBar = SingleDerivedPath::Opaque{StorePath{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}},
+    pathBarDev = SingleDerivedPath::Opaque{StorePath{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}},
+    pathBarDrvIA = SingleDerivedPath::Opaque{StorePath{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}},
+    pathBarDrvCA = SingleDerivedPath::Opaque{StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}},
+    placeholderFoo =
+        SingleDerivedPath::Built{
+            .drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
+            .output = "out",
+        },
+    placeholderFooDev =
+        SingleDerivedPath::Built{
+            .drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
+            .output = "dev",
+        },
+    placeholderBar =
+        SingleDerivedPath::Built{
+            .drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
+            .output = "out",
+        },
+    placeholderBarDev = SingleDerivedPath::Built{
+        .drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
+        .output = "dev",
+    };
 
-using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph);
+using ExportReferencesMap = decltype(DerivationOptions<SingleDerivedPath>::exportReferencesGraph);
 
-static const DerivationOptions advancedAttributes_defaults = {
+static const DerivationOptions<SingleDerivedPath> advancedAttributes_defaults = {
     .outputChecks =
-        DerivationOptions::OutputChecks{
+        DerivationOptions<SingleDerivedPath>::OutputChecks{
             .ignoreSelfRefs = true,
         },
     .unsafeDiscardReferences = {},
@@ -167,7 +187,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
     this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(!got.structuredAttrs);
 
@@ -192,9 +213,9 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
 
 TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
 {
-    DerivationOptions expected = {
+    DerivationOptions<SingleDerivedPath> expected = {
         .outputChecks =
-            DerivationOptions::OutputChecks{
+            DerivationOptions<SingleDerivedPath>::OutputChecks{
                 .ignoreSelfRefs = true,
             },
         .unsafeDiscardReferences = {},
@@ -212,12 +233,13 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
     this->readTest("advanced-attributes.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(!got.structuredAttrs);
 
         // Reset fields that vary between test cases to enable whole-object comparison
-        options.outputChecks = DerivationOptions::OutputChecks{.ignoreSelfRefs = true};
+        options.outputChecks = DerivationOptions<SingleDerivedPath>::OutputChecks{.ignoreSelfRefs = true};
         options.exportReferencesGraph = {};
 
         EXPECT_EQ(options, expected);
@@ -227,14 +249,14 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
     });
 };
 
-DerivationOptions advancedAttributes_ia = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_ia = {
     .outputChecks =
-        DerivationOptions::OutputChecks{
+        DerivationOptions<SingleDerivedPath>::OutputChecks{
             .ignoreSelfRefs = true,
-            .allowedReferences = StringSet{pathFoo},
-            .disallowedReferences = StringSet{pathBar, "dev"},
-            .allowedRequisites = StringSet{pathFooDev, "bin"},
-            .disallowedRequisites = StringSet{pathBarDev},
+            .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo},
+            .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}},
+            .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}},
+            .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev},
         },
     .unsafeDiscardReferences = {},
     .passAsFile = {},
@@ -257,14 +279,14 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_ia)
     testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"});
 };
 
-DerivationOptions advancedAttributes_ca = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_ca = {
     .outputChecks =
-        DerivationOptions::OutputChecks{
+        DerivationOptions<SingleDerivedPath>::OutputChecks{
             .ignoreSelfRefs = true,
-            .allowedReferences = StringSet{placeholderFoo},
-            .disallowedReferences = StringSet{placeholderBar, "dev"},
-            .allowedRequisites = StringSet{placeholderFooDev, "bin"},
-            .disallowedRequisites = StringSet{placeholderBarDev},
+            .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo},
+            .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}},
+            .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}},
+            .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev},
         },
     .unsafeDiscardReferences = {},
     .passAsFile = {},
@@ -287,8 +309,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
     testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"});
 };
 
-DerivationOptions advancedAttributes_structuredAttrs_defaults = {
-    .outputChecks = std::map<std::string, DerivationOptions::OutputChecks>{},
+DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_defaults = {
+    .outputChecks = std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{},
     .unsafeDiscardReferences = {},
     .passAsFile = {},
     .exportReferencesGraph = {},
@@ -307,7 +329,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
     this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(got.structuredAttrs);
 
@@ -332,11 +355,11 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
 
 TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
 {
-    DerivationOptions expected = {
+    DerivationOptions<SingleDerivedPath> expected = {
         .outputChecks =
-            std::map<std::string, DerivationOptions::OutputChecks>{
+            std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
                 {"dev",
-                 DerivationOptions::OutputChecks{
+                 DerivationOptions<SingleDerivedPath>::OutputChecks{
                      .maxSize = 789,
                      .maxClosureSize = 5909,
                  }},
@@ -357,7 +380,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
     this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
+        auto options = derivationOptionsFromStructuredAttrs(
+            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
 
         EXPECT_TRUE(got.structuredAttrs);
 
@@ -365,7 +389,8 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
         {
             // Delete all keys but "dev" in options.outputChecks
             auto * outputChecksMapP =
-                std::get_if<std::map<std::string, DerivationOptions::OutputChecks>>(&options.outputChecks);
+                std::get_if<std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>>(
+                    &options.outputChecks);
             ASSERT_TRUE(outputChecksMapP);
             auto & outputChecksMap = *outputChecksMapP;
             auto devEntry = outputChecksMap.find("dev");
@@ -385,21 +410,21 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
     });
 };
 
-DerivationOptions advancedAttributes_structuredAttrs_ia = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ia = {
     .outputChecks =
-        std::map<std::string, DerivationOptions::OutputChecks>{
+        std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
             {"out",
-             DerivationOptions::OutputChecks{
-                 .allowedReferences = StringSet{pathFoo},
-                 .allowedRequisites = StringSet{pathFooDev, "bin"},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo},
+                 .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}},
              }},
             {"bin",
-             DerivationOptions::OutputChecks{
-                 .disallowedReferences = StringSet{pathBar, "dev"},
-                 .disallowedRequisites = StringSet{pathBarDev},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}},
+                 .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev},
              }},
             {"dev",
-             DerivationOptions::OutputChecks{
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
                  .maxSize = 789,
                  .maxClosureSize = 5909,
              }},
@@ -427,21 +452,21 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
         "advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"});
 };
 
-DerivationOptions advancedAttributes_structuredAttrs_ca = {
+DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ca = {
     .outputChecks =
-        std::map<std::string, DerivationOptions::OutputChecks>{
+        std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
             {"out",
-             DerivationOptions::OutputChecks{
-                 .allowedReferences = StringSet{placeholderFoo},
-                 .allowedRequisites = StringSet{placeholderFooDev, "bin"},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo},
+                 .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}},
              }},
             {"bin",
-             DerivationOptions::OutputChecks{
-                 .disallowedReferences = StringSet{placeholderBar, "dev"},
-                 .disallowedRequisites = StringSet{placeholderBarDev},
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}},
+                 .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev},
              }},
             {"dev",
-             DerivationOptions::OutputChecks{
+             DerivationOptions<SingleDerivedPath>::OutputChecks{
                  .maxSize = 789,
                  .maxClosureSize = 5909,
              }},
@@ -471,14 +496,16 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
         {"rainbow", "uid-range", "ca-derivations"});
 };
 
-#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2)                                                             \
-    TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json)                                                  \
-    {                                                                                                      \
-        this->JsonCharacterizationTest<DerivationOptions>::readJsonTest(#VAR, advancedAttributes_##VAR2);  \
-    }                                                                                                      \
-    TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json)                                                    \
-    {                                                                                                      \
-        this->JsonCharacterizationTest<DerivationOptions>::writeJsonTest(#VAR, advancedAttributes_##VAR2); \
+#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2)                                               \
+    TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json)                                    \
+    {                                                                                        \
+        this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::readJsonTest(  \
+            #VAR, advancedAttributes_##VAR2);                                                \
+    }                                                                                        \
+    TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json)                                      \
+    {                                                                                        \
+        this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::writeJsonTest( \
+            #VAR, advancedAttributes_##VAR2);                                                \
     }
 
 TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults)

src/libstore-tests/references.cc

@@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep)
     // Create an in-memory file system with various reference patterns
     auto accessor = make_ref<MemorySourceAccessor>();
     accessor->root = File::Directory{
-        .contents{
+        .entries{
             {
                 // file1.txt: contains hash1
                 "file1.txt",
@@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep)
                 // subdir: a subdirectory
                 "subdir",
                 File::Directory{
-                    .contents{
+                    .entries{
                         {
                             // subdir/file4.txt: contains hash1 again
                             "file4.txt",

src/libstore/binary-cache-store.cc

@@ -8,7 +8,7 @@
 #include "nix/util/sync.hh"
 #include "nix/store/remote-fs-accessor.hh"
 #include "nix/store/nar-info-disk-cache.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 #include "nix/util/thread-pool.hh"
 #include "nix/util/callback.hh"
 #include "nix/util/signals.hh"
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
+            {"root", listNarDeep(*narAccessor, CanonPath::root)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");

src/libstore/build/derivation-building-goal.cc

@@ -32,14 +32,6 @@ DerivationBuildingGoal::DerivationBuildingGoal(
     , drv{std::make_unique<Derivation>(drv)}
     , buildMode(buildMode)
 {
-    try {
-        drvOptions =
-            std::make_unique<DerivationOptions>(DerivationOptions::fromStructuredAttrs(drv.env, drv.structuredAttrs));
-    } catch (Error & e) {
-        e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
-        throw;
-    }
-
     name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath));
     trace("created");
 
@@ -206,6 +198,38 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
 
 Goal::Co DerivationBuildingGoal::tryToBuild()
 {
+    auto drvOptions = [&] {
+        DerivationOptions<SingleDerivedPath> temp;
+        try {
+            temp =
+                derivationOptionsFromStructuredAttrs(worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
+        } catch (Error & e) {
+            e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
+            throw;
+        }
+
+        auto res = tryResolve(
+            temp,
+            [&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
+                try {
+                    return resolveDerivedPath(
+                        worker.store, SingleDerivedPath::Built{drvPath, outputName}, &worker.evalStore);
+                } catch (Error &) {
+                    return std::nullopt;
+                }
+            });
+
+        /* The derivation must have all of its inputs gotten this point,
+           so the resolution will surely succeed.
+
+           (Actually, we shouldn't even enter this goal until we have a
+           resolved derivation, or derivation with only input addressed
+           transitive inputs, so this should be a no-opt anyways.)
+         */
+        assert(res);
+        return *res;
+    }();
+
     std::map<std::string, InitialOutput> initialOutputs;
 
     /* Recheck at this point. In particular, whereas before we were
@@ -344,13 +368,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
         /* Don't do a remote build if the derivation has the attribute
            `preferLocalBuild' set.  Also, check and repair modes are only
            supported for local builds. */
-        bool buildLocally = (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
+        bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv))
                             && settings.maxBuildJobs.get() != 0;
 
         if (buildLocally) {
             useHook = false;
         } else {
-            switch (tryBuildHook(initialOutputs)) {
+            switch (tryBuildHook(initialOutputs, drvOptions)) {
             case rpAccept:
                 /* Yes, it has started doing so.  Wait until we get
                    EOF from the hook. */
@@ -379,7 +403,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
 
                 externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv);
 
-                if (!externalBuilder && !drvOptions->canBuildLocally(worker.store, *drv)) {
+                if (!externalBuilder && !drvOptions.canBuildLocally(worker.store, *drv)) {
                     auto msg =
                         fmt("Cannot build '%s'.\n"
                             "Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL
@@ -388,7 +412,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
                             "Current system: '%s' with features {%s}",
                             Magenta(worker.store.printStorePath(drvPath)),
                             Magenta(drv->platform),
-                            concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
+                            concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(*drv)),
                             Magenta(settings.thisSystem),
                             concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures));
 
@@ -586,7 +610,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
             }
 
             try {
-                desugaredEnv = DesugaredEnv::create(worker.store, *drv, *drvOptions, inputPaths);
+                desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths);
             } catch (BuildError & e) {
                 outputLocks.unlock();
                 worker.permanentFailure = true;
@@ -597,7 +621,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
                 .drvPath = drvPath,
                 .buildResult = buildResult,
                 .drv = *drv,
-                .drvOptions = *drvOptions,
+                .drvOptions = drvOptions,
                 .inputPaths = inputPaths,
                 .initialOutputs = initialOutputs,
                 .buildMode = buildMode,
@@ -803,7 +827,8 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
     return BuildError{e.status, msg};
 }
 
-HookReply DerivationBuildingGoal::tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs)
+HookReply DerivationBuildingGoal::tryBuildHook(
+    const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions)
 {
 #ifdef _WIN32 // TODO enable build hook on Windows
     return rpDecline;
@@ -820,7 +845,7 @@ HookReply DerivationBuildingGoal::tryBuildHook(const std::map<std::string, Initi
 
         /* Send the request to the hook. */
         worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
-                          << worker.store.printStorePath(drvPath) << drvOptions->getRequiredSystemFeatures(*drv);
+                          << worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv);
         worker.hook->sink.flush();
 
         /* Read the first line of input, which should be a word indicating

src/libstore/build/derivation-check.cc

@@ -11,7 +11,7 @@ void checkOutputs(
     Store & store,
     const StorePath & drvPath,
     const decltype(Derivation::outputs) & drvOutputs,
-    const decltype(DerivationOptions::outputChecks) & outputChecks,
+    const decltype(DerivationOptions<StorePath>::outputChecks) & outputChecks,
     const std::map<std::string, ValidPathInfo> & outputs)
 {
     std::map<Path, const ValidPathInfo &> outputsByPath;
@@ -85,7 +85,7 @@ void checkOutputs(
             return std::make_pair(std::move(pathsDone), closureSize);
         };
 
-        auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) {
+        auto applyChecks = [&](const DerivationOptions<StorePath>::OutputChecks & checks) {
             if (checks.maxSize && info.narSize > *checks.maxSize)
                 throw BuildError(
                     BuildResult::Failure::OutputRejected,
@@ -105,28 +105,33 @@ void checkOutputs(
                         *checks.maxClosureSize);
             }
 
-            auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) {
+            auto checkRefs = [&](const std::set<DrvRef<StorePath>> & value, bool allowed, bool recursive) {
                 /* Parse a list of reference specifiers.  Each element must
                    either be a store path, or the symbolic name of the output
                    of the derivation (such as `out'). */
                 StorePathSet spec;
                 for (auto & i : value) {
-                    if (store.isStorePath(i))
-                        spec.insert(store.parseStorePath(i));
-                    else if (auto output = get(outputs, i))
-                        spec.insert(output->path);
-                    else {
-                        std::string outputsListing =
-                            concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; });
-                        throw BuildError(
-                            BuildResult::Failure::OutputRejected,
-                            "derivation '%s' output check for '%s' contains an illegal reference specifier '%s',"
-                            " expected store path or output name (one of [%s])",
-                            store.printStorePath(drvPath),
-                            outputName,
-                            i,
-                            outputsListing);
-                    }
+                    std::visit(
+                        overloaded{
+                            [&](const StorePath & path) { spec.insert(path); },
+                            [&](const OutputName & refOutputName) {
+                                if (auto output = get(outputs, refOutputName))
+                                    spec.insert(output->path);
+                                else {
+                                    std::string outputsListing =
+                                        concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; });
+                                    throw BuildError(
+                                        BuildResult::Failure::OutputRejected,
+                                        "derivation '%s' output check for '%s' contains output name '%s',"
+                                        " but this is not a valid output of this derivation."
+                                        " (Valid outputs are [%s].)",
+                                        store.printStorePath(drvPath),
+                                        outputName,
+                                        refOutputName,
+                                        outputsListing);
+                                }
+                            }},
+                        i);
                 }
 
                 auto used = recursive ? getClosure(info.path).first : info.references;
@@ -180,8 +185,8 @@ void checkOutputs(
 
         std::visit(
             overloaded{
-                [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); },
-                [&](const std::map<std::string, DerivationOptions::OutputChecks> & checksPerOutput) {
+                [&](const DerivationOptions<StorePath>::OutputChecks & checks) { applyChecks(checks); },
+                [&](const std::map<std::string, DerivationOptions<StorePath>::OutputChecks> & checksPerOutput) {
                     if (auto outputChecks = get(checksPerOutput, outputName))
 
                         applyChecks(*outputChecks);

src/libstore/build/derivation-check.hh

@@ -21,7 +21,7 @@ void checkOutputs(
     Store & store,
     const StorePath & drvPath,
     const decltype(Derivation::outputs) & drvOutputs,
-    const decltype(DerivationOptions::outputChecks) & drvOptions,
+    const decltype(DerivationOptions<StorePath>::outputChecks) & drvOptions,
     const std::map<std::string, ValidPathInfo> & outputs);
 
 } // namespace nix

src/libstore/build/derivation-env-desugar.cc

@@ -18,7 +18,10 @@ std::string & DesugaredEnv::atFileEnvPair(std::string_view name, std::string fil
 }
 
 DesugaredEnv DesugaredEnv::create(
-    Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths)
+    Store & store,
+    const Derivation & drv,
+    const DerivationOptions<StorePath> & drvOptions,
+    const StorePathSet & inputPaths)
 {
     DesugaredEnv res;
 
@@ -46,7 +49,7 @@ DesugaredEnv DesugaredEnv::create(
         }
 
         /* Handle exportReferencesGraph(), if set. */
-        for (auto & [fileName, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) {
+        for (auto & [fileName, storePaths] : drvOptions.exportReferencesGraph) {
             /* Write closure info to <fileName>. */
             res.extraFiles.insert_or_assign(
                 fileName, store.makeValidityRegistration(store.exportReferences(storePaths, inputPaths), false, false));

src/libstore/build/derivation-goal.cc

@@ -64,9 +64,10 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
 {
     trace("have derivation");
 
-    auto drvOptions = [&]() -> DerivationOptions {
+    auto drvOptions = [&]() -> DerivationOptions<SingleDerivedPath> {
         try {
-            return DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
+            return derivationOptionsFromStructuredAttrs(
+                worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
         } catch (Error & e) {
             e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
             throw;

src/libstore/derivation-options.cc

@@ -2,15 +2,18 @@
 #include "nix/util/json-utils.hh"
 #include "nix/store/parsed-derivations.hh"
 #include "nix/store/derivations.hh"
+#include "nix/store/derived-path.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/types.hh"
 #include "nix/util/util.hh"
 #include "nix/store/globals.hh"
+#include "nix/util/variant-wrapper.hh"
 
 #include <optional>
 #include <string>
 #include <variant>
 #include <regex>
+#include <ranges>
 
 namespace nix {
 
@@ -90,14 +93,38 @@ getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const st
     return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{});
 }
 
-using OutputChecks = DerivationOptions::OutputChecks;
+template<typename Inputs>
+using OutputChecks = DerivationOptions<Inputs>::OutputChecks;
 
-using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
+template<typename Inputs>
+using OutputChecksVariant = std::variant<OutputChecks<Inputs>, std::map<std::string, OutputChecks<Inputs>>>;
 
-DerivationOptions DerivationOptions::fromStructuredAttrs(
-    const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn)
+DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs(
+    const StoreDirConfig & store,
+    const StringMap & env,
+    const StructuredAttrs * parsed,
+    bool shouldWarn,
+    const ExperimentalFeatureSettings & mockXpSettings)
 {
-    return fromStructuredAttrs(env, parsed ? &*parsed : nullptr);
+    /* Use the SingleDerivedPath version with empty inputDrvs, then
+       resolve. */
+    DerivedPathMap<StringSet> emptyInputDrvs{};
+    auto singleDerivedPathOptions =
+        derivationOptionsFromStructuredAttrs(store, emptyInputDrvs, env, parsed, shouldWarn, mockXpSettings);
+
+    /* "Resolve" all SingleDerivedPath inputs to StorePath. */
+    auto resolved = tryResolve(
+        singleDerivedPathOptions,
+        [&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
+            // there should be nothing to resolve
+            assert(false);
+        });
+
+    /* Since we should never need to call the call back, there should be
+       no way it fails. */
+    assert(resolved);
+
+    return *resolved;
 }
 
 static void flatten(const nlohmann::json & value, StringSet & res)
@@ -111,10 +138,63 @@ static void flatten(const nlohmann::json & value, StringSet & res)
         throw Error("'exportReferencesGraph' value is not an array or a string");
 }
 
-DerivationOptions
-DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
+DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
+    const StoreDirConfig & store,
+    const DerivedPathMap<StringSet> & inputDrvs,
+    const StringMap & env,
+    const StructuredAttrs * parsed,
+    bool shouldWarn,
+    const ExperimentalFeatureSettings & mockXpSettings)
 {
-    DerivationOptions defaults = {};
+    DerivationOptions<SingleDerivedPath> defaults = {};
+
+    std::map<std::string, SingleDerivedPath::Built> placeholders;
+    if (mockXpSettings.isEnabled(Xp::CaDerivations)) {
+        /* Initialize placeholder map from inputDrvs */
+        auto initPlaceholders = [&](this const auto & initPlaceholders,
+                                    ref<const SingleDerivedPath> basePath,
+                                    const DerivedPathMap<StringSet>::ChildNode & node) -> void {
+            for (const auto & outputName : node.value) {
+                auto built = SingleDerivedPath::Built{
+                    .drvPath = basePath,
+                    .output = outputName,
+                };
+                placeholders.insert_or_assign(
+                    DownstreamPlaceholder::fromSingleDerivedPathBuilt(built, mockXpSettings).render(),
+                    std::move(built));
+            }
+
+            for (const auto & [outputName, childNode] : node.childMap) {
+                initPlaceholders(
+                    make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{
+                        .drvPath = basePath,
+                        .output = outputName,
+                    }),
+                    childNode);
+            }
+        };
+
+        for (const auto & [drvPath, outputs] : inputDrvs.map) {
+            auto basePath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{drvPath});
+            initPlaceholders(basePath, outputs);
+        }
+    }
+
+    auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath {
+        if (auto it = placeholders.find(pathS); it != placeholders.end())
+            return it->second;
+        else
+            return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
+    };
+
+    auto parseRef = [&](const std::string & pathS) -> DrvRef<SingleDerivedPath> {
+        if (auto it = placeholders.find(pathS); it != placeholders.end())
+            return it->second;
+        if (store.isStorePath(pathS))
+            return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
+        else
+            return pathS;
+    };
 
     if (shouldWarn && parsed) {
         auto & structuredAttrs = parsed->structuredAttrs;
@@ -146,14 +226,14 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
     }
 
     return {
-        .outputChecks = [&]() -> OutputChecksVariant {
+        .outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> {
             if (parsed) {
                 auto & structuredAttrs = parsed->structuredAttrs;
 
-                std::map<std::string, OutputChecks> res;
+                std::map<std::string, OutputChecks<SingleDerivedPath>> res;
                 if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
                     for (auto & [outputName, output_] : getObject(*outputChecks)) {
-                        OutputChecks checks;
+                        OutputChecks<SingleDerivedPath> checks;
 
                         auto & output = getObject(output_);
 
@@ -163,13 +243,14 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
                         if (auto maxClosureSize = get(output, "maxClosureSize"))
                             checks.maxClosureSize = maxClosureSize->get<uint64_t>();
 
-                        auto get_ = [&output = output](const std::string & name) -> std::optional<StringSet> {
+                        auto get_ =
+                            [&](const std::string & name) -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
                             if (auto i = get(output, name)) {
-                                StringSet res;
+                                std::set<DrvRef<SingleDerivedPath>> res;
                                 for (auto j = i->begin(); j != i->end(); ++j) {
                                     if (!j->is_string())
                                         throw Error("attribute '%s' must be a list of strings", name);
-                                    res.insert(j->get<std::string>());
+                                    res.insert(parseRef(j->get<std::string>()));
                                 }
                                 return res;
                             }
@@ -178,7 +259,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
 
                         res.insert_or_assign(
                             outputName,
-                            OutputChecks{
+                            OutputChecks<SingleDerivedPath>{
                                 .maxSize = [&]() -> std::optional<uint64_t> {
                                     if (auto maxSize = get(output, "maxSize"))
                                         return maxSize->get<uint64_t>();
@@ -192,21 +273,32 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
                                         return std::nullopt;
                                 }(),
                                 .allowedReferences = get_("allowedReferences"),
-                                .disallowedReferences = get_("disallowedReferences").value_or(StringSet{}),
+                                .disallowedReferences =
+                                    get_("disallowedReferences").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
                                 .allowedRequisites = get_("allowedRequisites"),
-                                .disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}),
+                                .disallowedRequisites =
+                                    get_("disallowedRequisites").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
                             });
                     }
                 }
                 return res;
             } else {
-                return OutputChecks{
+                auto parseRefSet = [&](const std::optional<StringSet> optionalStringSet)
+                    -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
+                    if (!optionalStringSet)
+                        return std::nullopt;
+                    auto range = *optionalStringSet | std::views::transform(parseRef);
+                    return std::set<DrvRef<SingleDerivedPath>>(range.begin(), range.end());
+                };
+                return OutputChecks<SingleDerivedPath>{
                     // legacy non-structured-attributes case
                     .ignoreSelfRefs = true,
-                    .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"),
-                    .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}),
-                    .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"),
-                    .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}),
+                    .allowedReferences = parseRefSet(getStringSetAttr(env, parsed, "allowedReferences")),
+                    .disallowedReferences = parseRefSet(getStringSetAttr(env, parsed, "disallowedReferences"))
+                                                .value_or(std::set<DrvRef<SingleDerivedPath>>{}),
+                    .allowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "allowedRequisites")),
+                    .disallowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "disallowedRequisites"))
+                                                .value_or(std::set<DrvRef<SingleDerivedPath>>{}),
                 };
             }
         }(),
@@ -245,16 +337,19 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
             }(),
         .exportReferencesGraph =
             [&] {
-                std::map<std::string, StringSet> ret;
+                std::map<std::string, std::set<SingleDerivedPath>> ret;
 
                 if (parsed) {
                     auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
                     if (!e || !e->is_object())
                         return ret;
-                    for (auto & [key, value] : getObject(*e)) {
+                    for (auto & [key, storePathsJson] : getObject(*e)) {
                         StringSet ss;
-                        flatten(value, ss);
-                        ret.insert_or_assign(key, std::move(ss));
+                        flatten(storePathsJson, ss);
+                        std::set<SingleDerivedPath> storePaths;
+                        for (auto & s : ss)
+                            storePaths.insert(parseSingleDerivedPath(s));
+                        ret.insert_or_assign(key, std::move(storePaths));
                     }
                 } else {
                     auto s = getOr(env, "exportReferencesGraph", "");
@@ -268,7 +363,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
                             throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
 
                         auto & storePathS = *i++;
-                        ret.insert_or_assign(std::move(fileName), StringSet{storePathS});
+                        ret.insert_or_assign(std::move(fileName), std::set{parseSingleDerivedPath(storePathS)});
                     }
                 }
                 return ret;
@@ -286,28 +381,8 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
     };
 }
 
-std::map<std::string, StorePathSet>
-DerivationOptions::getParsedExportReferencesGraph(const StoreDirConfig & store) const
-{
-    std::map<std::string, StorePathSet> res;
-
-    for (auto & [fileName, ss] : exportReferencesGraph) {
-        StorePathSet storePaths;
-        for (auto & storePathS : ss) {
-            if (!store.isInStore(storePathS))
-                throw BuildError(
-                    BuildResult::Failure::InputRejected,
-                    "'exportReferencesGraph' contains a non-store path '%1%'",
-                    storePathS);
-            storePaths.insert(store.toStorePath(storePathS).first);
-        }
-        res.insert_or_assign(fileName, storePaths);
-    }
-
-    return res;
-}
-
-StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & drv) const
+template<typename Input>
+StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivation & drv) const
 {
     // FIXME: cache this?
     StringSet res;
@@ -318,7 +393,8 @@ StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & d
     return res;
 }
 
-bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
+template<typename Input>
+bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
 {
     if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform)
         && !drv.isBuiltin())
@@ -334,42 +410,194 @@ bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivatio
     return true;
 }
 
-bool DerivationOptions::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
+template<typename Input>
+bool DerivationOptions<Input>::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
 {
     return preferLocalBuild && canBuildLocally(localStore, drv);
 }
 
-bool DerivationOptions::substitutesAllowed() const
+template<typename Input>
+bool DerivationOptions<Input>::substitutesAllowed() const
 {
     return settings.alwaysAllowSubstitutes ? true : allowSubstitutes;
 }
 
-bool DerivationOptions::useUidRange(const BasicDerivation & drv) const
+template<typename Input>
+bool DerivationOptions<Input>::useUidRange(const BasicDerivation & drv) const
 {
     return getRequiredSystemFeatures(drv).count("uid-range");
 }
 
+std::optional<DerivationOptions<StorePath>> tryResolve(
+    const DerivationOptions<SingleDerivedPath> & drvOptions,
+    std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
+        queryResolutionChain)
+{
+    auto tryResolvePath = [&](const SingleDerivedPath & input) -> std::optional<StorePath> {
+        return std::visit(
+            overloaded{
+                [](const SingleDerivedPath::Opaque & p) -> std::optional<StorePath> { return p.path; },
+                [&](const SingleDerivedPath::Built & p) -> std::optional<StorePath> {
+                    return queryResolutionChain(p.drvPath, p.output);
+                }},
+            input.raw());
+    };
+
+    auto tryResolveRef = [&](const DrvRef<SingleDerivedPath> & ref) -> std::optional<DrvRef<StorePath>> {
+        return std::visit(
+            overloaded{
+                [](const OutputName & outputName) -> std::optional<DrvRef<StorePath>> { return outputName; },
+                [&](const SingleDerivedPath & input) -> std::optional<DrvRef<StorePath>> {
+                    return tryResolvePath(input);
+                }},
+            ref);
+    };
+
+    auto tryResolveRefSet =
+        [&](const std::set<DrvRef<SingleDerivedPath>> & refSet) -> std::optional<std::set<DrvRef<StorePath>>> {
+        std::set<DrvRef<StorePath>> resolvedSet;
+        for (const auto & ref : refSet) {
+            auto resolvedRef = tryResolveRef(ref);
+            if (!resolvedRef)
+                return std::nullopt;
+            resolvedSet.insert(*resolvedRef);
+        }
+        return resolvedSet;
+    };
+
+    // Helper function to try resolving OutputChecks using functional style
+    auto tryResolveOutputChecks = [&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
+        -> std::optional<DerivationOptions<StorePath>::OutputChecks> {
+        std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedReferences;
+        if (checks.allowedReferences) {
+            resolvedAllowedReferences = tryResolveRefSet(*checks.allowedReferences);
+            if (!resolvedAllowedReferences)
+                return std::nullopt;
+        }
+
+        std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedRequisites;
+        if (checks.allowedRequisites) {
+            resolvedAllowedRequisites = tryResolveRefSet(*checks.allowedRequisites);
+            if (!resolvedAllowedRequisites)
+                return std::nullopt;
+        }
+
+        auto resolvedDisallowedReferences = tryResolveRefSet(checks.disallowedReferences);
+        if (!resolvedDisallowedReferences)
+            return std::nullopt;
+
+        auto resolvedDisallowedRequisites = tryResolveRefSet(checks.disallowedRequisites);
+        if (!resolvedDisallowedRequisites)
+            return std::nullopt;
+
+        return DerivationOptions<StorePath>::OutputChecks{
+            .ignoreSelfRefs = checks.ignoreSelfRefs,
+            .maxSize = checks.maxSize,
+            .maxClosureSize = checks.maxClosureSize,
+            .allowedReferences = resolvedAllowedReferences,
+            .disallowedReferences = *resolvedDisallowedReferences,
+            .allowedRequisites = resolvedAllowedRequisites,
+            .disallowedRequisites = *resolvedDisallowedRequisites,
+        };
+    };
+
+    // Helper function to resolve exportReferencesGraph using functional style
+    auto tryResolveExportReferencesGraph = [&](const std::map<std::string, std::set<SingleDerivedPath>> & exportGraph)
+        -> std::optional<std::map<std::string, std::set<StorePath>>> {
+        std::map<std::string, std::set<StorePath>> resolved;
+        for (const auto & [name, inputPaths] : exportGraph) {
+            std::set<StorePath> resolvedPaths;
+            for (const auto & inputPath : inputPaths) {
+                auto resolvedPath = tryResolvePath(inputPath);
+                if (!resolvedPath)
+                    return std::nullopt;
+                resolvedPaths.insert(*resolvedPath);
+            }
+            resolved.emplace(name, std::move(resolvedPaths));
+        }
+        return resolved;
+    };
+
+    // Resolve outputChecks using functional style with std::visit
+    auto resolvedOutputChecks = std::visit(
+        overloaded{
+            [&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
+                -> std::optional<std::variant<
+                    DerivationOptions<StorePath>::OutputChecks,
+                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
+                auto resolved = tryResolveOutputChecks(checks);
+                if (!resolved)
+                    return std::nullopt;
+                return std::variant<
+                    DerivationOptions<StorePath>::OutputChecks,
+                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(*resolved);
+            },
+            [&](const std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks> & checksMap)
+                -> std::optional<std::variant<
+                    DerivationOptions<StorePath>::OutputChecks,
+                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
+                std::map<std::string, DerivationOptions<StorePath>::OutputChecks> resolvedMap;
+                for (const auto & [outputName, checks] : checksMap) {
+                    auto resolved = tryResolveOutputChecks(checks);
+                    if (!resolved)
+                        return std::nullopt;
+                    resolvedMap.emplace(outputName, *resolved);
+                }
+                return std::variant<
+                    DerivationOptions<StorePath>::OutputChecks,
+                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(resolvedMap);
+            }},
+        drvOptions.outputChecks);
+
+    if (!resolvedOutputChecks)
+        return std::nullopt;
+
+    // Resolve exportReferencesGraph
+    auto resolvedExportGraph = tryResolveExportReferencesGraph(drvOptions.exportReferencesGraph);
+    if (!resolvedExportGraph)
+        return std::nullopt;
+
+    // Return resolved DerivationOptions using designated initializers
+    return DerivationOptions<StorePath>{
+        .outputChecks = *resolvedOutputChecks,
+        .unsafeDiscardReferences = drvOptions.unsafeDiscardReferences,
+        .passAsFile = drvOptions.passAsFile,
+        .exportReferencesGraph = *resolvedExportGraph,
+        .additionalSandboxProfile = drvOptions.additionalSandboxProfile,
+        .noChroot = drvOptions.noChroot,
+        .impureHostDeps = drvOptions.impureHostDeps,
+        .impureEnvVars = drvOptions.impureEnvVars,
+        .allowLocalNetworking = drvOptions.allowLocalNetworking,
+        .requiredSystemFeatures = drvOptions.requiredSystemFeatures,
+        .preferLocalBuild = drvOptions.preferLocalBuild,
+        .allowSubstitutes = drvOptions.allowSubstitutes,
+    };
+}
+
+template struct DerivationOptions<StorePath>;
+template struct DerivationOptions<SingleDerivedPath>;
+
 } // namespace nix
 
 namespace nlohmann {
 
 using namespace nix;
 
-DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json_)
+DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDerivedPath>>::from_json(const json & json_)
 {
     auto & json = getObject(json_);
 
     return {
-        .outputChecks = [&]() -> OutputChecksVariant {
+        .outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> {
             auto outputChecks = getObject(valueAt(json, "outputChecks"));
 
             auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs");
             auto perOutputOpt = optionalValueAt(outputChecks, "perOutput");
 
             if (forAllOutputsOpt && !perOutputOpt) {
-                return static_cast<OutputChecks>(*forAllOutputsOpt);
+                return static_cast<OutputChecks<SingleDerivedPath>>(*forAllOutputsOpt);
             } else if (perOutputOpt && !forAllOutputsOpt) {
-                return static_cast<std::map<std::string, OutputChecks>>(*perOutputOpt);
+                return static_cast<std::map<std::string, OutputChecks<SingleDerivedPath>>>(*perOutputOpt);
             } else {
                 throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required");
             }
@@ -377,7 +605,7 @@ DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json
 
         .unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"),
         .passAsFile = getStringSet(valueAt(json, "passAsFile")),
-        .exportReferencesGraph = getMap<StringSet>(getObject(valueAt(json, "exportReferencesGraph")), getStringSet),
+        .exportReferencesGraph = valueAt(json, "exportReferencesGraph"),
 
         .additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")),
         .noChroot = getBoolean(valueAt(json, "noChroot")),
@@ -391,16 +619,17 @@ DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json
     };
 }
 
-void adl_serializer<DerivationOptions>::to_json(json & json, const DerivationOptions & o)
+void adl_serializer<DerivationOptions<SingleDerivedPath>>::to_json(
+    json & json, const DerivationOptions<SingleDerivedPath> & o)
 {
     json["outputChecks"] = std::visit(
         overloaded{
-            [&](const OutputChecks & checks) {
+            [&](const OutputChecks<SingleDerivedPath> & checks) {
                 nlohmann::json outputChecks;
                 outputChecks["forAllOutputs"] = checks;
                 return outputChecks;
             },
-            [&](const std::map<std::string, OutputChecks> & checksPerOutput) {
+            [&](const std::map<std::string, OutputChecks<SingleDerivedPath>> & checksPerOutput) {
                 nlohmann::json outputChecks;
                 outputChecks["perOutput"] = checksPerOutput;
                 return outputChecks;
@@ -423,7 +652,7 @@ void adl_serializer<DerivationOptions>::to_json(json & json, const DerivationOpt
     json["allowSubstitutes"] = o.allowSubstitutes;
 }
 
-DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json_)
+OutputChecks<SingleDerivedPath> adl_serializer<OutputChecks<SingleDerivedPath>>::from_json(const json & json_)
 {
     auto & json = getObject(json_);
 
@@ -431,14 +660,16 @@ DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>:
         .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")),
         .maxSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxSize"))),
         .maxClosureSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxClosureSize"))),
-        .allowedReferences = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedReferences"))),
-        .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")),
-        .allowedRequisites = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedRequisites"))),
-        .disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")),
+        .allowedReferences =
+            ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedReferences"))),
+        .disallowedReferences = valueAt(json, "disallowedReferences"),
+        .allowedRequisites =
+            ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedRequisites"))),
+        .disallowedRequisites = valueAt(json, "disallowedRequisites"),
     };
 }
 
-void adl_serializer<DerivationOptions::OutputChecks>::to_json(json & json, const DerivationOptions::OutputChecks & c)
+void adl_serializer<OutputChecks<SingleDerivedPath>>::to_json(json & json, const OutputChecks<SingleDerivedPath> & c)
 {
     json["ignoreSelfRefs"] = c.ignoreSelfRefs;
     json["maxSize"] = c.maxSize;

src/libstore/derivations.cc

@@ -13,6 +13,7 @@
 #include <boost/container/small_vector.hpp>
 #include <boost/unordered/concurrent_flat_map.hpp>
 #include <nlohmann/json.hpp>
+#include <optional>
 
 namespace nix {
 
@@ -791,71 +792,63 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName)
 
 DerivationType BasicDerivation::type() const
 {
-    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs,
-        impureOutputs;
     std::optional<HashAlgorithm> floatingHashAlgo;
+    std::optional<DerivationType> ty;
+
+    auto decide = [&](DerivationType newTy) {
+        if (!ty)
+            ty = newTy;
+        else if (ty.value() != newTy)
+            throw Error("can't mix derivation output types");
+        else if (ty.value() == DerivationType::ContentAddressed{.sandboxed = false, .fixed = true})
+            // FIXME: Experimental feature?
+            throw Error("only one fixed output is allowed for now");
+    };
 
     for (auto & i : outputs) {
         std::visit(
             overloaded{
-                [&](const DerivationOutput::InputAddressed &) { inputAddressedOutputs.insert(i.first); },
-                [&](const DerivationOutput::CAFixed &) { fixedCAOutputs.insert(i.first); },
-                [&](const DerivationOutput::CAFloating & dof) {
-                    floatingCAOutputs.insert(i.first);
-                    if (!floatingHashAlgo) {
-                        floatingHashAlgo = dof.hashAlgo;
-                    } else {
-                        if (*floatingHashAlgo != dof.hashAlgo)
-                            throw Error("all floating outputs must use the same hash algorithm");
-                    }
+                [&](const DerivationOutput::InputAddressed &) {
+                    decide(
+                        DerivationType::InputAddressed{
+                            .deferred = false,
+                        });
                 },
-                [&](const DerivationOutput::Deferred &) { deferredIAOutputs.insert(i.first); },
-                [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); },
+                [&](const DerivationOutput::CAFixed &) {
+                    decide(
+                        DerivationType::ContentAddressed{
+                            .sandboxed = false,
+                            .fixed = true,
+                        });
+                    if (i.first != "out"sv)
+                        throw Error("single fixed output must be named \"out\"");
+                },
+                [&](const DerivationOutput::CAFloating & dof) {
+                    decide(
+                        DerivationType::ContentAddressed{
+                            .sandboxed = true,
+                            .fixed = false,
+                        });
+                    if (!floatingHashAlgo)
+                        floatingHashAlgo = dof.hashAlgo;
+                    else if (*floatingHashAlgo != dof.hashAlgo)
+                        throw Error("all floating outputs must use the same hash algorithm");
+                },
+                [&](const DerivationOutput::Deferred &) {
+                    decide(
+                        DerivationType::InputAddressed{
+                            .deferred = true,
+                        });
+                },
+                [&](const DerivationOutput::Impure &) { decide(DerivationType::Impure{}); },
             },
             i.second.raw);
     }
 
-    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
-        && deferredIAOutputs.empty() && impureOutputs.empty())
+    if (!ty)
         throw Error("must have at least one output");
 
-    if (!inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
-        && deferredIAOutputs.empty() && impureOutputs.empty())
-        return DerivationType::InputAddressed{
-            .deferred = false,
-        };
-
-    if (inputAddressedOutputs.empty() && !fixedCAOutputs.empty() && floatingCAOutputs.empty()
-        && deferredIAOutputs.empty() && impureOutputs.empty()) {
-        if (fixedCAOutputs.size() > 1)
-            // FIXME: Experimental feature?
-            throw Error("only one fixed output is allowed for now");
-        if (*fixedCAOutputs.begin() != "out"sv)
-            throw Error("single fixed output must be named \"out\"");
-        return DerivationType::ContentAddressed{
-            .sandboxed = false,
-            .fixed = true,
-        };
-    }
-
-    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && !floatingCAOutputs.empty()
-        && deferredIAOutputs.empty() && impureOutputs.empty())
-        return DerivationType::ContentAddressed{
-            .sandboxed = true,
-            .fixed = false,
-        };
-
-    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
-        && !deferredIAOutputs.empty() && impureOutputs.empty())
-        return DerivationType::InputAddressed{
-            .deferred = true,
-        };
-
-    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
-        && deferredIAOutputs.empty() && !impureOutputs.empty())
-        return DerivationType::Impure{};
-
-    throw Error("can't mix derivation output types");
+    return ty.value();
 }
 
 DrvHashes drvHashes;

src/libstore/downstream-placeholder.cc

@@ -1,5 +1,6 @@
 #include "nix/store/downstream-placeholder.hh"
 #include "nix/store/derivations.hh"
+#include "nix/util/json-utils.hh"
 
 namespace nix {
 
@@ -49,3 +50,45 @@ DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
 }
 
 } // namespace nix
+
+namespace nlohmann {
+
+using namespace nix;
+
+template<typename Item>
+DrvRef<Item> adl_serializer<DrvRef<Item>>::from_json(const json & json)
+{
+    // OutputName case: { "drvPath": "self", "output": <output> }
+    if (json.type() == nlohmann::json::value_t::object) {
+        auto & obj = getObject(json);
+        if (auto * drvPath_ = get(obj, "drvPath")) {
+            auto & drvPath = *drvPath_;
+            if (drvPath.type() == nlohmann::json::value_t::string && getString(drvPath) == "self") {
+                return getString(valueAt(obj, "output"));
+            }
+        }
+    }
+
+    // Input case
+    return adl_serializer<Item>::from_json(json);
+}
+
+template<typename Item>
+void adl_serializer<DrvRef<Item>>::to_json(json & json, const DrvRef<Item> & ref)
+{
+    std::visit(
+        overloaded{
+            [&](const OutputName & outputName) {
+                json = nlohmann::json::object();
+                json["drvPath"] = "self";
+                json["output"] = outputName;
+            },
+            [&](const Item & item) { json = item; },
+        },
+        ref);
+}
+
+template struct adl_serializer<nix::DrvRef<StorePath>>;
+template struct adl_serializer<nix::DrvRef<SingleDerivedPath>>;
+
+} // namespace nlohmann

src/libstore/filetransfer.cc

@@ -48,7 +48,7 @@ struct curlFileTransfer : public FileTransfer
     std::random_device rd;
     std::mt19937 mt19937;
 
-    struct TransferItem : public std::enable_shared_from_this<TransferItem>
+    struct TransferItem : public std::enable_shared_from_this<TransferItem>, public FileTransfer::Item
     {
         curlFileTransfer & fileTransfer;
         FileTransferRequest request;
@@ -60,6 +60,7 @@ struct curlFileTransfer : public FileTransfer
         // buffer to accompany the `req` above
         char errbuf[CURL_ERROR_SIZE];
         bool active = false; // whether the handle has been added to the multi object
+        bool paused = false; // whether the request has been paused previously
         std::string statusMsg;
 
         unsigned int attempt = 0;
@@ -116,7 +117,13 @@ struct curlFileTransfer : public FileTransfer
                        successful response. */
                     if (successfulStatuses.count(httpStatus)) {
                         writtenToSink += data.size();
-                        this->request.dataCallback(data);
+                        PauseTransfer needsPause = this->request.dataCallback(data);
+                        if (needsPause == PauseTransfer::Yes) {
+                            /* Smuggle the boolean flag into writeCallback. Note that
+                               the finalSink might get called multiple times if there's
+                               decompression going on. */
+                            paused = true;
+                        }
                     }
                 } else
                     this->result.data.append(data);
@@ -151,15 +158,23 @@ struct curlFileTransfer : public FileTransfer
             }
         }
 
-        void failEx(std::exception_ptr ex)
+        void failEx(std::exception_ptr ex) noexcept
         {
             assert(!done);
             done = true;
+            try {
+                std::rethrow_exception(ex);
+            } catch (nix::Error & e) {
+                /* Add more context to the error message. */
+                e.addTrace({}, "during %s of '%s'", Uncolored(request.verb()), request.uri.to_string());
+            } catch (...) {
+                /* Can't add more context to the error. */
+            }
             callback.rethrow(ex);
         }
 
         template<class T>
-        void fail(T && e)
+        void fail(T && e) noexcept
         {
             failEx(std::make_exception_ptr(std::forward<T>(e)));
         }
@@ -168,32 +183,38 @@ struct curlFileTransfer : public FileTransfer
         std::shared_ptr<FinishSink> decompressionSink;
         std::optional<StringSink> errorSink;
 
-        std::exception_ptr writeException;
+        std::exception_ptr callbackException;
 
-        size_t writeCallback(void * contents, size_t size, size_t nmemb)
-        {
-            try {
-                size_t realSize = size * nmemb;
-                result.bodySize += realSize;
+        size_t writeCallback(void * contents, size_t size, size_t nmemb) noexcept
+        try {
+            size_t realSize = size * nmemb;
+            result.bodySize += realSize;
 
-                if (!decompressionSink) {
-                    decompressionSink = makeDecompressionSink(encoding, finalSink);
-                    if (!successfulStatuses.count(getHTTPStatus())) {
-                        // In this case we want to construct a TeeSink, to keep
-                        // the response around (which we figure won't be big
-                        // like an actual download should be) to improve error
-                        // messages.
-                        errorSink = StringSink{};
-                    }
+            if (!decompressionSink) {
+                decompressionSink = makeDecompressionSink(encoding, finalSink);
+                if (!successfulStatuses.count(getHTTPStatus())) {
+                    // In this case we want to construct a TeeSink, to keep
+                    // the response around (which we figure won't be big
+                    // like an actual download should be) to improve error
+                    // messages.
+                    errorSink = StringSink{};
                 }
-
-                (*decompressionSink)({(char *) contents, realSize});
-
-                return realSize;
-            } catch (...) {
-                writeException = std::current_exception();
-                return 0;
             }
+
+            (*decompressionSink)({(char *) contents, realSize});
+            if (paused) {
+                /* The callback has signaled that the transfer needs to be
+                   paused. Already consumed data won't be returned twice unlike
+                   when returning CURL_WRITEFUNC_PAUSE.
+                   https://curl-library.cool.haxx.narkive.com/larE1cRA/curl-easy-pause-documentation-question
+                   */
+                curl_easy_pause(req, CURLPAUSE_RECV);
+            }
+
+            return realSize;
+        } catch (...) {
+            callbackException = std::current_exception();
+            return 0;
         }
 
         static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -209,8 +230,8 @@ struct curlFileTransfer : public FileTransfer
                 result.urls.push_back(effectiveUriCStr);
         }
 
-        size_t headerCallback(void * contents, size_t size, size_t nmemb)
-        {
+        size_t headerCallback(void * contents, size_t size, size_t nmemb) noexcept
+        try {
             size_t realSize = size * nmemb;
             std::string line((char *) contents, realSize);
             printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
@@ -263,6 +284,15 @@ struct curlFileTransfer : public FileTransfer
                 }
             }
             return realSize;
+        } catch (...) {
+#if LIBCURL_VERSION_NUM >= 0x075700
+            /* https://curl.se/libcurl/c/CURLOPT_HEADERFUNCTION.html:
+               You can also abort the transfer by returning CURL_WRITEFUNC_ERROR. */
+            callbackException = std::current_exception();
+            return CURL_WRITEFUNC_ERROR;
+#else
+            return realSize;
+#endif
         }
 
         static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -270,14 +300,17 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
         }
 
-        int progressCallback(curl_off_t dltotal, curl_off_t dlnow)
-        {
-            try {
-                act.progress(dlnow, dltotal);
-            } catch (nix::Interrupted &) {
-                assert(getInterrupted());
-            }
+        int progressCallback(curl_off_t dltotal, curl_off_t dlnow) noexcept
+        try {
+            act.progress(dlnow, dltotal);
             return getInterrupted();
+        } catch (nix::Interrupted &) {
+            assert(getInterrupted());
+            return 1;
+        } catch (...) {
+            /* Something unexpected has happened like logger throwing an exception. */
+            callbackException = std::current_exception();
+            return 1;
         }
 
         static int progressCallbackWrapper(
@@ -288,11 +321,14 @@ struct curlFileTransfer : public FileTransfer
             return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow);
         }
 
-        static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
-        {
+        static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) noexcept
+        try {
             if (type == CURLINFO_TEXT)
                 vomit("curl: %s", chomp(std::string(data, size)));
             return 0;
+        } catch (...) {
+            /* Swallow the exception. Nothing left to do. */
+            return 0;
         }
 
         size_t readCallback(char * buffer, size_t size, size_t nitems) noexcept
@@ -302,6 +338,7 @@ struct curlFileTransfer : public FileTransfer
         } catch (EndOfFile &) {
             return 0;
         } catch (...) {
+            callbackException = std::current_exception();
             return CURL_READFUNC_ABORT;
         }
 
@@ -333,6 +370,7 @@ struct curlFileTransfer : public FileTransfer
             }
             return CURL_SEEKFUNC_OK;
         } catch (...) {
+            callbackException = std::current_exception();
             return CURL_SEEKFUNC_FAIL;
         }
 
@@ -341,6 +379,15 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) clientp)->seekCallback(offset, origin);
         }
 
+        void unpause()
+        {
+            /* Unpausing an already unpaused transfer is a no-op. */
+            if (paused) {
+                curl_easy_pause(req, CURLPAUSE_CONT);
+                paused = false;
+            }
+        }
+
         void init()
         {
             if (!req)
@@ -476,7 +523,7 @@ struct curlFileTransfer : public FileTransfer
                 try {
                     decompressionSink->finish();
                 } catch (...) {
-                    writeException = std::current_exception();
+                    callbackException = std::current_exception();
                 }
             }
 
@@ -485,8 +532,8 @@ struct curlFileTransfer : public FileTransfer
                 httpStatus = 304;
             }
 
-            if (writeException)
-                failEx(writeException);
+            if (callbackException)
+                failEx(callbackException);
 
             else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) {
                 result.cached = httpStatus == 304;
@@ -601,7 +648,7 @@ struct curlFileTransfer : public FileTransfer
                     errorSink.reset();
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
                     try {
-                        fileTransfer.enqueueItem(shared_from_this());
+                        fileTransfer.enqueueItem(ref{shared_from_this()});
                     } catch (const nix::Error & e) {
                         // If enqueue fails (e.g., during shutdown), fail the transfer properly
                         // instead of letting the exception propagate, which would leave done=false
@@ -618,24 +665,24 @@ struct curlFileTransfer : public FileTransfer
     {
         struct EmbargoComparator
         {
-            bool operator()(const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2)
+            bool operator()(const ref<TransferItem> & i1, const ref<TransferItem> & i2)
             {
                 return i1->embargo > i2->embargo;
             }
         };
 
-        std::
-            priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator>
-                incoming;
+        std::priority_queue<ref<TransferItem>, std::vector<ref<TransferItem>>, EmbargoComparator> incoming;
+        std::vector<ref<TransferItem>> unpause;
     private:
         bool quitting = false;
     public:
         void quit()
         {
             quitting = true;
-            /* We wil not be processing any more incoming requests */
+            /* We will not be processing any more incoming requests */
             while (!incoming.empty())
                 incoming.pop();
+            unpause.clear();
         }
 
         bool isQuitting()
@@ -804,6 +851,17 @@ struct curlFileTransfer : public FileTransfer
                 item->active = true;
                 items[item->req] = item;
             }
+
+            /* NOTE: Unpausing may invoke callbacks to flush all buffers. */
+            auto unpause = [&]() {
+                auto state(state_.lock());
+                auto res = state->unpause;
+                state->unpause.clear();
+                return res;
+            }();
+
+            for (auto & item : unpause)
+                item->unpause();
         }
 
         debug("download thread shutting down");
@@ -828,7 +886,7 @@ struct curlFileTransfer : public FileTransfer
         }
     }
 
-    void enqueueItem(std::shared_ptr<TransferItem> item)
+    ItemHandle enqueueItem(ref<TransferItem> item)
     {
         if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
             && item->request.uri.scheme() != "s3")
@@ -843,19 +901,34 @@ struct curlFileTransfer : public FileTransfer
 #ifndef _WIN32 // TODO need graceful async exit support on Windows?
         writeFull(wakeupPipe.writeSide.get(), " ");
 #endif
+
+        return ItemHandle(static_cast<Item &>(*item));
     }
 
-    void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
+    ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
     {
         /* Handle s3:// URIs by converting to HTTPS and optionally adding auth */
         if (request.uri.scheme() == "s3") {
             auto modifiedRequest = request;
             modifiedRequest.setupForS3();
-            enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
-            return;
+            return enqueueItem(make_ref<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
         }
 
-        enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
+        return enqueueItem(make_ref<TransferItem>(*this, request, std::move(callback)));
+    }
+
+    void unpauseTransfer(ref<TransferItem> item)
+    {
+        auto state(state_.lock());
+        state->unpause.push_back(std::move(item));
+#ifndef _WIN32 // TODO need graceful async exit support on Windows?
+        writeFull(wakeupPipe.writeSide.get(), " ");
+#endif
+    }
+
+    void unpauseTransfer(ItemHandle handle) override
+    {
+        unpauseTransfer(ref{static_cast<TransferItem &>(handle.item.get()).shared_from_this()});
     }
 };
 
@@ -952,6 +1025,7 @@ void FileTransfer::download(
     struct State
     {
         bool quit = false;
+        bool paused = false;
         std::exception_ptr exc;
         std::string data;
         std::condition_variable avail, request;
@@ -967,31 +1041,38 @@ void FileTransfer::download(
         state->request.notify_one();
     });
 
-    request.dataCallback = [_state](std::string_view data) {
+    request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer {
         auto state(_state->lock());
 
         if (state->quit)
-            return;
-
-        /* If the buffer is full, then go to sleep until the calling
-           thread wakes us up (i.e. when it has removed data from the
-           buffer). We don't wait forever to prevent stalling the
-           download thread. (Hopefully sleeping will throttle the
-           sender.) */
-        if (state->data.size() > fileTransferSettings.downloadBufferSize) {
-            debug("download buffer is full; going to sleep");
-            static bool haveWarned = false;
-            warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
-            state.wait_for(state->request, std::chrono::seconds(10));
-        }
+            return PauseTransfer::No;
 
         /* Append data to the buffer and wake up the calling
            thread. */
         state->data.append(data);
         state->avail.notify_one();
+
+        if (state->data.size() <= fileTransferSettings.downloadBufferSize)
+            return PauseTransfer::No;
+
+        /* dataCallback gets called multiple times by an intermediate sink. Only
+           issue the debug message the first time around. */
+        if (!state->paused)
+            debug(
+                "pausing transfer for '%s': download buffer is full (%d > %d)",
+                uri,
+                state->data.size(),
+                fileTransferSettings.downloadBufferSize);
+
+        state->paused = true;
+
+        /* Technically the buffer might become larger than
+           downloadBufferSize, but with sinks there's no way to avoid
+           consuming data. */
+        return PauseTransfer::Yes;
     };
 
-    enqueueFileTransfer(
+    auto handle = enqueueFileTransfer(
         request, {[_state, resultCallback{std::move(resultCallback)}](std::future<FileTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
@@ -1024,6 +1105,10 @@ void FileTransfer::download(
                     return;
                 }
 
+                if (state->paused) {
+                    unpauseTransfer(handle);
+                    state->paused = false;
+                }
                 state.wait(state->avail);
 
                 if (state->data.empty())

src/libstore/include/nix/store/build/derivation-builder.hh

@@ -69,7 +69,7 @@ struct DerivationBuilderParams
      *
      * @todo this should be part of `Derivation`.
      */
-    const DerivationOptions & drvOptions;
+    const DerivationOptions<StorePath> & drvOptions;
 
     // The remainder is state held during the build.
 

src/libstore/include/nix/store/build/derivation-building-goal.hh

@@ -52,8 +52,6 @@ private:
      */
     std::unique_ptr<Derivation> drv;
 
-    std::unique_ptr<DerivationOptions> drvOptions;
-
     /**
      * The remainder is state held during the build.
      */
@@ -115,7 +113,8 @@ private:
     /**
      * Is the build hook willing to perform the build?
      */
-    HookReply tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs);
+    HookReply tryBuildHook(
+        const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions);
 
     /**
      * Open a log file and a pipe to it.

src/libstore/include/nix/store/build/derivation-env-desugar.hh

@@ -8,6 +8,7 @@ namespace nix {
 
 class Store;
 struct Derivation;
+template<typename Input>
 struct DerivationOptions;
 
 /**
@@ -77,7 +78,10 @@ struct DesugaredEnv
      * just part of `Derivation`.
      */
     static DesugaredEnv create(
-        Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths);
+        Store & store,
+        const Derivation & drv,
+        const DerivationOptions<StorePath> & drvOptions,
+        const StorePathSet & inputPaths);
 };
 
 } // namespace nix

src/libstore/include/nix/store/derivation-options.hh

@@ -8,7 +8,8 @@
 
 #include "nix/util/types.hh"
 #include "nix/util/json-impls.hh"
-#include "nix/store/path.hh"
+#include "nix/store/store-dir-config.hh"
+#include "nix/store/downstream-placeholder.hh"
 
 namespace nix {
 
@@ -17,6 +18,9 @@ struct StoreDirConfig;
 struct BasicDerivation;
 struct StructuredAttrs;
 
+template<typename V>
+struct DerivedPathMap;
+
 /**
  * This represents all the special options on a `Derivation`.
  *
@@ -34,6 +38,7 @@ struct StructuredAttrs;
  * separately. That would be nice to separate concerns, and not make any
  * environment variable names magical.
  */
+template<typename Input>
 struct DerivationOptions
 {
     struct OutputChecks
@@ -41,13 +46,15 @@ struct DerivationOptions
         bool ignoreSelfRefs = false;
         std::optional<uint64_t> maxSize, maxClosureSize;
 
+        using DrvRef = nix::DrvRef<Input>;
+
         /**
          * env: allowedReferences
          *
          * A value of `nullopt` indicates that the check is skipped.
          * This means that all references are allowed.
          */
-        std::optional<StringSet> allowedReferences;
+        std::optional<std::set<DrvRef>> allowedReferences;
 
         /**
          * env: disallowedReferences
@@ -55,21 +62,21 @@ struct DerivationOptions
          * No needed for `std::optional`, because skipping the check is
          * the same as disallowing the references.
          */
-        StringSet disallowedReferences;
+        std::set<DrvRef> disallowedReferences;
 
         /**
          * env: allowedRequisites
          *
          * See `allowedReferences`
          */
-        std::optional<StringSet> allowedRequisites;
+        std::optional<std::set<DrvRef>> allowedRequisites;
 
         /**
          * env: disallowedRequisites
          *
          * See `disallowedReferences`
          */
-        StringSet disallowedRequisites;
+        std::set<DrvRef> disallowedRequisites;
 
         bool operator==(const OutputChecks &) const = default;
     };
@@ -116,23 +123,7 @@ struct DerivationOptions
      * attributes give to the builder. The set of paths in the original JSON
      * is replaced with a list of `PathInfo` in JSON format.
      */
-    std::map<std::string, StringSet> exportReferencesGraph;
-
-    /**
-     * Once a derivations is resolved, the strings in in
-     * `exportReferencesGraph` should all be store paths (with possible
-     * suffix paths, but those are discarded).
-     *
-     * @return The parsed path set for for each key in the map.
-     *
-     * @todo Ideally, `exportReferencesGraph` would just store
-     * `StorePath`s for this, but we can't just do that, because for CA
-     * derivations they is actually in general `DerivedPath`s (via
-     * placeholder strings) until the derivation is resolved and exact
-     * inputs store paths are known. We can use better types for that
-     * too, but that is a longer project.
-     */
-    std::map<std::string, StorePathSet> getParsedExportReferencesGraph(const StoreDirConfig & store) const;
+    std::map<std::string, std::set<Input>> exportReferencesGraph;
 
     /**
      * env: __sandboxProfile
@@ -185,18 +176,6 @@ struct DerivationOptions
 
     bool operator==(const DerivationOptions &) const = default;
 
-    /**
-     * Parse this information from its legacy encoding as part of the
-     * environment. This should not be used with nice greenfield formats
-     * (e.g. JSON) but is necessary for supporting old formats (e.g.
-     * ATerm).
-     */
-    static DerivationOptions
-    fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true);
-
-    static DerivationOptions
-    fromStructuredAttrs(const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn = true);
-
     /**
      * @param drv Must be the same derivation we parsed this from. In
      * the future we'll flip things around so a `BasicDerivation` has
@@ -222,7 +201,49 @@ struct DerivationOptions
     bool useUidRange(const BasicDerivation & drv) const;
 };
 
+extern template struct DerivationOptions<StorePath>;
+extern template struct DerivationOptions<SingleDerivedPath>;
+
+struct DerivationOutput;
+
+/**
+ * Parse this information from its legacy encoding as part of the
+ * environment. This should not be used with nice greenfield formats
+ * (e.g. JSON) but is necessary for supporting old formats (e.g.
+ * ATerm).
+ */
+DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
+    const StoreDirConfig & store,
+    const DerivedPathMap<StringSet> & inputDrvs,
+    const StringMap & env,
+    const StructuredAttrs * parsed,
+    bool shouldWarn = true,
+    const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
+
+DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs(
+    const StoreDirConfig & store,
+    const StringMap & env,
+    const StructuredAttrs * parsed,
+    bool shouldWarn = true,
+    const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
+
+/**
+ * This is the counterpart of `Derivation::tryResolve`. In particular,
+ * it takes the same sort of callback, which is used to reolve
+ * non-constant deriving paths.
+ *
+ * We need this function when resolving a derivation, and we will use
+ * this as part of that if/when `Derivation` includes
+ * `DerivationOptions`
+ */
+std::optional<DerivationOptions<StorePath>> tryResolve(
+    const DerivationOptions<SingleDerivedPath> & drvOptions,
+    std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
+        queryResolutionChain);
+
 }; // namespace nix
 
-JSON_IMPL(DerivationOptions);
-JSON_IMPL(DerivationOptions::OutputChecks)
+JSON_IMPL(nix::DerivationOptions<nix::StorePath>);
+JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>);
+JSON_IMPL(nix::DerivationOptions<nix::StorePath>::OutputChecks)
+JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>::OutputChecks)

src/libstore/include/nix/store/downstream-placeholder.hh

@@ -2,11 +2,23 @@
 ///@file
 
 #include "nix/util/hash.hh"
+#include "nix/util/json-impls.hh"
 #include "nix/store/path.hh"
 #include "nix/store/derived-path.hh"
 
 namespace nix {
 
+/**
+ * A reference is either to a to-be-registered output (by name),
+ * or to an already-registered store object (by `Input`).
+ *
+ * `Ref<SingleDerivedPath` is a representation of something that can be
+ * turned into a placeholder. (Regular own-output placeholder in the
+ * first case, `DownstreamPlaceholder` in the second case.)
+ */
+template<typename Input>
+using DrvRef = std::variant<OutputName, Input>;
+
 /**
  * Downstream Placeholders are opaque and almost certainly unique values
  * used to allow derivations to refer to store objects which are yet to
@@ -92,3 +104,17 @@ public:
 };
 
 } // namespace nix
+
+namespace nlohmann {
+
+template<typename Item>
+struct adl_serializer<nix::DrvRef<Item>>
+{
+    static nix::DrvRef<Item> from_json(const json & json);
+    static void to_json(json & json, const nix::DrvRef<Item> & t);
+};
+
+extern template struct adl_serializer<nix::DrvRef<nix::StorePath>>;
+extern template struct adl_serializer<nix::DrvRef<nix::SingleDerivedPath>>;
+
+} // namespace nlohmann

src/libstore/include/nix/store/filetransfer.hh

@@ -70,12 +70,12 @@ struct FileTransferSettings : Config
 
     Setting<size_t> downloadBufferSize{
         this,
-        64 * 1024 * 1024,
+        1 * 1024 * 1024,
         "download-buffer-size",
         R"(
           The size of Nix's internal download buffer in bytes during `curl` transfers. If data is
           not processed quickly enough to exceed the size of this buffer, downloads may stall.
-          The default is 67108864 (64 MiB).
+          The default is 1048576 (1 MiB).
         )"};
 };
 
@@ -105,6 +105,11 @@ struct UsernameAuth
     std::optional<std::string> password;
 };
 
+enum class PauseTransfer : bool {
+    No = false,
+    Yes = true,
+};
+
 struct FileTransferRequest
 {
     VerbatimURL uri;
@@ -136,7 +141,14 @@ struct FileTransferRequest
 
     std::optional<UploadData> data;
     std::string mimeType;
-    std::function<void(std::string_view data)> dataCallback;
+
+    /**
+     * Callbacked invoked with a chunk of received data.
+     * Can pause the transfer by returning PauseTransfer::Yes. No data must be consumed
+     * if transfer is paused.
+     */
+    std::function<PauseTransfer(std::string_view data)> dataCallback;
+
     /**
      * Optional username and password for HTTP basic authentication.
      * When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option.
@@ -226,6 +238,25 @@ class Store;
 
 struct FileTransfer
 {
+protected:
+    class Item
+    {};
+
+public:
+    /**
+     * An opaque handle to the file transfer. Can be used to reference an in-flight transfer operations.
+     */
+    struct ItemHandle
+    {
+        std::reference_wrapper<Item> item;
+        friend struct FileTransfer;
+
+        ItemHandle(Item & item)
+            : item(item)
+        {
+        }
+    };
+
     virtual ~FileTransfer() {}
 
     /**
@@ -233,7 +264,13 @@ struct FileTransfer
      * the download. The future may throw a FileTransferError
      * exception.
      */
-    virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
+    virtual ItemHandle
+    enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
+
+    /**
+     * Unpause a transfer that has been previously paused by a dataCallback.
+     */
+    virtual void unpauseTransfer(ItemHandle handle) = 0;
 
     std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
 

src/libstore/include/nix/store/meson.build

@@ -55,7 +55,6 @@ headers = [ config_pub_h ] + files(
   'machines.hh',
   'make-content-addressed.hh',
   'names.hh',
-  'nar-accessor.hh',
   'nar-info-disk-cache.hh',
   'nar-info.hh',
   'outputs-spec.hh',

src/libstore/include/nix/store/nar-accessor.hh

@@ -1,43 +0,0 @@
-#pragma once
-///@file
-
-#include "nix/util/source-accessor.hh"
-
-#include <functional>
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix {
-
-struct Source;
-
-/**
- * Return an object that provides access to the contents of a NAR
- * file.
- */
-ref<SourceAccessor> makeNarAccessor(std::string && nar);
-
-ref<SourceAccessor> makeNarAccessor(Source & source);
-
-/**
- * Create a NAR accessor from a NAR listing (in the format produced by
- * listNar()). The callback getNarBytes(offset, length) is used by the
- * readFile() method of the accessor to get the contents of files
- * inside the NAR.
- */
-using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
-
-/**
- * The canonical GetNarBytes function for a seekable Source.
- */
-GetNarBytes seekableGetNarBytes(const Path & path);
-
-ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
-
-/**
- * Write a JSON representation of the contents of a NAR (except file
- * contents).
- */
-nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
-
-} // namespace nix

src/libstore/include/nix/store/parsed-derivations.hh

@@ -9,6 +9,7 @@
 namespace nix {
 
 class Store;
+template<typename Input>
 struct DerivationOptions;
 struct DerivationOutput;
 
@@ -47,7 +48,7 @@ struct StructuredAttrs
 
     nlohmann::json::object_t prepareStructuredAttrs(
         Store & store,
-        const DerivationOptions & drvOptions,
+        const DerivationOptions<StorePath> & drvOptions,
         const StorePathSet & inputPaths,
         const DerivationOutputs & outputs) const;
 

src/libstore/meson.build

@@ -114,6 +114,17 @@ boost = dependency(
 deps_other += boost
 
 curl = dependency('libcurl', 'curl', version : '>= 7.75.0')
+if curl.version().version_compare('>=8.16.0') and curl.version().version_compare(
+  '<8.17.0',
+)
+  # Out of precaution, avoid building with libcurl version that suffer from https://github.com/curl/curl/issues/19334.
+  error(
+    'curl @0@ has issues with write pausing, please use libcurl < 8.16 or >= 8.17, see https://github.com/curl/curl/issues/19334'.format(
+      curl.version(),
+    ),
+  )
+endif
+
 deps_private += curl
 
 # seccomp only makes sense on Linux
@@ -300,7 +311,6 @@ sources = files(
   'make-content-addressed.cc',
   'misc.cc',
   'names.cc',
-  'nar-accessor.cc',
   'nar-info-disk-cache.cc',
   'nar-info.cc',
   'optimise-store.cc',

src/libstore/misc.cc

@@ -225,11 +225,12 @@ MissingPaths Store::queryMissing(const std::vector<DerivedPath> & targets)
                         return;
 
                     auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
-                    DerivationOptions drvOptions;
+                    DerivationOptions<SingleDerivedPath> drvOptions;
                     try {
                         // FIXME: this is a lot of work just to get the value
                         // of `allowSubstitutes`.
-                        drvOptions = DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
+                        drvOptions = derivationOptionsFromStructuredAttrs(
+                            *this, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
                     } catch (Error & e) {
                         e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath));
                         throw;

src/libstore/parsed-derivations.cc

@@ -100,7 +100,7 @@ static nlohmann::json pathInfoToJSON(Store & store, const StorePathSet & storePa
 
 nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
     Store & store,
-    const DerivationOptions & drvOptions,
+    const DerivationOptions<StorePath> & drvOptions,
     const StorePathSet & inputPaths,
     const DerivationOutputs & outputs) const
 {
@@ -114,8 +114,8 @@ nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
     json["outputs"] = std::move(outputsJson);
 
     /* Handle exportReferencesGraph. */
-    for (auto & [key, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) {
-        json[key] = pathInfoToJSON(store, store.exportReferences(storePaths, storePaths));
+    for (auto & [key, storePaths] : drvOptions.exportReferencesGraph) {
+        json[key] = pathInfoToJSON(store, store.exportReferences(storePaths, inputPaths));
     }
 
     return json;

src/libstore/remote-fs-accessor.cc

@@ -1,6 +1,6 @@
 #include <nlohmann/json.hpp>
 #include "nix/store/remote-fs-accessor.hh"
-#include "nix/store/nar-accessor.hh"
+#include "nix/util/nar-accessor.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -39,7 +39,7 @@ ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std:
 
     if (cacheDir != "") {
         try {
-            nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
+            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
             writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreExceptionExceptInterrupt();

src/libutil-c/nix_api_util.cc

@@ -36,7 +36,7 @@ nix_err nix_context_error(nix_c_context * context)
         const char * demangled = abi::__cxa_demangle(typeid(e).name(), 0, 0, &status);
         if (demangled) {
             context->name = demangled;
-            // todo: free(demangled);
+            free((void *) demangled);
         } else {
             context->name = typeid(e).name();
         }

src/libutil-tests/data/memory-source-accessor/complex.json

@@ -0,0 +1,24 @@
+{
+  "entries": {
+    "bar": {
+      "entries": {
+        "baz": {
+          "contents": "good day,\n\u0000\n\tworld!",
+          "executable": true,
+          "type": "regular"
+        },
+        "quux": {
+          "target": "/over/there",
+          "type": "symlink"
+        }
+      },
+      "type": "directory"
+    },
+    "foo": {
+      "contents": "hello\n\u0000\n\tworld!",
+      "executable": false,
+      "type": "regular"
+    }
+  },
+  "type": "directory"
+}

src/libutil-tests/data/memory-source-accessor/simple.json

@@ -0,0 +1,5 @@
+{
+  "contents": "asdf",
+  "executable": false,
+  "type": "regular"
+}

src/libutil-tests/data/nar-listing/deep.json

@@ -0,0 +1,23 @@
+{
+  "entries": {
+    "bar": {
+      "entries": {
+        "baz": {
+          "executable": true,
+          "size": 19,
+          "type": "regular"
+        },
+        "quux": {
+          "target": "/over/there",
+          "type": "symlink"
+        }
+      },
+      "type": "directory"
+    },
+    "foo": {
+      "size": 15,
+      "type": "regular"
+    }
+  },
+  "type": "directory"
+}

src/libutil-tests/data/nar-listing/shallow.json

@@ -0,0 +1,7 @@
+{
+  "entries": {
+    "bar": {},
+    "foo": {}
+  },
+  "type": "directory"
+}

src/libutil-tests/git.cc

@@ -224,42 +224,15 @@ TEST_F(GitTest, tree_sha256_write)
     });
 }
 
+namespace memory_source_accessor {
+
+extern ref<MemorySourceAccessor> exampleComplex();
+
+}
+
 TEST_F(GitTest, both_roundrip)
 {
-    using File = MemorySourceAccessor::File;
-
-    auto files = make_ref<MemorySourceAccessor>();
-    files->root = File::Directory{
-        .contents{
-            {
-                "foo",
-                File::Regular{
-                    .contents = "hello\n\0\n\tworld!",
-                },
-            },
-            {
-                "bar",
-                File::Directory{
-                    .contents =
-                        {
-                            {
-                                "baz",
-                                File::Regular{
-                                    .executable = true,
-                                    .contents = "good day,\n\0\n\tworld!",
-                                },
-                            },
-                            {
-                                "quux",
-                                File::Symlink{
-                                    .target = "/over/there",
-                                },
-                            },
-                        },
-                },
-            },
-        },
-    };
+    auto files = memory_source_accessor::exampleComplex();
 
     for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) {
         std::map<Hash, std::string> cas;

src/libutil-tests/memory-source-accessor.cc

@@ -0,0 +1,116 @@
+#include <string_view>
+
+#include "nix/util/memory-source-accessor.hh"
+#include "nix/util/tests/json-characterization.hh"
+
+namespace nix {
+
+namespace memory_source_accessor {
+
+using namespace std::literals;
+using File = MemorySourceAccessor::File;
+
+ref<MemorySourceAccessor> exampleSimple()
+{
+    auto sc = make_ref<MemorySourceAccessor>();
+    sc->root = File{File::Regular{
+        .executable = false,
+        .contents = "asdf",
+    }};
+    return sc;
+}
+
+ref<MemorySourceAccessor> exampleComplex()
+{
+    auto files = make_ref<MemorySourceAccessor>();
+    files->root = File::Directory{
+        .entries{
+            {
+                "foo",
+                File::Regular{
+                    .contents = "hello\n\0\n\tworld!"s,
+                },
+            },
+            {
+                "bar",
+                File::Directory{
+                    .entries =
+                        {
+                            {
+                                "baz",
+                                File::Regular{
+                                    .executable = true,
+                                    .contents = "good day,\n\0\n\tworld!"s,
+                                },
+                            },
+                            {
+                                "quux",
+                                File::Symlink{
+                                    .target = "/over/there",
+                                },
+                            },
+                        },
+                },
+            },
+        },
+    };
+    return files;
+}
+
+} // namespace memory_source_accessor
+
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+class MemorySourceAccessorTest : public virtual CharacterizationTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor";
+
+public:
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
+
+using nlohmann::json;
+
+struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest,
+                                      JsonCharacterizationTest<MemorySourceAccessor>,
+                                      ::testing::WithParamInterface<std::pair<std::string_view, MemorySourceAccessor>>
+{};
+
+TEST_P(MemorySourceAccessorJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    /* Cannot use `readJsonTest` because need to compare `root` field of
+       the source accessors for equality. */
+    readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
+        auto encoded = json::parse(encodedRaw);
+        auto decoded = static_cast<MemorySourceAccessor>(encoded);
+        ASSERT_EQ(decoded.root, expected.root);
+    });
+}
+
+TEST_P(MemorySourceAccessorJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    MemorySourceAccessorJSON,
+    MemorySourceAccessorJsonTest,
+    ::testing::Values(
+        std::pair{
+            "simple",
+            *memory_source_accessor::exampleSimple(),
+        },
+        std::pair{
+            "complex",
+            *memory_source_accessor::exampleComplex(),
+        }));
+
+} // namespace nix

src/libutil-tests/meson.build

@@ -63,7 +63,9 @@ sources = files(
   'json-utils.cc',
   'logging.cc',
   'lru-cache.cc',
+  'memory-source-accessor.cc',
   'monitorfdhup.cc',
+  'nar-listing.cc',
   'nix_api_util.cc',
   'nix_api_util_internal.cc',
   'pool.cc',

src/libutil-tests/nar-listing.cc

@@ -0,0 +1,83 @@
+#include <string_view>
+
+#include "nix/util/nar-accessor.hh"
+#include "nix/util/tests/json-characterization.hh"
+
+namespace nix {
+
+// Forward declaration from memory-source-accessor.cc
+namespace memory_source_accessor {
+ref<MemorySourceAccessor> exampleComplex();
+}
+
+/* ----------------------------------------------------------------------------
+ * JSON
+ * --------------------------------------------------------------------------*/
+
+class NarListingTest : public virtual CharacterizationTest
+{
+    std::filesystem::path unitTestData = getUnitTestData() / "nar-listing";
+
+public:
+
+    std::filesystem::path goldenMaster(std::string_view testStem) const override
+    {
+        return unitTestData / testStem;
+    }
+};
+
+using nlohmann::json;
+
+struct NarListingJsonTest : NarListingTest,
+                            JsonCharacterizationTest<NarListing>,
+                            ::testing::WithParamInterface<std::pair<std::string_view, NarListing>>
+{};
+
+TEST_P(NarListingJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    readJsonTest(name, expected);
+}
+
+TEST_P(NarListingJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    NarListingJSON,
+    NarListingJsonTest,
+    ::testing::Values(
+        std::pair{
+            "deep",
+            listNarDeep(*memory_source_accessor::exampleComplex(), CanonPath::root),
+        }));
+
+struct ShallowNarListingJsonTest : NarListingTest,
+                                   JsonCharacterizationTest<ShallowNarListing>,
+                                   ::testing::WithParamInterface<std::pair<std::string_view, ShallowNarListing>>
+{};
+
+TEST_P(ShallowNarListingJsonTest, from_json)
+{
+    auto & [name, expected] = GetParam();
+    readJsonTest(name, expected);
+}
+
+TEST_P(ShallowNarListingJsonTest, to_json)
+{
+    auto & [name, value] = GetParam();
+    writeJsonTest(name, value);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    ShallowNarListingJSON,
+    ShallowNarListingJsonTest,
+    ::testing::Values(
+        std::pair{
+            "shallow",
+            listNarShallow(*memory_source_accessor::exampleComplex(), CanonPath::root),
+        }));
+
+} // namespace nix

src/libutil/archive.cc

@@ -200,54 +200,54 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath
     }
 
     else if (type == "directory") {
-        sink.createDirectory(path);
+        sink.createDirectory(path, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) {
+            std::map<Path, int, CaseInsensitiveCompare> names;
 
-        std::map<Path, int, CaseInsensitiveCompare> names;
+            std::string prevName;
 
-        std::string prevName;
+            while (1) {
+                auto tag = getString();
 
-        while (1) {
-            auto tag = getString();
+                if (tag == ")")
+                    break;
 
-            if (tag == ")")
-                break;
+                if (tag != "entry")
+                    throw badArchive("expected tag 'entry' or ')', got '%s'", tag);
 
-            if (tag != "entry")
-                throw badArchive("expected tag 'entry' or ')', got '%s'", tag);
+                expectTag("(");
 
-            expectTag("(");
+                expectTag("name");
 
-            expectTag("name");
+                auto name = getString();
+                if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos
+                    || name.find((char) 0) != std::string::npos)
+                    throw badArchive("NAR contains invalid file name '%1%'", name);
+                if (name <= prevName)
+                    throw badArchive("NAR directory is not sorted");
+                prevName = name;
+                if (archiveSettings.useCaseHack) {
+                    auto i = names.find(name);
+                    if (i != names.end()) {
+                        debug("case collision between '%1%' and '%2%'", i->first, name);
+                        name += caseHackSuffix;
+                        name += std::to_string(++i->second);
+                        auto j = names.find(name);
+                        if (j != names.end())
+                            throw badArchive(
+                                "NAR contains file name '%s' that collides with case-hacked file name '%s'",
+                                prevName,
+                                j->first);
+                    } else
+                        names[name] = 0;
+                }
 
-            auto name = getString();
-            if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos
-                || name.find((char) 0) != std::string::npos)
-                throw badArchive("NAR contains invalid file name '%1%'", name);
-            if (name <= prevName)
-                throw badArchive("NAR directory is not sorted");
-            prevName = name;
-            if (archiveSettings.useCaseHack) {
-                auto i = names.find(name);
-                if (i != names.end()) {
-                    debug("case collision between '%1%' and '%2%'", i->first, name);
-                    name += caseHackSuffix;
-                    name += std::to_string(++i->second);
-                    auto j = names.find(name);
-                    if (j != names.end())
-                        throw badArchive(
-                            "NAR contains file name '%s' that collides with case-hacked file name '%s'",
-                            prevName,
-                            j->first);
-                } else
-                    names[name] = 0;
+                expectTag("node");
+
+                parse(dirSink, source, relDirPath / name);
+
+                expectTag(")");
             }
-
-            expectTag("node");
-
-            parse(sink, source, path / name);
-
-            expectTag(")");
-        }
+        });
     }
 
     else if (type == "symlink") {

src/libutil/current-process.cc

@@ -134,6 +134,11 @@ std::optional<Path> getSelfExe()
             return std::nullopt;
         }
 
+        // FreeBSD's sysctl(KERN_PROC_PATHNAME) includes the null terminator in
+        // pathLen. Strip it to prevent Nix evaluation errors when the path is
+        // serialized to JSON and evaluated as a Nix string.
+        path.pop_back();
+
         return Path(path.begin(), path.end());
 #else
         return std::nullopt;

src/libutil/fs-sink.cc

@@ -34,11 +34,11 @@ void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystem
     }
 
     case SourceAccessor::tDirectory: {
-        sink.createDirectory(to);
-        for (auto & [name, _] : accessor.readDirectory(from)) {
-            copyRecursive(accessor, from / name, sink, to / name);
-            break;
-        }
+        sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) {
+            for (auto & [name, _] : accessor.readDirectory(from)) {
+                copyRecursive(accessor, from / name, dirSink, relDirPath / name);
+            }
+        });
         break;
     }
 
@@ -70,11 +70,60 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can
     return dst;
 }
 
+#ifndef _WIN32
+void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
+{
+    if (path.isRoot()) {
+        createDirectory(path);
+        callback(*this, path);
+        return;
+    }
+
+    createDirectory(path);
+    assert(dirFd); // If that's not true the above call must have thrown an exception.
+
+    RestoreSink dirSink{startFsync};
+    dirSink.dstPath = append(dstPath, path);
+    dirSink.dirFd = ::openat(dirFd.get(), path.rel_c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
+
+    if (!dirSink.dirFd)
+        throw SysError("opening directory '%s'", dirSink.dstPath.string());
+
+    callback(dirSink, CanonPath::root);
+}
+#endif
+
 void RestoreSink::createDirectory(const CanonPath & path)
 {
     auto p = append(dstPath, path);
+
+#ifndef _WIN32
+    if (dirFd) {
+        if (path.isRoot())
+            /* Trying to create a directory that we already have a file descriptor for. */
+            throw Error("path '%s' already exists", p.string());
+
+        if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1)
+            throw SysError("creating directory '%s'", p.string());
+
+        return;
+    }
+#endif
+
     if (!std::filesystem::create_directory(p))
         throw Error("path '%s' already exists", p.string());
+
+#ifndef _WIN32
+    if (path.isRoot()) {
+        assert(!dirFd); // Handled above
+
+        /* Open directory for further *at operations relative to the sink root
+           directory. */
+        dirFd = open(p.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
+        if (!dirFd)
+            throw SysError("creating directory '%1%'", p.string());
+    }
+#endif
 };
 
 struct RestoreRegularFile : CreateRegularFileSink
@@ -114,7 +163,14 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function<void(C
             FILE_ATTRIBUTE_NORMAL,
             NULL)
 #else
-        open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666)
+        [&]() {
+            /* O_EXCL together with O_CREAT ensures symbolic links in the last
+              component are not followed. */
+            constexpr int flags = O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC;
+            if (!dirFd)
+                return ::open(p.c_str(), flags, 0666);
+            return ::openat(dirFd.get(), path.rel_c_str(), flags, 0666);
+        }();
 #endif
         ;
     if (!crf.fd)
@@ -161,6 +217,13 @@ void RestoreRegularFile::operator()(std::string_view data)
 void RestoreSink::createSymlink(const CanonPath & path, const std::string & target)
 {
     auto p = append(dstPath, path);
+#ifndef _WIN32
+    if (dirFd) {
+        if (::symlinkat(requireCString(target), dirFd.get(), path.rel_c_str()) == -1)
+            throw SysError("creating symlink from '%1%' -> '%2%'", p.string(), target);
+        return;
+    }
+#endif
     nix::createSymlink(target, p.string());
 }
 

src/libutil/include/nix/util/fs-sink.hh

@@ -36,6 +36,23 @@ struct FileSystemObjectSink
 
     virtual void createDirectory(const CanonPath & path) = 0;
 
+    using DirectoryCreatedCallback = std::function<void(FileSystemObjectSink & dirSink, const CanonPath & dirRelPath)>;
+
+    /**
+     * Create a directory and invoke a callback with a pair of sink + CanonPath
+     * of the created subdirectory relative to dirSink.
+     *
+     * @note This allows for UNIX RestoreSink implementations to implement
+     * *at-style accessors that always keep an open file descriptor for the
+     * freshly created directory. Use this when it's important to disallow any
+     * intermediate path components from being symlinks.
+     */
+    virtual void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
+    {
+        createDirectory(path);
+        callback(*this, path);
+    }
+
     /**
      * This function in general is no re-entrant. Only one file can be
      * written at a time.
@@ -82,6 +99,18 @@ struct NullFileSystemObjectSink : FileSystemObjectSink
 struct RestoreSink : FileSystemObjectSink
 {
     std::filesystem::path dstPath;
+#ifndef _WIN32
+    /**
+     * File descriptor for the directory located at dstPath. Used for *at
+     * operations relative to this file descriptor. This sink must *never*
+     * follow intermediate symlinks (starting from dstPath) in case a file
+     * collision is encountered for various reasons like case-insensitivity or
+     * other types on normalization. using appropriate *at system calls and traversing
+     * only one path component at a time ensures that writing is race-free and is
+     * is not susceptible to symlink replacement.
+     */
+    AutoCloseFD dirFd;
+#endif
     bool startFsync = false;
 
     explicit RestoreSink(bool startFsync)
@@ -91,6 +120,10 @@ struct RestoreSink : FileSystemObjectSink
 
     void createDirectory(const CanonPath & path) override;
 
+#ifndef _WIN32
+    void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override;
+#endif
+
     void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
 
     void createSymlink(const CanonPath & path, const std::string & target) override;

src/libutil/include/nix/util/json-impls.hh

@@ -6,15 +6,18 @@
 #include "nix/util/experimental-features.hh"
 
 // Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
-#define JSON_IMPL(TYPE)                                   \
-    namespace nlohmann {                                  \
-    using namespace nix;                                  \
-    template<>                                            \
+#define JSON_IMPL_INNER(TYPE)                             \
     struct adl_serializer<TYPE>                           \
     {                                                     \
         static TYPE from_json(const json & json);         \
         static void to_json(json & json, const TYPE & t); \
-    };                                                    \
+    };
+
+#define JSON_IMPL(TYPE)   \
+    namespace nlohmann {  \
+    using namespace nix;  \
+    template<>            \
+    JSON_IMPL_INNER(TYPE) \
     }
 
 #define JSON_IMPL_WITH_XP_FEATURES(TYPE)                                                                            \

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -4,59 +4,111 @@
 #include "nix/util/source-path.hh"
 #include "nix/util/fs-sink.hh"
 #include "nix/util/variant-wrapper.hh"
+#include "nix/util/json-impls.hh"
 
 namespace nix {
 
+/**
+ * File System Object definitions
+ *
+ * @see https://nix.dev/manual/nix/latest/store/file-system-object.html
+ */
+namespace fso {
+
+template<typename RegularContents>
+struct Regular
+{
+    bool executable = false;
+    RegularContents contents;
+
+    auto operator<=>(const Regular &) const = default;
+};
+
+/**
+ * Child parameter because sometimes we want "shallow" directories without
+ * full file children.
+ */
+template<typename Child>
+struct DirectoryT
+{
+    using Name = std::string;
+
+    std::map<Name, Child, std::less<>> entries;
+
+    inline bool operator==(const DirectoryT &) const noexcept;
+    inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept;
+};
+
+struct Symlink
+{
+    std::string target;
+
+    auto operator<=>(const Symlink &) const = default;
+};
+
+/**
+ * For when we know there is child, but don't know anything about it.
+ *
+ * This is not part of the core File System Object data model --- this
+ * represents not knowing, not an additional type of file.
+ */
+struct Opaque
+{
+    auto operator<=>(const Opaque &) const = default;
+};
+
+/**
+ * `File<std::string>` nicely defining what a "file system object"
+ * is in Nix.
+ *
+ * With a different type arugment, it is also can be a "skeletal"
+ * version is that abstract syntax for a "NAR listing".
+ */
+template<typename RegularContents, bool recur>
+struct VariantT
+{
+    bool operator==(const VariantT &) const noexcept;
+    std::strong_ordering operator<=>(const VariantT &) const noexcept;
+
+    using Regular = nix::fso::Regular<RegularContents>;
+
+    /**
+     * In the default case, we do want full file children for our directory.
+     */
+    using Directory = nix::fso::DirectoryT<std::conditional_t<recur, VariantT, Opaque>>;
+
+    using Symlink = nix::fso::Symlink;
+
+    using Raw = std::variant<Regular, Directory, Symlink>;
+    Raw raw;
+
+    MAKE_WRAPPER_CONSTRUCTOR(VariantT);
+
+    SourceAccessor::Stat lstat() const;
+};
+
+template<typename Child>
+inline bool DirectoryT<Child>::operator==(const DirectoryT &) const noexcept = default;
+
+template<typename Child>
+inline std::strong_ordering DirectoryT<Child>::operator<=>(const DirectoryT &) const noexcept = default;
+
+template<typename RegularContents, bool recur>
+inline bool
+VariantT<RegularContents, recur>::operator==(const VariantT<RegularContents, recur> &) const noexcept = default;
+
+template<typename RegularContents, bool recur>
+inline std::strong_ordering
+VariantT<RegularContents, recur>::operator<=>(const VariantT<RegularContents, recur> &) const noexcept = default;
+
+} // namespace fso
+
 /**
  * An source accessor for an in-memory file system.
  */
 struct MemorySourceAccessor : virtual SourceAccessor
 {
-    /**
-     * In addition to being part of the implementation of
-     * `MemorySourceAccessor`, this has a side benefit of nicely
-     * defining what a "file system object" is in Nix.
-     */
-    struct File
-    {
-        bool operator==(const File &) const noexcept;
-        std::strong_ordering operator<=>(const File &) const noexcept;
-
-        struct Regular
-        {
-            bool executable = false;
-            std::string contents;
-
-            bool operator==(const Regular &) const = default;
-            auto operator<=>(const Regular &) const = default;
-        };
-
-        struct Directory
-        {
-            using Name = std::string;
-
-            std::map<Name, File, std::less<>> contents;
-
-            bool operator==(const Directory &) const noexcept;
-            // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
-            bool operator<(const Directory &) const noexcept;
-        };
-
-        struct Symlink
-        {
-            std::string target;
-
-            bool operator==(const Symlink &) const = default;
-            auto operator<=>(const Symlink &) const = default;
-        };
-
-        using Raw = std::variant<Regular, Directory, Symlink>;
-        Raw raw;
-
-        MAKE_WRAPPER_CONSTRUCTOR(File);
-
-        Stat lstat() const;
-    };
+    using File = fso::VariantT<std::string, true>;
 
     std::optional<File> root;
 
@@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor
     SourcePath addFile(CanonPath path, std::string && contents);
 };
 
-inline bool MemorySourceAccessor::File::Directory::operator==(
-    const MemorySourceAccessor::File::Directory &) const noexcept = default;
-
-inline bool
-MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept
-{
-    return contents < other.contents;
-}
-
-inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default;
-inline std::strong_ordering
-MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default;
-
 /**
  * Write to a `MemorySourceAccessor` at the given path
  */
@@ -121,4 +160,53 @@ struct MemorySink : FileSystemObjectSink
     void createSymlink(const CanonPath & path, const std::string & target) override;
 };
 
+template<>
+struct json_avoids_null<MemorySourceAccessor::File::Regular> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<MemorySourceAccessor::File::Directory> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<MemorySourceAccessor::File::Symlink> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<MemorySourceAccessor::File> : std::true_type
+{};
+
+template<>
+struct json_avoids_null<MemorySourceAccessor> : std::true_type
+{};
+
 } // namespace nix
+
+namespace nlohmann {
+
+using namespace nix;
+
+#define ARG fso::Regular<RegularContents>
+template<typename RegularContents>
+JSON_IMPL_INNER(ARG)
+#undef ARG
+
+#define ARG fso::DirectoryT<Child>
+template<typename Child>
+JSON_IMPL_INNER(ARG)
+#undef ARG
+
+template<>
+JSON_IMPL_INNER(fso::Symlink)
+
+template<>
+JSON_IMPL_INNER(fso::Opaque)
+
+#define ARG fso::VariantT<RegularContents, recur>
+template<typename RegularContents, bool recur>
+JSON_IMPL_INNER(ARG)
+#undef ARG
+
+} // namespace nlohmann
+
+JSON_IMPL(MemorySourceAccessor)

src/libutil/include/nix/util/meson.build

@@ -50,6 +50,7 @@ headers = files(
   'memory-source-accessor.hh',
   'mounted-source-accessor.hh',
   'muxable-pipe.hh',
+  'nar-accessor.hh',
   'os-string.hh',
   'pool.hh',
   'pos-idx.hh',

src/libutil/include/nix/util/nar-accessor.hh

@@ -0,0 +1,80 @@
+#pragma once
+///@file
+
+#include "nix/util/memory-source-accessor.hh"
+
+#include <functional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+struct Source;
+
+/**
+ * Return an object that provides access to the contents of a NAR
+ * file.
+ */
+ref<SourceAccessor> makeNarAccessor(std::string && nar);
+
+ref<SourceAccessor> makeNarAccessor(Source & source);
+
+/**
+ * Create a NAR accessor from a NAR listing (in the format produced by
+ * listNar()). The callback getNarBytes(offset, length) is used by the
+ * readFile() method of the accessor to get the contents of files
+ * inside the NAR.
+ */
+using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
+
+/**
+ * The canonical GetNarBytes function for a seekable Source.
+ */
+GetNarBytes seekableGetNarBytes(const Path & path);
+
+ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
+
+struct NarListingRegularFile
+{
+    /**
+     * @see `SourceAccessor::Stat::fileSize`
+     */
+    std::optional<uint64_t> fileSize;
+
+    /**
+     * @see `SourceAccessor::Stat::narOffset`
+     *
+     * We only set to non-`std::nullopt` if it is also non-zero.
+     */
+    std::optional<uint64_t> narOffset;
+
+    auto operator<=>(const NarListingRegularFile &) const = default;
+};
+
+/**
+ * Abstract syntax for a "NAR listing".
+ */
+using NarListing = fso::VariantT<NarListingRegularFile, true>;
+
+/**
+ * Shallow NAR listing where directory children are not recursively expanded.
+ * Uses a variant that can hold Regular/Symlink fully, but Directory children
+ * are just unit types indicating presence without content.
+ */
+using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
+
+/**
+ * Return a deep structured representation of the contents of a NAR (except file
+ * contents), recursively listing all children.
+ */
+NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
+
+/**
+ * Return a shallow structured representation of the contents of a NAR (except file
+ * contents), only listing immediate children without recursing.
+ */
+ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
+
+// All json_avoids_null and JSON_IMPL covered by generic templates in memory-source-accessor.hh
+
+} // namespace nix

src/libutil/include/nix/util/strings.hh

@@ -166,4 +166,10 @@ public:
     }
 };
 
+/**
+ * Check that the string does not contain any NUL bytes and return c_str().
+ * @throws Error if str contains '\0' bytes.
+ */
+const char * requireCString(const std::string & str);
+
 } // namespace nix

src/libutil/include/nix/util/util.hh

@@ -242,6 +242,28 @@ std::string stripIndentation(std::string_view s);
  */
 std::pair<std::string_view, std::string_view> getLine(std::string_view s);
 
+/**
+ * Get a pointer to the contents of a `std::optional` if it is set, or a
+ * null pointer otherise.
+ *
+ * Const version.
+ */
+template<class T>
+const T * get(const std::optional<T> & opt)
+{
+    return opt ? &*opt : nullptr;
+}
+
+/**
+ * Non-const counterpart of `const T * get(const std::optional<T>)`.
+ * Takes a mutable reference, but returns a mutable pointer.
+ */
+template<class T>
+T * get(std::optional<T> & opt)
+{
+    return opt ? &*opt : nullptr;
+}
+
 /**
  * Get a value for the specified key from an associate container.
  */

src/libutil/memory-source-accessor.cc

@@ -1,4 +1,5 @@
 #include "nix/util/memory-source-accessor.hh"
+#include "nix/util/json-utils.hh"
 
 namespace nix {
 
@@ -29,13 +30,13 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path,
             return nullptr;
         auto & curDir = *curDirP;
 
-        auto i = curDir.contents.find(name);
-        if (i == curDir.contents.end()) {
+        auto i = curDir.entries.find(name);
+        if (i == curDir.entries.end()) {
             if (!create)
                 return nullptr;
             else {
                 newF = true;
-                i = curDir.contents.insert(
+                i = curDir.entries.insert(
                     i,
                     {
                         std::string{name},
@@ -68,25 +69,26 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path)
     return open(path, std::nullopt);
 }
 
-MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const
+template<>
+SourceAccessor::Stat MemorySourceAccessor::File::lstat() const
 {
     return std::visit(
         overloaded{
             [](const Regular & r) {
-                return Stat{
-                    .type = tRegular,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tRegular,
                     .fileSize = r.contents.size(),
                     .isExecutable = r.executable,
                 };
             },
             [](const Directory &) {
-                return Stat{
-                    .type = tDirectory,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tDirectory,
                 };
             },
             [](const Symlink &) {
-                return Stat{
-                    .type = tSymlink,
+                return SourceAccessor::Stat{
+                    .type = SourceAccessor::tSymlink,
                 };
             },
         },
@@ -106,7 +108,7 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon
         throw Error("file '%s' does not exist", path);
     if (auto * d = std::get_if<File::Directory>(&f->raw)) {
         DirEntries res;
-        for (auto & [name, file] : d->contents)
+        for (auto & [name, file] : d->entries)
             res.insert_or_assign(name, file.lstat().type);
         return res;
     } else