Move StringData to its own header

Now that it isn't just used for `Value`, it doesn't really belong in
there.

Rename `static-string-data.hh` to share the same prefix to keep them
close together when sorting lines, also.

.github/workflows/ci.yml

@@ -174,7 +174,7 @@ jobs:
         echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
         TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
         echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
-    - uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
+    - uses: cachix/install-nix-action@7ec16f2c061ab07b235a7245e06ed46fe9a1cab6 # v31.8.3
       if: ${{ !matrix.experimental-installer }}
       with:
         install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}

doc/manual/package.nix

@@ -37,7 +37,6 @@ mkMesonDerivation (finalAttrs: {
       (fileset.unions [
         ../../.version
         # For example JSON
-        ../../src/libutil-tests/data/memory-source-accessor
         ../../src/libutil-tests/data/hash
         ../../src/libstore-tests/data/content-address
         ../../src/libstore-tests/data/store-path

doc/manual/rl-next/libcurl-pausing.md

@@ -1,12 +0,0 @@
----
-synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
-prs: [14614]
-issues: [11728]
----
-
-The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
-[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
-
-This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
-
-Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).

doc/manual/source/SUMMARY.md.in

@@ -121,7 +121,6 @@
 - [Architecture and Design](architecture/architecture.md)
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
-    - [File System Object](protocols/json/file-system-object.md)
     - [Hash](protocols/json/hash.md)
     - [Content Address](protocols/json/content-address.md)
     - [Store Path](protocols/json/store-path.md)

doc/manual/source/command-ref/nix-build.md

@@ -36,7 +36,7 @@ to a temporary location. The tarball must include a single top-level
 directory containing at least a file named `default.nix`.
 
 `nix-build` is essentially a wrapper around
-[`nix-instantiate`](./nix-instantiate.md) (to translate a high-level Nix
+[`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix
 expression to a low-level [store derivation]) and [`nix-store
 --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
 derivation).
@@ -52,8 +52,8 @@ derivation).
 # Options
 
 All options not listed here are passed to
-[`nix-store --realise`](./nix-store/realise.md),
-except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](./nix-instantiate.md).
+[`nix-store --realise`](nix-store/realise.md),
+except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md).
 
 - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
 

doc/manual/source/command-ref/nix-instantiate.md

@@ -32,7 +32,7 @@ standard input.
 
 - `--add-root` *path*
 
-  See the [corresponding option](./nix-store.md) in `nix-store`.
+  See the [corresponding option](nix-store.md) in `nix-store`.
 
 - `--parse`
 

doc/manual/source/development/debugging.md

@@ -15,7 +15,7 @@ In the development shell, set the `mesonBuildType` environment variable to `debu
 Then, proceed to build Nix as described in [Building Nix](./building.md).
 This will build Nix with debug symbols, which are essential for effective debugging.
 
-It is also possible to build without optimization for faster build:
+It is also possible to build without debugging for faster build:
 
 ```console
 [nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify)

doc/manual/source/protocols/json/file-system-object.md

@@ -1,21 +0,0 @@
-{{#include file-system-object-v1-fixed.md}}
-
-## Examples
-
-### Simple
-
-```json
-{{#include schema/file-system-object-v1/simple.json}}
-```
-
-### Complex
-
-```json
-{{#include schema/file-system-object-v1/complex.json}}
-```
-
-<!-- need to convert YAML to JSON first
-## Raw Schema
-
-[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
--->

doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed

@@ -11,7 +11,6 @@ s/\\`/`/g
 #
 # As we have more such relative links, more replacements of this nature
 # should appear below.
-s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
 s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
 s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
 s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g

doc/manual/source/protocols/json/meson.build

@@ -9,7 +9,6 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
 json_schema_config = files('json-schema-for-humans-config.yaml')
 
 schemas = [
-  'file-system-object-v1',
   'hash-v1',
   'content-address-v1',
   'store-path-v1',

doc/manual/source/protocols/json/schema/file-system-object-v1

@@ -1 +0,0 @@
-../../../../../../src/libutil-tests/data/memory-source-accessor

doc/manual/source/protocols/json/schema/file-system-object-v1.yaml

@@ -1,71 +0,0 @@
-"$schema": http://json-schema.org/draft-04/schema#
-"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
-title: File System Object
-description: |
-  This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
-
-  The schema is recursive because file system objects contain other file system objects.
-type: object
-required: ["type"]
-properties:
-  type:
-    type: string
-    enum: ["regular", "symlink", "directory"]
-
-# Enforce conditional structure based on `type`
-anyOf:
-  - $ref: "#/$defs/regular"
-    required: ["type", "contents"]
-
-  - $ref: "#/$defs/directory"
-    required: ["type", "entries"]
-
-  - $ref: "#/$defs/symlink"
-    required: ["type", "target"]
-
-"$defs":
-  regular:
-    title: Regular File
-    description: |
-      See [Regular File](@docroot@/store/file-system-object.md#regular) in the manual for details.
-    required: ["contents"]
-    properties:
-      type:
-        const: "regular"
-      contents:
-        type: string
-        description: File contents
-      executable:
-        type: boolean
-        description: Whether the file is executable.
-        default: false
-    additionalProperties: false
-
-  directory:
-    title: Directory
-    description: |
-      See [Directory](@docroot@/store/file-system-object.md#directory) in the manual for details.
-    required: ["entries"]
-    properties:
-      type:
-        const: "directory"
-      entries:
-        type: object
-        description: |
-          Map of names to nested file system objects (for type=directory)
-        additionalProperties:
-          $ref: "#"
-    additionalProperties: false
-
-  symlink:
-    title: Symbolic Link
-    description: |
-      See [Symbolic Link](@docroot@/store/file-system-object.md#symlink) in the manual for details.
-    required: ["target"]
-    properties:
-      type:
-        const: "symlink"
-      target:
-        type: string
-        description: Target path of the symlink.
-    additionalProperties: false

doc/manual/source/store/file-system-object.md

@@ -3,23 +3,19 @@
 Nix uses a simplified model of the file system, which consists of file system objects.
 Every file system object is one of the following:
 
- - [**Regular File**]{#regular}
+ - File
 
    - A possibly empty sequence of bytes for contents
    - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission
 
- - [**Directory**]{#directory}
+ - Directory
 
    Mapping of names to child file system objects
 
- - [**Symbolic link**]{#symlink}
+ - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link)
 
-   An arbitrary string, known as the *target* of the symlink.
-
-   In general, Nix does not assign any semantics to symbolic links.
-   Certain operations however, may make additional assumptions and attempt to use the target to find another file system object.
-
-   > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept.
+   An arbitrary string.
+   Nix does not assign any semantics to symbolic links.
 
 File system objects and their children form a tree.
 A bare file or symlink can be a root file system object.

packaging/dev-shell.nix

@@ -130,19 +130,15 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
     havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
     ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
 
-    availableComponents = lib.filterAttrs (
-      k: v: lib.meta.availableOn pkgs.hostPlatform v
-    ) allComponents;
-
     activeComponents = buildInputsClosureCond isInternal (
-      lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents)
+      lib.attrValues (finalAttrs.passthru.config.getComponents allComponents)
     );
 
     allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2;
     internalDrvs = byDrvPath (
       # Drop the attr names (not present in buildInputs anyway)
-      lib.attrValues availableComponents
-      ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents)
+      lib.attrValues allComponents
+      ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues allComponents)
     );
 
     isInternal =
@@ -191,19 +187,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
         );
 
       small =
-        (finalAttrs.finalPackage.withActiveComponents (
-          c:
-          lib.intersectAttrs (lib.genAttrs [
-            "nix-cli"
-            "nix-util-tests"
-            "nix-store-tests"
-            "nix-expr-tests"
-            "nix-fetchers-tests"
-            "nix-flake-tests"
-            "nix-functional-tests"
-            "nix-perl-bindings"
-          ] (_: null)) c
-        )).overrideAttrs
+        (finalAttrs.finalPackage.withActiveComponents (c: {
+          inherit (c)
+            nix-cli
+            nix-util-tests
+            nix-store-tests
+            nix-expr-tests
+            nix-fetchers-tests
+            nix-flake-tests
+            nix-functional-tests
+            # Currently required
+            nix-perl-bindings
+            ;
+        })).overrideAttrs
           (o: {
             mesonFlags = o.mesonFlags ++ [
               # TODO: infer from activeComponents or vice versa
@@ -273,8 +269,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
       CXX_LD = "mold";
     };
 
-    dontUseCmakeConfigure = true;
-
     mesonFlags =
       map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
       ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)

src/json-schema-checks/file-system-object

@@ -1 +0,0 @@
-../../src/libutil-tests/data/memory-source-accessor

src/json-schema-checks/meson.build

@@ -20,14 +20,6 @@ schema_dir = meson.current_source_dir() / 'schema'
 
 # Get all example files
 schemas = [
-  {
-    'stem' : 'file-system-object',
-    'schema' : schema_dir / 'file-system-object-v1.yaml',
-    'files' : [
-      'simple.json',
-      'complex.json',
-    ],
-  },
   {
     'stem' : 'hash',
     'schema' : schema_dir / 'hash-v1.yaml',

src/json-schema-checks/package.nix

@@ -20,7 +20,6 @@ mkMesonDerivation (finalAttrs: {
   fileset = lib.fileset.unions [
     ../../.version
     ../../doc/manual/source/protocols/json/schema
-    ../../src/libutil-tests/data/memory-source-accessor
     ../../src/libutil-tests/data/hash
     ../../src/libstore-tests/data/content-address
     ../../src/libstore-tests/data/store-path

src/kaitai-struct-checks/package.nix

@@ -65,6 +65,6 @@ mkMesonDerivation (finalAttrs: {
   '';
 
   meta = {
-    platforms = lib.platforms.unix;
+    platforms = lib.platforms.all;
   };
 })

src/libcmd/common-eval-args.cc

@@ -34,7 +34,7 @@ EvalSettings evalSettings{
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
                 auto [accessor, lockedRef] =
-                    flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
+                    flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
                 auto storePath = nix::fetchToStore(
                     state.fetchSettings,
                     *state.store,
@@ -132,7 +132,7 @@ MixEvalArgs::MixEvalArgs()
             fetchers::Attrs extraAttrs;
             if (to.subdir != "")
                 extraAttrs["dir"] = to.subdir;
-            fetchers::overrideRegistry(from.input, to.input, extraAttrs);
+            fetchers::overrideRegistry(fetchSettings, from.input, to.input, extraAttrs);
         }},
         .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
             completeFlakeRef(completions, openStore(), prefix);
@@ -168,9 +168,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
                                 ? state.rootPath(absPath(getCommandBaseDir()))
                                 : state.rootPath(".")));
                 },
-                [&](const AutoArgString & arg) { v->mkString(arg.s, state.mem); },
-                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string()), state.mem); },
-                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO), state.mem); }},
+                [&](const AutoArgString & arg) { v->mkString(arg.s); },
+                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string())); },
+                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO)); }},
             arg);
         res.insert(state.symbols.create(name), v);
     }
@@ -180,7 +180,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
 {
     if (EvalSettings::isPseudoUrl(s)) {
-        auto accessor = fetchers::downloadTarball(*state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
+        auto accessor = fetchers::downloadTarball(state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
         auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy);
         return state.storePath(storePath);
     }
@@ -188,8 +188,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
-        auto [accessor, lockedRef] =
-            flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
+        auto [accessor, lockedRef] = flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
         auto storePath = nix::fetchToStore(
             state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
         state.allowPath(storePath);

src/libcmd/installables.cc

@@ -185,6 +185,7 @@ MixFlakeOptions::MixFlakeOptions()
                     }
 
                     overrideRegistry(
+                        fetchSettings,
                         fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
                         extraAttrs);
@@ -409,7 +410,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
     Args::completeDir(completions, 0, prefix);
 
     /* Look for registry entries that match the prefix. */
-    for (auto & registry : fetchers::getRegistries(fetchSettings, *store)) {
+    for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
         for (auto & entry : registry->entries) {
             auto from = entry.from.to_string();
             if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {

src/libcmd/repl.cc

@@ -656,7 +656,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                             + "\n\n";
             }
 
-            markdown += stripIndentation(doc->doc);
+            markdown += stripIndentation(doc->doc.view());
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
         } else if (fallbackPos) {
@@ -739,7 +739,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
 
     auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
     if (evalSettings.pureEval && !flakeRef.input.isLocked(fetchSettings))
-        throw Error("cannot use ':load-flake' on unlocked flake reference '%s' (use --impure to override)", flakeRefS);
+        throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
 
     Value v;
 

src/libexpr-c/nix_api_expr.cc

@@ -69,8 +69,8 @@ nix_err nix_expr_eval_from_string(
         context->last_err_code = NIX_OK;
     try {
         nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
-        state->state.eval(parsedExpr, *value->value);
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.eval(parsedExpr, value->value);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -80,8 +80,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.callFunction(*fn->value, *arg->value, *value->value, nix::noPos);
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -91,15 +91,9 @@ nix_err nix_value_call_multi(
 {
     if (context)
         context->last_err_code = NIX_OK;
-
-    std::vector<nix::Value *> internal_args;
-    internal_args.reserve(nargs);
-    for (size_t i = 0; i < nargs; i++)
-        internal_args.push_back(args[i]->value);
-
     try {
-        state->state.callFunction(*fn->value, {internal_args.data(), nargs}, *value->value, nix::noPos);
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.callFunction(fn->value, {(nix::Value **) args, nargs}, value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -109,7 +103,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -119,7 +113,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValueDeep(*value->value);
+        state->state.forceValueDeep(value->value);
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-c/nix_api_expr_internal.h

@@ -39,13 +39,7 @@ struct ListBuilder
 
 struct nix_value
 {
-    nix::Value * value;
-    /**
-     * As we move to a managed heap, we need EvalMemory in more places. Ideally, we would take in EvalState or
-     * EvalMemory as an argument when we need it, but we don't want to make changes to the stable C api, so we stuff it
-     * into the nix_value that will get passed in to the relevant functions.
-     */
-    nix::EvalMemory * mem;
+    nix::Value value;
 };
 
 struct nix_string_return

src/libexpr-c/nix_api_value.cc

@@ -20,7 +20,7 @@ static const nix::Value & check_value_not_null(const nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return *value->value;
+    return *((const nix::Value *) value);
 }
 
 static nix::Value & check_value_not_null(nix_value * value)
@@ -28,7 +28,7 @@ static nix::Value & check_value_not_null(nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return *value->value;
+    return value->value;
 }
 
 static const nix::Value & check_value_in(const nix_value * value)
@@ -58,14 +58,9 @@ static nix::Value & check_value_out(nix_value * value)
     return v;
 }
 
-static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
+static inline nix_value * as_nix_value_ptr(nix::Value * v)
 {
-    nix_value * ret = new (mem.allocBytes(sizeof(nix_value))) nix_value{
-        .value = v,
-        .mem = &mem,
-    };
-    nix_gc_incref(nullptr, ret);
-    return ret;
+    return reinterpret_cast<nix_value *>(v);
 }
 
 /**
@@ -74,13 +69,7 @@ static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
  * Deals with errors and converts arguments from C++ into C types.
  */
 static void nix_c_primop_wrapper(
-    PrimOpFun f,
-    void * userdata,
-    int arity,
-    nix::EvalState & state,
-    const nix::PosIdx pos,
-    nix::Value ** args,
-    nix::Value & v)
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
 {
     nix_c_context ctx;
 
@@ -96,15 +85,8 @@ static void nix_c_primop_wrapper(
     // ok because we don't see a need for this yet (e.g. inspecting thunks,
     // or maybe something to make blackholes work better; we don't know).
     nix::Value vTmp;
-    nix_value * vTmpPtr = new_nix_value(&vTmp, state.mem);
 
-    std::vector<nix_value *> external_args;
-    external_args.reserve(arity);
-    for (int i = 0; i < arity; i++) {
-        nix_value * external_arg = new_nix_value(args[i], state.mem);
-        external_args.push_back(external_arg);
-    }
-    f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr);
+    f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
 
     if (ctx.last_err_code != NIX_OK) {
         /* TODO: Throw different errors depending on the error code */
@@ -152,8 +134,8 @@ PrimOp * nix_alloc_primop(
                     .name = name,
                     .args = {},
                     .arity = (size_t) arity,
-                    .doc = doc,
-                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)};
+                    .doc = &nix::StringData::make(doc),
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
         if (args)
             for (size_t i = 0; args[i]; i++)
                 p->args.emplace_back(*args);
@@ -178,7 +160,8 @@ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        nix_value * res = new_nix_value(state->state.allocValue(), state->state.mem);
+        nix_value * res = as_nix_value_ptr(state->state.allocValue());
+        nix_gc_incref(nullptr, res);
         return res;
     }
     NIXC_CATCH_ERRS_NULL
@@ -348,10 +331,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
             return nullptr;
         }
         auto * p = v.listView()[ix];
-        if (p == nullptr)
-            return nullptr;
-        state->state.forceValue(*p, nix::noPos);
-        return new_nix_value(p, state->state.mem);
+        nix_gc_incref(nullptr, p);
+        if (p != nullptr)
+            state->state.forceValue(*p, nix::noPos);
+        return as_nix_value_ptr(p);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -369,8 +352,9 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt
             return nullptr;
         }
         auto * p = v.listView()[ix];
+        nix_gc_incref(nullptr, p);
         // Note: intentionally NOT calling forceValue() to keep the element lazy
-        return new_nix_value(p, state->state.mem);
+        return as_nix_value_ptr(p);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -385,8 +369,9 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
+            nix_gc_incref(nullptr, attr->value);
             state->state.forceValue(*attr->value, nix::noPos);
-            return new_nix_value(attr->value, state->state.mem);
+            return as_nix_value_ptr(attr->value);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -405,8 +390,9 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
+            nix_gc_incref(nullptr, attr->value);
             // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-            return new_nix_value(attr->value, state->state.mem);
+            return as_nix_value_ptr(attr->value);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -454,8 +440,9 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
         state->state.forceValue(*a.value, nix::noPos);
-        return new_nix_value(a.value, state->state.mem);
+        return as_nix_value_ptr(a.value);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -474,8 +461,9 @@ nix_value * nix_get_attr_byidx_lazy(
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
         // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-        return new_nix_value(a.value, state->state.mem);
+        return as_nix_value_ptr(a.value);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -515,7 +503,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkString(std::string_view(str), *value->mem);
+        v.mkString(std::string_view(str));
     }
     NIXC_CATCH_ERRS
 }
@@ -526,7 +514,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkPath(s->state.rootPath(nix::CanonPath(str)), s->state.mem);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-tests/json.cc

@@ -1,6 +1,6 @@
 #include "nix/expr/tests/libexpr.hh"
 #include "nix/expr/value-to-json.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 namespace nix {
 // Testing the conversion to JSON
@@ -73,7 +73,7 @@ TEST_F(JSONValueTest, StringQuotes)
 TEST_F(JSONValueTest, DISABLED_Path)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("/test")), state.mem);
+    v.mkPath(state.rootPath(CanonPath("/test")));
     ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
 }
 } /* namespace nix */

src/libexpr-tests/value/print.cc

@@ -1,5 +1,5 @@
 #include "nix/expr/tests/libexpr.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 #include "nix/expr/value.hh"
 #include "nix/expr/print.hh"
@@ -268,7 +268,7 @@ struct StringPrintingTests : LibExprTest
     void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args)
     {
         Value v;
-        v.mkString(literal, state.mem);
+        v.mkString(literal);
 
         std::stringstream out;
         printValue(state, out, v, PrintOptions{.maxStringLength = maxLength});
@@ -353,7 +353,7 @@ TEST_F(ValuePrintingTests, ansiColorsStringElided)
 TEST_F(ValuePrintingTests, ansiColorsPath)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("puppy")), state.mem);
+    v.mkPath(state.rootPath(CanonPath("puppy")));
 
     test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true});
 }

src/libexpr-tests/value/value.cc

@@ -1,5 +1,5 @@
 #include "nix/expr/value.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 #include "nix/store/tests/libstore.hh"
 #include <gtest/gtest.h>

src/libexpr/eval.cc

@@ -51,50 +51,20 @@ using json = nlohmann::json;
 
 namespace nix {
 
-/**
- * Just for doc strings. Not for regular string values.
- */
-static char * allocString(size_t size)
+StringData & StringData::alloc(size_t size)
 {
-    char * t;
-    t = (char *) GC_MALLOC_ATOMIC(size);
-    if (!t)
-        throw std::bad_alloc();
-    return t;
-}
-
-// When there's no need to write to the string, we can optimize away empty
-// string allocations.
-// This function handles makeImmutableString(std::string_view()) by returning
-// the empty string.
-/**
- * Just for doc strings. Not for regular string values.
- */
-static const char * makeImmutableString(std::string_view s)
-{
-    const size_t size = s.size();
-    if (size == 0)
-        return "";
-    auto t = allocString(size + 1);
-    memcpy(t, s.data(), size);
-    t[size] = '\0';
-    return t;
-}
-
-StringData & StringData::alloc(EvalMemory & mem, size_t size)
-{
-    void * t = mem.allocBytes(sizeof(StringData) + size + 1);
+    void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
     if (!t)
         throw std::bad_alloc();
     auto res = new (t) StringData(size);
     return *res;
 }
 
-const StringData & StringData::make(EvalMemory & mem, std::string_view s)
+const StringData & StringData::make(std::string_view s)
 {
     if (s.empty())
         return ""_sds;
-    auto & res = alloc(mem, s.size());
+    auto & res = alloc(s.size());
     std::memcpy(&res.data_, s.data(), s.size());
     res.data_[s.size()] = '\0';
     return res;
@@ -205,7 +175,7 @@ bool Value::isTrivial() const
 {
     return !isa<tApp, tPrimOpApp>()
            && (!isa<tThunk>()
-               || (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs->empty())
+               || (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs.empty())
                || dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr));
 }
 
@@ -517,16 +487,15 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     if (primOp.arity == 0) {
         primOp.arity = 1;
         auto vPrimOp = allocValue();
-        vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
+        vPrimOp->mkPrimOp(new PrimOp(primOp));
         Value v;
         v.mkApp(vPrimOp, vPrimOp);
-        auto & primOp1 = *vPrimOp->primOp();
         return addConstant(
-            primOp1.name,
+            primOp.name,
             v,
             {
                 .type = nThunk, // FIXME
-                .doc = primOp1.doc ? primOp1.doc->c_str() : nullptr,
+                .doc = primOp.doc,
             });
     }
 
@@ -566,14 +535,13 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        auto & primOp = *v2->primOp();
-        if (primOp.doc)
+        if (auto * doc = v2->primOp()->doc)
             return Doc{
                 .pos = {},
-                .name = primOp.name,
-                .arity = primOp.arity,
-                .args = primOp.args,
-                .doc = primOp.doc->c_str(),
+                .name = v2->primOp()->name,
+                .arity = v2->primOp()->arity,
+                .args = v2->primOp()->args,
+                .doc = *doc,
             };
     }
     if (v.isLambda()) {
@@ -615,9 +583,8 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
             .name = name,
             .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
             .args = {},
-            /* N.B. Can't use StringData here, because that would lead to an interior pointer.
-               NOTE: memory leak when compiled without GC. */
-            .doc = makeImmutableString(s.view()),
+            /* NOTE: memory leak when compiled without GC. */
+            .doc = StringData::make(s.view()),
         };
     }
     if (isFunctor(v)) {
@@ -849,9 +816,9 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
         evalState.runDebugRepl(nullptr, trace.env, trace.expr);
 }
 
-void Value::mkString(std::string_view s, EvalMemory & mem)
+void Value::mkString(std::string_view s)
 {
-    mkStringNoCopy(StringData::make(mem, s));
+    mkStringNoCopy(StringData::make(s));
 }
 
 Value::StringWithContext::Context *
@@ -862,13 +829,13 @@ Value::StringWithContext::Context::fromBuilder(const NixStringContext & context,
 
     auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
     std::ranges::transform(
-        context, ctx->elems, [&](const NixStringContextElem & elt) { return &StringData::make(mem, elt.to_string()); });
+        context, ctx->elems, [](const NixStringContextElem & elt) { return &StringData::make(elt.to_string()); });
     return ctx;
 }
 
 void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem)
 {
-    mkStringNoCopy(StringData::make(mem, s), Value::StringWithContext::Context::fromBuilder(context, mem));
+    mkStringNoCopy(StringData::make(s), Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
 void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem)
@@ -876,9 +843,9 @@ void Value::mkStringMove(const StringData & s, const NixStringContext & context,
     mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
-void Value::mkPath(const SourcePath & path, EvalMemory & mem)
+void Value::mkPath(const SourcePath & path)
 {
-    mkPath(&*path.accessor, StringData::make(mem, path.path.abs()));
+    mkPath(&*path.accessor, StringData::make(path.path.abs()));
 }
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@@ -943,7 +910,7 @@ void EvalState::mkPos(Value & v, PosIdx p)
     auto origin = positions.originOf(p);
     if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(s.file).mkString(path->path.abs(), mem);
+        attrs.alloc(s.file).mkString(path->path.abs());
         makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column));
         v.mkAttrs(attrs);
     } else
@@ -1226,26 +1193,26 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
 
 void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 {
-    auto bindings = state.buildBindings(attrs->size() + dynamicAttrs->size());
+    auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size());
     auto dynamicEnv = &env;
     bool sort = false;
 
     if (recursive) {
         /* Create a new environment that contains the attributes in
            this `rec'. */
-        Env & env2(state.mem.allocEnv(attrs->size()));
+        Env & env2(state.mem.allocEnv(attrs.size()));
         env2.up = &env;
         dynamicEnv = &env2;
         Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
 
-        AttrDefs::iterator overrides = attrs->find(state.s.overrides);
-        bool hasOverrides = overrides != attrs->end();
+        AttrDefs::iterator overrides = attrs.find(state.s.overrides);
+        bool hasOverrides = overrides != attrs.end();
 
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
         Displacement displ = 0;
-        for (auto & i : *attrs) {
+        for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
                 vAttr = state.allocValue();
@@ -1272,8 +1239,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
                 "while evaluating the `__overrides` attribute");
             bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size()));
             for (auto & i : *vOverrides->attrs()) {
-                AttrDefs::iterator j = attrs->find(i.name);
-                if (j != attrs->end()) {
+                AttrDefs::iterator j = attrs.find(i.name);
+                if (j != attrs.end()) {
                     (*bindings.bindings)[j->second.displ] = i;
                     env2.values[j->second.displ] = i.value;
                 } else
@@ -1285,13 +1252,13 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 
     else {
         Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
-        for (auto & i : *attrs)
+        for (auto & i : attrs)
             bindings.insert(
                 i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos);
     }
 
     /* Dynamic attrs apply *after* rec and __overrides. */
-    for (auto & i : *dynamicAttrs) {
+    for (auto & i : dynamicAttrs) {
         Value nameVal;
         i.nameExpr->eval(state, *dynamicEnv, nameVal);
         state.forceValue(nameVal, i.pos);
@@ -1325,7 +1292,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 {
     /* Create a new environment that contains the attributes in this
        `let'. */
-    Env & env2(state.mem.allocEnv(attrs->attrs->size()));
+    Env & env2(state.mem.allocEnv(attrs->attrs.size()));
     env2.up = &env;
 
     Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@@ -1334,7 +1301,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
        while the inherited attributes are evaluated in the original
        environment. */
     Displacement displ = 0;
-    for (auto & i : *attrs->attrs) {
+    for (auto & i : attrs->attrs) {
         env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
     }
 
@@ -1751,9 +1718,9 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    SmallValueVector<4> vArgs(args->size());
-    for (size_t i = 0; i < args->size(); ++i)
-        vArgs[i] = (*args)[i]->maybeThunk(state, env);
+    SmallValueVector<4> vArgs(args.size());
+    for (size_t i = 0; i < args.size(); ++i)
+        vArgs[i] = args[i]->maybeThunk(state, env);
 
     state.callFunction(vFun, vArgs, v, pos);
 }
@@ -2139,9 +2106,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         for (const auto & part : strings) {
             resultStr += *part;
         }
-        v.mkPath(state.rootPath(CanonPath(resultStr)), state.mem);
+        v.mkPath(state.rootPath(CanonPath(resultStr)));
     } else {
-        auto & resultStr = StringData::alloc(state.mem, sSize);
+        auto & resultStr = StringData::alloc(sSize);
         auto * tmp = resultStr.data();
         for (const auto & part : strings) {
             std::memcpy(tmp, part->data(), part->size());
@@ -2188,8 +2155,6 @@ void EvalState::forceValueDeep(Value & v)
     std::set<const Value *> seen;
 
     [&, &state(*this)](this const auto & recurse, Value & v) {
-        auto _level = state.addCallDepth(v.determinePos(noPos));
-
         if (!seen.insert(&v).second)
             return;
 
@@ -2216,15 +2181,8 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            size_t index = 0;
             for (auto v2 : v.listView())
-                try {
-                    recurse(*v2);
-                    index++;
-                } catch (Error & e) {
-                    state.addErrorTrace(e, "while evaluating list element at index %1%", index);
-                    throw;
-                }
+                recurse(*v2);
         }
     }(v);
 }
@@ -3199,7 +3157,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
 
     if (EvalSettings::isPseudoUrl(value)) {
         try {
-            auto accessor = fetchers::downloadTarball(*store, fetchSettings, EvalSettings::resolvePseudoUrl(value));
+            auto accessor = fetchers::downloadTarball(store, fetchSettings, EvalSettings::resolvePseudoUrl(value));
             auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
             return finish(this->storePath(storePath));
         } catch (Error & e) {

src/libexpr/include/nix/expr/eval.hh

@@ -108,7 +108,7 @@ struct PrimOp
     /**
      * Optional free-form documentation about the primop.
      */
-    std::optional<std::string> doc;
+    const StringData * doc = nullptr;
 
     /**
      * Add a trace item, while calling the `<name>` builtin.
@@ -156,7 +156,7 @@ struct Constant
     /**
      * Optional free-form documentation about the constant.
      */
-    const char * doc = nullptr;
+    const StringData * doc = nullptr;
 
     /**
      * Whether the constant is impure, and not available in pure mode.
@@ -837,11 +837,7 @@ public:
         std::optional<std::string> name;
         size_t arity;
         std::vector<std::string> args;
-        /**
-         * Unlike the other `doc` fields in this file, this one should never be
-         * `null`.
-         */
-        const char * doc;
+        const StringData & doc;
     };
 
     /**

src/libexpr/include/nix/expr/meson.build

@@ -31,7 +31,8 @@ headers = [ config_pub_h ] + files(
   'print.hh',
   'repl-exit-status.hh',
   'search-path.hh',
-  'static-string-data.hh',
+  'string-data.hh',
+  'string-data-static.hh',
   'symbol-table.hh',
   'value-to-json.hh',
   'value-to-xml.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -12,7 +12,7 @@
 #include "nix/expr/value.hh"
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/eval-error.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/util/pos-idx.hh"
 #include "nix/expr/counter.hh"
 #include "nix/util/pos-table.hh"
@@ -395,13 +395,9 @@ struct ExprAttrs : Expr
         }
     };
 
-    typedef std::pmr::map<Symbol, AttrDef> AttrDefs;
-    /**
-     * attrs will never be null. we use std::optional so that we can call emplace() to re-initialize the value with a
-     * new pmr::map using a different allocator (move assignment will copy into the old allocator)
-     */
-    std::optional<AttrDefs> attrs;
-    std::unique_ptr<std::pmr::vector<Expr *>> inheritFromExprs;
+    typedef std::map<Symbol, AttrDef> AttrDefs;
+    AttrDefs attrs;
+    std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
 
     struct DynamicAttrDef
     {
@@ -413,20 +409,13 @@ struct ExprAttrs : Expr
             , pos(pos) {};
     };
 
-    typedef std::pmr::vector<DynamicAttrDef> DynamicAttrDefs;
-    /**
-     * dynamicAttrs will never be null. See comment on AttrDefs above.
-     */
-    std::optional<DynamicAttrDefs> dynamicAttrs;
+    typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
+    DynamicAttrDefs dynamicAttrs;
     ExprAttrs(const PosIdx & pos)
         : recursive(false)
-        , pos(pos)
-        , attrs(AttrDefs{})
-        , dynamicAttrs(DynamicAttrDefs{}) {};
+        , pos(pos) {};
     ExprAttrs()
-        : recursive(false)
-        , attrs(AttrDefs{})
-        , dynamicAttrs(DynamicAttrDefs{}) {};
+        : recursive(false) {};
 
     PosIdx getPos() const override
     {
@@ -592,14 +581,11 @@ public:
 struct ExprCall : Expr
 {
     Expr * fun;
-    /**
-     * args will never be null. See comment on ExprAttrs::AttrDefs below.
-     */
-    std::optional<std::pmr::vector<Expr *>> args;
+    std::vector<Expr *> args;
     PosIdx pos;
     std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
+    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -607,7 +593,7 @@ struct ExprCall : Expr
     {
     }
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -839,7 +825,7 @@ public:
     // we define some calls to add explicitly so that the argument can be passed in as initializer lists
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
+    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args));
@@ -847,7 +833,7 @@ public:
 
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos));

src/libexpr/include/nix/expr/parser-state.hh

@@ -5,7 +5,7 @@
 
 #include "nix/expr/eval.hh"
 #include "nix/expr/value.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 namespace nix {
 
@@ -126,8 +126,8 @@ inline void ParserState::addAttr(
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
         ExprAttrs * nested;
         if (i->symbol) {
-            ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(i->symbol);
-            if (j != attrs->attrs->end()) {
+            ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
+            if (j != attrs->attrs.end()) {
                 nested = dynamic_cast<ExprAttrs *>(j->second.e);
                 if (!nested) {
                     attrPath.erase(i + 1, attrPath.end());
@@ -135,11 +135,11 @@ inline void ParserState::addAttr(
                 }
             } else {
                 nested = exprs.add<ExprAttrs>();
-                (*attrs->attrs)[i->symbol] = ExprAttrs::AttrDef(nested, pos);
+                attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
             }
         } else {
             nested = exprs.add<ExprAttrs>();
-            attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
+            attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
         }
         attrs = nested;
     }
@@ -148,7 +148,7 @@ inline void ParserState::addAttr(
     if (i->symbol) {
         addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
     } else {
-        attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
+        attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
     }
 
     auto it = lexerState.positionToDocComment.find(pos);
@@ -165,8 +165,8 @@ inline void ParserState::addAttr(
 inline void
 ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
 {
-    ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(symbol);
-    if (j != attrs->attrs->end()) {
+    ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
+    if (j != attrs->attrs.end()) {
         // This attr path is already defined. However, if both
         // e and the expr pointed by the attr path are two attribute sets,
         // we want to merge them.
@@ -181,8 +181,8 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
         // See https://github.com/NixOS/nix/issues/9020.
         if (jAttrs && ae) {
             if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                jAttrs->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>();
-            for (auto & ad : *ae->attrs) {
+                jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+            for (auto & ad : ae->attrs) {
                 if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
                     auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
                     auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
@@ -192,12 +192,12 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
                 addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
                 attrPath.pop_back();
             }
-            ae->attrs->clear();
-            jAttrs->dynamicAttrs->insert(
-                jAttrs->dynamicAttrs->end(),
-                std::make_move_iterator(ae->dynamicAttrs->begin()),
-                std::make_move_iterator(ae->dynamicAttrs->end()));
-            ae->dynamicAttrs->clear();
+            ae->attrs.clear();
+            jAttrs->dynamicAttrs.insert(
+                jAttrs->dynamicAttrs.end(),
+                std::make_move_iterator(ae->dynamicAttrs.begin()),
+                std::make_move_iterator(ae->dynamicAttrs.end()));
+            ae->dynamicAttrs.clear();
             if (ae->inheritFromExprs) {
                 jAttrs->inheritFromExprs->insert(
                     jAttrs->inheritFromExprs->end(),
@@ -210,7 +210,7 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
         }
     } else {
         // This attr path is not defined. Let's create it.
-        attrs->attrs->emplace(symbol, def);
+        attrs->attrs.emplace(symbol, def);
         def.e->setName(symbol);
     }
 }

src/libexpr/include/nix/expr/string-data-static.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "nix/expr/value.hh"
+#include "nix/expr/string-data.hh"
 
 namespace nix {
 

src/libexpr/include/nix/expr/string-data.hh

@@ -0,0 +1,96 @@
+#pragma once
+///@file
+
+#include <cstddef>
+#include <cstring>
+#include <memory_resource>
+#include <string_view>
+
+namespace nix {
+
+class StringData
+{
+public:
+    using size_type = std::size_t;
+
+    size_type size_;
+    char data_[];
+
+    /*
+     * This in particular ensures that we cannot have a `StringData`
+     * that we use by value, which is just what we want!
+     *
+     * Dynamically sized types aren't a thing in C++ and even flexible array
+     * members are a language extension and beyond the realm of standard C++.
+     * Technically, sizeof data_ member is 0 and the intended way to use flexible
+     * array members is to allocate sizeof(StrindData) + count * sizeof(char) bytes
+     * and the compiler will consider alignment restrictions for the FAM.
+     *
+     */
+
+    StringData(StringData &&) = delete;
+    StringData & operator=(StringData &&) = delete;
+    StringData(const StringData &) = delete;
+    StringData & operator=(const StringData &) = delete;
+    ~StringData() = default;
+
+private:
+    StringData() = delete;
+
+    explicit StringData(size_type size)
+        : size_(size)
+    {
+    }
+
+public:
+    /**
+     * Allocate StringData on the (possibly) GC-managed heap and copy
+     * the contents of s to it.
+     */
+    static const StringData & make(std::string_view s);
+
+    /**
+     * Allocate StringData on the (possibly) GC-managed heap.
+     * @param size Length of the string (without the NUL terminator).
+     */
+    static StringData & alloc(size_t size);
+
+    size_t size() const
+    {
+        return size_;
+    }
+
+    char * data() noexcept
+    {
+        return data_;
+    }
+
+    const char * data() const noexcept
+    {
+        return data_;
+    }
+
+    const char * c_str() const noexcept
+    {
+        return data_;
+    }
+
+    constexpr std::string_view view() const noexcept
+    {
+        return std::string_view(data_, size_);
+    }
+
+    template<size_t N>
+    struct Static;
+
+    static StringData & make(std::pmr::memory_resource & resource, std::string_view s)
+    {
+        auto & res =
+            *new (resource.allocate(sizeof(StringData) + s.size() + 1, alignof(StringData))) StringData(s.size());
+        std::memcpy(res.data_, s.data(), s.size());
+        res.data_[s.size()] = '\0';
+        return res;
+    }
+};
+
+} // namespace nix

src/libexpr/include/nix/expr/symbol-table.hh

@@ -3,7 +3,7 @@
 
 #include <memory_resource>
 #include "nix/expr/value.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/util/chunked-vector.hh"
 #include "nix/util/error.hh"
 

src/libexpr/include/nix/expr/value.hh

@@ -3,16 +3,14 @@
 
 #include <bit>
 #include <cassert>
-#include <cstddef>
-#include <cstring>
 #include <memory>
-#include <memory_resource>
 #include <span>
 #include <string_view>
 #include <type_traits>
 #include <concepts>
 
 #include "nix/expr/eval-gc.hh"
+#include "nix/expr/string-data.hh"
 #include "nix/expr/value/context.hh"
 #include "nix/util/source-path.hh"
 #include "nix/expr/print-options.hh"
@@ -193,91 +191,6 @@ public:
     friend struct Value;
 };
 
-class StringData
-{
-public:
-    using size_type = std::size_t;
-
-    size_type size_;
-    char data_[];
-
-    /*
-     * This in particular ensures that we cannot have a `StringData`
-     * that we use by value, which is just what we want!
-     *
-     * Dynamically sized types aren't a thing in C++ and even flexible array
-     * members are a language extension and beyond the realm of standard C++.
-     * Technically, sizeof data_ member is 0 and the intended way to use flexible
-     * array members is to allocate sizeof(StrindData) + count * sizeof(char) bytes
-     * and the compiler will consider alignment restrictions for the FAM.
-     *
-     */
-
-    StringData(StringData &&) = delete;
-    StringData & operator=(StringData &&) = delete;
-    StringData(const StringData &) = delete;
-    StringData & operator=(const StringData &) = delete;
-    ~StringData() = default;
-
-private:
-    StringData() = delete;
-
-    explicit StringData(size_type size)
-        : size_(size)
-    {
-    }
-
-public:
-    /**
-     * Allocate StringData on the (possibly) GC-managed heap and copy
-     * the contents of s to it.
-     */
-    static const StringData & make(EvalMemory & mem, std::string_view s);
-
-    /**
-     * Allocate StringData on the (possibly) GC-managed heap.
-     * @param size Length of the string (without the NUL terminator).
-     */
-    static StringData & alloc(EvalMemory & mem, size_t size);
-
-    size_t size() const
-    {
-        return size_;
-    }
-
-    char * data() noexcept
-    {
-        return data_;
-    }
-
-    const char * data() const noexcept
-    {
-        return data_;
-    }
-
-    const char * c_str() const noexcept
-    {
-        return data_;
-    }
-
-    constexpr std::string_view view() const noexcept
-    {
-        return std::string_view(data_, size_);
-    }
-
-    template<size_t N>
-    struct Static;
-
-    static StringData & make(std::pmr::memory_resource & resource, std::string_view s)
-    {
-        auto & res =
-            *new (resource.allocate(sizeof(StringData) + s.size() + 1, alignof(StringData))) StringData(s.size());
-        std::memcpy(res.data_, s.data(), s.size());
-        res.data_[s.size()] = '\0';
-        return res;
-    }
-};
-
 namespace detail {
 
 /**
@@ -1147,13 +1060,13 @@ public:
         setStorage(StringWithContext{.str = &s, .context = context});
     }
 
-    void mkString(std::string_view s, EvalMemory & mem);
+    void mkString(std::string_view s);
 
     void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem);
 
     void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem);
 
-    void mkPath(const SourcePath & path, EvalMemory & mem);
+    void mkPath(const SourcePath & path);
 
     inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept
     {

src/libexpr/json-to-value.cc

@@ -151,7 +151,7 @@ public:
     bool string(string_t & val) override
     {
         forceNoNullByte(val);
-        rs->value(state).mkString(val, state.mem);
+        rs->value(state).mkString(val);
         rs->add();
         return true;
     }

src/libexpr/nixexpr.cc

@@ -74,9 +74,9 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
 {
-    typedef const AttrDefs::value_type * Attr;
+    typedef const decltype(attrs)::value_type * Attr;
     std::vector<Attr> sorted;
-    for (auto & i : *attrs)
+    for (auto & i : attrs)
         sorted.push_back(&i);
     std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) {
         std::string_view sa = symbols[a->first], sb = symbols[b->first];
@@ -122,7 +122,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
             str << "; ";
         }
     }
-    for (auto & i : *dynamicAttrs) {
+    for (auto & i : dynamicAttrs) {
         str << "\"${";
         i.nameExpr->show(symbols, str);
         str << "}\" = ";
@@ -191,7 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << '(';
     fun->show(symbols, str);
-    for (auto e : *args) {
+    for (auto e : args) {
         str << ' ';
         e->show(symbols, str);
     }
@@ -401,26 +401,15 @@ ExprAttrs::bindInheritSources(EvalState & es, const std::shared_ptr<const Static
 
 void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    // Move storage into the Exprs arena
-    {
-        auto arena = es.mem.exprs.alloc;
-        AttrDefs newAttrs{std::move(*attrs), arena};
-        attrs.emplace(std::move(newAttrs), arena);
-        DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), arena};
-        dynamicAttrs.emplace(std::move(newDynamicAttrs), arena);
-        if (inheritFromExprs)
-            inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), arena);
-    }
-
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
         auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
-            auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->size());
+            auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs.size());
 
             Displacement displ = 0;
-            for (auto & i : *attrs)
+            for (auto & i : attrs)
                 newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
             return newEnv;
         }();
@@ -428,20 +417,20 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         // No need to sort newEnv since attrs is in sorted order.
 
         auto inheritFromEnv = bindInheritSources(es, newEnv);
-        for (auto & i : *attrs)
+        for (auto & i : attrs)
             i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
-        for (auto & i : *dynamicAttrs) {
+        for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, newEnv);
             i.valueExpr->bindVars(es, newEnv);
         }
     } else {
         auto inheritFromEnv = bindInheritSources(es, env);
 
-        for (auto & i : *attrs)
+        for (auto & i : attrs)
             i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
 
-        for (auto & i : *dynamicAttrs) {
+        for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, env);
             i.valueExpr->bindVars(es, env);
         }
@@ -486,27 +475,21 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
 
 void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    // Move storage into the Exprs arena
-    {
-        auto arena = es.mem.exprs.alloc;
-        std::pmr::vector<Expr *> newArgs{std::move(*args), arena};
-        args.emplace(std::move(newArgs), arena);
-    }
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     fun->bindVars(es, env);
-    for (auto e : *args)
+    for (auto e : args)
         e->bindVars(es, env);
 }
 
 void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
-        auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs->size());
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs.size());
 
         Displacement displ = 0;
-        for (auto & i : *attrs->attrs)
+        for (auto & i : attrs->attrs)
             newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
         return newEnv;
     }();
@@ -514,7 +497,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     // No need to sort newEnv since attrs->attrs is in sorted order.
 
     auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
-    for (auto & i : *attrs->attrs)
+    for (auto & i : attrs->attrs)
         i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
     if (es.debugRepl)

src/libexpr/parser.y

@@ -115,7 +115,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P
 
 static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
     if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
-        e2->args->push_back(arg);
+        e2->args.push_back(arg);
         return fn;
     }
     return exprs.add<ExprCall>(pos, fn, {arg});
@@ -129,7 +129,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
 %type <Expr *> start expr expr_function expr_if expr_op
 %type <Expr *> expr_select expr_simple expr_app
 %type <Expr *> expr_pipe_from expr_pipe_into
-%type <std::pmr::vector<Expr *>> list
+%type <std::vector<Expr *>> list
 %type <ExprAttrs *> binds binds1
 %type <FormalsBuilder> formals formal_set
 %type <Formal> formal
@@ -211,7 +211,7 @@ expr_function
   | WITH expr ';' expr_function
     { $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); }
   | LET binds IN_KW expr_function
-    { if (!$2->dynamicAttrs->empty())
+    { if (!$2->dynamicAttrs.empty())
         throw ParseError({
             .msg = HintFmt("dynamic attributes not allowed in let"),
             .pos = state->positions[CUR_POS]
@@ -413,9 +413,9 @@ binds1
   | binds[accum] INHERIT attrs ';'
     { $$ = $accum;
       for (auto & [i, iPos] : $attrs) {
-          if ($accum->attrs->find(i.symbol) != $accum->attrs->end())
-              state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos);
-          $accum->attrs->emplace(
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
       }
@@ -423,13 +423,13 @@ binds1
   | binds[accum] INHERIT '(' expr ')' attrs ';'
     { $$ = $accum;
       if (!$accum->inheritFromExprs)
-          $accum->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>();
+          $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
       $accum->inheritFromExprs->push_back($expr);
       auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1);
       for (auto & [i, iPos] : $attrs) {
-          if ($accum->attrs->find(i.symbol) != $accum->attrs->end())
-              state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos);
-          $accum->attrs->emplace(
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(
                   state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol),

src/libexpr/primops/context.cc

@@ -1,5 +1,6 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/store/derivations.hh"
 #include "nix/store/store-api.hh"
 #include "nix/store/globals.hh"
@@ -11,15 +12,15 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
     NixStringContext context;
     auto s = state.coerceToString(
         pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
-    v.mkString(*s, state.mem);
+    v.mkString(*s);
 }
 
 static RegisterPrimOp primop_unsafeDiscardStringContext({
     .name = "__unsafeDiscardStringContext",
     .args = {"s"},
-    .doc = R"(
+    .doc = &R"(
         Discard the [string context](@docroot@/language/string-context.md) from a value that can be coerced to a string.
-    )",
+    )"_sds,
     .fun = prim_unsafeDiscardStringContext,
 });
 
@@ -33,7 +34,7 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value ** args,
 static RegisterPrimOp primop_hasContext(
     {.name = "__hasContext",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Return `true` if string *s* has a non-empty context.
       The context can be obtained with
       [`getContext`](#builtins-getContext).
@@ -50,7 +51,7 @@ static RegisterPrimOp primop_hasContext(
       > then throw "package name cannot contain string context"
       > else { ${name} = meta; }
       > ```
-    )",
+    )"_sds,
      .fun = prim_hasContext});
 
 static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value ** args, Value & v)
@@ -75,7 +76,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
 static RegisterPrimOp primop_unsafeDiscardOutputDependency(
     {.name = "__unsafeDiscardOutputDependency",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Create a copy of the given string where every
       [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
       string context element is turned into a
@@ -91,7 +92,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency(
       Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything.
 
       [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies
-    )",
+    )"_sds,
      .fun = prim_unsafeDiscardOutputDependency});
 
 static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value ** args, Value & v)
@@ -143,7 +144,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
 static RegisterPrimOp primop_addDrvOutputDependencies(
     {.name = "__addDrvOutputDependencies",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Create a copy of the given string where a single
       [constant](@docroot@/language/string-context.md#string-context-constant)
       string context element is turned into a
@@ -156,7 +157,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies(
       The latter is supported so this function is idempotent.
 
       This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-unsafeDiscardOutputDependency).
-    )",
+    )"_sds,
      .fun = prim_addDrvOutputDependencies});
 
 /* Extract the context of a string as a structured Nix value.
@@ -218,7 +219,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
         if (!info.second.outputs.empty()) {
             auto list = state.buildList(info.second.outputs.size());
             for (const auto & [i, output] : enumerate(info.second.outputs))
-                (list[i] = state.allocValue())->mkString(output, state.mem);
+                (list[i] = state.allocValue())->mkString(output);
             infoAttrs.alloc(state.s.outputs).mkList(list);
         }
         attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
@@ -230,7 +231,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
 static RegisterPrimOp primop_getContext(
     {.name = "__getContext",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Return the string context of *s*.
 
       The string context tracks references to derivations within a string.
@@ -248,7 +249,7 @@ static RegisterPrimOp primop_getContext(
       ```
       { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; }
       ```
-    )",
+    )"_sds,
      .fun = prim_getContext});
 
 /* Append the given context to a given string.

src/libexpr/primops/fetchClosure.cc

@@ -1,4 +1,5 @@
 #include "nix/expr/primops.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/store/store-open.hh"
 #include "nix/store/realisation.hh"
 #include "nix/store/make-content-addressed.hh"
@@ -216,7 +217,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
 static RegisterPrimOp primop_fetchClosure({
     .name = "__fetchClosure",
     .args = {"args"},
-    .doc = R"(
+    .doc = &R"(
       Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context.
 
       This function can be invoked in three ways that we will discuss in order of preference.
@@ -284,7 +285,7 @@ static RegisterPrimOp primop_fetchClosure({
       `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression.
       However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched.
       Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
-    )",
+    )"_sds,
     .fun = prim_fetchClosure,
     .experimentalFeature = Xp::FetchClosure,
 });

src/libexpr/primops/fetchMercurial.cc

@@ -81,17 +81,17 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
         attrs.insert_or_assign("rev", rev->gitRev());
     auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
 
-    auto [storePath, input2] = input.fetchToStore(state.fetchSettings, *state.store);
+    auto [storePath, input2] = input.fetchToStore(state.fetchSettings, state.store);
 
     auto attrs2 = state.buildBindings(8);
     state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));
     if (input2.getRef())
-        attrs2.alloc("branch").mkString(*input2.getRef(), state.mem);
+        attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
     auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
-    attrs2.alloc("rev").mkString(rev2.gitRev(), state.mem);
-    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12), state.mem);
+    attrs2.alloc("rev").mkString(rev2.gitRev());
+    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
     if (auto revCount = input2.getRevCount())
         attrs2.alloc("revCount").mkInt(*revCount);
     v.mkAttrs(attrs2);

src/libexpr/primops/fetchTree.cc

@@ -2,6 +2,7 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
 #include "nix/expr/eval-settings.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/store/store-api.hh"
 #include "nix/fetchers/fetchers.hh"
 #include "nix/store/filetransfer.hh"
@@ -35,7 +36,7 @@ void emitTreeAttrs(
     // FIXME: support arbitrary input attributes.
 
     if (auto narHash = input.getNarHash())
-        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true), state.mem);
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
@@ -43,13 +44,13 @@ void emitTreeAttrs(
     if (!forceDirty) {
 
         if (auto rev = input.getRev()) {
-            attrs.alloc("rev").mkString(rev->gitRev(), state.mem);
-            attrs.alloc("shortRev").mkString(rev->gitShortRev(), state.mem);
+            attrs.alloc("rev").mkString(rev->gitRev());
+            attrs.alloc("shortRev").mkString(rev->gitShortRev());
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
             auto emptyHash = Hash(HashAlgorithm::SHA1);
-            attrs.alloc("rev").mkString(emptyHash.gitRev(), state.mem);
-            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem);
+            attrs.alloc("rev").mkString(emptyHash.gitRev());
+            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
         if (auto revCount = input.getRevCount())
@@ -59,14 +60,13 @@ void emitTreeAttrs(
     }
 
     if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) {
-        attrs.alloc("dirtyRev").mkString(*dirtyRev, state.mem);
-        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"), state.mem);
+        attrs.alloc("dirtyRev").mkString(*dirtyRev);
+        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"));
     }
 
     if (auto lastModified = input.getLastModified()) {
         attrs.alloc("lastModified").mkInt(*lastModified);
-        attrs.alloc("lastModifiedDate")
-            .mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")), state.mem);
+        attrs.alloc("lastModifiedDate").mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
     }
 
     v.mkAttrs(attrs);
@@ -151,6 +151,11 @@ static void fetchTree(
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
+        // fetchTree should fetch git repos with shallow = true by default
+        if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
+            attrs.emplace("shallow", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>("argument 'name' isn’t supported in call to '%s'", fetcher)
@@ -190,7 +195,7 @@ static void fetchTree(
     }
 
     if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
-        input = lookupInRegistries(state.fetchSettings, *state.store, input, fetchers::UseRegistries::Limited).first;
+        input = lookupInRegistries(state.fetchSettings, state.store, input, fetchers::UseRegistries::Limited).first;
 
     if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) {
         if (input.getNarHash())
@@ -216,7 +221,7 @@ static void fetchTree(
     }
 
     auto cachedInput =
-        state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No);
+        state.inputCache->getAccessor(state.fetchSettings, state.store, input, fetchers::UseRegistries::No);
 
     auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
 
@@ -231,127 +236,229 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, V
 static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
-    .doc = []() -> std::string {
-        std::string doc = stripIndentation(R"(
-          Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
+    .doc = &R"(
+      Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-          - the resulting fixed-output [store path](@docroot@/store/store-path.md)
-          - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
-          - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
+      - the resulting fixed-output [store path](@docroot@/store/store-path.md)
+      - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
+      - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
-          *input* must be an attribute set with the following attributes:
+      *input* must be an attribute set with the following attributes:
 
-          - `type` (String, required)
+      - `type` (String, required)
 
-            One of the [supported source types](#source-types).
-            This determines other required and allowed input attributes.
+        One of the [supported source types](#source-types).
+        This determines other required and allowed input attributes.
 
-          - `narHash` (String, optional)
+      - `narHash` (String, optional)
 
-            The `narHash` parameter can be used to substitute the source of the tree.
-            It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
-            If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
+        The `narHash` parameter can be used to substitute the source of the tree.
+        It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
+        If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
 
-          A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
-          That is, `fetchTree` is idempotent.
+      A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+      That is, `fetchTree` is idempotent.
 
-          Downloads are cached in `$XDG_CACHE_HOME/nix`.
-          The remote source is fetched from the network if both are true:
-          - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
+      Downloads are cached in `$XDG_CACHE_HOME/nix`.
+      The remote source is fetched from the network if both are true:
+      - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
 
-            > **Note**
+        > **Note**
+        >
+        > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
+
+      - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
+
+      ## Source types
+
+      The following source types and associated input attributes are supported.
+
+      <!-- TODO: It would be soooo much more predictable to work with (and
+      document) if `fetchTree` was a curried call with the first parameter for
+      `type` or an attribute like `builtins.fetchTree.git`! -->
+
+      - `"file"`
+
+        Place a plain file into the Nix store.
+        This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
+
+        - `url` (String, required)
+
+          Supported protocols:
+
+          - `https`
+
+            > **Example**
             >
-            > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "https://example.com/index.html";
+            > }
+            > ```
 
-          - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
+          - `http`
 
-          ## Source types
+            Insecure HTTP transfer for legacy sources.
 
-          The following source types and associated input attributes are supported.
+            > **Warning**
+            >
+            > HTTP performs no encryption or authentication.
+            > Use a `narHash` known in advance to ensure the output has expected contents.
 
-          <!-- TODO: It would be soooo much more predictable to work with (and
-          document) if `fetchTree` was a curried call with the first parameter for
-          `type` or an attribute like `builtins.fetchTree.git`! -->
-        )");
+          - `file`
 
-        auto indentString = [](std::string const & str, std::string const & indent) {
-            std::string result;
-            std::istringstream stream(str);
-            std::string line;
-            bool first = true;
-            while (std::getline(stream, line)) {
-                if (!first)
-                    result += "\n";
-                result += indent + line;
-                first = false;
-            }
-            return result;
-        };
+            A file on the local file system.
 
-        for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
-            doc += "\n- `" + quoteString(schemeName, '"') + "`\n\n";
-            doc += indentString(scheme->schemeDescription(), "  ");
-            if (!doc.empty() && doc.back() != '\n')
-                doc += "\n";
+            > **Example**
+            >
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "file:///home/eelco/nix/README.md";
+            > }
+            > ```
 
-            for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
-                doc += "\n  - `" + attrName + "` (" + attribute.type + ", "
-                       + (attribute.required ? "required" : "optional") + ")\n\n";
-                doc += indentString(stripIndentation(attribute.doc), "    ");
-                if (!doc.empty() && doc.back() != '\n')
-                    doc += "\n";
-            }
-        }
+      - `"git"`
 
-        doc += "\n" + stripIndentation(R"(
-          The following input types are still subject to change:
+        Fetch a Git tree and copy it to the Nix store.
+        This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
 
-          - `"path"`
-          - `"github"`
-          - `"gitlab"`
-          - `"sourcehut"`
-          - `"mercurial"`
+        - `allRefs` (Bool, optional)
 
-         *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
-         The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          Whether to fetch all references (eg. branches and tags) of the repository.
+          With this argument being true, it's possible to load a `rev` from *any* `ref`.
+          (Without setting this option, only `rev`s from the specified `ref` are supported).
+
+          Default: `false`
+
+        - `lastModified` (Integer, optional)
+
+          Unix timestamp of the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+        - `lfs` (Bool, optional)
+
+          Fetch any [Git LFS](https://git-lfs.com/) files.
+
+          Default: `false`
+
+        - `ref` (String, optional)
+
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
+
+          Default: `"HEAD"`
+
+        - `rev` (String, optional)
+
+          A Git revision; a commit hash.
+
+          Default: the tip of `ref`
+
+        - `revCount` (Integer, optional)
+
+          Number of revisions in the history of the Git repository before the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+        - `shallow` (Bool, optional)
+
+          Make a shallow clone when fetching the Git tree.
+          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
+
+          Default: `true`
+
+        - `submodules` (Bool, optional)
+
+          Also fetch submodules if available.
+
+          Default: `false`
+
+        - `url` (String, required)
+
+          The URL formats supported are the same as for Git itself.
 
           > **Example**
           >
-          > Fetch a GitHub repository using the attribute set representation:
-          >
           > ```nix
-          > builtins.fetchTree {
-          >   type = "github";
-          >   owner = "NixOS";
-          >   repo = "nixpkgs";
-          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-          > }
-          > ```
-          >
-          > This evaluates to the following attribute set:
-          >
-          > ```nix
-          > {
-          >   lastModified = 1686503798;
-          >   lastModifiedDate = "20230611171638";
-          >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
-          >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
-          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-          >   shortRev = "ae2e6b3";
+          > fetchTree {
+          >   type = "git";
+          >   url = "git@github.com:NixOS/nixpkgs.git";
           > }
           > ```
 
-          > **Example**
+          > **Note**
           >
-          > Fetch the same GitHub repository using the URL-like syntax:
-          >
-          >   ```nix
-          >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
-          >   ```
-        )");
+          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
 
-        return doc;
-    }(),
+      - `"tarball"`
+
+        Download a tar archive and extract it into the Nix store.
+        This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
+
+        - `url` (String, required)
+
+           > **Example**
+           >
+           > ```nix
+           > fetchTree {
+           >   type = "tarball";
+           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
+           > }
+           > ```
+
+      The following input types are still subject to change:
+
+      - `"path"`
+      - `"github"`
+      - `"gitlab"`
+      - `"sourcehut"`
+      - `"mercurial"`
+
+     *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
+     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
+
+      > **Example**
+      >
+      > Fetch a GitHub repository using the attribute set representation:
+      >
+      > ```nix
+      > builtins.fetchTree {
+      >   type = "github";
+      >   owner = "NixOS";
+      >   repo = "nixpkgs";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      > }
+      > ```
+      >
+      > This evaluates to the following attribute set:
+      >
+      > ```nix
+      > {
+      >   lastModified = 1686503798;
+      >   lastModifiedDate = "20230611171638";
+      >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
+      >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      >   shortRev = "ae2e6b3";
+      > }
+      > ```
+
+      > **Example**
+      >
+      > Fetch the same GitHub repository using the URL-like syntax:
+      >
+      >   ```nix
+      >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
+      >   ```
+    )"_sds,
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
 });
@@ -476,10 +583,10 @@ static void fetch(
     auto storePath = unpack ? fetchToStore(
                                   state.fetchSettings,
                                   *state.store,
-                                  fetchers::downloadTarball(*state.store, state.fetchSettings, *url),
+                                  fetchers::downloadTarball(state.store, state.fetchSettings, *url),
                                   FetchMode::Copy,
                                   name)
-                            : fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath;
+                            : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
 
     if (expectedHash) {
         auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash
@@ -511,7 +618,7 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value ** args, Va
 static RegisterPrimOp primop_fetchurl({
     .name = "__fetchurl",
     .args = {"arg"},
-    .doc = R"(
+    .doc = &R"(
       Download the specified URL and return the path of the downloaded file.
       `arg` can be either a string denoting the URL, or an attribute set with the following attributes:
 
@@ -525,7 +632,7 @@ static RegisterPrimOp primop_fetchurl({
         characters that are invalid for the store.
 
       Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
-    )",
+    )"_sds,
     .fun = prim_fetchurl,
 });
 
@@ -537,7 +644,7 @@ static void prim_fetchTarball(EvalState & state, const PosIdx pos, Value ** args
 static RegisterPrimOp primop_fetchTarball({
     .name = "fetchTarball",
     .args = {"args"},
-    .doc = R"(
+    .doc = &R"(
       Download the specified URL, unpack it and return the path of the
       unpacked tree. The file must be a tape archive (`.tar`) compressed
       with `gzip`, `bzip2` or `xz`. If the tarball consists of a
@@ -575,7 +682,7 @@ static RegisterPrimOp primop_fetchTarball({
       ```
 
       Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
-    )",
+    )"_sds,
     .fun = prim_fetchTarball,
 });
 
@@ -588,7 +695,7 @@ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value ** args, Va
 static RegisterPrimOp primop_fetchGit({
     .name = "fetchGit",
     .args = {"args"},
-    .doc = R"(
+    .doc = &R"(
       Fetch a path from git. *args* can be a URL, in which case the HEAD
       of the repo at that URL is fetched. Otherwise, it can be an
       attribute with the following attributes (all except `url` optional):
@@ -791,7 +898,7 @@ static RegisterPrimOp primop_fetchGit({
       given, `fetchGit` uses the current content of the checked-out
       files, even if they are not committed or added to Git's index. It
       only considers files added to the Git repository, as listed by `git ls-files`.
-    )",
+    )"_sds,
     .fun = prim_fetchGit,
 });
 

src/libexpr/primops/fromTOML.cc

@@ -1,6 +1,6 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 #include "expr-config-private.hh"
 
@@ -126,7 +126,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
         case toml::value_t::string: {
             auto s = toml::get<std::string_view>(t);
             forceNoNullByte(s);
-            v.mkString(s, state.mem);
+            v.mkString(s);
         } break;
         case toml::value_t::local_datetime:
         case toml::value_t::offset_datetime:
@@ -142,7 +142,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 s << t;
                 auto str = s.view();
                 forceNoNullByte(str);
-                attrs.alloc("value").mkString(str, state.mem);
+                attrs.alloc("value").mkString(str);
                 v.mkAttrs(attrs);
             } else {
                 throw std::runtime_error("Dates and times are not supported");
@@ -170,10 +170,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
     }
 }
 
-static RegisterPrimOp primop_fromTOML(
-    {.name = "fromTOML",
-     .args = {"e"},
-     .doc = R"(
+static RegisterPrimOp primop_fromTOML({
+    .name = "fromTOML",
+    .args = {"e"},
+    .doc = &R"(
       Convert a TOML string to a Nix value. For example,
 
       ```nix
@@ -186,7 +186,8 @@ static RegisterPrimOp primop_fromTOML(
       ```
 
       returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`.
-    )",
-     .fun = prim_fromTOML});
+    )"_sds,
+    .fun = prim_fromTOML,
+});
 
 } // namespace nix

src/libexpr/value-to-json.cc

@@ -16,8 +16,6 @@ json printValueAsJSON(
 {
     checkInterrupt();
 
-    auto _level = state.addCallDepth(pos);
-
     if (strict)
         state.forceValue(v, pos);
 

src/libfetchers-tests/git-utils.cc

@@ -92,7 +92,7 @@ TEST_F(GitUtilsTest, sink_basic)
     // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
 
     auto result = repo->dereferenceSingletonDirectory(sink->flush());
-    auto accessor = repo->getAccessor(result, {}, getRepoName());
+    auto accessor = repo->getAccessor(result, false, getRepoName());
     auto entries = accessor->readDirectory(CanonPath::root);
     ASSERT_EQ(entries.size(), 5u);
     ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");

src/libfetchers-tests/git.cc

@@ -196,7 +196,7 @@ TEST_F(GitTest, submodulePeriodSupport)
             {"ref", "main"},
         });
 
-    auto [accessor, i] = input.getAccessor(settings, *store);
+    auto [accessor, i] = input.getAccessor(settings, store);
 
     ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n");
 }

src/libfetchers/fetchers.cc

@@ -6,7 +6,6 @@
 #include "nix/fetchers/fetch-settings.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/util/url.hh"
-#include "nix/util/archive.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -27,9 +26,18 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
         throw Error("Input scheme with name %s already registered", schemeName);
 }
 
-const InputSchemeMap & getAllInputSchemes()
+nlohmann::json dumpRegisterInputSchemeInfo()
 {
-    return inputSchemes();
+    using nlohmann::json;
+
+    auto res = json::object();
+
+    for (auto & [name, scheme] : inputSchemes()) {
+        auto & r = res[name] = json::object();
+        r["allowedAttrs"] = scheme->allowedAttrs();
+    }
+
+    return res;
 }
 
 Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree)
@@ -111,7 +119,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
     return std::move(*res);
 }
 
-std::optional<std::string> Input::getFingerprint(Store & store) const
+std::optional<std::string> Input::getFingerprint(ref<Store> store) const
 {
     if (!scheme)
         return std::nullopt;
@@ -190,7 +198,7 @@ bool Input::contains(const Input & other) const
 }
 
 // FIXME: remove
-std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store & store) const
+std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, ref<Store> store) const
 {
     if (!scheme)
         throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
@@ -200,9 +208,9 @@ std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store
             auto [accessor, result] = getAccessorUnchecked(settings, store);
 
             auto storePath =
-                nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName());
+                nix::fetchToStore(settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
 
-            auto narHash = store.queryPathInfo(storePath)->narHash;
+            auto narHash = store->queryPathInfo(storePath)->narHash;
             result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
             result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@@ -269,13 +277,27 @@ void Input::checkLocks(Input specified, Input & result)
         }
     }
 
+    if (auto prevLastModified = specified.getLastModified()) {
+        if (result.getLastModified() != prevLastModified)
+            throw Error(
+                "'lastModified' attribute mismatch in input '%s', expected %d, got %d",
+                result.to_string(),
+                *prevLastModified,
+                result.getLastModified().value_or(-1));
+    }
+
     if (auto prevRev = specified.getRev()) {
         if (result.getRev() != prevRev)
             throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev());
     }
+
+    if (auto prevRevCount = specified.getRevCount()) {
+        if (result.getRevCount() != prevRevCount)
+            throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount);
+    }
 }
 
-std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, Store & store) const
+std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, ref<Store> store) const
 {
     try {
         auto [accessor, result] = getAccessorUnchecked(settings, store);
@@ -291,7 +313,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settin
     }
 }
 
-std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, Store & store) const
+std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, ref<Store> store) const
 {
     // FIXME: cache the accessor
 
@@ -311,13 +333,13 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
     */
     if (isFinal() && getNarHash()) {
         try {
-            auto storePath = computeStorePath(store);
+            auto storePath = computeStorePath(*store);
 
-            store.ensurePath(storePath);
+            store->ensurePath(storePath);
 
-            debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath));
+            debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
 
-            auto accessor = store.requireStoreObjectAccessor(storePath);
+            auto accessor = store->requireStoreObjectAccessor(storePath);
 
             accessor->fingerprint = getFingerprint(store);
 
@@ -327,7 +349,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
             if (accessor->fingerprint) {
                 ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
                 auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
-                settings.getCache()->upsert(cacheKey, store, {}, storePath);
+                settings.getCache()->upsert(cacheKey, *store, {}, storePath);
             }
 
             accessor->setPathDisplay("«" + to_string() + "»");
@@ -355,10 +377,10 @@ Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash>
     return scheme->applyOverrides(*this, ref, rev);
 }
 
-void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const
+void Input::clone(const Settings & settings, const Path & destDir) const
 {
     assert(scheme);
-    scheme->clone(settings, store, *this, destDir);
+    scheme->clone(settings, *this, destDir);
 }
 
 std::optional<std::filesystem::path> Input::getSourcePath() const
@@ -471,19 +493,9 @@ void InputScheme::putFile(
     throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
 }
 
-void InputScheme::clone(
-    const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
+void InputScheme::clone(const Settings & settings, const Input & input, const Path & destDir) const
 {
-    if (std::filesystem::exists(destDir))
-        throw Error("cannot clone into existing path %s", destDir);
-
-    auto [accessor, input2] = getAccessor(settings, store, input);
-
-    Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
-
-    RestoreSink sink(/*startFsync=*/false);
-    sink.dstPath = destDir;
-    copyRecursive(*accessor, CanonPath::root, sink, CanonPath::root);
+    throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const

src/libfetchers/git-lfs-fetch.cc

@@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
     auto url = api.endpoint + "/objects/batch";
     const auto & authHeader = api.authHeader;
     FileTransferRequest request(parseURL(url));
-    request.method = HttpMethod::Post;
+    request.method = HttpMethod::POST;
     Headers headers;
     if (authHeader.has_value())
         headers.push_back({"Authorization", *authHeader});

src/libfetchers/git-utils.cc

@@ -541,15 +541,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 
     /**
-     * A 'GitSourceAccessor' with no regard for export-ignore.
+     * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
      */
-    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, const GitAccessorOptions & options);
+    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, bool smudgeLfs = false);
 
     ref<SourceAccessor>
-    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) override;
+    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) override;
 
-    ref<SourceAccessor>
-    getAccessor(const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError e) override;
+    ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
 
     ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
 
@@ -669,7 +668,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
     {
-        auto accessor = getAccessor(treeHash, {}, "");
+        auto accessor = getAccessor(treeHash, false, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
@@ -717,17 +716,15 @@ struct GitSourceAccessor : SourceAccessor
         ref<GitRepoImpl> repo;
         Object root;
         std::optional<lfs::Fetch> lfsFetch = std::nullopt;
-        GitAccessorOptions options;
     };
 
     Sync<State> state_;
 
-    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, const GitAccessorOptions & options)
+    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
         : state_{State{
               .repo = repo_,
               .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
-              .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
-              .options = options,
+              .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
           }}
     {
     }
@@ -1257,26 +1254,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
+ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitSourceAccessor>(self, rev, options);
+    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
 }
 
 ref<SourceAccessor>
-GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix)
+GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options);
+    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
-    if (options.exportIgnore)
+    if (exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
     else
         return rawGitAccessor;
 }
 
-ref<SourceAccessor> GitRepoImpl::getAccessor(
-    const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError)
+ref<SourceAccessor>
+GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create(
@@ -1286,9 +1283,10 @@ ref<SourceAccessor> GitRepoImpl::getAccessor(
                                            boost::unordered_flat_set<CanonPath>{CanonPath::root},
                                            std::move(makeNotAllowedError))
                                            .cast<SourceAccessor>();
-    if (options.exportIgnore)
-        fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
-    return fileAccessor;
+    if (exportIgnore)
+        return make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
+    else
+        return fileAccessor;
 }
 
 ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()
@@ -1301,7 +1299,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
 
-    auto accessor = getAccessor(rev, {.exportIgnore = exportIgnore}, "");
+    auto accessor = getAccessor(rev, exportIgnore, "");
     if (!accessor->pathExists(modulesFile))
         return {};
 
@@ -1318,7 +1316,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
 
     std::vector<std::tuple<Submodule, Hash>> result;
 
-    auto rawAccessor = getRawAccessor(rev, {});
+    auto rawAccessor = getRawAccessor(rev);
 
     for (auto & submodule : parseSubmodules(pathTemp)) {
         /* Filter out .gitmodules entries that don't exist or are not
@@ -1334,8 +1332,10 @@ namespace fetchers {
 
 ref<GitRepo> Settings::getTarballCache() const
 {
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
-    return GitRepo::openRepo(repoDir, true, true);
+    auto tarballCache(_tarballCache.lock());
+    if (!*tarballCache)
+        *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
+    return ref<GitRepo>(*tarballCache);
 }
 
 } // namespace fetchers

src/libfetchers/git.cc

@@ -194,183 +194,28 @@ struct GitInputScheme : InputScheme
         return "git";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        return stripIndentation(R"(
-          Fetch a Git tree and copy it to the Nix store.
-          This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
-        )");
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "url",
-                {
-                    .type = "String",
-                    .required = true,
-                    .doc = R"(
-                      The URL formats supported are the same as for Git itself.
-
-                      > **Example**
-                      >
-                      > ```nix
-                      > fetchTree {
-                      >   type = "git";
-                      >   url = "git@github.com:NixOS/nixpkgs.git";
-                      > }
-                      > ```
-
-                      > **Note**
-                      >
-                      > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but uses the *current file contents* of the Git working directory.
-                    )",
-                },
-            },
-            {
-                "ref",
-                {
-                    .type = "String",
-                    .required = false,
-                    .doc = R"(
-                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-                      A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
-
-                      Default: `"HEAD"`
-                    )",
-                },
-            },
-            {
-                "rev",
-                {
-                    .type = "String",
-                    .required = false,
-                    .doc = R"(
-                      A Git revision; a commit hash.
-
-                      Default: the tip of `ref`
-                    )",
-                },
-            },
-            {
-                "shallow",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      Make a shallow clone when fetching the Git tree.
-                      When this is enabled, the options `ref` and `allRefs` have no effect anymore.
-
-                      Default: `true`
-                    )",
-                },
-            },
-            {
-                "submodules",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      Also fetch submodules if available.
-
-                      Default: `false`
-                    )",
-                },
-            },
-            {
-                "lfs",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      Fetch any [Git LFS](https://git-lfs.com/) files.
-
-                      Default: `false`
-                    )",
-                },
-            },
-            {
-                "exportIgnore",
-                {},
-            },
-            {
-                "lastModified",
-                {
-                    .type = "Integer",
-                    .required = false,
-                    .doc = R"(
-                      Unix timestamp of the fetched commit.
-
-                      If set, pass through the value to the output attribute set.
-                      Otherwise, generated from the fetched Git tree.
-                    )",
-                },
-            },
-            {
-                "revCount",
-                {
-                    .type = "Integer",
-                    .required = false,
-                    .doc = R"(
-                      Number of revisions in the history of the Git repository before the fetched commit.
-
-                      If set, pass through the value to the output attribute set.
-                      Otherwise, generated from the fetched Git tree.
-                    )",
-                },
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "allRefs",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-                      Whether to fetch all references (eg. branches and tags) of the repository.
-                      With this argument being true, it's possible to load a `rev` from *any* `ref`.
-                      (Without setting this option, only `rev`s from the specified `ref` are supported).
-
-                      Default: `false`
-                    )",
-                },
-            },
-            {
-                "name",
-                {},
-            },
-            {
-                "dirtyRev",
-                {},
-            },
-            {
-                "dirtyShortRev",
-                {},
-            },
-            {
-                "verifyCommit",
-                {},
-            },
-            {
-                "keytype",
-                {},
-            },
-            {
-                "publicKey",
-                {},
-            },
-            {
-                "publicKeys",
-                {},
-            },
+        return {
+            "url",
+            "ref",
+            "rev",
+            "shallow",
+            "submodules",
+            "lfs",
+            "exportIgnore",
+            "lastModified",
+            "revCount",
+            "narHash",
+            "allRefs",
+            "name",
+            "dirtyRev",
+            "dirtyShortRev",
+            "verifyCommit",
+            "keytype",
+            "publicKey",
+            "publicKeys",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -433,8 +278,7 @@ struct GitInputScheme : InputScheme
         return res;
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto repoInfo = getRepoInfo(input);
 
@@ -778,25 +622,15 @@ struct GitInputScheme : InputScheme
         }
     }
 
-    /**
-     * Decide whether we can do a shallow clone, which is faster. This is possible if the user explicitly specified
-     * `shallow = true`, or if we already have a `revCount`.
-     */
-    bool canDoShallow(const Input & input) const
-    {
-        bool shallow = getShallowAttr(input);
-        return shallow || input.getRevCount().has_value();
-    }
-
     std::pair<ref<SourceAccessor>, Input>
-    getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const
+    getAccessorFromCommit(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
     {
         assert(!repoInfo.workdirInfo.isDirty);
 
         auto origRev = input.getRev();
 
         auto originalRef = input.getRef();
-        bool shallow = canDoShallow(input);
+        bool shallow = getShallowAttr(input);
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
         input.attrs.insert_or_assign("ref", ref);
 
@@ -807,27 +641,11 @@ struct GitInputScheme : InputScheme
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
         } else {
-            auto rev = input.getRev();
             auto repoUrl = std::get<ParsedURL>(repoInfo.location);
             std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
             repoDir = cacheDir;
             repoInfo.gitDir = ".";
 
-            /* If shallow = false, but we have a non-shallow repo that already contains the desired rev, then use that
-             * repo instead. */
-            std::filesystem::path cacheDirNonShallow = getCachePath(repoUrl.to_string(), false);
-            if (rev && shallow && pathExists(cacheDirNonShallow)) {
-                auto nonShallowRepo = GitRepo::openRepo(cacheDirNonShallow, true, true);
-                if (nonShallowRepo->hasObject(*rev)) {
-                    debug(
-                        "using non-shallow cached repo for '%s' since it contains rev '%s'",
-                        repoUrl.to_string(),
-                        rev->gitRev());
-                    repoDir = cacheDirNonShallow;
-                    goto have_rev;
-                }
-            }
-
             std::filesystem::create_directories(cacheDir.parent_path());
             PathLocks cacheDirLock({cacheDir.string()});
 
@@ -843,7 +661,7 @@ struct GitInputScheme : InputScheme
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (rev) {
+            if (auto rev = input.getRev()) {
                 doFetch = !repo->hasObject(*rev);
             } else {
                 if (getAllRefsAttr(input)) {
@@ -857,6 +675,7 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
+                bool shallow = getShallowAttr(input);
                 try {
                     auto fetchRef = getAllRefsAttr(input)             ? "refs/*:refs/*"
                                     : input.getRev()                  ? input.getRev()->gitRev()
@@ -884,7 +703,7 @@ struct GitInputScheme : InputScheme
                     warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
             }
 
-            if (rev) {
+            if (auto rev = input.getRev()) {
                 if (!repo->hasObject(*rev))
                     throw Error(
                         "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
@@ -901,30 +720,23 @@ struct GitInputScheme : InputScheme
             // the remainder
         }
 
-    have_rev:
         auto repo = GitRepo::openRepo(repoDir);
 
+        auto isShallow = repo->isShallow();
+
+        if (isShallow && !getShallowAttr(input))
+            throw Error(
+                "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
+                repoInfo.locationToArg());
+
         // FIXME: check whether rev is an ancestor of ref?
 
         auto rev = *input.getRev();
 
-        /* Skip lastModified computation if it's already supplied by the caller.
-           We don't care if they specify an incorrect value; it doesn't
-           matter for security, unlike narHash. */
-        if (!input.attrs.contains("lastModified"))
-            input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
-
-        /* Like lastModified, skip revCount if supplied by the caller. */
-        if (!shallow && !input.attrs.contains("revCount")) {
-            auto isShallow = repo->isShallow();
-
-            if (isShallow && !shallow)
-                throw Error(
-                    "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
-                    repoInfo.locationToArg());
+        input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
 
+        if (!getShallowAttr(input))
             input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev));
-        }
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
 
@@ -932,8 +744,7 @@ struct GitInputScheme : InputScheme
 
         bool exportIgnore = getExportIgnoreAttr(input);
         bool smudgeLfs = getLfsAttr(input);
-        auto accessor = repo->getAccessor(
-            rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»");
+        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -986,7 +797,7 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessorFromWorkdir(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const
+    getAccessorFromWorkdir(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
     {
         auto repoPath = repoInfo.getPath().value();
 
@@ -1000,7 +811,7 @@ struct GitInputScheme : InputScheme
         auto exportIgnore = getExportIgnoreAttr(input);
 
         ref<SourceAccessor> accessor =
-            repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath));
+            repo->getAccessor(repoInfo.workdirInfo, exportIgnore, makeNotAllowedError(repoPath));
 
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
@@ -1070,7 +881,7 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         Input input(_input);
 
@@ -1092,7 +903,7 @@ struct GitInputScheme : InputScheme
         return {accessor, std::move(final)};
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         auto makeFingerprint = [&](const Hash & rev) {
             return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")

src/libfetchers/github.cc

@@ -110,43 +110,18 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
+    StringSet allowedAttrs() const override
     {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "owner",
-                {},
-            },
-            {
-                "repo",
-                {},
-            },
-            {
-                "ref",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "lastModified",
-                {},
-            },
-            {
-                "host",
-                {},
-            },
-            {
-                "treeHash",
-                {},
-            },
+        return {
+            "owner",
+            "repo",
+            "ref",
+            "rev",
+            "narHash",
+            "lastModified",
+            "host",
+            "treeHash",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const fetchers::Settings & settings, const Attrs & attrs) const override
@@ -258,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme
         std::optional<Hash> treeHash;
     };
 
-    virtual RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const = 0;
+    virtual RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const = 0;
 
     virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0;
 
@@ -268,7 +243,7 @@ struct GitArchiveInputScheme : InputScheme
         time_t lastModified;
     };
 
-    std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, Store & store, Input input) const
+    std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, ref<Store> store, Input input) const
     {
         if (!maybeGetStrAttr(input.attrs, "ref"))
             input.attrs.insert_or_assign("ref", "HEAD");
@@ -341,7 +316,7 @@ struct GitArchiveInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         auto [input, tarballInfo] = downloadArchive(settings, store, _input);
 
@@ -351,7 +326,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
         auto accessor =
-            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string() + "»");
+            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
 
         return {accessor, input};
     }
@@ -370,7 +345,7 @@ struct GitArchiveInputScheme : InputScheme
         return Xp::Flakes;
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto rev = input.getRev())
             return rev->gitRev();
@@ -386,12 +361,6 @@ struct GitHubInputScheme : GitArchiveInputScheme
         return "github";
     }
 
-    std::string schemeDescription() const override
-    {
-        // TODO
-        return "";
-    }
-
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Github supports PAT/OAuth2 tokens and HTTP Basic
@@ -418,7 +387,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         return getStrAttr(input.attrs, "repo");
     }
 
-    RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override
+    RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
     {
         auto host = getHost(input);
         auto url = fmt(
@@ -432,7 +401,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
         auto downloadResult = downloadFile(store, settings, url, "source", headers);
         auto json = nlohmann::json::parse(
-            store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
+            store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         return RefInfo{
             .rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1),
@@ -457,13 +426,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
         return DownloadUrl{parseURL(url), headers};
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto host = getHost(input);
         Input::fromURL(settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input)))
             .applyOverrides(input.getRef(), input.getRev())
-            .clone(settings, store, destDir);
+            .clone(settings, destDir);
     }
 };
 
@@ -474,12 +442,6 @@ struct GitLabInputScheme : GitArchiveInputScheme
         return "gitlab";
     }
 
-    std::string schemeDescription() const override
-    {
-        // TODO
-        return "";
-    }
-
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Gitlab supports 4 kinds of authorization, two of which are
@@ -499,7 +461,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1));
     }
 
-    RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override
+    RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // See rate limiting note below
@@ -514,7 +476,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
         auto downloadResult = downloadFile(store, settings, url, "source", headers);
         auto json = nlohmann::json::parse(
-            store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
+            store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
             return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)};
@@ -545,8 +507,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         return DownloadUrl{parseURL(url), headers};
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
@@ -554,7 +515,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             settings,
             fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef(), input.getRev())
-            .clone(settings, store, destDir);
+            .clone(settings, destDir);
     }
 };
 
@@ -565,12 +526,6 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         return "sourcehut";
     }
 
-    std::string schemeDescription() const override
-    {
-        // TODO
-        return "";
-    }
-
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // SourceHut supports both PAT and OAuth2. See
@@ -581,7 +536,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         // Once it is implemented, however, should work as expected.
     }
 
-    RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override
+    RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
     {
         // TODO: In the future, when the sourcehut graphql API is implemented for mercurial
         // and with anonymous access, this method should use it instead.
@@ -597,7 +552,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::string refUri;
         if (ref == "HEAD") {
             auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers);
-            auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
+            auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
 
             auto remoteLine = git::parseLsRemoteLine(getLine(contents).first);
             if (!remoteLine) {
@@ -610,7 +565,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::regex refRegex(refUri);
 
         auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers);
-        auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
+        auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
         std::istringstream is(contents);
 
         std::string line;
@@ -641,15 +596,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         return DownloadUrl{parseURL(url), headers};
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
         Input::fromURL(
             settings,
             fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef(), input.getRev())
-            .clone(settings, store, destDir);
+            .clone(settings, destDir);
     }
 };
 

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -135,6 +135,8 @@ struct Settings : public Config
 
 private:
     mutable Sync<std::shared_ptr<Cache>> _cache;
+
+    mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
 };
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/fetchers.hh

@@ -113,7 +113,7 @@ public:
      * Fetch the entire input into the Nix store, returning the
      * location in the Nix store and the locked input.
      */
-    std::pair<StorePath, Input> fetchToStore(const Settings & settings, Store & store) const;
+    std::pair<StorePath, Input> fetchToStore(const Settings & settings, ref<Store> store) const;
 
     /**
      * Check the locking attributes in `result` against
@@ -133,17 +133,17 @@ public:
      * input without copying it to the store. Also return a possibly
      * unlocked input.
      */
-    std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, Store & store) const;
+    std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, ref<Store> store) const;
 
 private:
 
-    std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, Store & store) const;
+    std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, ref<Store> store) const;
 
 public:
 
     Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const;
 
-    void clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const;
+    void clone(const Settings & settings, const Path & destDir) const;
 
     std::optional<std::filesystem::path> getSourcePath() const;
 
@@ -173,7 +173,7 @@ public:
      *
      * This is not a stable identifier between Nix versions, but not guaranteed to change either.
      */
-    std::optional<std::string> getFingerprint(Store & store) const;
+    std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
 
 /**
@@ -203,34 +203,20 @@ struct InputScheme
      */
     virtual std::string_view schemeName() const = 0;
 
-    /**
-     * Longform description of this scheme, for documentation purposes.
-     */
-    virtual std::string schemeDescription() const = 0;
-
-    // TODO remove these defaults
-    struct AttributeInfo
-    {
-        const char * type = "String";
-        bool required = true;
-        const char * doc = "";
-    };
-
     /**
      * Allowed attributes in an attribute set that is converted to an
-     * input, and documentation for each attribute.
+     * input.
      *
-     * `type` is not included from this map, because the `type` field is
+     * `type` is not included from this set, because the `type` field is
       parsed first to choose which scheme; `type` is always required.
      */
-    virtual const std::map<std::string, AttributeInfo> & allowedAttrs() const = 0;
+    virtual StringSet allowedAttrs() const = 0;
 
     virtual ParsedURL toURL(const Input & input) const;
 
     virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const;
 
-    virtual void
-    clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const;
+    virtual void clone(const Settings & settings, const Input & input, const Path & destDir) const;
 
     virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const;
 
@@ -241,7 +227,7 @@ struct InputScheme
         std::optional<std::string> commitMsg) const;
 
     virtual std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & input) const = 0;
+    getAccessor(const Settings & settings, ref<Store> store, const Input & input) const = 0;
 
     /**
      * Is this `InputScheme` part of an experimental feature?
@@ -253,7 +239,7 @@ struct InputScheme
         return true;
     }
 
-    virtual std::optional<std::string> getFingerprint(Store & store, const Input & input) const
+    virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     {
         return std::nullopt;
     }
@@ -277,12 +263,7 @@ struct InputScheme
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
-using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
-
-/**
- * Use this for docs, not for finding a specific scheme
- */
-const InputSchemeMap & getAllInputSchemes();
+nlohmann::json dumpRegisterInputSchemeInfo();
 
 struct PublicKey
 {

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -22,12 +22,6 @@ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
     virtual Hash flush() = 0;
 };
 
-struct GitAccessorOptions
-{
-    bool exportIgnore = false;
-    bool smudgeLfs = false;
-};
-
 struct GitRepo
 {
     virtual ~GitRepo() {}
@@ -95,10 +89,10 @@ struct GitRepo
     virtual bool hasObject(const Hash & oid) = 0;
 
     virtual ref<SourceAccessor>
-    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0;
+    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) = 0;
 
-    virtual ref<SourceAccessor> getAccessor(
-        const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) = 0;
+    virtual ref<SourceAccessor>
+    getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
 
     virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
 

src/libfetchers/include/nix/fetchers/input-cache.hh

@@ -16,7 +16,7 @@ struct InputCache
     };
 
     CachedResult
-    getAccessor(const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries);
+    getAccessor(const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
 
     struct CachedInput
     {

src/libfetchers/include/nix/fetchers/registry.hh

@@ -13,6 +13,8 @@ namespace nix::fetchers {
 
 struct Registry
 {
+    const Settings & settings;
+
     enum RegistryType {
         Flag = 0,
         User = 1,
@@ -32,8 +34,9 @@ struct Registry
 
     std::vector<Entry> entries;
 
-    Registry(RegistryType type)
-        : type{type}
+    Registry(const Settings & settings, RegistryType type)
+        : settings{settings}
+        , type{type}
     {
     }
 
@@ -54,9 +57,9 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
 
 Path getUserRegistryPath();
 
-Registries getRegistries(const Settings & settings, Store & store);
+Registries getRegistries(const Settings & settings, ref<Store> store);
 
-void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs);
+void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs);
 
 enum class UseRegistries : int {
     No,
@@ -69,6 +72,6 @@ enum class UseRegistries : int {
  * use the registries for which the filter function returns true.
  */
 std::pair<Input, Attrs>
-lookupInRegistries(const Settings & settings, Store & store, const Input & input, UseRegistries useRegistries);
+lookupInRegistries(const Settings & settings, ref<Store> store, const Input & input, UseRegistries useRegistries);
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/tarball.hh

@@ -25,7 +25,7 @@ struct DownloadFileResult
 };
 
 DownloadFileResult downloadFile(
-    Store & store,
+    ref<Store> store,
     const Settings & settings,
     const std::string & url,
     const std::string & name,
@@ -43,6 +43,6 @@ struct DownloadTarballResult
  * Download and import a tarball into the Git cache. The result is the
  * Git tree hash of the root directory.
  */
-ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url);
+ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url);
 
 } // namespace nix::fetchers

src/libfetchers/indirect.cc

@@ -60,33 +60,14 @@ struct IndirectInputScheme : InputScheme
         return "indirect";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        // TODO
-        return "";
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "id",
-                {},
-            },
-            {
-                "ref",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
+        return {
+            "id",
+            "ref",
+            "rev",
+            "narHash",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -126,7 +107,7 @@ struct IndirectInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & input) const override
     {
         throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
     }

src/libfetchers/input-cache.cc

@@ -6,7 +6,7 @@
 namespace nix::fetchers {
 
 InputCache::CachedResult InputCache::getAccessor(
-    const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries)
+    const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
 {
     auto fetched = lookup(originalInput);
     Input resolvedInput = originalInput;

src/libfetchers/mercurial.cc

@@ -68,41 +68,16 @@ struct MercurialInputScheme : InputScheme
         return "hg";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        // TODO
-        return "";
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "url",
-                {},
-            },
-            {
-                "ref",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "revCount",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "name",
-                {},
-            },
+        return {
+            "url",
+            "ref",
+            "rev",
+            "revCount",
+            "narHash",
+            "name",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -179,7 +154,7 @@ struct MercurialInputScheme : InputScheme
         return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()};
     }
 
-    StorePath fetchToStore(const Settings & settings, Store & store, Input & input) const
+    StorePath fetchToStore(const Settings & settings, ref<Store> store, Input & input) const
     {
         auto origRev = input.getRev();
 
@@ -230,7 +205,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store.addToStore(
+                auto storePath = store->addToStore(
                     input.getName(),
                     {getFSSourceAccessor(), CanonPath(actualPath)},
                     ContentAddressMethod::Raw::NixArchive,
@@ -251,7 +226,7 @@ struct MercurialInputScheme : InputScheme
                     "Hash '%s' is not supported by Mercurial. Only sha1 is supported.",
                     rev.to_string(HashFormat::Base16, true));
 
-            return Cache::Key{"hgRev", {{"store", store.storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}};
+            return Cache::Key{"hgRev", {{"store", store->storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}};
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath {
@@ -271,7 +246,7 @@ struct MercurialInputScheme : InputScheme
 
         /* If we have a rev, check if we have a cached store path. */
         if (auto rev = input.getRev()) {
-            if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), store))
+            if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), *store))
                 return makeResult(res->value, res->storePath);
         }
 
@@ -325,7 +300,7 @@ struct MercurialInputScheme : InputScheme
 
         /* Now that we have the rev, check the cache again for a
            cached store path. */
-        if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), store))
+        if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), *store))
             return makeResult(res->value, res->storePath);
 
         Path tmpDir = createTempDir();
@@ -335,7 +310,7 @@ struct MercurialInputScheme : InputScheme
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
-        auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
+        auto storePath = store->addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
 
         Attrs infoAttrs({
             {"revCount", (uint64_t) revCount},
@@ -344,18 +319,18 @@ struct MercurialInputScheme : InputScheme
         if (!origRev)
             settings.getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
 
-        settings.getCache()->upsert(revInfoKey(rev), store, infoAttrs, storePath);
+        settings.getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
 
         return makeResult(infoAttrs, std::move(storePath));
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         Input input(_input);
 
         auto storePath = fetchToStore(settings, store, input);
-        auto accessor = store.requireStoreObjectAccessor(storePath);
+        auto accessor = store->requireStoreObjectAccessor(storePath);
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 
@@ -367,7 +342,7 @@ struct MercurialInputScheme : InputScheme
         return (bool) input.getRev();
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto rev = input.getRev())
             return rev->gitRev();

src/libfetchers/path.cc

@@ -40,42 +40,20 @@ struct PathInputScheme : InputScheme
         return "path";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        // TODO
-        return "";
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "path",
-                {},
-            },
+        return {
+            "path",
             /* Allow the user to pass in "fake" tree info
                attributes. This is useful for making a pinned tree work
                the same as the repository from which is exported (e.g.
                path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
              */
-            {
-                "rev",
-                {},
-            },
-            {
-                "revCount",
-                {},
-            },
-            {
-                "lastModified",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
+            "rev",
+            "revCount",
+            "lastModified",
+            "narHash",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -139,7 +117,7 @@ struct PathInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         Input input(_input);
         auto path = getStrAttr(input.attrs, "path");
@@ -147,31 +125,31 @@ struct PathInputScheme : InputScheme
         auto absPath = getAbsPath(input);
 
         // FIXME: check whether access to 'path' is allowed.
-        auto storePath = store.maybeParseStorePath(absPath.string());
+        auto storePath = store->maybeParseStorePath(absPath.string());
 
         if (storePath)
-            store.addTempRoot(*storePath);
+            store->addTempRoot(*storePath);
 
         time_t mtime = 0;
-        if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) {
+        if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
             // FIXME: try to substitute storePath.
             auto src = sinkToSource(
                 [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
-            storePath = store.addToStoreFromDump(*src, "source");
+            storePath = store->addToStoreFromDump(*src, "source");
         }
 
-        auto accessor = store.requireStoreObjectAccessor(*storePath);
+        auto accessor = store->requireStoreObjectAccessor(*storePath);
 
         // To prevent `fetchToStore()` copying the path again to Nix
         // store, pre-create an entry in the fetcher cache.
-        auto info = store.queryPathInfo(*storePath);
+        auto info = store->queryPathInfo(*storePath);
         accessor->fingerprint =
-            fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
+            fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
         settings.getCache()->upsert(
             makeFetchToStoreCacheKey(
                 input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
-            store,
+            *store,
             {},
             *storePath);
 

src/libfetchers/registry.cc

@@ -14,10 +14,10 @@ std::shared_ptr<Registry> Registry::read(const Settings & settings, const Source
 {
     debug("reading registry '%s'", path);
 
-    auto registry = std::make_shared<Registry>(type);
+    auto registry = std::make_shared<Registry>(settings, type);
 
     if (!path.pathExists())
-        return std::make_shared<Registry>(type);
+        return std::make_shared<Registry>(settings, type);
 
     try {
 
@@ -125,23 +125,23 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
     return customRegistry;
 }
 
-std::shared_ptr<Registry> getFlagRegistry()
+std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
 {
-    static auto flagRegistry = std::make_shared<Registry>(Registry::Flag);
+    static auto flagRegistry = std::make_shared<Registry>(settings, Registry::Flag);
     return flagRegistry;
 }
 
-void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs)
+void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs)
 {
-    getFlagRegistry()->add(from, to, extraAttrs);
+    getFlagRegistry(settings)->add(from, to, extraAttrs);
 }
 
-static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, Store & store)
+static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, ref<Store> store)
 {
     static auto reg = [&]() {
         auto path = settings.flakeRegistry.get();
         if (path == "") {
-            return std::make_shared<Registry>(Registry::Global); // empty registry
+            return std::make_shared<Registry>(settings, Registry::Global); // empty registry
         }
 
         return Registry::read(
@@ -149,9 +149,9 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
             [&] -> SourcePath {
                 if (!isAbsolute(path)) {
                     auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
-                    if (auto store2 = dynamic_cast<LocalFSStore *>(&store))
+                    if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                         store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
-                    return {store.requireStoreObjectAccessor(storePath)};
+                    return {store->requireStoreObjectAccessor(storePath)};
                 } else {
                     return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
                 }
@@ -162,10 +162,10 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
     return reg;
 }
 
-Registries getRegistries(const Settings & settings, Store & store)
+Registries getRegistries(const Settings & settings, ref<Store> store)
 {
     Registries registries;
-    registries.push_back(getFlagRegistry());
+    registries.push_back(getFlagRegistry(settings));
     registries.push_back(getUserRegistry(settings));
     registries.push_back(getSystemRegistry(settings));
     registries.push_back(getGlobalRegistry(settings, store));
@@ -173,7 +173,7 @@ Registries getRegistries(const Settings & settings, Store & store)
 }
 
 std::pair<Input, Attrs>
-lookupInRegistries(const Settings & settings, Store & store, const Input & _input, UseRegistries useRegistries)
+lookupInRegistries(const Settings & settings, ref<Store> store, const Input & _input, UseRegistries useRegistries)
 {
     Attrs extraAttrs;
     int n = 0;

src/libfetchers/tarball.cc

@@ -13,7 +13,7 @@
 namespace nix::fetchers {
 
 DownloadFileResult downloadFile(
-    Store & store,
+    ref<Store> store,
     const Settings & settings,
     const std::string & url,
     const std::string & name,
@@ -28,7 +28,7 @@ DownloadFileResult downloadFile(
             {"name", name},
         }}};
 
-    auto cached = settings.getCache()->lookupStorePath(key, store);
+    auto cached = settings.getCache()->lookupStorePath(key, *store);
 
     auto useCached = [&]() -> DownloadFileResult {
         return {
@@ -74,7 +74,7 @@ DownloadFileResult downloadFile(
         dumpString(res.data, sink);
         auto hash = hashString(HashAlgorithm::SHA256, res.data);
         auto info = ValidPathInfo::makeFromCA(
-            store,
+            *store,
             name,
             FixedOutputInfo{
                 .method = FileIngestionMethod::Flat,
@@ -84,7 +84,7 @@ DownloadFileResult downloadFile(
             hashString(HashAlgorithm::SHA256, sink.s));
         info.narSize = sink.s.size();
         auto source = StringSource{sink.s};
-        store.addToStore(info, source, NoRepair, NoCheckSigs);
+        store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
     }
 
@@ -93,7 +93,7 @@ DownloadFileResult downloadFile(
         key.second.insert_or_assign("url", url);
         assert(!res.urls.empty());
         infoAttrs.insert_or_assign("url", *res.urls.rbegin());
-        settings.getCache()->upsert(key, store, infoAttrs, *storePath);
+        settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
     }
 
     return {
@@ -136,7 +136,7 @@ static DownloadTarballResult downloadTarball_(
             .treeHash = treeHash,
             .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
             .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
-            .accessor = settings.getTarballCache()->getAccessor(treeHash, {}, displayPrefix),
+            .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix),
         };
     };
 
@@ -214,7 +214,7 @@ static DownloadTarballResult downloadTarball_(
     return attrsToResult(infoAttrs);
 }
 
-ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url)
+ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url)
 {
     /* Go through Input::getAccessor() to ensure that the resulting
        accessor has a fingerprint. */
@@ -278,7 +278,7 @@ struct CurlInputScheme : InputScheme
            HTTP request. Now that we've processed the Nix-specific
            attributes above, remove them so we don't also send them as
            part of the HTTP request. */
-        for (auto & [param, _] : allowedAttrs())
+        for (auto & param : allowedAttrs())
             url.query.erase(param);
 
         input.attrs.insert_or_assign("type", std::string{schemeName()});
@@ -286,83 +286,18 @@ struct CurlInputScheme : InputScheme
         return input;
     }
 
-    static const std::map<std::string, AttributeInfo> & allowedAttrsImpl()
+    StringSet allowedAttrs() const override
     {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "url",
-                {
-                    .type = "String",
-                    .required = true,
-                    .doc = R"(
-                      Supported protocols:
-
-                      - `https`
-
-                        > **Example**
-                        >
-                        > ```nix
-                        > fetchTree {
-                        >   type = "file";
-                        >   url = "https://example.com/index.html";
-                        > }
-                        > ```
-
-                      - `http`
-
-                        Insecure HTTP transfer for legacy sources.
-
-                        > **Warning**
-                        >
-                        > HTTP performs no encryption or authentication.
-                        > Use a `narHash` known in advance to ensure the output has expected contents.
-
-                      - `file`
-
-                        A file on the local file system.
-
-                        > **Example**
-                        >
-                        > ```nix
-                        > fetchTree {
-                        >   type = "file";
-                        >   url = "file:///home/eelco/nix/README.md";
-                        > }
-                        > ```
-                    )",
-                },
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "name",
-                {},
-            },
-            {
-                "unpack",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "revCount",
-                {},
-            },
-            {
-                "lastModified",
-                {},
-            },
+        return {
+            "type",
+            "url",
+            "narHash",
+            "name",
+            "unpack",
+            "rev",
+            "revCount",
+            "lastModified",
         };
-        return attrs;
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        return allowedAttrsImpl();
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -397,14 +332,6 @@ struct FileInputScheme : CurlInputScheme
         return "file";
     }
 
-    std::string schemeDescription() const override
-    {
-        return stripIndentation(R"(
-          Place a plain file into the Nix store.
-          This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
-        )");
-    }
-
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -414,7 +341,7 @@ struct FileInputScheme : CurlInputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         auto input(_input);
 
@@ -424,10 +351,10 @@ struct FileInputScheme : CurlInputScheme
            tarballs. */
         auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName());
 
-        auto narHash = store.queryPathInfo(file.storePath)->narHash;
+        auto narHash = store->queryPathInfo(file.storePath)->narHash;
         input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-        auto accessor = ref{store.getFSAccessor(file.storePath)};
+        auto accessor = ref{store->getFSAccessor(file.storePath)};
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 
@@ -442,34 +369,6 @@ struct TarballInputScheme : CurlInputScheme
         return "tarball";
     }
 
-    std::string schemeDescription() const override
-    {
-        return stripIndentation(R"(
-          Download a tar archive and extract it into the Nix store.
-          This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
-        )");
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = [] {
-            auto attrs = CurlInputScheme::allowedAttrsImpl();
-            // Override the "url" attribute to add tarball-specific example
-            attrs["url"].doc = R"(
-              > **Example**
-              >
-              > ```nix
-              > fetchTree {
-              >   type = "tarball";
-              >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
-              > }
-              > ```
-            )";
-            return attrs;
-        }();
-        return attrs;
-    }
-
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -480,7 +379,7 @@ struct TarballInputScheme : CurlInputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         auto input(_input);
 
@@ -505,7 +404,7 @@ struct TarballInputScheme : CurlInputScheme
         return {result.accessor, input};
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto narHash = input.getNarHash())
             return narHash->to_string(HashFormat::SRI, true);

src/libflake-c/nix_api_flake.cc

@@ -200,7 +200,7 @@ nix_value * nix_locked_flake_get_output_attrs(
     nix_clear_err(context);
     try {
         auto v = nix_alloc_value(context, evalState);
-        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value);
+        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value);
         return v;
     }
     NIXC_CATCH_ERRS_NULL

src/libflake/flake-primops.cc

@@ -19,6 +19,7 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/value.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/fetchers/attrs.hh"
 #include "nix/fetchers/fetchers.hh"
 #include "nix/util/configuration.hh"
@@ -62,7 +63,7 @@ PrimOp getFlake(const Settings & settings)
     return PrimOp{
         .name = "__getFlake",
         .args = {"args"},
-        .doc = R"(
+        .doc = &R"(
           Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
 
           ```nix
@@ -76,7 +77,7 @@ PrimOp getFlake(const Settings & settings)
           ```nix
           (builtins.getFlake "github:edolstra/dwarffs").rev
           ```
-        )",
+        )"_sds,
         .fun = prim_getFlake,
         .experimentalFeature = Xp::Flakes,
     };
@@ -93,7 +94,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
         auto & vv = binds.alloc(s);
         std::visit(
             overloaded{
-                [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); },
+                [&vv](const std::string & value) { vv.mkString(value); },
                 [&vv](const uint64_t & value) { vv.mkInt(value); },
                 [&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }},
             value);
@@ -104,7 +105,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
 nix::PrimOp parseFlakeRef({
     .name = "__parseFlakeRef",
     .args = {"flake-ref"},
-    .doc = R"(
+    .doc = &R"(
       Parse a flake reference, and return its exploded form.
 
       For example:
@@ -118,7 +119,7 @@ nix::PrimOp parseFlakeRef({
       ```nix
       { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; }
       ```
-    )",
+    )"_sds,
     .fun = prim_parseFlakeRef,
     .experimentalFeature = Xp::Flakes,
 });
@@ -156,13 +157,13 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
         }
     }
     auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs);
-    v.mkString(flakeRef.to_string(), state.mem);
+    v.mkString(flakeRef.to_string());
 }
 
 nix::PrimOp flakeRefToString({
     .name = "__flakeRefToString",
     .args = {"attrs"},
-    .doc = R"(
+    .doc = &R"(
       Convert a flake reference from attribute set format to URL format.
 
       For example:
@@ -178,7 +179,7 @@ nix::PrimOp flakeRefToString({
       ```nix
       "github:NixOS/nixpkgs/23.05?dir=lib"
       ```
-    )",
+    )"_sds,
     .fun = prim_flakeRefToString,
     .experimentalFeature = Xp::Flakes,
 });

src/libflake/flake.cc

@@ -373,7 +373,7 @@ static Flake getFlake(
 {
     // Fetch a lazy tree first.
     auto cachedInput =
-        state.inputCache->getAccessor(state.fetchSettings, *state.store, originalRef.input, useRegistries);
+        state.inputCache->getAccessor(state.fetchSettings, state.store, originalRef.input, useRegistries);
 
     auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
     auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
@@ -390,7 +390,7 @@ static Flake getFlake(
         // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
         newLockedRef.input.attrs.erase("narHash");
         auto cachedInput2 = state.inputCache->getAccessor(
-            state.fetchSettings, *state.store, newLockedRef.input, fetchers::UseRegistries::No);
+            state.fetchSettings, state.store, newLockedRef.input, fetchers::UseRegistries::No);
         cachedInput.accessor = cachedInput2.accessor;
         lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
     }
@@ -756,7 +756,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
                                     return {*resolvedPath, *input.ref};
                                 } else {
                                     auto cachedInput = state.inputCache->getAccessor(
-                                        state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs);
+                                        state.fetchSettings, state.store, input.ref->input, useRegistriesInputs);
 
                                     auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
 
@@ -956,7 +956,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
         auto key = keyMap.find(node);
         assert(key != keyMap.end());
 
-        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem);
+        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel());
 
         overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
     }
@@ -966,7 +966,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
     Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix"));
 
     auto vLocks = state.allocValue();
-    vLocks->mkString(lockFileStr, state.mem);
+    vLocks->mkString(lockFileStr);
 
     auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree");
     assert(vFetchFinalTree);
@@ -975,7 +975,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
     state.callFunction(*vCallFlake, args, vRes, noPos);
 }
 
-std::optional<Fingerprint> LockedFlake::getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const
+std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const
 {
     if (lockFile.isUnlocked(fetchSettings))
         return std::nullopt;
@@ -1005,7 +1005,7 @@ Flake::~Flake() {}
 ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake)
 {
     auto fingerprint = state.settings.useEvalCache && state.settings.pureEval
-                           ? lockedFlake->getFingerprint(*state.store, state.fetchSettings)
+                           ? lockedFlake->getFingerprint(state.store, state.fetchSettings)
                            : std::nullopt;
     auto rootLoader = [&state, lockedFlake]() {
         /* For testing whether the evaluation cache is

src/libflake/flakeref.cc

@@ -64,8 +64,8 @@ std::ostream & operator<<(std::ostream & str, const FlakeRef & flakeRef)
     return str;
 }
 
-FlakeRef
-FlakeRef::resolve(const fetchers::Settings & fetchSettings, Store & store, fetchers::UseRegistries useRegistries) const
+FlakeRef FlakeRef::resolve(
+    const fetchers::Settings & fetchSettings, ref<Store> store, fetchers::UseRegistries useRegistries) const
 {
     auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries);
     return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
@@ -289,7 +289,7 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Settings & fetchSettings, const fet
 }
 
 std::pair<ref<SourceAccessor>, FlakeRef>
-FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const
+FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const
 {
     auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store);
     return {accessor, FlakeRef(std::move(lockedInput), subdir)};

src/libflake/include/nix/flake/flake.hh

@@ -135,7 +135,7 @@ struct LockedFlake
      */
     std::map<ref<Node>, SourcePath> nodePaths;
 
-    std::optional<Fingerprint> getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const;
+    std::optional<Fingerprint> getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const;
 };
 
 struct LockFlags

src/libflake/include/nix/flake/flakeref.hh

@@ -73,12 +73,13 @@ struct FlakeRef
 
     FlakeRef resolve(
         const fetchers::Settings & fetchSettings,
-        Store & store,
+        ref<Store> store,
         fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
 
     static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs);
 
-    std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const;
+    std::pair<ref<SourceAccessor>, FlakeRef>
+    lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const;
 
     /**
      * Canonicalize a flakeref for the purpose of comparing "old" and

src/libstore-c/meson.build

@@ -35,8 +35,6 @@ include_dirs = [ include_directories('.') ]
 
 headers = files(
   'nix_api_store.h',
-  'nix_api_store/derivation.h',
-  'nix_api_store/store_path.h',
 )
 
 # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c`

src/libstore-c/nix_api_store.h

@@ -12,8 +12,6 @@
  */
 
 #include "nix_api_util.h"
-#include "nix_api_store/store_path.h"
-#include "nix_api_store/derivation.h"
 #include <stdbool.h>
 
 #ifdef __cplusplus
@@ -23,6 +21,10 @@ extern "C" {
 
 /** @brief Reference to a Nix store */
 typedef struct Store Store;
+/** @brief Nix store path */
+typedef struct StorePath StorePath;
+/** @brief Nix Derivation */
+typedef struct nix_derivation nix_derivation;
 
 /**
  * @brief Initializes the Nix store library
@@ -116,6 +118,30 @@ nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_ca
  */
 StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path);
 
+/**
+ * @brief Get the path name (e.g. "name" in /nix/store/...-name)
+ *
+ * @param[in] store_path the path to get the name from
+ * @param[in] callback called with the name
+ * @param[in] user_data arbitrary data, passed to the callback when it's called.
+ */
+void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
+
+/**
+ * @brief Copy a StorePath
+ *
+ * @param[in] p the path to copy
+ * @return a new StorePath
+ */
+StorePath * nix_store_path_clone(const StorePath * p);
+
+/** @brief Deallocate a StorePath
+ *
+ * Does not fail.
+ * @param[in] p the path to free
+ */
+void nix_store_path_free(StorePath * p);
+
 /**
  * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in
  * the store)
@@ -203,6 +229,14 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
  */
 StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation);
 
+/**
+ * @brief Deallocate a `nix_derivation`
+ *
+ * Does not fail.
+ * @param[in] drv the derivation to free
+ */
+void nix_derivation_free(nix_derivation * drv);
+
 /**
  * @brief Copy the closure of `path` from `srcStore` to `dstStore`.
  *

src/libstore-c/nix_api_store/derivation.h

@@ -1,38 +0,0 @@
-#ifndef NIX_API_STORE_DERIVATION_H
-#define NIX_API_STORE_DERIVATION_H
-/**
- * @defgroup libstore_derivation Derivation
- * @ingroup libstore
- * @brief Derivation operations that don't require a Store
- * @{
- */
-/** @file
- * @brief Derivation operations
- */
-
-#include "nix_api_util.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-// cffi start
-
-/** @brief Nix Derivation */
-typedef struct nix_derivation nix_derivation;
-
-/**
- * @brief Deallocate a `nix_derivation`
- *
- * Does not fail.
- * @param[in] drv the derivation to free
- */
-void nix_derivation_free(nix_derivation * drv);
-
-// cffi end
-#ifdef __cplusplus
-}
-#endif
-/**
- * @}
- */
-#endif // NIX_API_STORE_DERIVATION_H

src/libstore-c/nix_api_store/store_path.h

@@ -1,54 +0,0 @@
-#ifndef NIX_API_STORE_STORE_PATH_H
-#define NIX_API_STORE_STORE_PATH_H
-/**
- * @defgroup libstore_storepath StorePath
- * @ingroup libstore
- * @brief Store path operations that don't require a Store
- * @{
- */
-/** @file
- * @brief Store path operations
- */
-
-#include "nix_api_util.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-// cffi start
-
-/** @brief Nix store path */
-typedef struct StorePath StorePath;
-
-/**
- * @brief Copy a StorePath
- *
- * @param[in] p the path to copy
- * @return a new StorePath
- */
-StorePath * nix_store_path_clone(const StorePath * p);
-
-/** @brief Deallocate a StorePath
- *
- * Does not fail.
- * @param[in] p the path to free
- */
-void nix_store_path_free(StorePath * p);
-
-/**
- * @brief Get the path name (e.g. "<name>" in /nix/store/<hash>-<name>)
- *
- * @param[in] store_path the path to get the name from
- * @param[in] callback called with the name
- * @param[in] user_data arbitrary data, passed to the callback when it's called.
- */
-void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
-
-// cffi end
-#ifdef __cplusplus
-}
-#endif
-/**
- * @}
- */
-#endif // NIX_API_STORE_STORE_PATH_H

src/libstore-tests/data/derivation/ca/all_set.json

@@ -4,13 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      {
-        "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
-        "output": "out"
-      }
+      "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
     ],
     "refs2": [
-      "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+      "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -23,36 +20,18 @@
   "outputChecks": {
     "forAllOutputs": {
       "allowedReferences": [
-        {
-          "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
-          "output": "out"
-        }
+        "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
       ],
       "allowedRequisites": [
-        {
-          "drvPath": "self",
-          "output": "bin"
-        },
-        {
-          "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
-          "output": "dev"
-        }
+        "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+        "bin"
       ],
       "disallowedReferences": [
-        {
-          "drvPath": "self",
-          "output": "dev"
-        },
-        {
-          "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
-          "output": "out"
-        }
+        "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+        "dev"
       ],
       "disallowedRequisites": [
-        {
-          "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
-          "output": "dev"
-        }
+        "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
       ],
       "ignoreSelfRefs": true,
       "maxClosureSize": null,

src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json

@@ -4,13 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      {
-        "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
-        "output": "out"
-      }
+      "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
     ],
     "refs2": [
-      "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+      "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -26,20 +23,11 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          {
-            "drvPath": "self",
-            "output": "dev"
-          },
-          {
-            "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
-            "output": "out"
-          }
+          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+          "dev"
         ],
         "disallowedRequisites": [
-          {
-            "drvPath": "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
-            "output": "dev"
-          }
+          "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
         ],
         "ignoreSelfRefs": false,
         "maxClosureSize": null,
@@ -56,20 +44,11 @@
       },
       "out": {
         "allowedReferences": [
-          {
-            "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
-            "output": "out"
-          }
+          "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
         ],
         "allowedRequisites": [
-          {
-            "drvPath": "self",
-            "output": "bin"
-          },
-          {
-            "drvPath": "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",
-            "output": "dev"
-          }
+          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+          "bin"
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/data/derivation/ia/all_set.json

@@ -4,10 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+      "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
     ],
     "refs2": [
-      "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+      "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -20,24 +20,18 @@
   "outputChecks": {
     "forAllOutputs": {
       "allowedReferences": [
-        "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+        "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
       ],
       "allowedRequisites": [
-        {
-          "drvPath": "self",
-          "output": "bin"
-        },
-        "z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
+        "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+        "bin"
       ],
       "disallowedReferences": [
-        {
-          "drvPath": "self",
-          "output": "dev"
-        },
-        "r5cff30838majxk5mp3ip2diffi8vpaj-bar"
+        "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+        "dev"
       ],
       "disallowedRequisites": [
-        "9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
+        "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
       ],
       "ignoreSelfRefs": true,
       "maxClosureSize": null,

src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json

@@ -4,10 +4,10 @@
   "allowSubstitutes": false,
   "exportReferencesGraph": {
     "refs1": [
-      "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+      "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
     ],
     "refs2": [
-      "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+      "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
     ]
   },
   "impureEnvVars": [
@@ -23,14 +23,11 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          {
-            "drvPath": "self",
-            "output": "dev"
-          },
-          "r5cff30838majxk5mp3ip2diffi8vpaj-bar"
+          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+          "dev"
         ],
         "disallowedRequisites": [
-          "9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
+          "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
         ],
         "ignoreSelfRefs": false,
         "maxClosureSize": null,
@@ -47,14 +44,11 @@
       },
       "out": {
         "allowedReferences": [
-          "p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+          "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
         ],
         "allowedRequisites": [
-          {
-            "drvPath": "self",
-            "output": "bin"
-          },
-          "z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
+          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+          "bin"
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/derivation-advanced-attrs.cc

@@ -3,7 +3,7 @@
 
 #include "nix/util/experimental-features.hh"
 #include "nix/store/derivations.hh"
-#include "nix/store/derived-path.hh"
+#include "nix/store/derivations.hh"
 #include "nix/store/derivation-options.hh"
 #include "nix/store/parsed-derivations.hh"
 #include "nix/util/types.hh"
@@ -17,7 +17,7 @@ namespace nix {
 using namespace nlohmann;
 
 class DerivationAdvancedAttrsTest : public JsonCharacterizationTest<Derivation>,
-                                    public JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>,
+                                    public JsonCharacterizationTest<DerivationOptions>,
                                     public LibStoreTest
 {
 protected:
@@ -42,8 +42,7 @@ public:
     {
         this->readTest(fileName, [&](auto encoded) {
             auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
-            auto options = derivationOptionsFromStructuredAttrs(
-                *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
+            DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
             EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedFeatures);
         });
     }
@@ -52,14 +51,11 @@ public:
      * Helper function to test DerivationOptions parsing and comparison
      */
     void testDerivationOptions(
-        const std::string & fileName,
-        const DerivationOptions<SingleDerivedPath> & expected,
-        const StringSet & expectedSystemFeatures)
+        const std::string & fileName, const DerivationOptions & expected, const StringSet & expectedSystemFeatures)
     {
         this->readTest(fileName, [&](auto encoded) {
             auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
-            auto options = derivationOptionsFromStructuredAttrs(
-                *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
+            DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
 
             EXPECT_EQ(options, expected);
             EXPECT_EQ(options.getRequiredSystemFeatures(got), expectedSystemFeatures);
@@ -135,38 +131,22 @@ TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attribute
  * Since these are both repeated and sensative opaque values, it makes
  * sense to give them names in this file.
  */
-static SingleDerivedPath
-    pathFoo = SingleDerivedPath::Opaque{StorePath{"p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}},
-    pathFooDev = SingleDerivedPath::Opaque{StorePath{"z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}},
-    pathBar = SingleDerivedPath::Opaque{StorePath{"r5cff30838majxk5mp3ip2diffi8vpaj-bar"}},
-    pathBarDev = SingleDerivedPath::Opaque{StorePath{"9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}},
-    pathBarDrvIA = SingleDerivedPath::Opaque{StorePath{"vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"}},
-    pathBarDrvCA = SingleDerivedPath::Opaque{StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}},
-    placeholderFoo =
-        SingleDerivedPath::Built{
-            .drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
-            .output = "out",
-        },
-    placeholderFooDev =
-        SingleDerivedPath::Built{
-            .drvPath = makeConstantStorePathRef(StorePath{"j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv"}),
-            .output = "dev",
-        },
-    placeholderBar =
-        SingleDerivedPath::Built{
-            .drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
-            .output = "out",
-        },
-    placeholderBarDev = SingleDerivedPath::Built{
-        .drvPath = makeConstantStorePathRef(StorePath{"qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"}),
-        .output = "dev",
-    };
+static std::string pathFoo = "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
+                   pathFooDev = "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+                   pathBar = "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+                   pathBarDev = "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
+                   pathBarDrvIA = "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
+                   pathBarDrvCA = "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
+                   placeholderFoo = "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
+                   placeholderFooDev = "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+                   placeholderBar = "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+                   placeholderBarDev = "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8";
 
-using ExportReferencesMap = decltype(DerivationOptions<SingleDerivedPath>::exportReferencesGraph);
+using ExportReferencesMap = decltype(DerivationOptions::exportReferencesGraph);
 
-static const DerivationOptions<SingleDerivedPath> advancedAttributes_defaults = {
+static const DerivationOptions advancedAttributes_defaults = {
     .outputChecks =
-        DerivationOptions<SingleDerivedPath>::OutputChecks{
+        DerivationOptions::OutputChecks{
             .ignoreSelfRefs = true,
         },
     .unsafeDiscardReferences = {},
@@ -187,8 +167,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
     this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto options = derivationOptionsFromStructuredAttrs(
-            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
 
         EXPECT_TRUE(!got.structuredAttrs);
 
@@ -213,9 +192,9 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
 
 TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
 {
-    DerivationOptions<SingleDerivedPath> expected = {
+    DerivationOptions expected = {
         .outputChecks =
-            DerivationOptions<SingleDerivedPath>::OutputChecks{
+            DerivationOptions::OutputChecks{
                 .ignoreSelfRefs = true,
             },
         .unsafeDiscardReferences = {},
@@ -233,13 +212,12 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
     this->readTest("advanced-attributes.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto options = derivationOptionsFromStructuredAttrs(
-            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
 
         EXPECT_TRUE(!got.structuredAttrs);
 
         // Reset fields that vary between test cases to enable whole-object comparison
-        options.outputChecks = DerivationOptions<SingleDerivedPath>::OutputChecks{.ignoreSelfRefs = true};
+        options.outputChecks = DerivationOptions::OutputChecks{.ignoreSelfRefs = true};
         options.exportReferencesGraph = {};
 
         EXPECT_EQ(options, expected);
@@ -249,14 +227,14 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
     });
 };
 
-DerivationOptions<SingleDerivedPath> advancedAttributes_ia = {
+DerivationOptions advancedAttributes_ia = {
     .outputChecks =
-        DerivationOptions<SingleDerivedPath>::OutputChecks{
+        DerivationOptions::OutputChecks{
             .ignoreSelfRefs = true,
-            .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo},
-            .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}},
-            .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}},
-            .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev},
+            .allowedReferences = StringSet{pathFoo},
+            .disallowedReferences = StringSet{pathBar, "dev"},
+            .allowedRequisites = StringSet{pathFooDev, "bin"},
+            .disallowedRequisites = StringSet{pathBarDev},
         },
     .unsafeDiscardReferences = {},
     .passAsFile = {},
@@ -279,14 +257,14 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_ia)
     testDerivationOptions("advanced-attributes.drv", advancedAttributes_ia, {"rainbow", "uid-range"});
 };
 
-DerivationOptions<SingleDerivedPath> advancedAttributes_ca = {
+DerivationOptions advancedAttributes_ca = {
     .outputChecks =
-        DerivationOptions<SingleDerivedPath>::OutputChecks{
+        DerivationOptions::OutputChecks{
             .ignoreSelfRefs = true,
-            .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo},
-            .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}},
-            .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}},
-            .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev},
+            .allowedReferences = StringSet{placeholderFoo},
+            .disallowedReferences = StringSet{placeholderBar, "dev"},
+            .allowedRequisites = StringSet{placeholderFooDev, "bin"},
+            .disallowedRequisites = StringSet{placeholderBarDev},
         },
     .unsafeDiscardReferences = {},
     .passAsFile = {},
@@ -309,8 +287,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
     testDerivationOptions("advanced-attributes.drv", advancedAttributes_ca, {"rainbow", "uid-range", "ca-derivations"});
 };
 
-DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_defaults = {
-    .outputChecks = std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{},
+DerivationOptions advancedAttributes_structuredAttrs_defaults = {
+    .outputChecks = std::map<std::string, DerivationOptions::OutputChecks>{},
     .unsafeDiscardReferences = {},
     .passAsFile = {},
     .exportReferencesGraph = {},
@@ -329,8 +307,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
     this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto options = derivationOptionsFromStructuredAttrs(
-            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
 
         EXPECT_TRUE(got.structuredAttrs);
 
@@ -355,11 +332,11 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
 
 TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
 {
-    DerivationOptions<SingleDerivedPath> expected = {
+    DerivationOptions expected = {
         .outputChecks =
-            std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
+            std::map<std::string, DerivationOptions::OutputChecks>{
                 {"dev",
-                 DerivationOptions<SingleDerivedPath>::OutputChecks{
+                 DerivationOptions::OutputChecks{
                      .maxSize = 789,
                      .maxClosureSize = 5909,
                  }},
@@ -380,8 +357,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
     this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) {
         auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings);
 
-        auto options = derivationOptionsFromStructuredAttrs(
-            *this->store, got.inputDrvs, got.env, get(got.structuredAttrs), true, this->mockXpSettings);
+        DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, got.structuredAttrs);
 
         EXPECT_TRUE(got.structuredAttrs);
 
@@ -389,8 +365,7 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
         {
             // Delete all keys but "dev" in options.outputChecks
             auto * outputChecksMapP =
-                std::get_if<std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>>(
-                    &options.outputChecks);
+                std::get_if<std::map<std::string, DerivationOptions::OutputChecks>>(&options.outputChecks);
             ASSERT_TRUE(outputChecksMapP);
             auto & outputChecksMap = *outputChecksMapP;
             auto devEntry = outputChecksMap.find("dev");
@@ -410,21 +385,21 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
     });
 };
 
-DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ia = {
+DerivationOptions advancedAttributes_structuredAttrs_ia = {
     .outputChecks =
-        std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
+        std::map<std::string, DerivationOptions::OutputChecks>{
             {"out",
-             DerivationOptions<SingleDerivedPath>::OutputChecks{
-                 .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathFoo},
-                 .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathFooDev, OutputName{"bin"}},
+             DerivationOptions::OutputChecks{
+                 .allowedReferences = StringSet{pathFoo},
+                 .allowedRequisites = StringSet{pathFooDev, "bin"},
              }},
             {"bin",
-             DerivationOptions<SingleDerivedPath>::OutputChecks{
-                 .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{pathBar, OutputName{"dev"}},
-                 .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{pathBarDev},
+             DerivationOptions::OutputChecks{
+                 .disallowedReferences = StringSet{pathBar, "dev"},
+                 .disallowedRequisites = StringSet{pathBarDev},
              }},
             {"dev",
-             DerivationOptions<SingleDerivedPath>::OutputChecks{
+             DerivationOptions::OutputChecks{
                  .maxSize = 789,
                  .maxClosureSize = 5909,
              }},
@@ -452,21 +427,21 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
         "advanced-attributes-structured-attrs.drv", advancedAttributes_structuredAttrs_ia, {"rainbow", "uid-range"});
 };
 
-DerivationOptions<SingleDerivedPath> advancedAttributes_structuredAttrs_ca = {
+DerivationOptions advancedAttributes_structuredAttrs_ca = {
     .outputChecks =
-        std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks>{
+        std::map<std::string, DerivationOptions::OutputChecks>{
             {"out",
-             DerivationOptions<SingleDerivedPath>::OutputChecks{
-                 .allowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderFoo},
-                 .allowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderFooDev, OutputName{"bin"}},
+             DerivationOptions::OutputChecks{
+                 .allowedReferences = StringSet{placeholderFoo},
+                 .allowedRequisites = StringSet{placeholderFooDev, "bin"},
              }},
             {"bin",
-             DerivationOptions<SingleDerivedPath>::OutputChecks{
-                 .disallowedReferences = std::set<DrvRef<SingleDerivedPath>>{placeholderBar, OutputName{"dev"}},
-                 .disallowedRequisites = std::set<DrvRef<SingleDerivedPath>>{placeholderBarDev},
+             DerivationOptions::OutputChecks{
+                 .disallowedReferences = StringSet{placeholderBar, "dev"},
+                 .disallowedRequisites = StringSet{placeholderBarDev},
              }},
             {"dev",
-             DerivationOptions<SingleDerivedPath>::OutputChecks{
+             DerivationOptions::OutputChecks{
                  .maxSize = 789,
                  .maxClosureSize = 5909,
              }},
@@ -496,16 +471,14 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
         {"rainbow", "uid-range", "ca-derivations"});
 };
 
-#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2)                                               \
-    TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json)                                    \
-    {                                                                                        \
-        this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::readJsonTest(  \
-            #VAR, advancedAttributes_##VAR2);                                                \
-    }                                                                                        \
-    TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json)                                      \
-    {                                                                                        \
-        this->JsonCharacterizationTest<DerivationOptions<SingleDerivedPath>>::writeJsonTest( \
-            #VAR, advancedAttributes_##VAR2);                                                \
+#define TEST_JSON_OPTIONS(FIXUTURE, VAR, VAR2)                                                             \
+    TEST_F(FIXUTURE, DerivationOptions_##VAR##_from_json)                                                  \
+    {                                                                                                      \
+        this->JsonCharacterizationTest<DerivationOptions>::readJsonTest(#VAR, advancedAttributes_##VAR2);  \
+    }                                                                                                      \
+    TEST_F(FIXUTURE, DerivationOptions_##VAR##_to_json)                                                    \
+    {                                                                                                      \
+        this->JsonCharacterizationTest<DerivationOptions>::writeJsonTest(#VAR, advancedAttributes_##VAR2); \
     }
 
 TEST_JSON_OPTIONS(DerivationAdvancedAttrsTest, defaults, defaults)

src/libstore-tests/references.cc

@@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep)
     // Create an in-memory file system with various reference patterns
     auto accessor = make_ref<MemorySourceAccessor>();
     accessor->root = File::Directory{
-        .entries{
+        .contents{
             {
                 // file1.txt: contains hash1
                 "file1.txt",
@@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep)
                 // subdir: a subdirectory
                 "subdir",
                 File::Directory{
-                    .entries{
+                    .contents{
                         {
                             // subdir/file4.txt: contains hash1 again
                             "file4.txt",

src/libstore/binary-cache-store.cc

@@ -8,7 +8,7 @@
 #include "nix/util/sync.hh"
 #include "nix/store/remote-fs-accessor.hh"
 #include "nix/store/nar-info-disk-cache.hh"
-#include "nix/util/nar-accessor.hh"
+#include "nix/store/nar-accessor.hh"
 #include "nix/util/thread-pool.hh"
 #include "nix/util/callback.hh"
 #include "nix/util/signals.hh"
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNarDeep(*narAccessor, CanonPath::root)},
+            {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");

src/libstore/build/derivation-building-goal.cc

@@ -32,6 +32,14 @@ DerivationBuildingGoal::DerivationBuildingGoal(
     , drv{std::make_unique<Derivation>(drv)}
     , buildMode(buildMode)
 {
+    try {
+        drvOptions =
+            std::make_unique<DerivationOptions>(DerivationOptions::fromStructuredAttrs(drv.env, drv.structuredAttrs));
+    } catch (Error & e) {
+        e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
+        throw;
+    }
+
     name = fmt("building derivation '%s'", worker.store.printStorePath(drvPath));
     trace("created");
 
@@ -198,38 +206,6 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution(bool storeDerivation)
 
 Goal::Co DerivationBuildingGoal::tryToBuild()
 {
-    auto drvOptions = [&] {
-        DerivationOptions<SingleDerivedPath> temp;
-        try {
-            temp =
-                derivationOptionsFromStructuredAttrs(worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
-        } catch (Error & e) {
-            e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
-            throw;
-        }
-
-        auto res = tryResolve(
-            temp,
-            [&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
-                try {
-                    return resolveDerivedPath(
-                        worker.store, SingleDerivedPath::Built{drvPath, outputName}, &worker.evalStore);
-                } catch (Error &) {
-                    return std::nullopt;
-                }
-            });
-
-        /* The derivation must have all of its inputs gotten this point,
-           so the resolution will surely succeed.
-
-           (Actually, we shouldn't even enter this goal until we have a
-           resolved derivation, or derivation with only input addressed
-           transitive inputs, so this should be a no-opt anyways.)
-         */
-        assert(res);
-        return *res;
-    }();
-
     std::map<std::string, InitialOutput> initialOutputs;
 
     /* Recheck at this point. In particular, whereas before we were
@@ -368,13 +344,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
         /* Don't do a remote build if the derivation has the attribute
            `preferLocalBuild' set.  Also, check and repair modes are only
            supported for local builds. */
-        bool buildLocally = (buildMode != bmNormal || drvOptions.willBuildLocally(worker.store, *drv))
+        bool buildLocally = (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
                             && settings.maxBuildJobs.get() != 0;
 
         if (buildLocally) {
             useHook = false;
         } else {
-            switch (tryBuildHook(initialOutputs, drvOptions)) {
+            switch (tryBuildHook(initialOutputs)) {
             case rpAccept:
                 /* Yes, it has started doing so.  Wait until we get
                    EOF from the hook. */
@@ -403,7 +379,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
 
                 externalBuilder = settings.findExternalDerivationBuilderIfSupported(*drv);
 
-                if (!externalBuilder && !drvOptions.canBuildLocally(worker.store, *drv)) {
+                if (!externalBuilder && !drvOptions->canBuildLocally(worker.store, *drv)) {
                     auto msg =
                         fmt("Cannot build '%s'.\n"
                             "Reason: " ANSI_RED "required system or feature not available" ANSI_NORMAL
@@ -412,7 +388,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
                             "Current system: '%s' with features {%s}",
                             Magenta(worker.store.printStorePath(drvPath)),
                             Magenta(drv->platform),
-                            concatStringsSep(", ", drvOptions.getRequiredSystemFeatures(*drv)),
+                            concatStringsSep(", ", drvOptions->getRequiredSystemFeatures(*drv)),
                             Magenta(settings.thisSystem),
                             concatStringsSep<StringSet>(", ", worker.store.Store::config.systemFeatures));
 
@@ -610,7 +586,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
             }
 
             try {
-                desugaredEnv = DesugaredEnv::create(worker.store, *drv, drvOptions, inputPaths);
+                desugaredEnv = DesugaredEnv::create(worker.store, *drv, *drvOptions, inputPaths);
             } catch (BuildError & e) {
                 outputLocks.unlock();
                 worker.permanentFailure = true;
@@ -621,7 +597,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
                 .drvPath = drvPath,
                 .buildResult = buildResult,
                 .drv = *drv,
-                .drvOptions = drvOptions,
+                .drvOptions = *drvOptions,
                 .inputPaths = inputPaths,
                 .initialOutputs = initialOutputs,
                 .buildMode = buildMode,
@@ -827,8 +803,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
     return BuildError{e.status, msg};
 }
 
-HookReply DerivationBuildingGoal::tryBuildHook(
-    const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions)
+HookReply DerivationBuildingGoal::tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs)
 {
 #ifdef _WIN32 // TODO enable build hook on Windows
     return rpDecline;
@@ -845,7 +820,7 @@ HookReply DerivationBuildingGoal::tryBuildHook(
 
         /* Send the request to the hook. */
         worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform
-                          << worker.store.printStorePath(drvPath) << drvOptions.getRequiredSystemFeatures(*drv);
+                          << worker.store.printStorePath(drvPath) << drvOptions->getRequiredSystemFeatures(*drv);
         worker.hook->sink.flush();
 
         /* Read the first line of input, which should be a word indicating

src/libstore/build/derivation-check.cc

@@ -11,7 +11,7 @@ void checkOutputs(
     Store & store,
     const StorePath & drvPath,
     const decltype(Derivation::outputs) & drvOutputs,
-    const decltype(DerivationOptions<StorePath>::outputChecks) & outputChecks,
+    const decltype(DerivationOptions::outputChecks) & outputChecks,
     const std::map<std::string, ValidPathInfo> & outputs)
 {
     std::map<Path, const ValidPathInfo &> outputsByPath;
@@ -85,7 +85,7 @@ void checkOutputs(
             return std::make_pair(std::move(pathsDone), closureSize);
         };
 
-        auto applyChecks = [&](const DerivationOptions<StorePath>::OutputChecks & checks) {
+        auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) {
             if (checks.maxSize && info.narSize > *checks.maxSize)
                 throw BuildError(
                     BuildResult::Failure::OutputRejected,
@@ -105,33 +105,28 @@ void checkOutputs(
                         *checks.maxClosureSize);
             }
 
-            auto checkRefs = [&](const std::set<DrvRef<StorePath>> & value, bool allowed, bool recursive) {
+            auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) {
                 /* Parse a list of reference specifiers.  Each element must
                    either be a store path, or the symbolic name of the output
                    of the derivation (such as `out'). */
                 StorePathSet spec;
                 for (auto & i : value) {
-                    std::visit(
-                        overloaded{
-                            [&](const StorePath & path) { spec.insert(path); },
-                            [&](const OutputName & refOutputName) {
-                                if (auto output = get(outputs, refOutputName))
-                                    spec.insert(output->path);
-                                else {
-                                    std::string outputsListing =
-                                        concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; });
-                                    throw BuildError(
-                                        BuildResult::Failure::OutputRejected,
-                                        "derivation '%s' output check for '%s' contains output name '%s',"
-                                        " but this is not a valid output of this derivation."
-                                        " (Valid outputs are [%s].)",
-                                        store.printStorePath(drvPath),
-                                        outputName,
-                                        refOutputName,
-                                        outputsListing);
-                                }
-                            }},
-                        i);
+                    if (store.isStorePath(i))
+                        spec.insert(store.parseStorePath(i));
+                    else if (auto output = get(outputs, i))
+                        spec.insert(output->path);
+                    else {
+                        std::string outputsListing =
+                            concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; });
+                        throw BuildError(
+                            BuildResult::Failure::OutputRejected,
+                            "derivation '%s' output check for '%s' contains an illegal reference specifier '%s',"
+                            " expected store path or output name (one of [%s])",
+                            store.printStorePath(drvPath),
+                            outputName,
+                            i,
+                            outputsListing);
+                    }
                 }
 
                 auto used = recursive ? getClosure(info.path).first : info.references;
@@ -185,8 +180,8 @@ void checkOutputs(
 
         std::visit(
             overloaded{
-                [&](const DerivationOptions<StorePath>::OutputChecks & checks) { applyChecks(checks); },
-                [&](const std::map<std::string, DerivationOptions<StorePath>::OutputChecks> & checksPerOutput) {
+                [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); },
+                [&](const std::map<std::string, DerivationOptions::OutputChecks> & checksPerOutput) {
                     if (auto outputChecks = get(checksPerOutput, outputName))
 
                         applyChecks(*outputChecks);

src/libstore/build/derivation-check.hh

@@ -21,7 +21,7 @@ void checkOutputs(
     Store & store,
     const StorePath & drvPath,
     const decltype(Derivation::outputs) & drvOutputs,
-    const decltype(DerivationOptions<StorePath>::outputChecks) & drvOptions,
+    const decltype(DerivationOptions::outputChecks) & drvOptions,
     const std::map<std::string, ValidPathInfo> & outputs);
 
 } // namespace nix

src/libstore/build/derivation-env-desugar.cc

@@ -18,10 +18,7 @@ std::string & DesugaredEnv::atFileEnvPair(std::string_view name, std::string fil
 }
 
 DesugaredEnv DesugaredEnv::create(
-    Store & store,
-    const Derivation & drv,
-    const DerivationOptions<StorePath> & drvOptions,
-    const StorePathSet & inputPaths)
+    Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths)
 {
     DesugaredEnv res;
 
@@ -49,7 +46,7 @@ DesugaredEnv DesugaredEnv::create(
         }
 
         /* Handle exportReferencesGraph(), if set. */
-        for (auto & [fileName, storePaths] : drvOptions.exportReferencesGraph) {
+        for (auto & [fileName, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) {
             /* Write closure info to <fileName>. */
             res.extraFiles.insert_or_assign(
                 fileName, store.makeValidityRegistration(store.exportReferences(storePaths, inputPaths), false, false));

src/libstore/build/derivation-goal.cc

@@ -64,10 +64,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
 {
     trace("have derivation");
 
-    auto drvOptions = [&]() -> DerivationOptions<SingleDerivedPath> {
+    auto drvOptions = [&]() -> DerivationOptions {
         try {
-            return derivationOptionsFromStructuredAttrs(
-                worker.store, drv->inputDrvs, drv->env, get(drv->structuredAttrs));
+            return DerivationOptions::fromStructuredAttrs(drv->env, drv->structuredAttrs);
         } catch (Error & e) {
             e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
             throw;

src/libstore/derivation-options.cc

@@ -2,18 +2,15 @@
 #include "nix/util/json-utils.hh"
 #include "nix/store/parsed-derivations.hh"
 #include "nix/store/derivations.hh"
-#include "nix/store/derived-path.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/types.hh"
 #include "nix/util/util.hh"
 #include "nix/store/globals.hh"
-#include "nix/util/variant-wrapper.hh"
 
 #include <optional>
 #include <string>
 #include <variant>
 #include <regex>
-#include <ranges>
 
 namespace nix {
 
@@ -93,38 +90,14 @@ getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const st
     return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{});
 }
 
-template<typename Inputs>
-using OutputChecks = DerivationOptions<Inputs>::OutputChecks;
+using OutputChecks = DerivationOptions::OutputChecks;
 
-template<typename Inputs>
-using OutputChecksVariant = std::variant<OutputChecks<Inputs>, std::map<std::string, OutputChecks<Inputs>>>;
+using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
 
-DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs(
-    const StoreDirConfig & store,
-    const StringMap & env,
-    const StructuredAttrs * parsed,
-    bool shouldWarn,
-    const ExperimentalFeatureSettings & mockXpSettings)
+DerivationOptions DerivationOptions::fromStructuredAttrs(
+    const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn)
 {
-    /* Use the SingleDerivedPath version with empty inputDrvs, then
-       resolve. */
-    DerivedPathMap<StringSet> emptyInputDrvs{};
-    auto singleDerivedPathOptions =
-        derivationOptionsFromStructuredAttrs(store, emptyInputDrvs, env, parsed, shouldWarn, mockXpSettings);
-
-    /* "Resolve" all SingleDerivedPath inputs to StorePath. */
-    auto resolved = tryResolve(
-        singleDerivedPathOptions,
-        [&](ref<const SingleDerivedPath> drvPath, const std::string & outputName) -> std::optional<StorePath> {
-            // there should be nothing to resolve
-            assert(false);
-        });
-
-    /* Since we should never need to call the call back, there should be
-       no way it fails. */
-    assert(resolved);
-
-    return *resolved;
+    return fromStructuredAttrs(env, parsed ? &*parsed : nullptr);
 }
 
 static void flatten(const nlohmann::json & value, StringSet & res)
@@ -138,63 +111,10 @@ static void flatten(const nlohmann::json & value, StringSet & res)
         throw Error("'exportReferencesGraph' value is not an array or a string");
 }
 
-DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
-    const StoreDirConfig & store,
-    const DerivedPathMap<StringSet> & inputDrvs,
-    const StringMap & env,
-    const StructuredAttrs * parsed,
-    bool shouldWarn,
-    const ExperimentalFeatureSettings & mockXpSettings)
+DerivationOptions
+DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
 {
-    DerivationOptions<SingleDerivedPath> defaults = {};
-
-    std::map<std::string, SingleDerivedPath::Built> placeholders;
-    if (mockXpSettings.isEnabled(Xp::CaDerivations)) {
-        /* Initialize placeholder map from inputDrvs */
-        auto initPlaceholders = [&](this const auto & initPlaceholders,
-                                    ref<const SingleDerivedPath> basePath,
-                                    const DerivedPathMap<StringSet>::ChildNode & node) -> void {
-            for (const auto & outputName : node.value) {
-                auto built = SingleDerivedPath::Built{
-                    .drvPath = basePath,
-                    .output = outputName,
-                };
-                placeholders.insert_or_assign(
-                    DownstreamPlaceholder::fromSingleDerivedPathBuilt(built, mockXpSettings).render(),
-                    std::move(built));
-            }
-
-            for (const auto & [outputName, childNode] : node.childMap) {
-                initPlaceholders(
-                    make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{
-                        .drvPath = basePath,
-                        .output = outputName,
-                    }),
-                    childNode);
-            }
-        };
-
-        for (const auto & [drvPath, outputs] : inputDrvs.map) {
-            auto basePath = make_ref<const SingleDerivedPath>(SingleDerivedPath::Opaque{drvPath});
-            initPlaceholders(basePath, outputs);
-        }
-    }
-
-    auto parseSingleDerivedPath = [&](const std::string & pathS) -> SingleDerivedPath {
-        if (auto it = placeholders.find(pathS); it != placeholders.end())
-            return it->second;
-        else
-            return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
-    };
-
-    auto parseRef = [&](const std::string & pathS) -> DrvRef<SingleDerivedPath> {
-        if (auto it = placeholders.find(pathS); it != placeholders.end())
-            return it->second;
-        if (store.isStorePath(pathS))
-            return SingleDerivedPath::Opaque{store.toStorePath(pathS).first};
-        else
-            return pathS;
-    };
+    DerivationOptions defaults = {};
 
     if (shouldWarn && parsed) {
         auto & structuredAttrs = parsed->structuredAttrs;
@@ -226,14 +146,14 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
     }
 
     return {
-        .outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> {
+        .outputChecks = [&]() -> OutputChecksVariant {
             if (parsed) {
                 auto & structuredAttrs = parsed->structuredAttrs;
 
-                std::map<std::string, OutputChecks<SingleDerivedPath>> res;
+                std::map<std::string, OutputChecks> res;
                 if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
                     for (auto & [outputName, output_] : getObject(*outputChecks)) {
-                        OutputChecks<SingleDerivedPath> checks;
+                        OutputChecks checks;
 
                         auto & output = getObject(output_);
 
@@ -243,14 +163,13 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
                         if (auto maxClosureSize = get(output, "maxClosureSize"))
                             checks.maxClosureSize = maxClosureSize->get<uint64_t>();
 
-                        auto get_ =
-                            [&](const std::string & name) -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
+                        auto get_ = [&output = output](const std::string & name) -> std::optional<StringSet> {
                             if (auto i = get(output, name)) {
-                                std::set<DrvRef<SingleDerivedPath>> res;
+                                StringSet res;
                                 for (auto j = i->begin(); j != i->end(); ++j) {
                                     if (!j->is_string())
                                         throw Error("attribute '%s' must be a list of strings", name);
-                                    res.insert(parseRef(j->get<std::string>()));
+                                    res.insert(j->get<std::string>());
                                 }
                                 return res;
                             }
@@ -259,7 +178,7 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
 
                         res.insert_or_assign(
                             outputName,
-                            OutputChecks<SingleDerivedPath>{
+                            OutputChecks{
                                 .maxSize = [&]() -> std::optional<uint64_t> {
                                     if (auto maxSize = get(output, "maxSize"))
                                         return maxSize->get<uint64_t>();
@@ -273,32 +192,21 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
                                         return std::nullopt;
                                 }(),
                                 .allowedReferences = get_("allowedReferences"),
-                                .disallowedReferences =
-                                    get_("disallowedReferences").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
+                                .disallowedReferences = get_("disallowedReferences").value_or(StringSet{}),
                                 .allowedRequisites = get_("allowedRequisites"),
-                                .disallowedRequisites =
-                                    get_("disallowedRequisites").value_or(std::set<DrvRef<SingleDerivedPath>>{}),
+                                .disallowedRequisites = get_("disallowedRequisites").value_or(StringSet{}),
                             });
                     }
                 }
                 return res;
             } else {
-                auto parseRefSet = [&](const std::optional<StringSet> optionalStringSet)
-                    -> std::optional<std::set<DrvRef<SingleDerivedPath>>> {
-                    if (!optionalStringSet)
-                        return std::nullopt;
-                    auto range = *optionalStringSet | std::views::transform(parseRef);
-                    return std::set<DrvRef<SingleDerivedPath>>(range.begin(), range.end());
-                };
-                return OutputChecks<SingleDerivedPath>{
+                return OutputChecks{
                     // legacy non-structured-attributes case
                     .ignoreSelfRefs = true,
-                    .allowedReferences = parseRefSet(getStringSetAttr(env, parsed, "allowedReferences")),
-                    .disallowedReferences = parseRefSet(getStringSetAttr(env, parsed, "disallowedReferences"))
-                                                .value_or(std::set<DrvRef<SingleDerivedPath>>{}),
-                    .allowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "allowedRequisites")),
-                    .disallowedRequisites = parseRefSet(getStringSetAttr(env, parsed, "disallowedRequisites"))
-                                                .value_or(std::set<DrvRef<SingleDerivedPath>>{}),
+                    .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"),
+                    .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}),
+                    .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"),
+                    .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}),
                 };
             }
         }(),
@@ -337,19 +245,16 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
             }(),
         .exportReferencesGraph =
             [&] {
-                std::map<std::string, std::set<SingleDerivedPath>> ret;
+                std::map<std::string, StringSet> ret;
 
                 if (parsed) {
                     auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
                     if (!e || !e->is_object())
                         return ret;
-                    for (auto & [key, storePathsJson] : getObject(*e)) {
+                    for (auto & [key, value] : getObject(*e)) {
                         StringSet ss;
-                        flatten(storePathsJson, ss);
-                        std::set<SingleDerivedPath> storePaths;
-                        for (auto & s : ss)
-                            storePaths.insert(parseSingleDerivedPath(s));
-                        ret.insert_or_assign(key, std::move(storePaths));
+                        flatten(value, ss);
+                        ret.insert_or_assign(key, std::move(ss));
                     }
                 } else {
                     auto s = getOr(env, "exportReferencesGraph", "");
@@ -363,7 +268,7 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
                             throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
 
                         auto & storePathS = *i++;
-                        ret.insert_or_assign(std::move(fileName), std::set{parseSingleDerivedPath(storePathS)});
+                        ret.insert_or_assign(std::move(fileName), StringSet{storePathS});
                     }
                 }
                 return ret;
@@ -381,8 +286,28 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
     };
 }
 
-template<typename Input>
-StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivation & drv) const
+std::map<std::string, StorePathSet>
+DerivationOptions::getParsedExportReferencesGraph(const StoreDirConfig & store) const
+{
+    std::map<std::string, StorePathSet> res;
+
+    for (auto & [fileName, ss] : exportReferencesGraph) {
+        StorePathSet storePaths;
+        for (auto & storePathS : ss) {
+            if (!store.isInStore(storePathS))
+                throw BuildError(
+                    BuildResult::Failure::InputRejected,
+                    "'exportReferencesGraph' contains a non-store path '%1%'",
+                    storePathS);
+            storePaths.insert(store.toStorePath(storePathS).first);
+        }
+        res.insert_or_assign(fileName, storePaths);
+    }
+
+    return res;
+}
+
+StringSet DerivationOptions::getRequiredSystemFeatures(const BasicDerivation & drv) const
 {
     // FIXME: cache this?
     StringSet res;
@@ -393,8 +318,7 @@ StringSet DerivationOptions<Input>::getRequiredSystemFeatures(const BasicDerivat
     return res;
 }
 
-template<typename Input>
-bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
+bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDerivation & drv) const
 {
     if (drv.platform != settings.thisSystem.get() && !settings.extraPlatforms.get().count(drv.platform)
         && !drv.isBuiltin())
@@ -410,194 +334,42 @@ bool DerivationOptions<Input>::canBuildLocally(Store & localStore, const BasicDe
     return true;
 }
 
-template<typename Input>
-bool DerivationOptions<Input>::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
+bool DerivationOptions::willBuildLocally(Store & localStore, const BasicDerivation & drv) const
 {
     return preferLocalBuild && canBuildLocally(localStore, drv);
 }
 
-template<typename Input>
-bool DerivationOptions<Input>::substitutesAllowed() const
+bool DerivationOptions::substitutesAllowed() const
 {
     return settings.alwaysAllowSubstitutes ? true : allowSubstitutes;
 }
 
-template<typename Input>
-bool DerivationOptions<Input>::useUidRange(const BasicDerivation & drv) const
+bool DerivationOptions::useUidRange(const BasicDerivation & drv) const
 {
     return getRequiredSystemFeatures(drv).count("uid-range");
 }
 
-std::optional<DerivationOptions<StorePath>> tryResolve(
-    const DerivationOptions<SingleDerivedPath> & drvOptions,
-    std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
-        queryResolutionChain)
-{
-    auto tryResolvePath = [&](const SingleDerivedPath & input) -> std::optional<StorePath> {
-        return std::visit(
-            overloaded{
-                [](const SingleDerivedPath::Opaque & p) -> std::optional<StorePath> { return p.path; },
-                [&](const SingleDerivedPath::Built & p) -> std::optional<StorePath> {
-                    return queryResolutionChain(p.drvPath, p.output);
-                }},
-            input.raw());
-    };
-
-    auto tryResolveRef = [&](const DrvRef<SingleDerivedPath> & ref) -> std::optional<DrvRef<StorePath>> {
-        return std::visit(
-            overloaded{
-                [](const OutputName & outputName) -> std::optional<DrvRef<StorePath>> { return outputName; },
-                [&](const SingleDerivedPath & input) -> std::optional<DrvRef<StorePath>> {
-                    return tryResolvePath(input);
-                }},
-            ref);
-    };
-
-    auto tryResolveRefSet =
-        [&](const std::set<DrvRef<SingleDerivedPath>> & refSet) -> std::optional<std::set<DrvRef<StorePath>>> {
-        std::set<DrvRef<StorePath>> resolvedSet;
-        for (const auto & ref : refSet) {
-            auto resolvedRef = tryResolveRef(ref);
-            if (!resolvedRef)
-                return std::nullopt;
-            resolvedSet.insert(*resolvedRef);
-        }
-        return resolvedSet;
-    };
-
-    // Helper function to try resolving OutputChecks using functional style
-    auto tryResolveOutputChecks = [&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
-        -> std::optional<DerivationOptions<StorePath>::OutputChecks> {
-        std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedReferences;
-        if (checks.allowedReferences) {
-            resolvedAllowedReferences = tryResolveRefSet(*checks.allowedReferences);
-            if (!resolvedAllowedReferences)
-                return std::nullopt;
-        }
-
-        std::optional<std::set<DrvRef<StorePath>>> resolvedAllowedRequisites;
-        if (checks.allowedRequisites) {
-            resolvedAllowedRequisites = tryResolveRefSet(*checks.allowedRequisites);
-            if (!resolvedAllowedRequisites)
-                return std::nullopt;
-        }
-
-        auto resolvedDisallowedReferences = tryResolveRefSet(checks.disallowedReferences);
-        if (!resolvedDisallowedReferences)
-            return std::nullopt;
-
-        auto resolvedDisallowedRequisites = tryResolveRefSet(checks.disallowedRequisites);
-        if (!resolvedDisallowedRequisites)
-            return std::nullopt;
-
-        return DerivationOptions<StorePath>::OutputChecks{
-            .ignoreSelfRefs = checks.ignoreSelfRefs,
-            .maxSize = checks.maxSize,
-            .maxClosureSize = checks.maxClosureSize,
-            .allowedReferences = resolvedAllowedReferences,
-            .disallowedReferences = *resolvedDisallowedReferences,
-            .allowedRequisites = resolvedAllowedRequisites,
-            .disallowedRequisites = *resolvedDisallowedRequisites,
-        };
-    };
-
-    // Helper function to resolve exportReferencesGraph using functional style
-    auto tryResolveExportReferencesGraph = [&](const std::map<std::string, std::set<SingleDerivedPath>> & exportGraph)
-        -> std::optional<std::map<std::string, std::set<StorePath>>> {
-        std::map<std::string, std::set<StorePath>> resolved;
-        for (const auto & [name, inputPaths] : exportGraph) {
-            std::set<StorePath> resolvedPaths;
-            for (const auto & inputPath : inputPaths) {
-                auto resolvedPath = tryResolvePath(inputPath);
-                if (!resolvedPath)
-                    return std::nullopt;
-                resolvedPaths.insert(*resolvedPath);
-            }
-            resolved.emplace(name, std::move(resolvedPaths));
-        }
-        return resolved;
-    };
-
-    // Resolve outputChecks using functional style with std::visit
-    auto resolvedOutputChecks = std::visit(
-        overloaded{
-            [&](const DerivationOptions<SingleDerivedPath>::OutputChecks & checks)
-                -> std::optional<std::variant<
-                    DerivationOptions<StorePath>::OutputChecks,
-                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
-                auto resolved = tryResolveOutputChecks(checks);
-                if (!resolved)
-                    return std::nullopt;
-                return std::variant<
-                    DerivationOptions<StorePath>::OutputChecks,
-                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(*resolved);
-            },
-            [&](const std::map<std::string, DerivationOptions<SingleDerivedPath>::OutputChecks> & checksMap)
-                -> std::optional<std::variant<
-                    DerivationOptions<StorePath>::OutputChecks,
-                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>> {
-                std::map<std::string, DerivationOptions<StorePath>::OutputChecks> resolvedMap;
-                for (const auto & [outputName, checks] : checksMap) {
-                    auto resolved = tryResolveOutputChecks(checks);
-                    if (!resolved)
-                        return std::nullopt;
-                    resolvedMap.emplace(outputName, *resolved);
-                }
-                return std::variant<
-                    DerivationOptions<StorePath>::OutputChecks,
-                    std::map<std::string, DerivationOptions<StorePath>::OutputChecks>>(resolvedMap);
-            }},
-        drvOptions.outputChecks);
-
-    if (!resolvedOutputChecks)
-        return std::nullopt;
-
-    // Resolve exportReferencesGraph
-    auto resolvedExportGraph = tryResolveExportReferencesGraph(drvOptions.exportReferencesGraph);
-    if (!resolvedExportGraph)
-        return std::nullopt;
-
-    // Return resolved DerivationOptions using designated initializers
-    return DerivationOptions<StorePath>{
-        .outputChecks = *resolvedOutputChecks,
-        .unsafeDiscardReferences = drvOptions.unsafeDiscardReferences,
-        .passAsFile = drvOptions.passAsFile,
-        .exportReferencesGraph = *resolvedExportGraph,
-        .additionalSandboxProfile = drvOptions.additionalSandboxProfile,
-        .noChroot = drvOptions.noChroot,
-        .impureHostDeps = drvOptions.impureHostDeps,
-        .impureEnvVars = drvOptions.impureEnvVars,
-        .allowLocalNetworking = drvOptions.allowLocalNetworking,
-        .requiredSystemFeatures = drvOptions.requiredSystemFeatures,
-        .preferLocalBuild = drvOptions.preferLocalBuild,
-        .allowSubstitutes = drvOptions.allowSubstitutes,
-    };
-}
-
-template struct DerivationOptions<StorePath>;
-template struct DerivationOptions<SingleDerivedPath>;
-
 } // namespace nix
 
 namespace nlohmann {
 
 using namespace nix;
 
-DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDerivedPath>>::from_json(const json & json_)
+DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json_)
 {
     auto & json = getObject(json_);
 
     return {
-        .outputChecks = [&]() -> OutputChecksVariant<SingleDerivedPath> {
+        .outputChecks = [&]() -> OutputChecksVariant {
             auto outputChecks = getObject(valueAt(json, "outputChecks"));
 
             auto forAllOutputsOpt = optionalValueAt(outputChecks, "forAllOutputs");
             auto perOutputOpt = optionalValueAt(outputChecks, "perOutput");
 
             if (forAllOutputsOpt && !perOutputOpt) {
-                return static_cast<OutputChecks<SingleDerivedPath>>(*forAllOutputsOpt);
+                return static_cast<OutputChecks>(*forAllOutputsOpt);
             } else if (perOutputOpt && !forAllOutputsOpt) {
-                return static_cast<std::map<std::string, OutputChecks<SingleDerivedPath>>>(*perOutputOpt);
+                return static_cast<std::map<std::string, OutputChecks>>(*perOutputOpt);
             } else {
                 throw Error("Exactly one of 'perOutput' or 'forAllOutputs' is required");
             }
@@ -605,7 +377,7 @@ DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDeri
 
         .unsafeDiscardReferences = valueAt(json, "unsafeDiscardReferences"),
         .passAsFile = getStringSet(valueAt(json, "passAsFile")),
-        .exportReferencesGraph = valueAt(json, "exportReferencesGraph"),
+        .exportReferencesGraph = getMap<StringSet>(getObject(valueAt(json, "exportReferencesGraph")), getStringSet),
 
         .additionalSandboxProfile = getString(valueAt(json, "additionalSandboxProfile")),
         .noChroot = getBoolean(valueAt(json, "noChroot")),
@@ -619,17 +391,16 @@ DerivationOptions<SingleDerivedPath> adl_serializer<DerivationOptions<SingleDeri
     };
 }
 
-void adl_serializer<DerivationOptions<SingleDerivedPath>>::to_json(
-    json & json, const DerivationOptions<SingleDerivedPath> & o)
+void adl_serializer<DerivationOptions>::to_json(json & json, const DerivationOptions & o)
 {
     json["outputChecks"] = std::visit(
         overloaded{
-            [&](const OutputChecks<SingleDerivedPath> & checks) {
+            [&](const OutputChecks & checks) {
                 nlohmann::json outputChecks;
                 outputChecks["forAllOutputs"] = checks;
                 return outputChecks;
             },
-            [&](const std::map<std::string, OutputChecks<SingleDerivedPath>> & checksPerOutput) {
+            [&](const std::map<std::string, OutputChecks> & checksPerOutput) {
                 nlohmann::json outputChecks;
                 outputChecks["perOutput"] = checksPerOutput;
                 return outputChecks;
@@ -652,7 +423,7 @@ void adl_serializer<DerivationOptions<SingleDerivedPath>>::to_json(
     json["allowSubstitutes"] = o.allowSubstitutes;
 }
 
-OutputChecks<SingleDerivedPath> adl_serializer<OutputChecks<SingleDerivedPath>>::from_json(const json & json_)
+DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json_)
 {
     auto & json = getObject(json_);
 
@@ -660,16 +431,14 @@ OutputChecks<SingleDerivedPath> adl_serializer<OutputChecks<SingleDerivedPath>>:
         .ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")),
         .maxSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxSize"))),
         .maxClosureSize = ptrToOwned<uint64_t>(getNullable(valueAt(json, "maxClosureSize"))),
-        .allowedReferences =
-            ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedReferences"))),
-        .disallowedReferences = valueAt(json, "disallowedReferences"),
-        .allowedRequisites =
-            ptrToOwned<std::set<DrvRef<SingleDerivedPath>>>(getNullable(valueAt(json, "allowedRequisites"))),
-        .disallowedRequisites = valueAt(json, "disallowedRequisites"),
+        .allowedReferences = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedReferences"))),
+        .disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")),
+        .allowedRequisites = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedRequisites"))),
+        .disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")),
     };
 }
 
-void adl_serializer<OutputChecks<SingleDerivedPath>>::to_json(json & json, const OutputChecks<SingleDerivedPath> & c)
+void adl_serializer<DerivationOptions::OutputChecks>::to_json(json & json, const DerivationOptions::OutputChecks & c)
 {
     json["ignoreSelfRefs"] = c.ignoreSelfRefs;
     json["maxSize"] = c.maxSize;

src/libstore/derivations.cc

@@ -13,7 +13,6 @@
 #include <boost/container/small_vector.hpp>
 #include <boost/unordered/concurrent_flat_map.hpp>
 #include <nlohmann/json.hpp>
-#include <optional>
 
 namespace nix {
 
@@ -792,63 +791,71 @@ std::string outputPathName(std::string_view drvName, OutputNameView outputName)
 
 DerivationType BasicDerivation::type() const
 {
+    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs,
+        impureOutputs;
     std::optional<HashAlgorithm> floatingHashAlgo;
-    std::optional<DerivationType> ty;
-
-    auto decide = [&](DerivationType newTy) {
-        if (!ty)
-            ty = newTy;
-        else if (ty.value() != newTy)
-            throw Error("can't mix derivation output types");
-        else if (ty.value() == DerivationType::ContentAddressed{.sandboxed = false, .fixed = true})
-            // FIXME: Experimental feature?
-            throw Error("only one fixed output is allowed for now");
-    };
 
     for (auto & i : outputs) {
         std::visit(
             overloaded{
-                [&](const DerivationOutput::InputAddressed &) {
-                    decide(
-                        DerivationType::InputAddressed{
-                            .deferred = false,
-                        });
-                },
-                [&](const DerivationOutput::CAFixed &) {
-                    decide(
-                        DerivationType::ContentAddressed{
-                            .sandboxed = false,
-                            .fixed = true,
-                        });
-                    if (i.first != "out"sv)
-                        throw Error("single fixed output must be named \"out\"");
-                },
+                [&](const DerivationOutput::InputAddressed &) { inputAddressedOutputs.insert(i.first); },
+                [&](const DerivationOutput::CAFixed &) { fixedCAOutputs.insert(i.first); },
                 [&](const DerivationOutput::CAFloating & dof) {
-                    decide(
-                        DerivationType::ContentAddressed{
-                            .sandboxed = true,
-                            .fixed = false,
-                        });
-                    if (!floatingHashAlgo)
+                    floatingCAOutputs.insert(i.first);
+                    if (!floatingHashAlgo) {
                         floatingHashAlgo = dof.hashAlgo;
-                    else if (*floatingHashAlgo != dof.hashAlgo)
-                        throw Error("all floating outputs must use the same hash algorithm");
+                    } else {
+                        if (*floatingHashAlgo != dof.hashAlgo)
+                            throw Error("all floating outputs must use the same hash algorithm");
+                    }
                 },
-                [&](const DerivationOutput::Deferred &) {
-                    decide(
-                        DerivationType::InputAddressed{
-                            .deferred = true,
-                        });
-                },
-                [&](const DerivationOutput::Impure &) { decide(DerivationType::Impure{}); },
+                [&](const DerivationOutput::Deferred &) { deferredIAOutputs.insert(i.first); },
+                [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); },
             },
             i.second.raw);
     }
 
-    if (!ty)
+    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty() && impureOutputs.empty())
         throw Error("must have at least one output");
 
-    return ty.value();
+    if (!inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty() && impureOutputs.empty())
+        return DerivationType::InputAddressed{
+            .deferred = false,
+        };
+
+    if (inputAddressedOutputs.empty() && !fixedCAOutputs.empty() && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty() && impureOutputs.empty()) {
+        if (fixedCAOutputs.size() > 1)
+            // FIXME: Experimental feature?
+            throw Error("only one fixed output is allowed for now");
+        if (*fixedCAOutputs.begin() != "out"sv)
+            throw Error("single fixed output must be named \"out\"");
+        return DerivationType::ContentAddressed{
+            .sandboxed = false,
+            .fixed = true,
+        };
+    }
+
+    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && !floatingCAOutputs.empty()
+        && deferredIAOutputs.empty() && impureOutputs.empty())
+        return DerivationType::ContentAddressed{
+            .sandboxed = true,
+            .fixed = false,
+        };
+
+    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
+        && !deferredIAOutputs.empty() && impureOutputs.empty())
+        return DerivationType::InputAddressed{
+            .deferred = true,
+        };
+
+    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty() && !impureOutputs.empty())
+        return DerivationType::Impure{};
+
+    throw Error("can't mix derivation output types");
 }
 
 DrvHashes drvHashes;

src/libstore/downstream-placeholder.cc

@@ -1,6 +1,5 @@
 #include "nix/store/downstream-placeholder.hh"
 #include "nix/store/derivations.hh"
-#include "nix/util/json-utils.hh"
 
 namespace nix {
 
@@ -50,45 +49,3 @@ DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
 }
 
 } // namespace nix
-
-namespace nlohmann {
-
-using namespace nix;
-
-template<typename Item>
-DrvRef<Item> adl_serializer<DrvRef<Item>>::from_json(const json & json)
-{
-    // OutputName case: { "drvPath": "self", "output": <output> }
-    if (json.type() == nlohmann::json::value_t::object) {
-        auto & obj = getObject(json);
-        if (auto * drvPath_ = get(obj, "drvPath")) {
-            auto & drvPath = *drvPath_;
-            if (drvPath.type() == nlohmann::json::value_t::string && getString(drvPath) == "self") {
-                return getString(valueAt(obj, "output"));
-            }
-        }
-    }
-
-    // Input case
-    return adl_serializer<Item>::from_json(json);
-}
-
-template<typename Item>
-void adl_serializer<DrvRef<Item>>::to_json(json & json, const DrvRef<Item> & ref)
-{
-    std::visit(
-        overloaded{
-            [&](const OutputName & outputName) {
-                json = nlohmann::json::object();
-                json["drvPath"] = "self";
-                json["output"] = outputName;
-            },
-            [&](const Item & item) { json = item; },
-        },
-        ref);
-}
-
-template struct adl_serializer<nix::DrvRef<StorePath>>;
-template struct adl_serializer<nix::DrvRef<SingleDerivedPath>>;
-
-} // namespace nlohmann

src/libstore/filetransfer.cc

@@ -48,7 +48,7 @@ struct curlFileTransfer : public FileTransfer
     std::random_device rd;
     std::mt19937 mt19937;
 
-    struct TransferItem : public std::enable_shared_from_this<TransferItem>, public FileTransfer::Item
+    struct TransferItem : public std::enable_shared_from_this<TransferItem>
     {
         curlFileTransfer & fileTransfer;
         FileTransferRequest request;
@@ -60,7 +60,6 @@ struct curlFileTransfer : public FileTransfer
         // buffer to accompany the `req` above
         char errbuf[CURL_ERROR_SIZE];
         bool active = false; // whether the handle has been added to the multi object
-        bool paused = false; // whether the request has been paused previously
         std::string statusMsg;
 
         unsigned int attempt = 0;
@@ -117,13 +116,7 @@ struct curlFileTransfer : public FileTransfer
                        successful response. */
                     if (successfulStatuses.count(httpStatus)) {
                         writtenToSink += data.size();
-                        PauseTransfer needsPause = this->request.dataCallback(data);
-                        if (needsPause == PauseTransfer::Yes) {
-                            /* Smuggle the boolean flag into writeCallback. Note that
-                               the finalSink might get called multiple times if there's
-                               decompression going on. */
-                            paused = true;
-                        }
+                        this->request.dataCallback(data);
                     }
                 } else
                     this->result.data.append(data);
@@ -158,23 +151,15 @@ struct curlFileTransfer : public FileTransfer
             }
         }
 
-        void failEx(std::exception_ptr ex) noexcept
+        void failEx(std::exception_ptr ex)
         {
             assert(!done);
             done = true;
-            try {
-                std::rethrow_exception(ex);
-            } catch (nix::Error & e) {
-                /* Add more context to the error message. */
-                e.addTrace({}, "during %s of '%s'", Uncolored(request.verb()), request.uri.to_string());
-            } catch (...) {
-                /* Can't add more context to the error. */
-            }
             callback.rethrow(ex);
         }
 
         template<class T>
-        void fail(T && e) noexcept
+        void fail(T && e)
         {
             failEx(std::make_exception_ptr(std::forward<T>(e)));
         }
@@ -183,38 +168,32 @@ struct curlFileTransfer : public FileTransfer
         std::shared_ptr<FinishSink> decompressionSink;
         std::optional<StringSink> errorSink;
 
-        std::exception_ptr callbackException;
+        std::exception_ptr writeException;
 
-        size_t writeCallback(void * contents, size_t size, size_t nmemb) noexcept
-        try {
-            size_t realSize = size * nmemb;
-            result.bodySize += realSize;
+        size_t writeCallback(void * contents, size_t size, size_t nmemb)
+        {
+            try {
+                size_t realSize = size * nmemb;
+                result.bodySize += realSize;
 
-            if (!decompressionSink) {
-                decompressionSink = makeDecompressionSink(encoding, finalSink);
-                if (!successfulStatuses.count(getHTTPStatus())) {
-                    // In this case we want to construct a TeeSink, to keep
-                    // the response around (which we figure won't be big
-                    // like an actual download should be) to improve error
-                    // messages.
-                    errorSink = StringSink{};
+                if (!decompressionSink) {
+                    decompressionSink = makeDecompressionSink(encoding, finalSink);
+                    if (!successfulStatuses.count(getHTTPStatus())) {
+                        // In this case we want to construct a TeeSink, to keep
+                        // the response around (which we figure won't be big
+                        // like an actual download should be) to improve error
+                        // messages.
+                        errorSink = StringSink{};
+                    }
                 }
-            }
 
-            (*decompressionSink)({(char *) contents, realSize});
-            if (paused) {
-                /* The callback has signaled that the transfer needs to be
-                   paused. Already consumed data won't be returned twice unlike
-                   when returning CURL_WRITEFUNC_PAUSE.
-                   https://curl-library.cool.haxx.narkive.com/larE1cRA/curl-easy-pause-documentation-question
-                   */
-                curl_easy_pause(req, CURLPAUSE_RECV);
-            }
+                (*decompressionSink)({(char *) contents, realSize});
 
-            return realSize;
-        } catch (...) {
-            callbackException = std::current_exception();
-            return 0;
+                return realSize;
+            } catch (...) {
+                writeException = std::current_exception();
+                return 0;
+            }
         }
 
         static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -230,8 +209,8 @@ struct curlFileTransfer : public FileTransfer
                 result.urls.push_back(effectiveUriCStr);
         }
 
-        size_t headerCallback(void * contents, size_t size, size_t nmemb) noexcept
-        try {
+        size_t headerCallback(void * contents, size_t size, size_t nmemb)
+        {
             size_t realSize = size * nmemb;
             std::string line((char *) contents, realSize);
             printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
@@ -284,15 +263,6 @@ struct curlFileTransfer : public FileTransfer
                 }
             }
             return realSize;
-        } catch (...) {
-#if LIBCURL_VERSION_NUM >= 0x075700
-            /* https://curl.se/libcurl/c/CURLOPT_HEADERFUNCTION.html:
-               You can also abort the transfer by returning CURL_WRITEFUNC_ERROR. */
-            callbackException = std::current_exception();
-            return CURL_WRITEFUNC_ERROR;
-#else
-            return realSize;
-#endif
         }
 
         static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
@@ -300,17 +270,14 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
         }
 
-        int progressCallback(curl_off_t dltotal, curl_off_t dlnow) noexcept
-        try {
-            act.progress(dlnow, dltotal);
+        int progressCallback(curl_off_t dltotal, curl_off_t dlnow)
+        {
+            try {
+                act.progress(dlnow, dltotal);
+            } catch (nix::Interrupted &) {
+                assert(getInterrupted());
+            }
             return getInterrupted();
-        } catch (nix::Interrupted &) {
-            assert(getInterrupted());
-            return 1;
-        } catch (...) {
-            /* Something unexpected has happened like logger throwing an exception. */
-            callbackException = std::current_exception();
-            return 1;
         }
 
         static int progressCallbackWrapper(
@@ -321,14 +288,11 @@ struct curlFileTransfer : public FileTransfer
             return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow);
         }
 
-        static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) noexcept
-        try {
+        static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
+        {
             if (type == CURLINFO_TEXT)
                 vomit("curl: %s", chomp(std::string(data, size)));
             return 0;
-        } catch (...) {
-            /* Swallow the exception. Nothing left to do. */
-            return 0;
         }
 
         size_t readCallback(char * buffer, size_t size, size_t nitems) noexcept
@@ -338,7 +302,6 @@ struct curlFileTransfer : public FileTransfer
         } catch (EndOfFile &) {
             return 0;
         } catch (...) {
-            callbackException = std::current_exception();
             return CURL_READFUNC_ABORT;
         }
 
@@ -370,7 +333,6 @@ struct curlFileTransfer : public FileTransfer
             }
             return CURL_SEEKFUNC_OK;
         } catch (...) {
-            callbackException = std::current_exception();
             return CURL_SEEKFUNC_FAIL;
         }
 
@@ -379,15 +341,6 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) clientp)->seekCallback(offset, origin);
         }
 
-        void unpause()
-        {
-            /* Unpausing an already unpaused transfer is a no-op. */
-            if (paused) {
-                curl_easy_pause(req, CURLPAUSE_CONT);
-                paused = false;
-            }
-        }
-
         void init()
         {
             if (!req)
@@ -433,17 +386,17 @@ struct curlFileTransfer : public FileTransfer
             if (settings.downloadSpeed.get() > 0)
                 curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
 
-            if (request.method == HttpMethod::Head)
+            if (request.method == HttpMethod::HEAD)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
-            if (request.method == HttpMethod::Delete)
+            if (request.method == HttpMethod::DELETE)
                 curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
 
             if (request.data) {
-                if (request.method == HttpMethod::Post) {
+                if (request.method == HttpMethod::POST) {
                     curl_easy_setopt(req, CURLOPT_POST, 1L);
                     curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);
-                } else if (request.method == HttpMethod::Put) {
+                } else if (request.method == HttpMethod::PUT) {
                     curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
                     curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->sizeHint);
                 } else {
@@ -523,7 +476,7 @@ struct curlFileTransfer : public FileTransfer
                 try {
                     decompressionSink->finish();
                 } catch (...) {
-                    callbackException = std::current_exception();
+                    writeException = std::current_exception();
                 }
             }
 
@@ -532,8 +485,8 @@ struct curlFileTransfer : public FileTransfer
                 httpStatus = 304;
             }
 
-            if (callbackException)
-                failEx(callbackException);
+            if (writeException)
+                failEx(writeException);
 
             else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) {
                 result.cached = httpStatus == 304;
@@ -648,7 +601,7 @@ struct curlFileTransfer : public FileTransfer
                     errorSink.reset();
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
                     try {
-                        fileTransfer.enqueueItem(ref{shared_from_this()});
+                        fileTransfer.enqueueItem(shared_from_this());
                     } catch (const nix::Error & e) {
                         // If enqueue fails (e.g., during shutdown), fail the transfer properly
                         // instead of letting the exception propagate, which would leave done=false
@@ -665,24 +618,24 @@ struct curlFileTransfer : public FileTransfer
     {
         struct EmbargoComparator
         {
-            bool operator()(const ref<TransferItem> & i1, const ref<TransferItem> & i2)
+            bool operator()(const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2)
             {
                 return i1->embargo > i2->embargo;
             }
         };
 
-        std::priority_queue<ref<TransferItem>, std::vector<ref<TransferItem>>, EmbargoComparator> incoming;
-        std::vector<ref<TransferItem>> unpause;
+        std::
+            priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator>
+                incoming;
     private:
         bool quitting = false;
     public:
         void quit()
         {
             quitting = true;
-            /* We will not be processing any more incoming requests */
+            /* We wil not be processing any more incoming requests */
             while (!incoming.empty())
                 incoming.pop();
-            unpause.clear();
         }
 
         bool isQuitting()
@@ -851,17 +804,6 @@ struct curlFileTransfer : public FileTransfer
                 item->active = true;
                 items[item->req] = item;
             }
-
-            /* NOTE: Unpausing may invoke callbacks to flush all buffers. */
-            auto unpause = [&]() {
-                auto state(state_.lock());
-                auto res = state->unpause;
-                state->unpause.clear();
-                return res;
-            }();
-
-            for (auto & item : unpause)
-                item->unpause();
         }
 
         debug("download thread shutting down");
@@ -886,7 +828,7 @@ struct curlFileTransfer : public FileTransfer
         }
     }
 
-    ItemHandle enqueueItem(ref<TransferItem> item)
+    void enqueueItem(std::shared_ptr<TransferItem> item)
     {
         if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
             && item->request.uri.scheme() != "s3")
@@ -901,34 +843,19 @@ struct curlFileTransfer : public FileTransfer
 #ifndef _WIN32 // TODO need graceful async exit support on Windows?
         writeFull(wakeupPipe.writeSide.get(), " ");
 #endif
-
-        return ItemHandle(static_cast<Item &>(*item));
     }
 
-    ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
+    void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
     {
         /* Handle s3:// URIs by converting to HTTPS and optionally adding auth */
         if (request.uri.scheme() == "s3") {
             auto modifiedRequest = request;
             modifiedRequest.setupForS3();
-            return enqueueItem(make_ref<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
+            enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
+            return;
         }
 
-        return enqueueItem(make_ref<TransferItem>(*this, request, std::move(callback)));
-    }
-
-    void unpauseTransfer(ref<TransferItem> item)
-    {
-        auto state(state_.lock());
-        state->unpause.push_back(std::move(item));
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ");
-#endif
-    }
-
-    void unpauseTransfer(ItemHandle handle) override
-    {
-        unpauseTransfer(ref{static_cast<TransferItem &>(handle.item.get()).shared_from_this()});
+        enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
     }
 };
 
@@ -1025,7 +952,6 @@ void FileTransfer::download(
     struct State
     {
         bool quit = false;
-        bool paused = false;
         std::exception_ptr exc;
         std::string data;
         std::condition_variable avail, request;
@@ -1041,38 +967,31 @@ void FileTransfer::download(
         state->request.notify_one();
     });
 
-    request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer {
+    request.dataCallback = [_state](std::string_view data) {
         auto state(_state->lock());
 
         if (state->quit)
-            return PauseTransfer::No;
+            return;
+
+        /* If the buffer is full, then go to sleep until the calling
+           thread wakes us up (i.e. when it has removed data from the
+           buffer). We don't wait forever to prevent stalling the
+           download thread. (Hopefully sleeping will throttle the
+           sender.) */
+        if (state->data.size() > fileTransferSettings.downloadBufferSize) {
+            debug("download buffer is full; going to sleep");
+            static bool haveWarned = false;
+            warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
+            state.wait_for(state->request, std::chrono::seconds(10));
+        }
 
         /* Append data to the buffer and wake up the calling
            thread. */
         state->data.append(data);
         state->avail.notify_one();
-
-        if (state->data.size() <= fileTransferSettings.downloadBufferSize)
-            return PauseTransfer::No;
-
-        /* dataCallback gets called multiple times by an intermediate sink. Only
-           issue the debug message the first time around. */
-        if (!state->paused)
-            debug(
-                "pausing transfer for '%s': download buffer is full (%d > %d)",
-                uri,
-                state->data.size(),
-                fileTransferSettings.downloadBufferSize);
-
-        state->paused = true;
-
-        /* Technically the buffer might become larger than
-           downloadBufferSize, but with sinks there's no way to avoid
-           consuming data. */
-        return PauseTransfer::Yes;
     };
 
-    auto handle = enqueueFileTransfer(
+    enqueueFileTransfer(
         request, {[_state, resultCallback{std::move(resultCallback)}](std::future<FileTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
@@ -1105,10 +1024,6 @@ void FileTransfer::download(
                     return;
                 }
 
-                if (state->paused) {
-                    unpauseTransfer(handle);
-                    state->paused = false;
-                }
                 state.wait(state->avail);
 
                 if (state->data.empty())

src/libstore/http-binary-cache-store.cc

@@ -120,7 +120,7 @@ bool HttpBinaryCacheStore::fileExists(const std::string & path)
 
     try {
         FileTransferRequest request(makeRequest(path));
-        request.method = HttpMethod::Head;
+        request.method = HttpMethod::HEAD;
         getFileTransfer()->download(request);
         return true;
     } catch (FileTransferError & e) {
@@ -141,7 +141,7 @@ void HttpBinaryCacheStore::upload(
     std::optional<Headers> headers)
 {
     auto req = makeRequest(path);
-    req.method = HttpMethod::Put;
+    req.method = HttpMethod::PUT;
 
     if (headers) {
         req.headers.reserve(req.headers.size() + headers->size());

src/libstore/include/nix/store/build/derivation-builder.hh

@@ -69,7 +69,7 @@ struct DerivationBuilderParams
      *
      * @todo this should be part of `Derivation`.
      */
-    const DerivationOptions<StorePath> & drvOptions;
+    const DerivationOptions & drvOptions;
 
     // The remainder is state held during the build.
 

src/libstore/include/nix/store/build/derivation-building-goal.hh

@@ -52,6 +52,8 @@ private:
      */
     std::unique_ptr<Derivation> drv;
 
+    std::unique_ptr<DerivationOptions> drvOptions;
+
     /**
      * The remainder is state held during the build.
      */
@@ -113,8 +115,7 @@ private:
     /**
      * Is the build hook willing to perform the build?
      */
-    HookReply tryBuildHook(
-        const std::map<std::string, InitialOutput> & initialOutputs, const DerivationOptions<StorePath> & drvOptions);
+    HookReply tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs);
 
     /**
      * Open a log file and a pipe to it.

src/libstore/include/nix/store/build/derivation-env-desugar.hh

@@ -8,7 +8,6 @@ namespace nix {
 
 class Store;
 struct Derivation;
-template<typename Input>
 struct DerivationOptions;
 
 /**
@@ -78,10 +77,7 @@ struct DesugaredEnv
      * just part of `Derivation`.
      */
     static DesugaredEnv create(
-        Store & store,
-        const Derivation & drv,
-        const DerivationOptions<StorePath> & drvOptions,
-        const StorePathSet & inputPaths);
+        Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths);
 };
 
 } // namespace nix

src/libstore/include/nix/store/derivation-options.hh

@@ -8,8 +8,7 @@
 
 #include "nix/util/types.hh"
 #include "nix/util/json-impls.hh"
-#include "nix/store/store-dir-config.hh"
-#include "nix/store/downstream-placeholder.hh"
+#include "nix/store/path.hh"
 
 namespace nix {
 
@@ -18,9 +17,6 @@ struct StoreDirConfig;
 struct BasicDerivation;
 struct StructuredAttrs;
 
-template<typename V>
-struct DerivedPathMap;
-
 /**
  * This represents all the special options on a `Derivation`.
  *
@@ -38,7 +34,6 @@ struct DerivedPathMap;
  * separately. That would be nice to separate concerns, and not make any
  * environment variable names magical.
  */
-template<typename Input>
 struct DerivationOptions
 {
     struct OutputChecks
@@ -46,15 +41,13 @@ struct DerivationOptions
         bool ignoreSelfRefs = false;
         std::optional<uint64_t> maxSize, maxClosureSize;
 
-        using DrvRef = nix::DrvRef<Input>;
-
         /**
          * env: allowedReferences
          *
          * A value of `nullopt` indicates that the check is skipped.
          * This means that all references are allowed.
          */
-        std::optional<std::set<DrvRef>> allowedReferences;
+        std::optional<StringSet> allowedReferences;
 
         /**
          * env: disallowedReferences
@@ -62,21 +55,21 @@ struct DerivationOptions
          * No needed for `std::optional`, because skipping the check is
          * the same as disallowing the references.
          */
-        std::set<DrvRef> disallowedReferences;
+        StringSet disallowedReferences;
 
         /**
          * env: allowedRequisites
          *
          * See `allowedReferences`
          */
-        std::optional<std::set<DrvRef>> allowedRequisites;
+        std::optional<StringSet> allowedRequisites;
 
         /**
          * env: disallowedRequisites
          *
          * See `disallowedReferences`
          */
-        std::set<DrvRef> disallowedRequisites;
+        StringSet disallowedRequisites;
 
         bool operator==(const OutputChecks &) const = default;
     };
@@ -123,7 +116,23 @@ struct DerivationOptions
      * attributes give to the builder. The set of paths in the original JSON
      * is replaced with a list of `PathInfo` in JSON format.
      */
-    std::map<std::string, std::set<Input>> exportReferencesGraph;
+    std::map<std::string, StringSet> exportReferencesGraph;
+
+    /**
+     * Once a derivations is resolved, the strings in in
+     * `exportReferencesGraph` should all be store paths (with possible
+     * suffix paths, but those are discarded).
+     *
+     * @return The parsed path set for for each key in the map.
+     *
+     * @todo Ideally, `exportReferencesGraph` would just store
+     * `StorePath`s for this, but we can't just do that, because for CA
+     * derivations they is actually in general `DerivedPath`s (via
+     * placeholder strings) until the derivation is resolved and exact
+     * inputs store paths are known. We can use better types for that
+     * too, but that is a longer project.
+     */
+    std::map<std::string, StorePathSet> getParsedExportReferencesGraph(const StoreDirConfig & store) const;
 
     /**
      * env: __sandboxProfile
@@ -176,6 +185,18 @@ struct DerivationOptions
 
     bool operator==(const DerivationOptions &) const = default;
 
+    /**
+     * Parse this information from its legacy encoding as part of the
+     * environment. This should not be used with nice greenfield formats
+     * (e.g. JSON) but is necessary for supporting old formats (e.g.
+     * ATerm).
+     */
+    static DerivationOptions
+    fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true);
+
+    static DerivationOptions
+    fromStructuredAttrs(const StringMap & env, const std::optional<StructuredAttrs> & parsed, bool shouldWarn = true);
+
     /**
      * @param drv Must be the same derivation we parsed this from. In
      * the future we'll flip things around so a `BasicDerivation` has
@@ -201,49 +222,7 @@ struct DerivationOptions
     bool useUidRange(const BasicDerivation & drv) const;
 };
 
-extern template struct DerivationOptions<StorePath>;
-extern template struct DerivationOptions<SingleDerivedPath>;
-
-struct DerivationOutput;
-
-/**
- * Parse this information from its legacy encoding as part of the
- * environment. This should not be used with nice greenfield formats
- * (e.g. JSON) but is necessary for supporting old formats (e.g.
- * ATerm).
- */
-DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
-    const StoreDirConfig & store,
-    const DerivedPathMap<StringSet> & inputDrvs,
-    const StringMap & env,
-    const StructuredAttrs * parsed,
-    bool shouldWarn = true,
-    const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
-
-DerivationOptions<StorePath> derivationOptionsFromStructuredAttrs(
-    const StoreDirConfig & store,
-    const StringMap & env,
-    const StructuredAttrs * parsed,
-    bool shouldWarn = true,
-    const ExperimentalFeatureSettings & mockXpSettings = experimentalFeatureSettings);
-
-/**
- * This is the counterpart of `Derivation::tryResolve`. In particular,
- * it takes the same sort of callback, which is used to reolve
- * non-constant deriving paths.
- *
- * We need this function when resolving a derivation, and we will use
- * this as part of that if/when `Derivation` includes
- * `DerivationOptions`
- */
-std::optional<DerivationOptions<StorePath>> tryResolve(
-    const DerivationOptions<SingleDerivedPath> & drvOptions,
-    std::function<std::optional<StorePath>(ref<const SingleDerivedPath> drvPath, const std::string & outputName)>
-        queryResolutionChain);
-
 }; // namespace nix
 
-JSON_IMPL(nix::DerivationOptions<nix::StorePath>);
-JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>);
-JSON_IMPL(nix::DerivationOptions<nix::StorePath>::OutputChecks)
-JSON_IMPL(nix::DerivationOptions<nix::SingleDerivedPath>::OutputChecks)
+JSON_IMPL(DerivationOptions);
+JSON_IMPL(DerivationOptions::OutputChecks)

src/libstore/include/nix/store/downstream-placeholder.hh

@@ -2,23 +2,11 @@
 ///@file
 
 #include "nix/util/hash.hh"
-#include "nix/util/json-impls.hh"
 #include "nix/store/path.hh"
 #include "nix/store/derived-path.hh"
 
 namespace nix {
 
-/**
- * A reference is either to a to-be-registered output (by name),
- * or to an already-registered store object (by `Input`).
- *
- * `Ref<SingleDerivedPath` is a representation of something that can be
- * turned into a placeholder. (Regular own-output placeholder in the
- * first case, `DownstreamPlaceholder` in the second case.)
- */
-template<typename Input>
-using DrvRef = std::variant<OutputName, Input>;
-
 /**
  * Downstream Placeholders are opaque and almost certainly unique values
  * used to allow derivations to refer to store objects which are yet to
@@ -104,17 +92,3 @@ public:
 };
 
 } // namespace nix
-
-namespace nlohmann {
-
-template<typename Item>
-struct adl_serializer<nix::DrvRef<Item>>
-{
-    static nix::DrvRef<Item> from_json(const json & json);
-    static void to_json(json & json, const nix::DrvRef<Item> & t);
-};
-
-extern template struct adl_serializer<nix::DrvRef<nix::StorePath>>;
-extern template struct adl_serializer<nix::DrvRef<nix::SingleDerivedPath>>;
-
-} // namespace nlohmann

src/libstore/include/nix/store/filetransfer.hh

@@ -70,12 +70,12 @@ struct FileTransferSettings : Config
 
     Setting<size_t> downloadBufferSize{
         this,
-        1 * 1024 * 1024,
+        64 * 1024 * 1024,
         "download-buffer-size",
         R"(
           The size of Nix's internal download buffer in bytes during `curl` transfers. If data is
           not processed quickly enough to exceed the size of this buffer, downloads may stall.
-          The default is 1048576 (1 MiB).
+          The default is 67108864 (64 MiB).
         )"};
 };
 
@@ -87,11 +87,11 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT;
  * HTTP methods supported by FileTransfer.
  */
 enum struct HttpMethod {
-    Get,
-    Put,
-    Head,
-    Post,
-    Delete,
+    GET,
+    PUT,
+    HEAD,
+    POST,
+    DELETE,
 };
 
 /**
@@ -105,17 +105,12 @@ struct UsernameAuth
     std::optional<std::string> password;
 };
 
-enum class PauseTransfer : bool {
-    No = false,
-    Yes = true,
-};
-
 struct FileTransferRequest
 {
     VerbatimURL uri;
     Headers headers;
     std::string expectedETag;
-    HttpMethod method = HttpMethod::Get;
+    HttpMethod method = HttpMethod::GET;
     size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT;
     ActivityId parentAct;
@@ -141,14 +136,7 @@ struct FileTransferRequest
 
     std::optional<UploadData> data;
     std::string mimeType;
-
-    /**
-     * Callbacked invoked with a chunk of received data.
-     * Can pause the transfer by returning PauseTransfer::Yes. No data must be consumed
-     * if transfer is paused.
-     */
-    std::function<PauseTransfer(std::string_view data)> dataCallback;
-
+    std::function<void(std::string_view data)> dataCallback;
     /**
      * Optional username and password for HTTP basic authentication.
      * When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option.
@@ -176,14 +164,14 @@ struct FileTransferRequest
     std::string verb() const
     {
         switch (method) {
-        case HttpMethod::Head:
-        case HttpMethod::Get:
+        case HttpMethod::HEAD:
+        case HttpMethod::GET:
             return "download";
-        case HttpMethod::Put:
-        case HttpMethod::Post:
+        case HttpMethod::PUT:
+        case HttpMethod::POST:
             assert(data);
             return "upload";
-        case HttpMethod::Delete:
+        case HttpMethod::DELETE:
             return "delet";
         }
         unreachable();
@@ -238,25 +226,6 @@ class Store;
 
 struct FileTransfer
 {
-protected:
-    class Item
-    {};
-
-public:
-    /**
-     * An opaque handle to the file transfer. Can be used to reference an in-flight transfer operations.
-     */
-    struct ItemHandle
-    {
-        std::reference_wrapper<Item> item;
-        friend struct FileTransfer;
-
-        ItemHandle(Item & item)
-            : item(item)
-        {
-        }
-    };
-
     virtual ~FileTransfer() {}
 
     /**
@@ -264,13 +233,7 @@ public:
      * the download. The future may throw a FileTransferError
      * exception.
      */
-    virtual ItemHandle
-    enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
-
-    /**
-     * Unpause a transfer that has been previously paused by a dataCallback.
-     */
-    virtual void unpauseTransfer(ItemHandle handle) = 0;
+    virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
 
     std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
 

src/libstore/include/nix/store/meson.build

@@ -55,6 +55,7 @@ headers = [ config_pub_h ] + files(
   'machines.hh',
   'make-content-addressed.hh',
   'names.hh',
+  'nar-accessor.hh',
   'nar-info-disk-cache.hh',
   'nar-info.hh',
   'outputs-spec.hh',

src/libstore/include/nix/store/nar-accessor.hh

@@ -0,0 +1,43 @@
+#pragma once
+///@file
+
+#include "nix/util/source-accessor.hh"
+
+#include <functional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+struct Source;
+
+/**
+ * Return an object that provides access to the contents of a NAR
+ * file.
+ */
+ref<SourceAccessor> makeNarAccessor(std::string && nar);
+
+ref<SourceAccessor> makeNarAccessor(Source & source);
+
+/**
+ * Create a NAR accessor from a NAR listing (in the format produced by
+ * listNar()). The callback getNarBytes(offset, length) is used by the
+ * readFile() method of the accessor to get the contents of files
+ * inside the NAR.
+ */
+using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
+
+/**
+ * The canonical GetNarBytes function for a seekable Source.
+ */
+GetNarBytes seekableGetNarBytes(const Path & path);
+
+ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
+
+/**
+ * Write a JSON representation of the contents of a NAR (except file
+ * contents).
+ */
+nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
+
+} // namespace nix