libutil: Remove really dead code for git hashing

It's not being excersised by anything and already contains
bugs (like the fact that copyRecursive only handles the first entry
in a directory). Evidently, it can just be removed and was never used
by anything.

doc/manual/package.nix

@@ -37,7 +37,6 @@ mkMesonDerivation (finalAttrs: {
       (fileset.unions [
         ../../.version
         # For example JSON
-        ../../src/libutil-tests/data/memory-source-accessor
         ../../src/libutil-tests/data/hash
         ../../src/libstore-tests/data/content-address
         ../../src/libstore-tests/data/store-path

doc/manual/rl-next/libcurl-pausing.md

@@ -1,12 +0,0 @@
----
-synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
-prs: [14614]
-issues: [11728]
----
-
-The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
-[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
-
-This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
-
-Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).

doc/manual/source/SUMMARY.md.in

@@ -121,7 +121,6 @@
 - [Architecture and Design](architecture/architecture.md)
 - [Formats and Protocols](protocols/index.md)
   - [JSON Formats](protocols/json/index.md)
-    - [File System Object](protocols/json/file-system-object.md)
     - [Hash](protocols/json/hash.md)
     - [Content Address](protocols/json/content-address.md)
     - [Store Path](protocols/json/store-path.md)

doc/manual/source/development/debugging.md

@@ -15,7 +15,7 @@ In the development shell, set the `mesonBuildType` environment variable to `debu
 Then, proceed to build Nix as described in [Building Nix](./building.md).
 This will build Nix with debug symbols, which are essential for effective debugging.
 
-It is also possible to build without optimization for faster build:
+It is also possible to build without debugging for faster build:
 
 ```console
 [nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify)

doc/manual/source/protocols/json/file-system-object.md

@@ -1,21 +0,0 @@
-{{#include file-system-object-v1-fixed.md}}
-
-## Examples
-
-### Simple
-
-```json
-{{#include schema/file-system-object-v1/simple.json}}
-```
-
-### Complex
-
-```json
-{{#include schema/file-system-object-v1/complex.json}}
-```
-
-<!-- need to convert YAML to JSON first
-## Raw Schema
-
-[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
--->

doc/manual/source/protocols/json/fixup-json-schema-generated-doc.sed

@@ -11,7 +11,6 @@ s/\\`/`/g
 #
 # As we have more such relative links, more replacements of this nature
 # should appear below.
-s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
 s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
 s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
 s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g

doc/manual/source/protocols/json/meson.build

@@ -9,7 +9,6 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
 json_schema_config = files('json-schema-for-humans-config.yaml')
 
 schemas = [
-  'file-system-object-v1',
   'hash-v1',
   'content-address-v1',
   'store-path-v1',

doc/manual/source/protocols/json/schema/file-system-object-v1

@@ -1 +0,0 @@
-../../../../../../src/libutil-tests/data/memory-source-accessor

doc/manual/source/protocols/json/schema/file-system-object-v1.yaml

@@ -1,71 +0,0 @@
-"$schema": http://json-schema.org/draft-04/schema#
-"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
-title: File System Object
-description: |
-  This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
-
-  The schema is recursive because file system objects contain other file system objects.
-type: object
-required: ["type"]
-properties:
-  type:
-    type: string
-    enum: ["regular", "symlink", "directory"]
-
-# Enforce conditional structure based on `type`
-anyOf:
-  - $ref: "#/$defs/regular"
-    required: ["type", "contents"]
-
-  - $ref: "#/$defs/directory"
-    required: ["type", "entries"]
-
-  - $ref: "#/$defs/symlink"
-    required: ["type", "target"]
-
-"$defs":
-  regular:
-    title: Regular File
-    description: |
-      See [Regular File](@docroot@/store/file-system-object.md#regular) in the manual for details.
-    required: ["contents"]
-    properties:
-      type:
-        const: "regular"
-      contents:
-        type: string
-        description: File contents
-      executable:
-        type: boolean
-        description: Whether the file is executable.
-        default: false
-    additionalProperties: false
-
-  directory:
-    title: Directory
-    description: |
-      See [Directory](@docroot@/store/file-system-object.md#directory) in the manual for details.
-    required: ["entries"]
-    properties:
-      type:
-        const: "directory"
-      entries:
-        type: object
-        description: |
-          Map of names to nested file system objects (for type=directory)
-        additionalProperties:
-          $ref: "#"
-    additionalProperties: false
-
-  symlink:
-    title: Symbolic Link
-    description: |
-      See [Symbolic Link](@docroot@/store/file-system-object.md#symlink) in the manual for details.
-    required: ["target"]
-    properties:
-      type:
-        const: "symlink"
-      target:
-        type: string
-        description: Target path of the symlink.
-    additionalProperties: false

doc/manual/source/store/file-system-object.md

@@ -3,23 +3,19 @@
 Nix uses a simplified model of the file system, which consists of file system objects.
 Every file system object is one of the following:
 
- - [**Regular File**]{#regular}
+ - File
 
    - A possibly empty sequence of bytes for contents
    - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission
 
- - [**Directory**]{#directory}
+ - Directory
 
    Mapping of names to child file system objects
 
- - [**Symbolic link**]{#symlink}
+ - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link)
 
-   An arbitrary string, known as the *target* of the symlink.
+   An arbitrary string.
-
+   Nix does not assign any semantics to symbolic links.
-   In general, Nix does not assign any semantics to symbolic links.
-   Certain operations however, may make additional assumptions and attempt to use the target to find another file system object.
-
-   > See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept.
 
 File system objects and their children form a tree.
 A bare file or symlink can be a root file system object.

src/json-schema-checks/file-system-object

@@ -1 +0,0 @@
-../../src/libutil-tests/data/memory-source-accessor

src/json-schema-checks/meson.build

@@ -20,14 +20,6 @@ schema_dir = meson.current_source_dir() / 'schema'
 
 # Get all example files
 schemas = [
-  {
-    'stem' : 'file-system-object',
-    'schema' : schema_dir / 'file-system-object-v1.yaml',
-    'files' : [
-      'simple.json',
-      'complex.json',
-    ],
-  },
   {
     'stem' : 'hash',
     'schema' : schema_dir / 'hash-v1.yaml',

src/json-schema-checks/package.nix

@@ -20,7 +20,6 @@ mkMesonDerivation (finalAttrs: {
   fileset = lib.fileset.unions [
     ../../.version
     ../../doc/manual/source/protocols/json/schema
-    ../../src/libutil-tests/data/memory-source-accessor
     ../../src/libutil-tests/data/hash
     ../../src/libstore-tests/data/content-address
     ../../src/libstore-tests/data/store-path

src/libcmd/common-eval-args.cc

@@ -168,9 +168,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
                                 ? state.rootPath(absPath(getCommandBaseDir()))
                                 : state.rootPath(".")));
                 },
-                [&](const AutoArgString & arg) { v->mkString(arg.s, state.mem); },
+                [&](const AutoArgString & arg) { v->mkString(arg.s); },
-                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string()), state.mem); },
+                [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string())); },
-                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO), state.mem); }},
+                [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO)); }},
             arg);
         res.insert(state.symbols.create(name), v);
     }

src/libexpr-c/nix_api_expr.cc

@@ -69,8 +69,8 @@ nix_err nix_expr_eval_from_string(
         context->last_err_code = NIX_OK;
     try {
         nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
-        state->state.eval(parsedExpr, *value->value);
+        state->state.eval(parsedExpr, value->value);
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -80,8 +80,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.callFunction(*fn->value, *arg->value, *value->value, nix::noPos);
+        state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -91,15 +91,9 @@ nix_err nix_value_call_multi(
 {
     if (context)
         context->last_err_code = NIX_OK;
-
-    std::vector<nix::Value *> internal_args;
-    internal_args.reserve(nargs);
-    for (size_t i = 0; i < nargs; i++)
-        internal_args.push_back(args[i]->value);
-
     try {
-        state->state.callFunction(*fn->value, {internal_args.data(), nargs}, *value->value, nix::noPos);
+        state->state.callFunction(fn->value, {(nix::Value **) args, nargs}, value->value, nix::noPos);
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -109,7 +103,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValue(*value->value, nix::noPos);
+        state->state.forceValue(value->value, nix::noPos);
     }
     NIXC_CATCH_ERRS
 }
@@ -119,7 +113,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        state->state.forceValueDeep(*value->value);
+        state->state.forceValueDeep(value->value);
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-c/nix_api_expr_internal.h

@@ -39,13 +39,7 @@ struct ListBuilder
 
 struct nix_value
 {
-    nix::Value * value;
+    nix::Value value;
-    /**
-     * As we move to a managed heap, we need EvalMemory in more places. Ideally, we would take in EvalState or
-     * EvalMemory as an argument when we need it, but we don't want to make changes to the stable C api, so we stuff it
-     * into the nix_value that will get passed in to the relevant functions.
-     */
-    nix::EvalMemory * mem;
 };
 
 struct nix_string_return

src/libexpr-c/nix_api_value.cc

@@ -20,7 +20,7 @@ static const nix::Value & check_value_not_null(const nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return *value->value;
+    return *((const nix::Value *) value);
 }
 
 static nix::Value & check_value_not_null(nix_value * value)
@@ -28,7 +28,7 @@ static nix::Value & check_value_not_null(nix_value * value)
     if (!value) {
         throw std::runtime_error("nix_value is null");
     }
-    return *value->value;
+    return value->value;
 }
 
 static const nix::Value & check_value_in(const nix_value * value)
@@ -58,14 +58,9 @@ static nix::Value & check_value_out(nix_value * value)
     return v;
 }
 
-static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
+static inline nix_value * as_nix_value_ptr(nix::Value * v)
 {
-    nix_value * ret = new (mem.allocBytes(sizeof(nix_value))) nix_value{
+    return reinterpret_cast<nix_value *>(v);
-        .value = v,
-        .mem = &mem,
-    };
-    nix_gc_incref(nullptr, ret);
-    return ret;
 }
 
 /**
@@ -74,13 +69,7 @@ static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
  * Deals with errors and converts arguments from C++ into C types.
  */
 static void nix_c_primop_wrapper(
-    PrimOpFun f,
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
-    void * userdata,
-    int arity,
-    nix::EvalState & state,
-    const nix::PosIdx pos,
-    nix::Value ** args,
-    nix::Value & v)
 {
     nix_c_context ctx;
 
@@ -96,15 +85,8 @@ static void nix_c_primop_wrapper(
     // ok because we don't see a need for this yet (e.g. inspecting thunks,
     // or maybe something to make blackholes work better; we don't know).
     nix::Value vTmp;
-    nix_value * vTmpPtr = new_nix_value(&vTmp, state.mem);
 
-    std::vector<nix_value *> external_args;
+    f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
-    external_args.reserve(arity);
-    for (int i = 0; i < arity; i++) {
-        nix_value * external_arg = new_nix_value(args[i], state.mem);
-        external_args.push_back(external_arg);
-    }
-    f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr);
 
     if (ctx.last_err_code != NIX_OK) {
         /* TODO: Throw different errors depending on the error code */
@@ -153,7 +135,7 @@ PrimOp * nix_alloc_primop(
                     .args = {},
                     .arity = (size_t) arity,
                     .doc = doc,
-                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)};
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
         if (args)
             for (size_t i = 0; args[i]; i++)
                 p->args.emplace_back(*args);
@@ -178,7 +160,8 @@ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
     if (context)
         context->last_err_code = NIX_OK;
     try {
-        nix_value * res = new_nix_value(state->state.allocValue(), state->state.mem);
+        nix_value * res = as_nix_value_ptr(state->state.allocValue());
+        nix_gc_incref(nullptr, res);
         return res;
     }
     NIXC_CATCH_ERRS_NULL
@@ -348,10 +331,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
             return nullptr;
         }
         auto * p = v.listView()[ix];
-        if (p == nullptr)
+        nix_gc_incref(nullptr, p);
-            return nullptr;
+        if (p != nullptr)
-        state->state.forceValue(*p, nix::noPos);
+            state->state.forceValue(*p, nix::noPos);
-        return new_nix_value(p, state->state.mem);
+        return as_nix_value_ptr(p);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -369,8 +352,9 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt
             return nullptr;
         }
         auto * p = v.listView()[ix];
+        nix_gc_incref(nullptr, p);
         // Note: intentionally NOT calling forceValue() to keep the element lazy
-        return new_nix_value(p, state->state.mem);
+        return as_nix_value_ptr(p);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -385,8 +369,9 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
+            nix_gc_incref(nullptr, attr->value);
             state->state.forceValue(*attr->value, nix::noPos);
-            return new_nix_value(attr->value, state->state.mem);
+            return as_nix_value_ptr(attr->value);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -405,8 +390,9 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS
         nix::Symbol s = state->state.symbols.create(name);
         auto attr = v.attrs()->get(s);
         if (attr) {
+            nix_gc_incref(nullptr, attr->value);
             // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-            return new_nix_value(attr->value, state->state.mem);
+            return as_nix_value_ptr(attr->value);
         }
         nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
         return nullptr;
@@ -454,8 +440,9 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
         state->state.forceValue(*a.value, nix::noPos);
-        return new_nix_value(a.value, state->state.mem);
+        return as_nix_value_ptr(a.value);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -474,8 +461,9 @@ nix_value * nix_get_attr_byidx_lazy(
         }
         const nix::Attr & a = (*v.attrs())[i];
         *name = state->state.symbols[a.name].c_str();
+        nix_gc_incref(nullptr, a.value);
         // Note: intentionally NOT calling forceValue() to keep the attribute lazy
-        return new_nix_value(a.value, state->state.mem);
+        return as_nix_value_ptr(a.value);
     }
     NIXC_CATCH_ERRS_NULL
 }
@@ -515,7 +503,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkString(std::string_view(str), *value->mem);
+        v.mkString(std::string_view(str));
     }
     NIXC_CATCH_ERRS
 }
@@ -526,7 +514,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value *
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_out(value);
-        v.mkPath(s->state.rootPath(nix::CanonPath(str)), s->state.mem);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
     }
     NIXC_CATCH_ERRS
 }

src/libexpr-tests/json.cc

@@ -73,7 +73,7 @@ TEST_F(JSONValueTest, StringQuotes)
 TEST_F(JSONValueTest, DISABLED_Path)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("/test")), state.mem);
+    v.mkPath(state.rootPath(CanonPath("/test")));
     ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
 }
 } /* namespace nix */

src/libexpr-tests/value/print.cc

@@ -268,7 +268,7 @@ struct StringPrintingTests : LibExprTest
     void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args)
     {
         Value v;
-        v.mkString(literal, state.mem);
+        v.mkString(literal);
 
         std::stringstream out;
         printValue(state, out, v, PrintOptions{.maxStringLength = maxLength});
@@ -353,7 +353,7 @@ TEST_F(ValuePrintingTests, ansiColorsStringElided)
 TEST_F(ValuePrintingTests, ansiColorsPath)
 {
     Value v;
-    v.mkPath(state.rootPath(CanonPath("puppy")), state.mem);
+    v.mkPath(state.rootPath(CanonPath("puppy")));
 
     test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true});
 }

src/libexpr/eval.cc

@@ -81,20 +81,20 @@ static const char * makeImmutableString(std::string_view s)
     return t;
 }
 
-StringData & StringData::alloc(EvalMemory & mem, size_t size)
+StringData & StringData::alloc(size_t size)
 {
-    void * t = mem.allocBytes(sizeof(StringData) + size + 1);
+    void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
     if (!t)
         throw std::bad_alloc();
     auto res = new (t) StringData(size);
     return *res;
 }
 
-const StringData & StringData::make(EvalMemory & mem, std::string_view s)
+const StringData & StringData::make(std::string_view s)
 {
     if (s.empty())
         return ""_sds;
-    auto & res = alloc(mem, s.size());
+    auto & res = alloc(s.size());
     std::memcpy(&res.data_, s.data(), s.size());
     res.data_[s.size()] = '\0';
     return res;
@@ -849,9 +849,9 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
         evalState.runDebugRepl(nullptr, trace.env, trace.expr);
 }
 
-void Value::mkString(std::string_view s, EvalMemory & mem)
+void Value::mkString(std::string_view s)
 {
-    mkStringNoCopy(StringData::make(mem, s));
+    mkStringNoCopy(StringData::make(s));
 }
 
 Value::StringWithContext::Context *
@@ -862,13 +862,13 @@ Value::StringWithContext::Context::fromBuilder(const NixStringContext & context,
 
     auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
     std::ranges::transform(
-        context, ctx->elems, [&](const NixStringContextElem & elt) { return &StringData::make(mem, elt.to_string()); });
+        context, ctx->elems, [](const NixStringContextElem & elt) { return &StringData::make(elt.to_string()); });
     return ctx;
 }
 
 void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem)
 {
-    mkStringNoCopy(StringData::make(mem, s), Value::StringWithContext::Context::fromBuilder(context, mem));
+    mkStringNoCopy(StringData::make(s), Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
 void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem)
@@ -876,9 +876,9 @@ void Value::mkStringMove(const StringData & s, const NixStringContext & context,
     mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
-void Value::mkPath(const SourcePath & path, EvalMemory & mem)
+void Value::mkPath(const SourcePath & path)
 {
-    mkPath(&*path.accessor, StringData::make(mem, path.path.abs()));
+    mkPath(&*path.accessor, StringData::make(path.path.abs()));
 }
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@@ -943,7 +943,7 @@ void EvalState::mkPos(Value & v, PosIdx p)
     auto origin = positions.originOf(p);
     if (auto path = std::get_if<SourcePath>(&origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(s.file).mkString(path->path.abs(), mem);
+        attrs.alloc(s.file).mkString(path->path.abs());
         makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column));
         v.mkAttrs(attrs);
     } else
@@ -1751,9 +1751,9 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
     // 4: about 60
     // 5: under 10
     // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
-    SmallValueVector<4> vArgs(args->size());
+    SmallValueVector<4> vArgs(args.size());
-    for (size_t i = 0; i < args->size(); ++i)
+    for (size_t i = 0; i < args.size(); ++i)
-        vArgs[i] = (*args)[i]->maybeThunk(state, env);
+        vArgs[i] = args[i]->maybeThunk(state, env);
 
     state.callFunction(vFun, vArgs, v, pos);
 }
@@ -2139,9 +2139,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
         for (const auto & part : strings) {
             resultStr += *part;
         }
-        v.mkPath(state.rootPath(CanonPath(resultStr)), state.mem);
+        v.mkPath(state.rootPath(CanonPath(resultStr)));
     } else {
-        auto & resultStr = StringData::alloc(state.mem, sSize);
+        auto & resultStr = StringData::alloc(sSize);
         auto * tmp = resultStr.data();
         for (const auto & part : strings) {
             std::memcpy(tmp, part->data(), part->size());
@@ -2188,8 +2188,6 @@ void EvalState::forceValueDeep(Value & v)
     std::set<const Value *> seen;
 
     [&, &state(*this)](this const auto & recurse, Value & v) {
-        auto _level = state.addCallDepth(v.determinePos(noPos));
-
         if (!seen.insert(&v).second)
             return;
 
@@ -2216,15 +2214,8 @@ void EvalState::forceValueDeep(Value & v)
         }
 
         else if (v.isList()) {
-            size_t index = 0;
             for (auto v2 : v.listView())
-                try {
+                recurse(*v2);
-                    recurse(*v2);
-                    index++;
-                } catch (Error & e) {
-                    state.addErrorTrace(e, "while evaluating list element at index %1%", index);
-                    throw;
-                }
         }
     }(v);
 }

src/libexpr/include/nix/expr/nixexpr.hh

@@ -592,14 +592,11 @@ public:
 struct ExprCall : Expr
 {
     Expr * fun;
-    /**
+    std::vector<Expr *> args;
-     * args will never be null. See comment on ExprAttrs::AttrDefs below.
-     */
-    std::optional<std::pmr::vector<Expr *>> args;
     PosIdx pos;
     std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
+    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -607,7 +604,7 @@ struct ExprCall : Expr
     {
     }
 
-    ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         : fun(fun)
         , args(args)
         , pos(pos)
@@ -839,7 +836,7 @@ public:
     // we define some calls to add explicitly so that the argument can be passed in as initializer lists
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args)
+    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args));
@@ -847,7 +844,7 @@ public:
 
     template<class C>
     [[gnu::always_inline]]
-    C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos)
+    C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
         requires(std::same_as<C, ExprCall>)
     {
         return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos));

src/libexpr/include/nix/expr/value.hh

@@ -232,13 +232,13 @@ public:
      * Allocate StringData on the (possibly) GC-managed heap and copy
      * the contents of s to it.
      */
-    static const StringData & make(EvalMemory & mem, std::string_view s);
+    static const StringData & make(std::string_view s);
 
     /**
      * Allocate StringData on the (possibly) GC-managed heap.
      * @param size Length of the string (without the NUL terminator).
      */
-    static StringData & alloc(EvalMemory & mem, size_t size);
+    static StringData & alloc(size_t size);
 
     size_t size() const
     {
@@ -1147,13 +1147,13 @@ public:
         setStorage(StringWithContext{.str = &s, .context = context});
     }
 
-    void mkString(std::string_view s, EvalMemory & mem);
+    void mkString(std::string_view s);
 
     void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem);
 
     void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem);
 
-    void mkPath(const SourcePath & path, EvalMemory & mem);
+    void mkPath(const SourcePath & path);
 
     inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept
     {

src/libexpr/json-to-value.cc

@@ -151,7 +151,7 @@ public:
     bool string(string_t & val) override
     {
         forceNoNullByte(val);
-        rs->value(state).mkString(val, state.mem);
+        rs->value(state).mkString(val);
         rs->add();
         return true;
     }

src/libexpr/nixexpr.cc

@@ -191,7 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << '(';
     fun->show(symbols, str);
-    for (auto e : *args) {
+    for (auto e : args) {
         str << ' ';
         e->show(symbols, str);
     }
@@ -486,17 +486,11 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
 
 void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    // Move storage into the Exprs arena
-    {
-        auto arena = es.mem.exprs.alloc;
-        std::pmr::vector<Expr *> newArgs{std::move(*args), arena};
-        args.emplace(std::move(newArgs), arena);
-    }
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     fun->bindVars(es, env);
-    for (auto e : *args)
+    for (auto e : args)
         e->bindVars(es, env);
 }
 

src/libexpr/parser.y

@@ -115,7 +115,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P
 
 static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
     if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
-        e2->args->push_back(arg);
+        e2->args.push_back(arg);
         return fn;
     }
     return exprs.add<ExprCall>(pos, fn, {arg});
@@ -129,7 +129,7 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
 %type <Expr *> start expr expr_function expr_if expr_op
 %type <Expr *> expr_select expr_simple expr_app
 %type <Expr *> expr_pipe_from expr_pipe_into
-%type <std::pmr::vector<Expr *>> list
+%type <std::vector<Expr *>> list
 %type <ExprAttrs *> binds binds1
 %type <FormalsBuilder> formals formal_set
 %type <Formal> formal

src/libexpr/primops.cc

@@ -53,7 +53,7 @@ RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
 static inline Value * mkString(EvalState & state, const std::csub_match & match)
 {
     Value * v = state.allocValue();
-    v->mkString({match.first, match.second}, state.mem);
+    v->mkString({match.first, match.second});
     return v;
 }
 
@@ -230,12 +230,12 @@ void derivationToValue(
                 NixStringContextElem::DrvDeep{.drvPath = storePath},
             },
             state.mem);
-    attrs.alloc(state.s.name).mkString(drv.env["name"], state.mem);
+    attrs.alloc(state.s.name).mkString(drv.env["name"]);
 
     auto list = state.buildList(drv.outputs.size());
     for (const auto & [i, o] : enumerate(drv.outputs)) {
         mkOutputString(state, attrs, storePath, o);
-        (list[i] = state.allocValue())->mkString(o.first, state.mem);
+        (list[i] = state.allocValue())->mkString(o.first);
     }
     attrs.alloc(state.s.outputs).mkList(list);
 
@@ -519,7 +519,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
         v.mkStringNoCopy("lambda"_sds);
         break;
     case nExternal:
-        v.mkString(args[0]->external()->typeOf(), state.mem);
+        v.mkString(args[0]->external()->typeOf());
         break;
     case nFloat:
         v.mkStringNoCopy("float"_sds);
@@ -1176,7 +1176,7 @@ static void prim_getEnv(EvalState & state, const PosIdx pos, Value ** args, Valu
 {
     std::string name(
         state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv"));
-    v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""), state.mem);
+    v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""));
 }
 
 static RegisterPrimOp primop_getEnv({
@@ -1842,10 +1842,8 @@ static RegisterPrimOp primop_derivationStrict(
    ‘out’. */
 static void prim_placeholder(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
-    v.mkString(
+    v.mkString(hashPlaceholder(
-        hashPlaceholder(state.forceStringNoCtx(
+        state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder")));
-            *args[0], pos, "while evaluating the first argument passed to builtins.placeholder")),
-        state.mem);
 }
 
 static RegisterPrimOp primop_placeholder({
@@ -2029,7 +2027,7 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
     state.forceValue(*args[0], pos);
     if (args[0]->type() == nPath) {
         auto path = args[0]->path();
-        v.mkPath(path.path.isRoot() ? path : path.parent(), state.mem);
+        v.mkPath(path.path.isRoot() ? path : path.parent());
     } else {
         NixStringContext context;
         auto path = state.coerceToString(
@@ -2146,7 +2144,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
     auto path =
         state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
 
-    v.mkPath(state.findFile(lookupPath, path, pos), state.mem);
+    v.mkPath(state.findFile(lookupPath, path, pos));
 }
 
 static RegisterPrimOp primop_findFile(
@@ -2295,7 +2293,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
 
     auto path = realisePath(state, pos, *args[1]);
 
-    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem);
+    v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashFile({
@@ -2384,7 +2382,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Val
             // detailed node info quickly in this case we produce a thunk to
             // query the file type lazily.
             auto epath = state.allocValue();
-            epath->mkPath(path / name, state.mem);
+            epath->mkPath(path / name);
             if (!readFileType)
                 readFileType = &state.getBuiltin("readFileType");
             attr.mkApp(readFileType, epath);
@@ -2765,7 +2763,7 @@ bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosId
     /* Call the filter function.  The first argument is the path, the
        second is a string indicating the type of the file. */
     Value arg1;
-    arg1.mkString(path.path.abs(), mem);
+    arg1.mkString(path.path.abs());
 
     // assert that type is not "unknown"
     Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))};
@@ -4543,7 +4541,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value ** args,
     auto s =
         state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
 
-    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false), state.mem);
+    v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
 }
 
 static RegisterPrimOp primop_hashString({
@@ -4576,7 +4574,7 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value ** args,
     HashFormat hf = parseHashFormat(
         state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
 
-    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI), state.mem);
+    v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
 }
 
 static RegisterPrimOp primop_convertHash({
@@ -4994,8 +4992,8 @@ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value ** args
         state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName");
     DrvName parsed(name);
     auto attrs = state.buildBindings(2);
-    attrs.alloc(state.s.name).mkString(parsed.name, state.mem);
+    attrs.alloc(state.s.name).mkString(parsed.name);
-    attrs.alloc("version").mkString(parsed.version, state.mem);
+    attrs.alloc("version").mkString(parsed.version);
     v.mkAttrs(attrs);
 }
 
@@ -5050,7 +5048,7 @@ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value ** args
     }
     auto list = state.buildList(components.size());
     for (const auto & [n, component] : enumerate(components))
-        (list[n] = state.allocValue())->mkString(std::move(component), state.mem);
+        (list[n] = state.allocValue())->mkString(std::move(component));
     v.mkList(list);
 }
 
@@ -5194,7 +5192,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
         });
 
     if (!settings.pureEval)
-        v.mkString(settings.getCurrentSystem(), mem);
+        v.mkString(settings.getCurrentSystem());
     addConstant(
         "__currentSystem",
         v,
@@ -5226,7 +5224,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
             .impureOnly = true,
         });
 
-    v.mkString(nixVersion, mem);
+    v.mkString(nixVersion);
     addConstant(
         "__nixVersion",
         v,
@@ -5251,7 +5249,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
         )",
         });
 
-    v.mkString(store->storeDir, mem);
+    v.mkString(store->storeDir);
     addConstant(
         "__storeDir",
         v,
@@ -5316,8 +5314,8 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
     auto list = buildList(lookupPath.elements.size());
     for (const auto & [n, i] : enumerate(lookupPath.elements)) {
         auto attrs = buildBindings(2);
-        attrs.alloc("path").mkString(i.path.s, mem);
+        attrs.alloc("path").mkString(i.path.s);
-        attrs.alloc("prefix").mkString(i.prefix.s, mem);
+        attrs.alloc("prefix").mkString(i.prefix.s);
         (list[n] = allocValue())->mkAttrs(attrs);
     }
     v.mkList(list);

src/libexpr/primops/context.cc

@@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
     NixStringContext context;
     auto s = state.coerceToString(
         pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
-    v.mkString(*s, state.mem);
+    v.mkString(*s);
 }
 
 static RegisterPrimOp primop_unsafeDiscardStringContext({
@@ -218,7 +218,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
         if (!info.second.outputs.empty()) {
             auto list = state.buildList(info.second.outputs.size());
             for (const auto & [i, output] : enumerate(info.second.outputs))
-                (list[i] = state.allocValue())->mkString(output, state.mem);
+                (list[i] = state.allocValue())->mkString(output);
             infoAttrs.alloc(state.s.outputs).mkList(list);
         }
         attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);

src/libexpr/primops/fetchMercurial.cc

@@ -86,12 +86,12 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
     auto attrs2 = state.buildBindings(8);
     state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));
     if (input2.getRef())
-        attrs2.alloc("branch").mkString(*input2.getRef(), state.mem);
+        attrs2.alloc("branch").mkString(*input2.getRef());
     // Backward compatibility: set 'rev' to
     // 0000000000000000000000000000000000000000 for a dirty tree.
     auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
-    attrs2.alloc("rev").mkString(rev2.gitRev(), state.mem);
+    attrs2.alloc("rev").mkString(rev2.gitRev());
-    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12), state.mem);
+    attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
     if (auto revCount = input2.getRevCount())
         attrs2.alloc("revCount").mkInt(*revCount);
     v.mkAttrs(attrs2);

src/libexpr/primops/fetchTree.cc

@@ -35,7 +35,7 @@ void emitTreeAttrs(
     // FIXME: support arbitrary input attributes.
 
     if (auto narHash = input.getNarHash())
-        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true), state.mem);
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
@@ -43,13 +43,13 @@ void emitTreeAttrs(
     if (!forceDirty) {
 
         if (auto rev = input.getRev()) {
-            attrs.alloc("rev").mkString(rev->gitRev(), state.mem);
+            attrs.alloc("rev").mkString(rev->gitRev());
-            attrs.alloc("shortRev").mkString(rev->gitShortRev(), state.mem);
+            attrs.alloc("shortRev").mkString(rev->gitShortRev());
         } else if (emptyRevFallback) {
             // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
             auto emptyHash = Hash(HashAlgorithm::SHA1);
-            attrs.alloc("rev").mkString(emptyHash.gitRev(), state.mem);
+            attrs.alloc("rev").mkString(emptyHash.gitRev());
-            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem);
+            attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
         if (auto revCount = input.getRevCount())
@@ -59,14 +59,13 @@ void emitTreeAttrs(
     }
 
     if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) {
-        attrs.alloc("dirtyRev").mkString(*dirtyRev, state.mem);
+        attrs.alloc("dirtyRev").mkString(*dirtyRev);
-        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"), state.mem);
+        attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"));
     }
 
     if (auto lastModified = input.getLastModified()) {
         attrs.alloc("lastModified").mkInt(*lastModified);
-        attrs.alloc("lastModifiedDate")
+        attrs.alloc("lastModifiedDate").mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
-            .mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")), state.mem);
     }
 
     v.mkAttrs(attrs);
@@ -151,6 +150,11 @@ static void fetchTree(
             attrs.emplace("exportIgnore", Explicit<bool>{true});
         }
 
+        // fetchTree should fetch git repos with shallow = true by default
+        if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
+            attrs.emplace("shallow", Explicit<bool>{true});
+        }
+
         if (!params.allowNameArgument)
             if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
                 state.error<EvalError>("argument 'name' isn’t supported in call to '%s'", fetcher)
@@ -362,13 +366,10 @@ void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value ** args, Val
 }
 
 static RegisterPrimOp primop_fetchFinalTree({
-    .name = "__fetchFinalTree",
+    .name = "fetchFinalTree",
     .args = {"input"},
-    .doc = R"(
-      Like `fetchTree`, but does not return any additional fetcher attributes (like `revCount`).
-      This allows inputs to be substituted if `narHash` is specified.
-    )",
     .fun = prim_fetchFinalTree,
+    .internal = true,
 });
 
 static void fetch(

src/libexpr/primops/fromTOML.cc

@@ -126,7 +126,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
         case toml::value_t::string: {
             auto s = toml::get<std::string_view>(t);
             forceNoNullByte(s);
-            v.mkString(s, state.mem);
+            v.mkString(s);
         } break;
         case toml::value_t::local_datetime:
         case toml::value_t::offset_datetime:
@@ -142,7 +142,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 s << t;
                 auto str = s.view();
                 forceNoNullByte(str);
-                attrs.alloc("value").mkString(str, state.mem);
+                attrs.alloc("value").mkString(str);
                 v.mkAttrs(attrs);
             } else {
                 throw std::runtime_error("Dates and times are not supported");

src/libexpr/value-to-json.cc

@@ -16,8 +16,6 @@ json printValueAsJSON(
 {
     checkInterrupt();
 
-    auto _level = state.addCallDepth(pos);
-
     if (strict)
         state.forceValue(v, pos);
 

src/libfetchers-tests/git-utils.cc

@@ -92,7 +92,7 @@ TEST_F(GitUtilsTest, sink_basic)
     // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
 
     auto result = repo->dereferenceSingletonDirectory(sink->flush());
-    auto accessor = repo->getAccessor(result, {}, getRepoName());
+    auto accessor = repo->getAccessor(result, false, getRepoName());
     auto entries = accessor->readDirectory(CanonPath::root);
     ASSERT_EQ(entries.size(), 5u);
     ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");

src/libfetchers/fetchers.cc

@@ -269,10 +269,24 @@ void Input::checkLocks(Input specified, Input & result)
         }
     }
 
+    if (auto prevLastModified = specified.getLastModified()) {
+        if (result.getLastModified() != prevLastModified)
+            throw Error(
+                "'lastModified' attribute mismatch in input '%s', expected %d, got %d",
+                result.to_string(),
+                *prevLastModified,
+                result.getLastModified().value_or(-1));
+    }
+
     if (auto prevRev = specified.getRev()) {
         if (result.getRev() != prevRev)
             throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev());
     }
+
+    if (auto prevRevCount = specified.getRevCount()) {
+        if (result.getRevCount() != prevRevCount)
+            throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount);
+    }
 }
 
 std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, Store & store) const
@@ -481,9 +495,9 @@ void InputScheme::clone(
 
     Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
 
-    RestoreSink sink(/*startFsync=*/false);
+    auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); });
-    sink.dstPath = destDir;
+
-    copyRecursive(*accessor, CanonPath::root, sink, CanonPath::root);
+    restorePath(destDir, *source);
 }
 
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const

src/libfetchers/git-utils.cc

@@ -541,15 +541,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 
     /**
-     * A 'GitSourceAccessor' with no regard for export-ignore.
+     * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
      */
-    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, const GitAccessorOptions & options);
+    ref<GitSourceAccessor> getRawAccessor(const Hash & rev, bool smudgeLfs = false);
 
     ref<SourceAccessor>
-    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) override;
+    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) override;
 
-    ref<SourceAccessor>
+    ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
-    getAccessor(const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError e) override;
 
     ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
 
@@ -669,7 +668,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
     {
-        auto accessor = getAccessor(treeHash, {}, "");
+        auto accessor = getAccessor(treeHash, false, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
@@ -717,17 +716,15 @@ struct GitSourceAccessor : SourceAccessor
         ref<GitRepoImpl> repo;
         Object root;
         std::optional<lfs::Fetch> lfsFetch = std::nullopt;
-        GitAccessorOptions options;
     };
 
     Sync<State> state_;
 
-    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, const GitAccessorOptions & options)
+    GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
         : state_{State{
               .repo = repo_,
               .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
-              .lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
+              .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
-              .options = options,
           }}
     {
     }
@@ -1257,26 +1254,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
+ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitSourceAccessor>(self, rev, options);
+    return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
 }
 
 ref<SourceAccessor>
-GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix)
+GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options);
+    ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
     rawGitAccessor->setPathDisplay(std::move(displayPrefix));
-    if (options.exportIgnore)
+    if (exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
     else
         return rawGitAccessor;
 }
 
-ref<SourceAccessor> GitRepoImpl::getAccessor(
+ref<SourceAccessor>
-    const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError)
+GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create(
@@ -1286,9 +1283,10 @@ ref<SourceAccessor> GitRepoImpl::getAccessor(
                                            boost::unordered_flat_set<CanonPath>{CanonPath::root},
                                            std::move(makeNotAllowedError))
                                            .cast<SourceAccessor>();
-    if (options.exportIgnore)
+    if (exportIgnore)
-        fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
+        return make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
-    return fileAccessor;
+    else
+        return fileAccessor;
 }
 
 ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()
@@ -1301,7 +1299,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
 
-    auto accessor = getAccessor(rev, {.exportIgnore = exportIgnore}, "");
+    auto accessor = getAccessor(rev, exportIgnore, "");
     if (!accessor->pathExists(modulesFile))
         return {};
 
@@ -1318,7 +1316,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
 
     std::vector<std::tuple<Submodule, Hash>> result;
 
-    auto rawAccessor = getRawAccessor(rev, {});
+    auto rawAccessor = getRawAccessor(rev);
 
     for (auto & submodule : parseSubmodules(pathTemp)) {
         /* Filter out .gitmodules entries that don't exist or are not
@@ -1334,8 +1332,10 @@ namespace fetchers {
 
 ref<GitRepo> Settings::getTarballCache() const
 {
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
+    auto tarballCache(_tarballCache.lock());
-    return GitRepo::openRepo(repoDir, true, true);
+    if (!*tarballCache)
+        *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
+    return ref<GitRepo>(*tarballCache);
 }
 
 } // namespace fetchers

src/libfetchers/git.cc

@@ -778,16 +778,6 @@ struct GitInputScheme : InputScheme
         }
     }
 
-    /**
-     * Decide whether we can do a shallow clone, which is faster. This is possible if the user explicitly specified
-     * `shallow = true`, or if we already have a `revCount`.
-     */
-    bool canDoShallow(const Input & input) const
-    {
-        bool shallow = getShallowAttr(input);
-        return shallow || input.getRevCount().has_value();
-    }
-
     std::pair<ref<SourceAccessor>, Input>
     getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const
     {
@@ -796,7 +786,7 @@ struct GitInputScheme : InputScheme
         auto origRev = input.getRev();
 
         auto originalRef = input.getRef();
-        bool shallow = canDoShallow(input);
+        bool shallow = getShallowAttr(input);
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
         input.attrs.insert_or_assign("ref", ref);
 
@@ -807,27 +797,11 @@ struct GitInputScheme : InputScheme
             if (!input.getRev())
                 input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
         } else {
-            auto rev = input.getRev();
             auto repoUrl = std::get<ParsedURL>(repoInfo.location);
             std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
             repoDir = cacheDir;
             repoInfo.gitDir = ".";
 
-            /* If shallow = false, but we have a non-shallow repo that already contains the desired rev, then use that
-             * repo instead. */
-            std::filesystem::path cacheDirNonShallow = getCachePath(repoUrl.to_string(), false);
-            if (rev && shallow && pathExists(cacheDirNonShallow)) {
-                auto nonShallowRepo = GitRepo::openRepo(cacheDirNonShallow, true, true);
-                if (nonShallowRepo->hasObject(*rev)) {
-                    debug(
-                        "using non-shallow cached repo for '%s' since it contains rev '%s'",
-                        repoUrl.to_string(),
-                        rev->gitRev());
-                    repoDir = cacheDirNonShallow;
-                    goto have_rev;
-                }
-            }
-
             std::filesystem::create_directories(cacheDir.parent_path());
             PathLocks cacheDirLock({cacheDir.string()});
 
@@ -843,7 +817,7 @@ struct GitInputScheme : InputScheme
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (rev) {
+            if (auto rev = input.getRev()) {
                 doFetch = !repo->hasObject(*rev);
             } else {
                 if (getAllRefsAttr(input)) {
@@ -857,6 +831,7 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
+                bool shallow = getShallowAttr(input);
                 try {
                     auto fetchRef = getAllRefsAttr(input)             ? "refs/*:refs/*"
                                     : input.getRev()                  ? input.getRev()->gitRev()
@@ -884,7 +859,7 @@ struct GitInputScheme : InputScheme
                     warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
             }
 
-            if (rev) {
+            if (auto rev = input.getRev()) {
                 if (!repo->hasObject(*rev))
                     throw Error(
                         "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
@@ -901,30 +876,23 @@ struct GitInputScheme : InputScheme
             // the remainder
         }
 
-    have_rev:
         auto repo = GitRepo::openRepo(repoDir);
 
+        auto isShallow = repo->isShallow();
+
+        if (isShallow && !getShallowAttr(input))
+            throw Error(
+                "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
+                repoInfo.locationToArg());
+
         // FIXME: check whether rev is an ancestor of ref?
 
         auto rev = *input.getRev();
 
-        /* Skip lastModified computation if it's already supplied by the caller.
+        input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
-           We don't care if they specify an incorrect value; it doesn't
-           matter for security, unlike narHash. */
-        if (!input.attrs.contains("lastModified"))
-            input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
-
-        /* Like lastModified, skip revCount if supplied by the caller. */
-        if (!shallow && !input.attrs.contains("revCount")) {
-            auto isShallow = repo->isShallow();
-
-            if (isShallow && !shallow)
-                throw Error(
-                    "'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
-                    repoInfo.locationToArg());
 
+        if (!getShallowAttr(input))
             input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev));
-        }
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
 
@@ -932,8 +900,7 @@ struct GitInputScheme : InputScheme
 
         bool exportIgnore = getExportIgnoreAttr(input);
         bool smudgeLfs = getLfsAttr(input);
-        auto accessor = repo->getAccessor(
+        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
-            rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»");
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -1000,7 +967,7 @@ struct GitInputScheme : InputScheme
         auto exportIgnore = getExportIgnoreAttr(input);
 
         ref<SourceAccessor> accessor =
-            repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath));
+            repo->getAccessor(repoInfo.workdirInfo, exportIgnore, makeNotAllowedError(repoPath));
 
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the

src/libfetchers/github.cc

@@ -351,7 +351,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
         auto accessor =
-            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string() + "»");
+            settings.getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
 
         return {accessor, input};
     }

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -135,6 +135,8 @@ struct Settings : public Config
 
 private:
     mutable Sync<std::shared_ptr<Cache>> _cache;
+
+    mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
 };
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -22,12 +22,6 @@ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
     virtual Hash flush() = 0;
 };
 
-struct GitAccessorOptions
-{
-    bool exportIgnore = false;
-    bool smudgeLfs = false;
-};
-
 struct GitRepo
 {
     virtual ~GitRepo() {}
@@ -95,10 +89,10 @@ struct GitRepo
     virtual bool hasObject(const Hash & oid) = 0;
 
     virtual ref<SourceAccessor>
-    getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0;
+    getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) = 0;
 
-    virtual ref<SourceAccessor> getAccessor(
+    virtual ref<SourceAccessor>
-        const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) = 0;
+    getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
 
     virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
 

src/libfetchers/tarball.cc

@@ -136,7 +136,7 @@ static DownloadTarballResult downloadTarball_(
             .treeHash = treeHash,
             .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
             .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
-            .accessor = settings.getTarballCache()->getAccessor(treeHash, {}, displayPrefix),
+            .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix),
         };
     };
 

src/libflake-c/nix_api_flake.cc

@@ -200,7 +200,7 @@ nix_value * nix_locked_flake_get_output_attrs(
     nix_clear_err(context);
     try {
         auto v = nix_alloc_value(context, evalState);
-        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value);
+        nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value);
         return v;
     }
     NIXC_CATCH_ERRS_NULL

src/libflake/call-flake.nix

@@ -10,6 +10,9 @@ lockFileStr:
 # unlocked trees.
 overrides:
 
+# This is `prim_fetchFinalTree`.
+fetchTreeFinal:
+
 let
   inherit (builtins) mapAttrs;
 
@@ -49,7 +52,7 @@ let
         else
           # FIXME: remove obsolete node.info.
           # Note: lock file entries are always final.
-          builtins.fetchFinalTree (node.info or { } // removeAttrs node.locked [ "dir" ]);
+          fetchTreeFinal (node.info or { } // removeAttrs node.locked [ "dir" ]);
 
       subdir = overrides.${key}.dir or node.locked.dir or "";
 

src/libflake/flake-primops.cc

@@ -93,7 +93,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
         auto & vv = binds.alloc(s);
         std::visit(
             overloaded{
-                [&vv, &state](const std::string & value) { vv.mkString(value, state.mem); },
+                [&vv](const std::string & value) { vv.mkString(value); },
                 [&vv](const uint64_t & value) { vv.mkInt(value); },
                 [&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }},
             value);
@@ -156,7 +156,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
         }
     }
     auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs);
-    v.mkString(flakeRef.to_string(), state.mem);
+    v.mkString(flakeRef.to_string());
 }
 
 nix::PrimOp flakeRefToString({

src/libflake/flake.cc

@@ -956,7 +956,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
         auto key = keyMap.find(node);
         assert(key != keyMap.end());
 
-        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem);
+        override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel());
 
         overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
     }
@@ -966,9 +966,12 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
     Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix"));
 
     auto vLocks = state.allocValue();
-    vLocks->mkString(lockFileStr, state.mem);
+    vLocks->mkString(lockFileStr);
 
-    Value * args[] = {vLocks, &vOverrides};
+    auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree");
+    assert(vFetchFinalTree);
+
+    Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree};
     state.callFunction(*vCallFlake, args, vRes, noPos);
 }
 

src/libstore-tests/references.cc

@@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep)
     // Create an in-memory file system with various reference patterns
     auto accessor = make_ref<MemorySourceAccessor>();
     accessor->root = File::Directory{
-        .entries{
+        .contents{
             {
                 // file1.txt: contains hash1
                 "file1.txt",
@@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep)
                 // subdir: a subdirectory
                 "subdir",
                 File::Directory{
-                    .entries{
+                    .contents{
                         {
                             // subdir/file4.txt: contains hash1 again
                             "file4.txt",

src/libstore/binary-cache-store.cc

@@ -8,7 +8,7 @@
 #include "nix/util/sync.hh"
 #include "nix/store/remote-fs-accessor.hh"
 #include "nix/store/nar-info-disk-cache.hh"
-#include "nix/util/nar-accessor.hh"
+#include "nix/store/nar-accessor.hh"
 #include "nix/util/thread-pool.hh"
 #include "nix/util/callback.hh"
 #include "nix/util/signals.hh"
@@ -208,7 +208,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     if (config.writeNARListing) {
         nlohmann::json j = {
             {"version", 1},
-            {"root", listNarDeep(*narAccessor, CanonPath::root)},
+            {"root", listNar(ref<SourceAccessor>(narAccessor), CanonPath::root, true)},
         };
 
         upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json");

src/libstore/filetransfer.cc

@@ -48,7 +48,7 @@ struct curlFileTransfer : public FileTransfer
     std::random_device rd;
     std::mt19937 mt19937;
 
-    struct TransferItem : public std::enable_shared_from_this<TransferItem>, public FileTransfer::Item
+    struct TransferItem : public std::enable_shared_from_this<TransferItem>
     {
         curlFileTransfer & fileTransfer;
         FileTransferRequest request;
@@ -60,7 +60,6 @@ struct curlFileTransfer : public FileTransfer
         // buffer to accompany the `req` above
         char errbuf[CURL_ERROR_SIZE];
         bool active = false; // whether the handle has been added to the multi object
-        bool paused = false; // whether the request has been paused previously
         std::string statusMsg;
 
         unsigned int attempt = 0;
@@ -117,13 +116,7 @@ struct curlFileTransfer : public FileTransfer
                        successful response. */
                     if (successfulStatuses.count(httpStatus)) {
                         writtenToSink += data.size();
-                        PauseTransfer needsPause = this->request.dataCallback(data);
+                        this->request.dataCallback(data);
-                        if (needsPause == PauseTransfer::Yes) {
-                            /* Smuggle the boolean flag into writeCallback. Note that
-                               the finalSink might get called multiple times if there's
-                               decompression going on. */
-                            paused = true;
-                        }
                     }
                 } else
                     this->result.data.append(data);
@@ -202,14 +195,6 @@ struct curlFileTransfer : public FileTransfer
             }
 
             (*decompressionSink)({(char *) contents, realSize});
-            if (paused) {
-                /* The callback has signaled that the transfer needs to be
-                   paused. Already consumed data won't be returned twice unlike
-                   when returning CURL_WRITEFUNC_PAUSE.
-                   https://curl-library.cool.haxx.narkive.com/larE1cRA/curl-easy-pause-documentation-question
-                   */
-                curl_easy_pause(req, CURLPAUSE_RECV);
-            }
 
             return realSize;
         } catch (...) {
@@ -379,15 +364,6 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) clientp)->seekCallback(offset, origin);
         }
 
-        void unpause()
-        {
-            /* Unpausing an already unpaused transfer is a no-op. */
-            if (paused) {
-                curl_easy_pause(req, CURLPAUSE_CONT);
-                paused = false;
-            }
-        }
-
         void init()
         {
             if (!req)
@@ -648,7 +624,7 @@ struct curlFileTransfer : public FileTransfer
                     errorSink.reset();
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
                     try {
-                        fileTransfer.enqueueItem(ref{shared_from_this()});
+                        fileTransfer.enqueueItem(shared_from_this());
                     } catch (const nix::Error & e) {
                         // If enqueue fails (e.g., during shutdown), fail the transfer properly
                         // instead of letting the exception propagate, which would leave done=false
@@ -665,24 +641,24 @@ struct curlFileTransfer : public FileTransfer
     {
         struct EmbargoComparator
         {
-            bool operator()(const ref<TransferItem> & i1, const ref<TransferItem> & i2)
+            bool operator()(const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2)
             {
                 return i1->embargo > i2->embargo;
             }
         };
 
-        std::priority_queue<ref<TransferItem>, std::vector<ref<TransferItem>>, EmbargoComparator> incoming;
+        std::
-        std::vector<ref<TransferItem>> unpause;
+            priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator>
+                incoming;
     private:
         bool quitting = false;
     public:
         void quit()
         {
             quitting = true;
-            /* We will not be processing any more incoming requests */
+            /* We wil not be processing any more incoming requests */
             while (!incoming.empty())
                 incoming.pop();
-            unpause.clear();
         }
 
         bool isQuitting()
@@ -851,17 +827,6 @@ struct curlFileTransfer : public FileTransfer
                 item->active = true;
                 items[item->req] = item;
             }
-
-            /* NOTE: Unpausing may invoke callbacks to flush all buffers. */
-            auto unpause = [&]() {
-                auto state(state_.lock());
-                auto res = state->unpause;
-                state->unpause.clear();
-                return res;
-            }();
-
-            for (auto & item : unpause)
-                item->unpause();
         }
 
         debug("download thread shutting down");
@@ -886,7 +851,7 @@ struct curlFileTransfer : public FileTransfer
         }
     }
 
-    ItemHandle enqueueItem(ref<TransferItem> item)
+    void enqueueItem(std::shared_ptr<TransferItem> item)
     {
         if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https"
             && item->request.uri.scheme() != "s3")
@@ -901,34 +866,19 @@ struct curlFileTransfer : public FileTransfer
 #ifndef _WIN32 // TODO need graceful async exit support on Windows?
         writeFull(wakeupPipe.writeSide.get(), " ");
 #endif
-
-        return ItemHandle(static_cast<Item &>(*item));
     }
 
-    ItemHandle enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
+    void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override
     {
         /* Handle s3:// URIs by converting to HTTPS and optionally adding auth */
         if (request.uri.scheme() == "s3") {
             auto modifiedRequest = request;
             modifiedRequest.setupForS3();
-            return enqueueItem(make_ref<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
+            enqueueItem(std::make_shared<TransferItem>(*this, std::move(modifiedRequest), std::move(callback)));
+            return;
         }
 
-        return enqueueItem(make_ref<TransferItem>(*this, request, std::move(callback)));
+        enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
-    }
-
-    void unpauseTransfer(ref<TransferItem> item)
-    {
-        auto state(state_.lock());
-        state->unpause.push_back(std::move(item));
-#ifndef _WIN32 // TODO need graceful async exit support on Windows?
-        writeFull(wakeupPipe.writeSide.get(), " ");
-#endif
-    }
-
-    void unpauseTransfer(ItemHandle handle) override
-    {
-        unpauseTransfer(ref{static_cast<TransferItem &>(handle.item.get()).shared_from_this()});
     }
 };
 
@@ -1025,7 +975,6 @@ void FileTransfer::download(
     struct State
     {
         bool quit = false;
-        bool paused = false;
         std::exception_ptr exc;
         std::string data;
         std::condition_variable avail, request;
@@ -1041,38 +990,31 @@ void FileTransfer::download(
         state->request.notify_one();
     });
 
-    request.dataCallback = [_state, uri = request.uri.to_string()](std::string_view data) -> PauseTransfer {
+    request.dataCallback = [_state](std::string_view data) {
         auto state(_state->lock());
 
         if (state->quit)
-            return PauseTransfer::No;
+            return;
+
+        /* If the buffer is full, then go to sleep until the calling
+           thread wakes us up (i.e. when it has removed data from the
+           buffer). We don't wait forever to prevent stalling the
+           download thread. (Hopefully sleeping will throttle the
+           sender.) */
+        if (state->data.size() > fileTransferSettings.downloadBufferSize) {
+            debug("download buffer is full; going to sleep");
+            static bool haveWarned = false;
+            warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
+            state.wait_for(state->request, std::chrono::seconds(10));
+        }
 
         /* Append data to the buffer and wake up the calling
            thread. */
         state->data.append(data);
         state->avail.notify_one();
-
-        if (state->data.size() <= fileTransferSettings.downloadBufferSize)
-            return PauseTransfer::No;
-
-        /* dataCallback gets called multiple times by an intermediate sink. Only
-           issue the debug message the first time around. */
-        if (!state->paused)
-            debug(
-                "pausing transfer for '%s': download buffer is full (%d > %d)",
-                uri,
-                state->data.size(),
-                fileTransferSettings.downloadBufferSize);
-
-        state->paused = true;
-
-        /* Technically the buffer might become larger than
-           downloadBufferSize, but with sinks there's no way to avoid
-           consuming data. */
-        return PauseTransfer::Yes;
     };
 
-    auto handle = enqueueFileTransfer(
+    enqueueFileTransfer(
         request, {[_state, resultCallback{std::move(resultCallback)}](std::future<FileTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
@@ -1105,10 +1047,6 @@ void FileTransfer::download(
                     return;
                 }
 
-                if (state->paused) {
-                    unpauseTransfer(handle);
-                    state->paused = false;
-                }
                 state.wait(state->avail);
 
                 if (state->data.empty())

src/libstore/include/nix/store/filetransfer.hh

@@ -70,12 +70,12 @@ struct FileTransferSettings : Config
 
     Setting<size_t> downloadBufferSize{
         this,
-        1 * 1024 * 1024,
+        64 * 1024 * 1024,
         "download-buffer-size",
         R"(
           The size of Nix's internal download buffer in bytes during `curl` transfers. If data is
           not processed quickly enough to exceed the size of this buffer, downloads may stall.
-          The default is 1048576 (1 MiB).
+          The default is 67108864 (64 MiB).
         )"};
 };
 
@@ -105,11 +105,6 @@ struct UsernameAuth
     std::optional<std::string> password;
 };
 
-enum class PauseTransfer : bool {
-    No = false,
-    Yes = true,
-};
-
 struct FileTransferRequest
 {
     VerbatimURL uri;
@@ -141,14 +136,7 @@ struct FileTransferRequest
 
     std::optional<UploadData> data;
     std::string mimeType;
-
+    std::function<void(std::string_view data)> dataCallback;
-    /**
-     * Callbacked invoked with a chunk of received data.
-     * Can pause the transfer by returning PauseTransfer::Yes. No data must be consumed
-     * if transfer is paused.
-     */
-    std::function<PauseTransfer(std::string_view data)> dataCallback;
-
     /**
      * Optional username and password for HTTP basic authentication.
      * When provided, these credentials will be used with curl's CURLOPT_USERNAME/PASSWORD option.
@@ -238,25 +226,6 @@ class Store;
 
 struct FileTransfer
 {
-protected:
-    class Item
-    {};
-
-public:
-    /**
-     * An opaque handle to the file transfer. Can be used to reference an in-flight transfer operations.
-     */
-    struct ItemHandle
-    {
-        std::reference_wrapper<Item> item;
-        friend struct FileTransfer;
-
-        ItemHandle(Item & item)
-            : item(item)
-        {
-        }
-    };
-
     virtual ~FileTransfer() {}
 
     /**
@@ -264,13 +233,7 @@ public:
      * the download. The future may throw a FileTransferError
      * exception.
      */
-    virtual ItemHandle
+    virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
-    enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0;
-
-    /**
-     * Unpause a transfer that has been previously paused by a dataCallback.
-     */
-    virtual void unpauseTransfer(ItemHandle handle) = 0;
 
     std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
 

src/libstore/include/nix/store/meson.build

@@ -55,6 +55,7 @@ headers = [ config_pub_h ] + files(
   'machines.hh',
   'make-content-addressed.hh',
   'names.hh',
+  'nar-accessor.hh',
   'nar-info-disk-cache.hh',
   'nar-info.hh',
   'outputs-spec.hh',

src/libstore/include/nix/store/nar-accessor.hh

@@ -0,0 +1,43 @@
+#pragma once
+///@file
+
+#include "nix/util/source-accessor.hh"
+
+#include <functional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+struct Source;
+
+/**
+ * Return an object that provides access to the contents of a NAR
+ * file.
+ */
+ref<SourceAccessor> makeNarAccessor(std::string && nar);
+
+ref<SourceAccessor> makeNarAccessor(Source & source);
+
+/**
+ * Create a NAR accessor from a NAR listing (in the format produced by
+ * listNar()). The callback getNarBytes(offset, length) is used by the
+ * readFile() method of the accessor to get the contents of files
+ * inside the NAR.
+ */
+using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
+
+/**
+ * The canonical GetNarBytes function for a seekable Source.
+ */
+GetNarBytes seekableGetNarBytes(const Path & path);
+
+ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
+
+/**
+ * Write a JSON representation of the contents of a NAR (except file
+ * contents).
+ */
+nlohmann::json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse);
+
+} // namespace nix

src/libstore/meson.build

@@ -114,17 +114,6 @@ boost = dependency(
 deps_other += boost
 
 curl = dependency('libcurl', 'curl', version : '>= 7.75.0')
-if curl.version().version_compare('>=8.16.0') and curl.version().version_compare(
-  '<8.17.0',
-)
-  # Out of precaution, avoid building with libcurl version that suffer from https://github.com/curl/curl/issues/19334.
-  error(
-    'curl @0@ has issues with write pausing, please use libcurl < 8.16 or >= 8.17, see https://github.com/curl/curl/issues/19334'.format(
-      curl.version(),
-    ),
-  )
-endif
-
 deps_private += curl
 
 # seccomp only makes sense on Linux
@@ -311,6 +300,7 @@ sources = files(
   'make-content-addressed.cc',
   'misc.cc',
   'names.cc',
+  'nar-accessor.cc',
   'nar-info-disk-cache.cc',
   'nar-info.cc',
   'optimise-store.cc',

src/libstore/nar-accessor.cc

@@ -1,4 +1,4 @@
-#include "nix/util/nar-accessor.hh"
+#include "nix/store/nar-accessor.hh"
 #include "nix/util/archive.hh"
 
 #include <map>
@@ -272,39 +272,41 @@ GetNarBytes seekableGetNarBytes(const Path & path)
     };
 }
 
-template<bool deep>
+using nlohmann::json;
-using ListNarResult = std::conditional_t<deep, NarListing, ShallowNarListing>;
 
-template<bool deep>
+json listNar(ref<SourceAccessor> accessor, const CanonPath & path, bool recurse)
-static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPath & path)
 {
-    auto st = accessor.lstat(path);
+    auto st = accessor->lstat(path);
+
+    json obj = json::object();
 
     switch (st.type) {
     case SourceAccessor::Type::tRegular:
-        return typename ListNarResult<deep>::Regular{
+        obj["type"] = "regular";
-            .executable = st.isExecutable,
+        if (st.fileSize)
-            .contents =
+            obj["size"] = *st.fileSize;
-                NarListingRegularFile{
+        if (st.isExecutable)
-                    .fileSize = st.fileSize,
+            obj["executable"] = true;
-                    .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt,
+        if (st.narOffset && *st.narOffset)
-                },
+            obj["narOffset"] = *st.narOffset;
-        };
+        break;
-    case SourceAccessor::Type::tDirectory: {
+    case SourceAccessor::Type::tDirectory:
-        typename ListNarResult<deep>::Directory dir;
+        obj["type"] = "directory";
-        for (const auto & [name, type] : accessor.readDirectory(path)) {
+        {
-            if constexpr (deep) {
+            obj["entries"] = json::object();
-                dir.entries.emplace(name, listNarImpl<true>(accessor, path / name));
+            json & res2 = obj["entries"];
-            } else {
+            for (const auto & [name, type] : accessor->readDirectory(path)) {
-                dir.entries.emplace(name, fso::Opaque{});
+                if (recurse) {
+                    res2[name] = listNar(accessor, path / name, true);
+                } else
+                    res2[name] = json::object();
             }
         }
-        return dir;
+        break;
-    }
     case SourceAccessor::Type::tSymlink:
-        return typename ListNarResult<deep>::Symlink{
+        obj["type"] = "symlink";
-            .target = accessor.readLink(path),
+        obj["target"] = accessor->readLink(path);
-        };
+        break;
     case SourceAccessor::Type::tBlock:
     case SourceAccessor::Type::tChar:
     case SourceAccessor::Type::tSocket:
@@ -312,16 +314,7 @@ static ListNarResult<deep> listNarImpl(SourceAccessor & accessor, const CanonPat
     case SourceAccessor::Type::tUnknown:
         assert(false); // cannot happen for NARs
     }
-}
+    return obj;
-
-NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path)
-{
-    return listNarImpl<true>(accessor, path);
-}
-
-ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path)
-{
-    return listNarImpl<false>(accessor, path);
 }
 
 } // namespace nix

src/libstore/remote-fs-accessor.cc

@@ -1,6 +1,6 @@
 #include <nlohmann/json.hpp>
 #include "nix/store/remote-fs-accessor.hh"
-#include "nix/util/nar-accessor.hh"
+#include "nix/store/nar-accessor.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -39,7 +39,7 @@ ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std:
 
     if (cacheDir != "") {
         try {
-            nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root);
+            nlohmann::json j = listNar(narAccessor, CanonPath::root, true);
             writeFile(makeCacheFile(hashPart, "ls"), j.dump());
         } catch (...) {
             ignoreExceptionExceptInterrupt();

src/libutil-tests/data/memory-source-accessor/complex.json

@@ -1,24 +0,0 @@
-{
-  "entries": {
-    "bar": {
-      "entries": {
-        "baz": {
-          "contents": "good day,\n\u0000\n\tworld!",
-          "executable": true,
-          "type": "regular"
-        },
-        "quux": {
-          "target": "/over/there",
-          "type": "symlink"
-        }
-      },
-      "type": "directory"
-    },
-    "foo": {
-      "contents": "hello\n\u0000\n\tworld!",
-      "executable": false,
-      "type": "regular"
-    }
-  },
-  "type": "directory"
-}

src/libutil-tests/data/memory-source-accessor/simple.json

@@ -1,5 +0,0 @@
-{
-  "contents": "asdf",
-  "executable": false,
-  "type": "regular"
-}

src/libutil-tests/data/nar-listing/deep.json

@@ -1,23 +0,0 @@
-{
-  "entries": {
-    "bar": {
-      "entries": {
-        "baz": {
-          "executable": true,
-          "size": 19,
-          "type": "regular"
-        },
-        "quux": {
-          "target": "/over/there",
-          "type": "symlink"
-        }
-      },
-      "type": "directory"
-    },
-    "foo": {
-      "size": 15,
-      "type": "regular"
-    }
-  },
-  "type": "directory"
-}

src/libutil-tests/data/nar-listing/shallow.json

@@ -1,7 +0,0 @@
-{
-  "entries": {
-    "bar": {},
-    "foo": {}
-  },
-  "type": "directory"
-}

src/libutil-tests/git.cc

@@ -224,15 +224,42 @@ TEST_F(GitTest, tree_sha256_write)
     });
 }
 
-namespace memory_source_accessor {
-
-extern ref<MemorySourceAccessor> exampleComplex();
-
-}
-
 TEST_F(GitTest, both_roundrip)
 {
-    auto files = memory_source_accessor::exampleComplex();
+    using File = MemorySourceAccessor::File;
+
+    auto files = make_ref<MemorySourceAccessor>();
+    files->root = File::Directory{
+        .contents{
+            {
+                "foo",
+                File::Regular{
+                    .contents = "hello\n\0\n\tworld!",
+                },
+            },
+            {
+                "bar",
+                File::Directory{
+                    .contents =
+                        {
+                            {
+                                "baz",
+                                File::Regular{
+                                    .executable = true,
+                                    .contents = "good day,\n\0\n\tworld!",
+                                },
+                            },
+                            {
+                                "quux",
+                                File::Symlink{
+                                    .target = "/over/there",
+                                },
+                            },
+                        },
+                },
+            },
+        },
+    };
 
     for (const auto hashAlgo : {HashAlgorithm::SHA1, HashAlgorithm::SHA256}) {
         std::map<Hash, std::string> cas;

src/libutil-tests/memory-source-accessor.cc

@@ -1,116 +0,0 @@
-#include <string_view>
-
-#include "nix/util/memory-source-accessor.hh"
-#include "nix/util/tests/json-characterization.hh"
-
-namespace nix {
-
-namespace memory_source_accessor {
-
-using namespace std::literals;
-using File = MemorySourceAccessor::File;
-
-ref<MemorySourceAccessor> exampleSimple()
-{
-    auto sc = make_ref<MemorySourceAccessor>();
-    sc->root = File{File::Regular{
-        .executable = false,
-        .contents = "asdf",
-    }};
-    return sc;
-}
-
-ref<MemorySourceAccessor> exampleComplex()
-{
-    auto files = make_ref<MemorySourceAccessor>();
-    files->root = File::Directory{
-        .entries{
-            {
-                "foo",
-                File::Regular{
-                    .contents = "hello\n\0\n\tworld!"s,
-                },
-            },
-            {
-                "bar",
-                File::Directory{
-                    .entries =
-                        {
-                            {
-                                "baz",
-                                File::Regular{
-                                    .executable = true,
-                                    .contents = "good day,\n\0\n\tworld!"s,
-                                },
-                            },
-                            {
-                                "quux",
-                                File::Symlink{
-                                    .target = "/over/there",
-                                },
-                            },
-                        },
-                },
-            },
-        },
-    };
-    return files;
-}
-
-} // namespace memory_source_accessor
-
-/* ----------------------------------------------------------------------------
- * JSON
- * --------------------------------------------------------------------------*/
-
-class MemorySourceAccessorTest : public virtual CharacterizationTest
-{
-    std::filesystem::path unitTestData = getUnitTestData() / "memory-source-accessor";
-
-public:
-
-    std::filesystem::path goldenMaster(std::string_view testStem) const override
-    {
-        return unitTestData / testStem;
-    }
-};
-
-using nlohmann::json;
-
-struct MemorySourceAccessorJsonTest : MemorySourceAccessorTest,
-                                      JsonCharacterizationTest<MemorySourceAccessor>,
-                                      ::testing::WithParamInterface<std::pair<std::string_view, MemorySourceAccessor>>
-{};
-
-TEST_P(MemorySourceAccessorJsonTest, from_json)
-{
-    auto & [name, expected] = GetParam();
-    /* Cannot use `readJsonTest` because need to compare `root` field of
-       the source accessors for equality. */
-    readTest(Path{name} + ".json", [&](const auto & encodedRaw) {
-        auto encoded = json::parse(encodedRaw);
-        auto decoded = static_cast<MemorySourceAccessor>(encoded);
-        ASSERT_EQ(decoded.root, expected.root);
-    });
-}
-
-TEST_P(MemorySourceAccessorJsonTest, to_json)
-{
-    auto & [name, value] = GetParam();
-    writeJsonTest(name, value);
-}
-
-INSTANTIATE_TEST_SUITE_P(
-    MemorySourceAccessorJSON,
-    MemorySourceAccessorJsonTest,
-    ::testing::Values(
-        std::pair{
-            "simple",
-            *memory_source_accessor::exampleSimple(),
-        },
-        std::pair{
-            "complex",
-            *memory_source_accessor::exampleComplex(),
-        }));
-
-} // namespace nix

src/libutil-tests/meson.build

@@ -63,9 +63,7 @@ sources = files(
   'json-utils.cc',
   'logging.cc',
   'lru-cache.cc',
-  'memory-source-accessor.cc',
   'monitorfdhup.cc',
-  'nar-listing.cc',
   'nix_api_util.cc',
   'nix_api_util_internal.cc',
   'pool.cc',

src/libutil-tests/nar-listing.cc

@@ -1,83 +0,0 @@
-#include <string_view>
-
-#include "nix/util/nar-accessor.hh"
-#include "nix/util/tests/json-characterization.hh"
-
-namespace nix {
-
-// Forward declaration from memory-source-accessor.cc
-namespace memory_source_accessor {
-ref<MemorySourceAccessor> exampleComplex();
-}
-
-/* ----------------------------------------------------------------------------
- * JSON
- * --------------------------------------------------------------------------*/
-
-class NarListingTest : public virtual CharacterizationTest
-{
-    std::filesystem::path unitTestData = getUnitTestData() / "nar-listing";
-
-public:
-
-    std::filesystem::path goldenMaster(std::string_view testStem) const override
-    {
-        return unitTestData / testStem;
-    }
-};
-
-using nlohmann::json;
-
-struct NarListingJsonTest : NarListingTest,
-                            JsonCharacterizationTest<NarListing>,
-                            ::testing::WithParamInterface<std::pair<std::string_view, NarListing>>
-{};
-
-TEST_P(NarListingJsonTest, from_json)
-{
-    auto & [name, expected] = GetParam();
-    readJsonTest(name, expected);
-}
-
-TEST_P(NarListingJsonTest, to_json)
-{
-    auto & [name, value] = GetParam();
-    writeJsonTest(name, value);
-}
-
-INSTANTIATE_TEST_SUITE_P(
-    NarListingJSON,
-    NarListingJsonTest,
-    ::testing::Values(
-        std::pair{
-            "deep",
-            listNarDeep(*memory_source_accessor::exampleComplex(), CanonPath::root),
-        }));
-
-struct ShallowNarListingJsonTest : NarListingTest,
-                                   JsonCharacterizationTest<ShallowNarListing>,
-                                   ::testing::WithParamInterface<std::pair<std::string_view, ShallowNarListing>>
-{};
-
-TEST_P(ShallowNarListingJsonTest, from_json)
-{
-    auto & [name, expected] = GetParam();
-    readJsonTest(name, expected);
-}
-
-TEST_P(ShallowNarListingJsonTest, to_json)
-{
-    auto & [name, value] = GetParam();
-    writeJsonTest(name, value);
-}
-
-INSTANTIATE_TEST_SUITE_P(
-    ShallowNarListingJSON,
-    ShallowNarListingJsonTest,
-    ::testing::Values(
-        std::pair{
-            "shallow",
-            listNarShallow(*memory_source_accessor::exampleComplex(), CanonPath::root),
-        }));
-
-} // namespace nix

src/libutil/current-process.cc

@@ -134,11 +134,6 @@ std::optional<Path> getSelfExe()
             return std::nullopt;
         }
 
-        // FreeBSD's sysctl(KERN_PROC_PATHNAME) includes the null terminator in
-        // pathLen. Strip it to prevent Nix evaluation errors when the path is
-        // serialized to JSON and evaluated as a Nix string.
-        path.pop_back();
-
         return Path(path.begin(), path.end());
 #else
         return std::nullopt;

src/libutil/fs-sink.cc

@@ -14,44 +14,6 @@
 
 namespace nix {
 
-void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystemObjectSink & sink, const CanonPath & to)
-{
-    auto stat = accessor.lstat(from);
-
-    switch (stat.type) {
-    case SourceAccessor::tSymlink: {
-        sink.createSymlink(to, accessor.readLink(from));
-        break;
-    }
-
-    case SourceAccessor::tRegular: {
-        sink.createRegularFile(to, [&](CreateRegularFileSink & crf) {
-            if (stat.isExecutable)
-                crf.isExecutable();
-            accessor.readFile(from, crf, [&](uint64_t size) { crf.preallocateContents(size); });
-        });
-        break;
-    }
-
-    case SourceAccessor::tDirectory: {
-        sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) {
-            for (auto & [name, _] : accessor.readDirectory(from)) {
-                copyRecursive(accessor, from / name, dirSink, relDirPath / name);
-            }
-        });
-        break;
-    }
-
-    case SourceAccessor::tChar:
-    case SourceAccessor::tBlock:
-    case SourceAccessor::tSocket:
-    case SourceAccessor::tFifo:
-    case SourceAccessor::tUnknown:
-    default:
-        throw Error("file '%1%' has an unsupported type of %2%", from, stat.typeString());
-    }
-}
-
 struct RestoreSinkSettings : Config
 {
     Setting<bool> preallocateContents{

src/libutil/git.cc

@@ -217,29 +217,6 @@ std::optional<Mode> convertMode(SourceAccessor::Type type)
     }
 }
 
-void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function<RestoreHook> hook)
-{
-    parse(sink, CanonPath::root, source, BlobMode::Regular, hashAlgo, [&](CanonPath name, TreeEntry entry) {
-        auto [accessor, from] = hook(entry.hash);
-        auto stat = accessor->lstat(from);
-        auto gotOpt = convertMode(stat.type);
-        if (!gotOpt)
-            throw Error(
-                "file '%s' (git hash %s) has an unsupported type",
-                from,
-                entry.hash.to_string(HashFormat::Base16, false));
-        auto & got = *gotOpt;
-        if (got != entry.mode)
-            throw Error(
-                "git mode of file '%s' (git hash %s) is %o but expected %o",
-                from,
-                entry.hash.to_string(HashFormat::Base16, false),
-                (RawMode) got,
-                (RawMode) entry.mode);
-        copyRecursive(*accessor, from, sink, name);
-    });
-}
-
 void dumpBlobPrefix(uint64_t size, Sink & sink, const ExperimentalFeatureSettings & xpSettings)
 {
     xpSettings.require(Xp::GitHashing);

src/libutil/include/nix/util/git.hh

@@ -136,13 +136,6 @@ std::optional<Mode> convertMode(SourceAccessor::Type type);
  */
 using RestoreHook = SourcePath(Hash);
 
-/**
- * Wrapper around `parse` and `RestoreSink`
- *
- * @param hashAlgo must be `HashAlgo::SHA1` or `HashAlgo::SHA256` for now.
- */
-void restore(FileSystemObjectSink & sink, Source & source, HashAlgorithm hashAlgo, std::function<RestoreHook> hook);
-
 /**
  * Dumps a single file to a sink
  *

src/libutil/include/nix/util/json-impls.hh

@@ -6,18 +6,15 @@
 #include "nix/util/experimental-features.hh"
 
 // Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types
-#define JSON_IMPL_INNER(TYPE)                             \
+#define JSON_IMPL(TYPE)                                   \
+    namespace nlohmann {                                  \
+    using namespace nix;                                  \
+    template<>                                            \
     struct adl_serializer<TYPE>                           \
     {                                                     \
         static TYPE from_json(const json & json);         \
         static void to_json(json & json, const TYPE & t); \
-    };
+    };                                                    \
-
-#define JSON_IMPL(TYPE)   \
-    namespace nlohmann {  \
-    using namespace nix;  \
-    template<>            \
-    JSON_IMPL_INNER(TYPE) \
     }
 
 #define JSON_IMPL_WITH_XP_FEATURES(TYPE)                                                                            \

src/libutil/include/nix/util/memory-source-accessor.hh

@@ -4,111 +4,59 @@
 #include "nix/util/source-path.hh"
 #include "nix/util/fs-sink.hh"
 #include "nix/util/variant-wrapper.hh"
-#include "nix/util/json-impls.hh"
 
 namespace nix {
 
-/**
- * File System Object definitions
- *
- * @see https://nix.dev/manual/nix/latest/store/file-system-object.html
- */
-namespace fso {
-
-template<typename RegularContents>
-struct Regular
-{
-    bool executable = false;
-    RegularContents contents;
-
-    auto operator<=>(const Regular &) const = default;
-};
-
-/**
- * Child parameter because sometimes we want "shallow" directories without
- * full file children.
- */
-template<typename Child>
-struct DirectoryT
-{
-    using Name = std::string;
-
-    std::map<Name, Child, std::less<>> entries;
-
-    inline bool operator==(const DirectoryT &) const noexcept;
-    inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept;
-};
-
-struct Symlink
-{
-    std::string target;
-
-    auto operator<=>(const Symlink &) const = default;
-};
-
-/**
- * For when we know there is child, but don't know anything about it.
- *
- * This is not part of the core File System Object data model --- this
- * represents not knowing, not an additional type of file.
- */
-struct Opaque
-{
-    auto operator<=>(const Opaque &) const = default;
-};
-
-/**
- * `File<std::string>` nicely defining what a "file system object"
- * is in Nix.
- *
- * With a different type arugment, it is also can be a "skeletal"
- * version is that abstract syntax for a "NAR listing".
- */
-template<typename RegularContents, bool recur>
-struct VariantT
-{
-    bool operator==(const VariantT &) const noexcept;
-    std::strong_ordering operator<=>(const VariantT &) const noexcept;
-
-    using Regular = nix::fso::Regular<RegularContents>;
-
-    /**
-     * In the default case, we do want full file children for our directory.
-     */
-    using Directory = nix::fso::DirectoryT<std::conditional_t<recur, VariantT, Opaque>>;
-
-    using Symlink = nix::fso::Symlink;
-
-    using Raw = std::variant<Regular, Directory, Symlink>;
-    Raw raw;
-
-    MAKE_WRAPPER_CONSTRUCTOR(VariantT);
-
-    SourceAccessor::Stat lstat() const;
-};
-
-template<typename Child>
-inline bool DirectoryT<Child>::operator==(const DirectoryT &) const noexcept = default;
-
-template<typename Child>
-inline std::strong_ordering DirectoryT<Child>::operator<=>(const DirectoryT &) const noexcept = default;
-
-template<typename RegularContents, bool recur>
-inline bool
-VariantT<RegularContents, recur>::operator==(const VariantT<RegularContents, recur> &) const noexcept = default;
-
-template<typename RegularContents, bool recur>
-inline std::strong_ordering
-VariantT<RegularContents, recur>::operator<=>(const VariantT<RegularContents, recur> &) const noexcept = default;
-
-} // namespace fso
-
 /**
  * An source accessor for an in-memory file system.
  */
 struct MemorySourceAccessor : virtual SourceAccessor
 {
-    using File = fso::VariantT<std::string, true>;
+    /**
+     * In addition to being part of the implementation of
+     * `MemorySourceAccessor`, this has a side benefit of nicely
+     * defining what a "file system object" is in Nix.
+     */
+    struct File
+    {
+        bool operator==(const File &) const noexcept;
+        std::strong_ordering operator<=>(const File &) const noexcept;
+
+        struct Regular
+        {
+            bool executable = false;
+            std::string contents;
+
+            bool operator==(const Regular &) const = default;
+            auto operator<=>(const Regular &) const = default;
+        };
+
+        struct Directory
+        {
+            using Name = std::string;
+
+            std::map<Name, File, std::less<>> contents;
+
+            bool operator==(const Directory &) const noexcept;
+            // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet.
+            bool operator<(const Directory &) const noexcept;
+        };
+
+        struct Symlink
+        {
+            std::string target;
+
+            bool operator==(const Symlink &) const = default;
+            auto operator<=>(const Symlink &) const = default;
+        };
+
+        using Raw = std::variant<Regular, Directory, Symlink>;
+        Raw raw;
+
+        MAKE_WRAPPER_CONSTRUCTOR(File);
+
+        Stat lstat() const;
+    };
 
     std::optional<File> root;
 
@@ -141,6 +89,19 @@ struct MemorySourceAccessor : virtual SourceAccessor
     SourcePath addFile(CanonPath path, std::string && contents);
 };
 
+inline bool MemorySourceAccessor::File::Directory::operator==(
+    const MemorySourceAccessor::File::Directory &) const noexcept = default;
+
+inline bool
+MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept
+{
+    return contents < other.contents;
+}
+
+inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default;
+inline std::strong_ordering
+MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default;
+
 /**
  * Write to a `MemorySourceAccessor` at the given path
  */
@@ -160,53 +121,4 @@ struct MemorySink : FileSystemObjectSink
     void createSymlink(const CanonPath & path, const std::string & target) override;
 };
 
-template<>
-struct json_avoids_null<MemorySourceAccessor::File::Regular> : std::true_type
-{};
-
-template<>
-struct json_avoids_null<MemorySourceAccessor::File::Directory> : std::true_type
-{};
-
-template<>
-struct json_avoids_null<MemorySourceAccessor::File::Symlink> : std::true_type
-{};
-
-template<>
-struct json_avoids_null<MemorySourceAccessor::File> : std::true_type
-{};
-
-template<>
-struct json_avoids_null<MemorySourceAccessor> : std::true_type
-{};
-
 } // namespace nix
-
-namespace nlohmann {
-
-using namespace nix;
-
-#define ARG fso::Regular<RegularContents>
-template<typename RegularContents>
-JSON_IMPL_INNER(ARG)
-#undef ARG
-
-#define ARG fso::DirectoryT<Child>
-template<typename Child>
-JSON_IMPL_INNER(ARG)
-#undef ARG
-
-template<>
-JSON_IMPL_INNER(fso::Symlink)
-
-template<>
-JSON_IMPL_INNER(fso::Opaque)
-
-#define ARG fso::VariantT<RegularContents, recur>
-template<typename RegularContents, bool recur>
-JSON_IMPL_INNER(ARG)
-#undef ARG
-
-} // namespace nlohmann
-
-JSON_IMPL(MemorySourceAccessor)

src/libutil/include/nix/util/meson.build

@@ -50,7 +50,6 @@ headers = files(
   'memory-source-accessor.hh',
   'mounted-source-accessor.hh',
   'muxable-pipe.hh',
-  'nar-accessor.hh',
   'os-string.hh',
   'pool.hh',
   'pos-idx.hh',

src/libutil/include/nix/util/nar-accessor.hh

@@ -1,80 +0,0 @@
-#pragma once
-///@file
-
-#include "nix/util/memory-source-accessor.hh"
-
-#include <functional>
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix {
-
-struct Source;
-
-/**
- * Return an object that provides access to the contents of a NAR
- * file.
- */
-ref<SourceAccessor> makeNarAccessor(std::string && nar);
-
-ref<SourceAccessor> makeNarAccessor(Source & source);
-
-/**
- * Create a NAR accessor from a NAR listing (in the format produced by
- * listNar()). The callback getNarBytes(offset, length) is used by the
- * readFile() method of the accessor to get the contents of files
- * inside the NAR.
- */
-using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
-
-/**
- * The canonical GetNarBytes function for a seekable Source.
- */
-GetNarBytes seekableGetNarBytes(const Path & path);
-
-ref<SourceAccessor> makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes);
-
-struct NarListingRegularFile
-{
-    /**
-     * @see `SourceAccessor::Stat::fileSize`
-     */
-    std::optional<uint64_t> fileSize;
-
-    /**
-     * @see `SourceAccessor::Stat::narOffset`
-     *
-     * We only set to non-`std::nullopt` if it is also non-zero.
-     */
-    std::optional<uint64_t> narOffset;
-
-    auto operator<=>(const NarListingRegularFile &) const = default;
-};
-
-/**
- * Abstract syntax for a "NAR listing".
- */
-using NarListing = fso::VariantT<NarListingRegularFile, true>;
-
-/**
- * Shallow NAR listing where directory children are not recursively expanded.
- * Uses a variant that can hold Regular/Symlink fully, but Directory children
- * are just unit types indicating presence without content.
- */
-using ShallowNarListing = fso::VariantT<NarListingRegularFile, false>;
-
-/**
- * Return a deep structured representation of the contents of a NAR (except file
- * contents), recursively listing all children.
- */
-NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path);
-
-/**
- * Return a shallow structured representation of the contents of a NAR (except file
- * contents), only listing immediate children without recursing.
- */
-ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path);
-
-// All json_avoids_null and JSON_IMPL covered by generic templates in memory-source-accessor.hh
-
-} // namespace nix

src/libutil/memory-source-accessor.cc

@@ -1,5 +1,4 @@
 #include "nix/util/memory-source-accessor.hh"
-#include "nix/util/json-utils.hh"
 
 namespace nix {
 
@@ -30,13 +29,13 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path,
             return nullptr;
         auto & curDir = *curDirP;
 
-        auto i = curDir.entries.find(name);
+        auto i = curDir.contents.find(name);
-        if (i == curDir.entries.end()) {
+        if (i == curDir.contents.end()) {
             if (!create)
                 return nullptr;
             else {
                 newF = true;
-                i = curDir.entries.insert(
+                i = curDir.contents.insert(
                     i,
                     {
                         std::string{name},
@@ -69,26 +68,25 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path)
     return open(path, std::nullopt);
 }
 
-template<>
+MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const
-SourceAccessor::Stat MemorySourceAccessor::File::lstat() const
 {
     return std::visit(
         overloaded{
             [](const Regular & r) {
-                return SourceAccessor::Stat{
+                return Stat{
-                    .type = SourceAccessor::tRegular,
+                    .type = tRegular,
                     .fileSize = r.contents.size(),
                     .isExecutable = r.executable,
                 };
             },
             [](const Directory &) {
-                return SourceAccessor::Stat{
+                return Stat{
-                    .type = SourceAccessor::tDirectory,
+                    .type = tDirectory,
                 };
             },
             [](const Symlink &) {
-                return SourceAccessor::Stat{
+                return Stat{
-                    .type = SourceAccessor::tSymlink,
+                    .type = tSymlink,
                 };
             },
         },
@@ -108,7 +106,7 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon
         throw Error("file '%s' does not exist", path);
     if (auto * d = std::get_if<File::Directory>(&f->raw)) {
         DirEntries res;
-        for (auto & [name, file] : d->entries)
+        for (auto & [name, file] : d->contents)
             res.insert_or_assign(name, file.lstat().type);
         return res;
     } else

src/libutil/memory-source-accessor/json.cc

@@ -1,162 +0,0 @@
-#include "nix/util/memory-source-accessor.hh"
-#include "nix/util/nar-accessor.hh"
-#include "nix/util/json-utils.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nlohmann {
-
-using namespace nix;
-
-// fso::Regular<RegularContents>
-template<>
-MemorySourceAccessor::File::Regular adl_serializer<MemorySourceAccessor::File::Regular>::from_json(const json & json)
-{
-    auto & obj = getObject(json);
-    return MemorySourceAccessor::File::Regular{
-        .executable = getBoolean(valueAt(obj, "executable")),
-        .contents = getString(valueAt(obj, "contents")),
-    };
-}
-
-template<>
-void adl_serializer<MemorySourceAccessor::File::Regular>::to_json(
-    json & json, const MemorySourceAccessor::File::Regular & r)
-{
-    json = {
-        {"executable", r.executable},
-        {"contents", r.contents},
-    };
-}
-
-template<>
-NarListing::Regular adl_serializer<NarListing::Regular>::from_json(const json & json)
-{
-    auto & obj = getObject(json);
-    auto * execPtr = optionalValueAt(obj, "executable");
-    auto * sizePtr = optionalValueAt(obj, "size");
-    auto * offsetPtr = optionalValueAt(obj, "narOffset");
-    return NarListing::Regular{
-        .executable = execPtr ? getBoolean(*execPtr) : false,
-        .contents{
-            .fileSize = ptrToOwned<uint64_t>(sizePtr),
-            .narOffset = ptrToOwned<uint64_t>(offsetPtr).and_then(
-                [](auto v) { return v != 0 ? std::optional{v} : std::nullopt; }),
-        },
-    };
-}
-
-template<>
-void adl_serializer<NarListing::Regular>::to_json(json & j, const NarListing::Regular & r)
-{
-    if (r.contents.fileSize)
-        j["size"] = *r.contents.fileSize;
-    if (r.executable)
-        j["executable"] = true;
-    if (r.contents.narOffset)
-        j["narOffset"] = *r.contents.narOffset;
-}
-
-template<typename Child>
-void adl_serializer<fso::DirectoryT<Child>>::to_json(json & j, const fso::DirectoryT<Child> & d)
-{
-    j["entries"] = d.entries;
-}
-
-template<typename Child>
-fso::DirectoryT<Child> adl_serializer<fso::DirectoryT<Child>>::from_json(const json & json)
-{
-    auto & obj = getObject(json);
-    return fso::DirectoryT<Child>{
-        .entries = valueAt(obj, "entries"),
-    };
-}
-
-// fso::Symlink
-fso::Symlink adl_serializer<fso::Symlink>::from_json(const json & json)
-{
-    auto & obj = getObject(json);
-    return fso::Symlink{
-        .target = getString(valueAt(obj, "target")),
-    };
-}
-
-void adl_serializer<fso::Symlink>::to_json(json & json, const fso::Symlink & s)
-{
-    json = {
-        {"target", s.target},
-    };
-}
-
-// fso::Opaque
-fso::Opaque adl_serializer<fso::Opaque>::from_json(const json &)
-{
-    return fso::Opaque{};
-}
-
-void adl_serializer<fso::Opaque>::to_json(json & j, const fso::Opaque &)
-{
-    j = nlohmann::json::object();
-}
-
-// fso::VariantT<RegularContents, recur> - generic implementation
-template<typename RegularContents, bool recur>
-void adl_serializer<fso::VariantT<RegularContents, recur>>::to_json(
-    json & j, const fso::VariantT<RegularContents, recur> & val)
-{
-    using Variant = fso::VariantT<RegularContents, recur>;
-    j = nlohmann::json::object();
-    std::visit(
-        overloaded{
-            [&](const typename Variant::Regular & r) {
-                j = r;
-                j["type"] = "regular";
-            },
-            [&](const typename Variant::Directory & d) {
-                j = d;
-                j["type"] = "directory";
-            },
-            [&](const typename Variant::Symlink & s) {
-                j = s;
-                j["type"] = "symlink";
-            },
-        },
-        val.raw);
-}
-
-template<typename RegularContents, bool recur>
-fso::VariantT<RegularContents, recur>
-adl_serializer<fso::VariantT<RegularContents, recur>>::from_json(const json & json)
-{
-    using Variant = fso::VariantT<RegularContents, recur>;
-    auto & obj = getObject(json);
-    auto type = getString(valueAt(obj, "type"));
-    if (type == "regular")
-        return static_cast<typename Variant::Regular>(json);
-    if (type == "directory")
-        return static_cast<typename Variant::Directory>(json);
-    if (type == "symlink")
-        return static_cast<typename Variant::Symlink>(json);
-    else
-        throw Error("unknown type of file '%s'", type);
-}
-
-// Explicit instantiations for VariantT types we use
-template struct adl_serializer<MemorySourceAccessor::File>;
-template struct adl_serializer<NarListing>;
-template struct adl_serializer<ShallowNarListing>;
-
-// MemorySourceAccessor
-MemorySourceAccessor adl_serializer<MemorySourceAccessor>::from_json(const json & json)
-{
-    MemorySourceAccessor res;
-    res.root = json;
-    return res;
-}
-
-void adl_serializer<MemorySourceAccessor>::to_json(json & json, const MemorySourceAccessor & val)
-{
-    json = val.root;
-}
-
-} // namespace nlohmann

src/libutil/meson.build

@@ -146,9 +146,7 @@ sources = [ config_priv_h ] + files(
   'json-utils.cc',
   'logging.cc',
   'memory-source-accessor.cc',
-  'memory-source-accessor/json.cc',
   'mounted-source-accessor.cc',
-  'nar-accessor.cc',
   'pos-table.cc',
   'position.cc',
   'posix-source-accessor.cc',

src/libutil/unix/file-system.cc

@@ -16,7 +16,7 @@ namespace nix {
 
 Descriptor openDirectory(const std::filesystem::path & path)
 {
-    return open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
+    return open(path.c_str(), O_RDONLY | O_DIRECTORY);
 }
 
 void setWriteTime(

src/nix/build-remote/build-remote.cc

@@ -322,7 +322,7 @@ static int main_build_remote(int argc, char ** argv)
             //    output ids, which break CA derivations
             if (!drv.inputDrvs.map.empty())
                 drv.inputSrcs = store->parseStorePathSet(inputs);
-            optResult = sshStore->buildDerivation(*drvPath, static_cast<const BasicDerivation &>(drv));
+            optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv);
             auto & result = *optResult;
             if (auto * failureP = result.tryGetFailure()) {
                 if (settings.keepFailed) {

src/nix/cat.cc

@@ -1,6 +1,6 @@
 #include "nix/cmd/command.hh"
 #include "nix/store/store-api.hh"
-#include "nix/util/nar-accessor.hh"
+#include "nix/store/nar-accessor.hh"
 #include "nix/util/serialise.hh"
 #include "nix/util/source-accessor.hh"
 
@@ -80,7 +80,7 @@ struct CmdCatNar : StoreCommand, MixCat
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
         auto narAccessor = makeNarAccessor(source);
-        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
+        auto listing = listNar(narAccessor, CanonPath::root, true);
         cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };

src/nix/ls.cc

@@ -1,6 +1,6 @@
 #include "nix/cmd/command.hh"
 #include "nix/store/store-api.hh"
-#include "nix/util/nar-accessor.hh"
+#include "nix/store/nar-accessor.hh"
 #include "nix/main/common-args.hh"
 #include <nlohmann/json.hpp>
 
@@ -85,12 +85,7 @@ struct MixLs : virtual Args, MixJSON
         if (json) {
             if (showDirectory)
                 throw UsageError("'--directory' is useless with '--json'");
-            nlohmann::json j;
+            logger->cout("%s", listNar(accessor, path, recursive));
-            if (recursive)
-                j = listNarDeep(*accessor, path);
-            else
-                j = listNarShallow(*accessor, path);
-            logger->cout("%s", j.dump());
         } else
             listText(accessor, std::move(path));
     }
@@ -155,7 +150,7 @@ struct CmdLsNar : Command, MixLs
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};
         auto narAccessor = makeNarAccessor(source);
-        nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root);
+        auto listing = listNar(narAccessor, CanonPath::root, true);
         list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path});
     }
 };

src/nix/main.cc

@@ -271,7 +271,7 @@ static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel)
     );
 
     auto vDump = state.allocValue();
-    vDump->mkString(toplevel.dumpCli(), state.mem);
+    vDump->mkString(toplevel.dumpCli());
 
     auto vRes = state.allocValue();
     Value * args[]{&state.getBuiltin("false"), vDump};

src/nix/meson.build

@@ -229,7 +229,6 @@ foreach linkname : nix_symlinks
     env : {'MSYS' : 'winsymlinks:lnk'},
     # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
     build_by_default : true,
-    depends : this_exe,
   )
   # TODO(Ericson3214): Doesn't yet work
   #meson.override_find_program(linkname, t)
@@ -251,7 +250,6 @@ custom_target(
   env : {'MSYS' : 'winsymlinks:lnk'},
   # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working)
   build_by_default : true,
-  depends : this_exe,
 )
 # TODO(Ericson3214): Doesn't yet work
 #meson.override_find_program(linkname, t)

src/nix/nix-env/nix-env.cc

@@ -132,7 +132,7 @@ static void getAllExprs(EvalState & state, const SourcePath & path, StringSet &
             }
             /* Load the expression on demand. */
             auto vArg = state.allocValue();
-            vArg->mkPath(path2, state.mem);
+            vArg->mkPath(path2);
             if (seen.size() == maxAttrs)
                 throw Error("too many Nix expressions in directory '%1%'", path);
             attrs.alloc(attrName).mkApp(&state.getBuiltin("import"), vArg);
@@ -483,7 +483,7 @@ static bool keep(PackageInfo & drv)
 static void setMetaFlag(EvalState & state, PackageInfo & drv, const std::string & name, const std::string & value)
 {
     auto v = state.allocValue();
-    v->mkString(value, state.mem);
+    v->mkString(value);
     drv.setMeta(name, v);
 }
 

src/nix/nix-env/user-env.cc

@@ -58,20 +58,20 @@ bool createUserEnv(
         auto attrs = state.buildBindings(7 + outputs.size());
 
         attrs.alloc(state.s.type).mkStringNoCopy("derivation"_sds);
-        attrs.alloc(state.s.name).mkString(i.queryName(), state.mem);
+        attrs.alloc(state.s.name).mkString(i.queryName());
         auto system = i.querySystem();
         if (!system.empty())
-            attrs.alloc(state.s.system).mkString(system, state.mem);
+            attrs.alloc(state.s.system).mkString(system);
-        attrs.alloc(state.s.outPath).mkString(state.store->printStorePath(i.queryOutPath()), state.mem);
+        attrs.alloc(state.s.outPath).mkString(state.store->printStorePath(i.queryOutPath()));
         if (drvPath)
-            attrs.alloc(state.s.drvPath).mkString(state.store->printStorePath(*drvPath), state.mem);
+            attrs.alloc(state.s.drvPath).mkString(state.store->printStorePath(*drvPath));
 
         // Copy each output meant for installation.
         auto outputsList = state.buildList(outputs.size());
         for (const auto & [m, j] : enumerate(outputs)) {
-            (outputsList[m] = state.allocValue())->mkString(j.first, state.mem);
+            (outputsList[m] = state.allocValue())->mkString(j.first);
             auto outputAttrs = state.buildBindings(2);
-            outputAttrs.alloc(state.s.outPath).mkString(state.store->printStorePath(*j.second), state.mem);
+            outputAttrs.alloc(state.s.outPath).mkString(state.store->printStorePath(*j.second));
             attrs.alloc(j.first).mkAttrs(outputAttrs);
 
             /* This is only necessary when installing store paths, e.g.,

src/nix/prefetch.cc

@@ -59,7 +59,7 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url)
 std::tuple<StorePath, Hash> prefetchFile(
     ref<Store> store,
     const VerbatimURL & url,
-    std::optional<std::string> maybeName,
+    std::optional<std::string> name,
     HashAlgorithm hashAlgo,
     std::optional<Hash> expectedHash,
     bool unpack,
@@ -68,21 +68,16 @@ std::tuple<StorePath, Hash> prefetchFile(
     ContentAddressMethod method =
         unpack || executable ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat;
 
-    std::string name = maybeName
+    /* Figure out a name in the Nix store. */
-                           .or_else([&]() {
+    if (!name) {
-                               /* Figure out a name in the Nix store. */
+        name = url.lastPathSegment();
-                               auto derivedFromUrl = url.lastPathSegment();
+        if (!name || name->empty())
-                               if (!derivedFromUrl || derivedFromUrl->empty())
+            throw Error("cannot figure out file name for '%s'", url.to_string());
-                                   throw Error("cannot figure out file name for '%s'", url.to_string());
+    }
-                               return derivedFromUrl;
-                           })
-                           .value();
-
     try {
-        checkName(name);
+        checkName(*name);
     } catch (BadStorePathName & e) {
-        if (!maybeName)
+        e.addTrace({}, "file name '%s' was extracted from URL '%s'", *name, url.to_string());
-            e.addTrace({}, "file name '%s' was extracted from URL '%s'", name, url.to_string());
         throw;
     }
 
@@ -94,7 +89,7 @@ std::tuple<StorePath, Hash> prefetchFile(
     if (expectedHash) {
         hashAlgo = expectedHash->algo;
         storePath =
-            store->makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(method, *expectedHash, {}));
+            store->makeFixedOutputPathFromCA(*name, ContentAddressWithReferences::fromParts(method, *expectedHash, {}));
         if (store->isValidPath(*storePath))
             hash = expectedHash;
         else
@@ -143,7 +138,7 @@ std::tuple<StorePath, Hash> prefetchFile(
 
         Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url.to_string()));
 
-        auto info = store->addToStoreSlow(name, makeFSSourceAccessor(tmpFile), method, hashAlgo, {}, expectedHash);
+        auto info = store->addToStoreSlow(*name, makeFSSourceAccessor(tmpFile), method, hashAlgo, {}, expectedHash);
         storePath = info.path;
         assert(info.ca);
         hash = info.ca->hash;

tests/functional/flakes/common.sh

@@ -32,8 +32,6 @@ writeSimpleFlake() {
     baseName = builtins.baseNameOf ./.;
 
     root = ./.;
-
-    number = 123;
   };
 }
 EOF

tests/functional/flakes/meson.build

@@ -34,7 +34,6 @@ suites += {
     'source-paths.sh',
     'old-lockfiles.sh',
     'trace-ifd.sh',
-    'shallow.sh',
   ],
   'workdir' : meson.current_source_dir(),
 }

tests/functional/flakes/shallow.sh

@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-export _NIX_FORCE_HTTP=1
-
-source ./common.sh
-
-requireGit
-TODO_NixOS
-
-createFlake1
-
-repoDir="$TEST_ROOT/repo"
-mkdir -p "$repoDir"
-echo "# foo" >> "$flake1Dir/flake.nix"
-git -C "$flake1Dir" commit -a -m bla
-
-cat > "$repoDir"/flake.nix <<EOF
-{
-  inputs.dep = {
-    type = "git";
-    url = "file://$flake1Dir";
-  };
-  outputs = inputs: rec {
-    revs = assert inputs.dep.number == 123; inputs.dep.revCount;
-  };
-}
-EOF
-
-# This will do a non-shallow fetch.
-[[ $(nix eval "path:$repoDir#revs") = 2 ]]
-
-# This should re-use the existing non-shallow clone.
-clearStore
-mv "$flake1Dir" "$flake1Dir.moved"
-[[ $(nix eval "path:$repoDir#revs") = 2 ]]

tests/functional/lang/eval-fail-deepseq-list-attr.err.exp

@@ -1,25 +0,0 @@
-error:
-       … while calling the 'deepSeq' builtin
-         at /pwd/lang/eval-fail-deepseq-list-attr.nix:3:1:
-            2|
-            3| builtins.deepSeq [
-             | ^
-            4|   1
-
-       … while evaluating list element at index 1
-
-       … while evaluating the attribute 'b'
-         at /pwd/lang/eval-fail-deepseq-list-attr.nix:7:5:
-            6|     a = 2;
-            7|     b = throw "error in attr in list element";
-             |     ^
-            8|   }
-
-       … while calling the 'throw' builtin
-         at /pwd/lang/eval-fail-deepseq-list-attr.nix:7:9:
-            6|     a = 2;
-            7|     b = throw "error in attr in list element";
-             |         ^
-            8|   }
-
-       error: error in attr in list element

tests/functional/lang/eval-fail-deepseq-list-attr.nix

@@ -1,10 +0,0 @@
-# Test that deepSeq reports list index and attribute name in error traces.
-
-builtins.deepSeq [
-  1
-  {
-    a = 2;
-    b = throw "error in attr in list element";
-  }
-  3
-] "unexpected success"

tests/functional/lang/eval-fail-deepseq-stack-overflow.err.exp

@@ -1,30 +0,0 @@
-error:
-       … while calling the 'deepSeq' builtin
-         at /pwd/lang/eval-fail-deepseq-stack-overflow.nix:8:1:
-            7| in
-            8| builtins.deepSeq reverseLinkedList (
-             | ^
-            9|   throw "unexpected success; expected a controlled stack overflow instead"
-
-       … while evaluating the attribute 'tail'
-         at /pwd/lang/eval-fail-deepseq-stack-overflow.nix:6:67:
-            5|   long = builtins.genList (x: x) 100000;
-            6|   reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                                   ^
-            7| in
-
-       (9997 duplicate frames omitted)
-
-       … while evaluating the attribute 'head'
-         at /pwd/lang/eval-fail-deepseq-stack-overflow.nix:6:62:
-            5|   long = builtins.genList (x: x) 100000;
-            6|   reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                              ^
-            7| in
-
-       error: stack overflow; max-call-depth exceeded
-       at /pwd/lang/eval-fail-deepseq-stack-overflow.nix:5:28:
-            4| let
-            5|   long = builtins.genList (x: x) 100000;
-             |                            ^
-            6|   reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;

tests/functional/lang/eval-fail-deepseq-stack-overflow.nix

@@ -1,10 +0,0 @@
-# Test that deepSeq on a deeply nested structure produces a controlled
-# stack overflow error rather than a segfault.
-
-let
-  long = builtins.genList (x: x) 100000;
-  reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-in
-builtins.deepSeq reverseLinkedList (
-  throw "unexpected success; expected a controlled stack overflow instead"
-)

tests/functional/lang/eval-fail-derivation-structuredAttrs-stack-overflow.err.exp

@@ -1,54 +0,0 @@
-error:
-       … while evaluating the attribute 'outPath'
-         at «nix-internal»/derivation-internal.nix:50:7:
-           49|     value = commonAttrs // {
-           50|       outPath = builtins.getAttr outputName strict;
-             |       ^
-           51|       drvPath = strict.drvPath;
-
-       … while calling the 'getAttr' builtin
-         at «nix-internal»/derivation-internal.nix:50:17:
-           49|     value = commonAttrs // {
-           50|       outPath = builtins.getAttr outputName strict;
-             |                 ^
-           51|       drvPath = strict.drvPath;
-
-       … while calling the 'derivationStrict' builtin
-         at «nix-internal»/derivation-internal.nix:37:12:
-           36|
-           37|   strict = derivationStrict drvAttrs;
-             |            ^
-           38|
-
-       … while evaluating derivation 'test'
-         whose name attribute is located at /pwd/lang/eval-fail-derivation-structuredAttrs-stack-overflow.nix:5:3
-
-       … while evaluating attribute 'nested' of derivation 'test'
-         at /pwd/lang/eval-fail-derivation-structuredAttrs-stack-overflow.nix:9:3:
-            8|   __structuredAttrs = true;
-            9|   nested =
-             |   ^
-           10|     let
-
-       … while evaluating attribute 'tail'
-         at /pwd/lang/eval-fail-derivation-structuredAttrs-stack-overflow.nix:12:71:
-           11|       long = builtins.genList (x: x) 100000;
-           12|       reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                                       ^
-           13|     in
-
-       (9994 duplicate frames omitted)
-
-       … while evaluating attribute 'head'
-         at /pwd/lang/eval-fail-derivation-structuredAttrs-stack-overflow.nix:12:66:
-           11|       long = builtins.genList (x: x) 100000;
-           12|       reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                                  ^
-           13|     in
-
-       error: stack overflow; max-call-depth exceeded
-       at /pwd/lang/eval-fail-derivation-structuredAttrs-stack-overflow.nix:12:66:
-           11|       long = builtins.genList (x: x) 100000;
-           12|       reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                                  ^
-           13|     in

tests/functional/lang/eval-fail-derivation-structuredAttrs-stack-overflow.nix

@@ -1,15 +0,0 @@
-# Test that derivations with __structuredAttrs and deeply nested structures
-# produce a controlled stack overflow error rather than a segfault.
-
-derivation {
-  name = "test";
-  system = "x86_64-linux";
-  builder = "/bin/sh";
-  __structuredAttrs = true;
-  nested =
-    let
-      long = builtins.genList (x: x) 100000;
-      reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-    in
-    reverseLinkedList;
-}

tests/functional/lang/eval-fail-toJSON-stack-overflow.err.exp

@@ -1,30 +0,0 @@
-error:
-       … while calling the 'toJSON' builtin
-         at /pwd/lang/eval-fail-toJSON-stack-overflow.nix:8:1:
-            7| in
-            8| builtins.toJSON reverseLinkedList
-             | ^
-            9|
-
-       … while evaluating attribute 'tail'
-         at /pwd/lang/eval-fail-toJSON-stack-overflow.nix:6:67:
-            5|   long = builtins.genList (x: x) 100000;
-            6|   reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                                   ^
-            7| in
-
-       (9997 duplicate frames omitted)
-
-       … while evaluating attribute 'head'
-         at /pwd/lang/eval-fail-toJSON-stack-overflow.nix:6:62:
-            5|   long = builtins.genList (x: x) 100000;
-            6|   reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                              ^
-            7| in
-
-       error: stack overflow; max-call-depth exceeded
-       at /pwd/lang/eval-fail-toJSON-stack-overflow.nix:6:62:
-            5|   long = builtins.genList (x: x) 100000;
-            6|   reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-             |                                                              ^
-            7| in

tests/functional/lang/eval-fail-toJSON-stack-overflow.nix

@@ -1,8 +0,0 @@
-# Test that toJSON on a deeply nested structure produces a controlled
-# stack overflow error rather than a segfault.
-
-let
-  long = builtins.genList (x: x) 100000;
-  reverseLinkedList = builtins.foldl' (tail: head: { inherit head tail; }) null long;
-in
-builtins.toJSON reverseLinkedList

tests/nixos/content-encoding.nix

@@ -131,7 +131,6 @@ in
     start_all()
 
     machine.wait_for_unit("nginx.service")
-    machine.wait_for_open_port(80)
 
     # Original test: zstd archive with gzip content-encoding
     # Make sure that the file is properly compressed as the test would be meaningless otherwise

tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix

@@ -1,5 +1,5 @@
 {
-  description = "fetchTree fetches git repos shallowly if possible";
+  description = "fetchTree fetches git repos shallowly by default";
   script = ''
     # purge nix git cache to make sure we start with a clean slate
     client.succeed("rm -rf ~/.cache/nix")
@@ -28,7 +28,6 @@
         type = "git";
         url = "{repo.remote}";
         rev = "{commit2_rev}";
-        revCount = 1234;
       }}
     """
 

tests/nixos/tarball-flakes.nix

@@ -99,6 +99,7 @@ in
 
       # Check that fetching fails if we provide incorrect attributes.
       machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0")
+      machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=789")
       machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=")
     '';