Add rl-next/c-api-recoverable-errors

This provides an explicit API for call-fail-retry-succeed evaluation
flows, such as currently used in NixOps4.

An alternative design would simply reset the `Value` to the original
thunk instead of `tFailed` under the condition of catching a
`RecoverableEvalError`.
That is somewhat simpler, but I believe the presence of `tFailed` is
beneficial for possible use in the repl; being able to show the error
sooner, without re-evaluation.

The hasPos method and isEmpty function are required in order to avoid
an include loop.

.github/workflows/labels.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-24.04
     if: github.repository_owner == 'NixOS'
     steps:
-    - uses: actions/labeler@v5
+    - uses: actions/labeler@v6
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         sync-labels: false

doc/manual/rl-next/c-api-recoverable-errors.md

@@ -0,0 +1,23 @@
+---
+synopsis: "C API: Errors returned from your primops are not treated as recoverable by default"
+prs: [13930]
+---
+
+Nix 2.32 by default remembers the error in the thunk that triggered it.
+
+Previously the following sequence of events worked:
+
+1. Have a thunk that invokes a primop that's defined through the C API
+2. The primop returns an error
+3. Force the thunk again
+4. The primop returns a value
+5. The thunk evaluated successfully
+
+**Resolution**
+
+C API consumers that rely on this must change their recoverable error calls:
+
+```diff
+-nix_set_err_msg(context, NIX_ERR_*, msg);
++nix_set_err_msg(context, NIX_ERR_RECOVERABLE, msg);
+```

doc/manual/rl-next/dropped-compat.md

@@ -0,0 +1,6 @@
+---
+synopsis: "Removed support for daemons and clients older than Nix 2.0"
+prs: [13951]
+---
+
+We have dropped support in the daemon worker protocol for daemons and clients that don't speak at least version 18 of the protocol. This first Nix release that supports this version is Nix 2.0, released in February 2018.

doc/manual/rl-next/shorter-build-dir-names.md

@@ -0,0 +1,6 @@
+---
+synopsis: "Temporary build directories no longer include derivation names"
+prs: [13839]
+---
+
+Temporary build directories created during derivation builds no longer include the derivation name in their path to avoid build failures when the derivation name is too long. This change ensures predictable prefix lengths for build directories under `/nix/var/nix/builds`.

src/libcmd/installables.cc

@@ -178,10 +178,16 @@ MixFlakeOptions::MixFlakeOptions()
             for (auto & [inputName, input] : flake.lockFile.root->inputs) {
                 auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
                 if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
+                    fetchers::Attrs extraAttrs;
+
+                    if (!input3->lockedRef.subdir.empty()) {
+                        extraAttrs["dir"] = input3->lockedRef.subdir;
+                    }
+
                     overrideRegistry(
                         fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}),
                         input3->lockedRef.input,
-                        {});
+                        extraAttrs);
                 }
             }
         }},

src/libexpr-c/nix_api_expr.cc

@@ -40,6 +40,8 @@ static T * unsafe_new_with_self(F && init)
     return new (p) T(init(static_cast<T *>(p)));
 }
 
+extern "C" {
+
 nix_err nix_libexpr_init(nix_c_context * context)
 {
     if (context)
@@ -287,3 +289,5 @@ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * o
     GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
 #endif
 }
+
+} // extern "C"

src/libexpr-c/nix_api_expr_internal.h

@@ -8,6 +8,8 @@
 #include "nix_api_value.h"
 #include "nix/expr/search-path.hh"
 
+extern "C" {
+
 struct nix_eval_state_builder
 {
     nix::ref<nix::Store> store;
@@ -61,4 +63,6 @@ struct nix_realised_string
     std::vector<StorePath> storePaths;
 };
 
+} // extern "C"
+
 #endif // NIX_API_EXPR_INTERNAL_H

src/libexpr-c/nix_api_external.cc

@@ -14,6 +14,8 @@
 
 #include <nlohmann/json.hpp>
 
+extern "C" {
+
 void nix_set_string_return(nix_string_return * str, const char * c)
 {
     str->str = c;
@@ -40,6 +42,8 @@ nix_err nix_external_add_string_context(nix_c_context * context, nix_string_cont
     NIXC_CATCH_ERRS
 }
 
+} // extern "C"
+
 class NixCExternalValue : public nix::ExternalValueBase
 {
     NixCExternalValueDesc & desc;
@@ -170,6 +174,8 @@ public:
     virtual ~NixCExternalValue() override {};
 };
 
+extern "C" {
+
 ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v)
 {
     if (context)
@@ -198,3 +204,5 @@ void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b
     }
     NIXC_CATCH_ERRS_NULL
 }
+
+} // extern "C"

src/libexpr-c/nix_api_value.cc

@@ -1,4 +1,5 @@
 #include "nix/expr/attr-set.hh"
+#include "nix/expr/eval-error.hh"
 #include "nix/util/configuration.hh"
 #include "nix/expr/eval.hh"
 #include "nix/store/globals.hh"
@@ -89,8 +90,13 @@ static void nix_c_primop_wrapper(
     f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
 
     if (ctx.last_err_code != NIX_OK) {
-        /* TODO: Throw different errors depending on the error code */
-        state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
+        if (ctx.last_err_code == NIX_ERR_RECOVERABLE) {
+            state.error<nix::RecoverableEvalError>("Recoverable error from custom function: %s", *ctx.last_err)
+                .atPos(pos)
+                .debugThrow();
+        } else {
+            state.error<nix::EvalError>("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow();
+        }
     }
 
     if (!vTmp.isValid()) {
@@ -111,6 +117,8 @@ static void nix_c_primop_wrapper(
     v = vTmp;
 }
 
+extern "C" {
+
 PrimOp * nix_alloc_primop(
     nix_c_context * context,
     PrimOpFun fun,
@@ -175,6 +183,8 @@ ValueType nix_get_type(nix_c_context * context, const nix_value * value)
         switch (v.type()) {
         case nThunk:
             return NIX_TYPE_THUNK;
+        case nFailed:
+            return NIX_TYPE_FAILED;
         case nInt:
             return NIX_TYPE_INT;
         case nFloat:
@@ -592,7 +602,7 @@ nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * b
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_not_null(value);
-        nix::Symbol s = bb->builder.state.symbols.create(name);
+        nix::Symbol s = bb->builder.state.get().symbols.create(name);
         bb->builder.insert(s, &v);
     }
     NIXC_CATCH_ERRS
@@ -651,3 +661,5 @@ const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, si
 {
     return &s->storePaths[i];
 }
+
+} // extern "C"

src/libexpr-c/nix_api_value.h

@@ -32,7 +32,8 @@ typedef enum {
     NIX_TYPE_ATTRS,
     NIX_TYPE_LIST,
     NIX_TYPE_FUNCTION,
-    NIX_TYPE_EXTERNAL
+    NIX_TYPE_EXTERNAL,
+    NIX_TYPE_FAILED,
 } ValueType;
 
 // forward declarations

src/libexpr-tests/json.cc

@@ -54,7 +54,7 @@ TEST_F(JSONValueTest, IntNegative)
 TEST_F(JSONValueTest, String)
 {
     Value v;
-    v.mkString("test");
+    v.mkStringNoCopy("test");
     ASSERT_EQ(getJSONValue(v), "\"test\"");
 }
 
@@ -62,7 +62,7 @@ TEST_F(JSONValueTest, StringQuotes)
 {
     Value v;
 
-    v.mkString("test\"");
+    v.mkStringNoCopy("test\"");
     ASSERT_EQ(getJSONValue(v), "\"test\\\"\"");
 }
 

src/libexpr-tests/meson.build

@@ -55,6 +55,7 @@ sources = files(
   'nix_api_expr.cc',
   'nix_api_external.cc',
   'nix_api_value.cc',
+  'nix_api_value_internal.cc',
   'primops.cc',
   'search-path.cc',
   'trivial.cc',

src/libexpr-tests/nix_api_expr.cc

@@ -1,7 +1,5 @@
 #include "nix_api_store.h"
-#include "nix_api_store_internal.h"
 #include "nix_api_util.h"
-#include "nix_api_util_internal.h"
 #include "nix_api_expr.h"
 #include "nix_api_value.h"
 
@@ -151,8 +149,8 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_value)
     assert_ctx_ok();
     auto r = nix_string_realise(ctx, state, value, false);
     ASSERT_EQ(nullptr, r);
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR);
-    ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("cannot coerce")));
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("cannot coerce"));
 }
 
 TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build)
@@ -168,8 +166,8 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build)
     assert_ctx_ok();
     auto r = nix_string_realise(ctx, state, value, false);
     ASSERT_EQ(nullptr, r);
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR);
-    ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("failed with exit code 1")));
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("failed with exit code 1"));
 }
 
 TEST_F(nix_api_expr_test, nix_expr_realise_context)
@@ -381,12 +379,11 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return)
     nix_value * result = nix_alloc_value(ctx, state);
     assert_ctx_ok();
     nix_value_call(ctx, state, primopValue, three, result);
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR);
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
     ASSERT_THAT(
-        ctx->last_err,
-        testing::Optional(
-            testing::HasSubstr("Implementation error in custom function: return value was not initialized")));
-    ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badNoReturn")));
+        nix_err_msg(nullptr, ctx, nullptr),
+        testing::HasSubstr("Implementation error in custom function: return value was not initialized"));
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("badNoReturn"));
 }
 
 static void primop_bad_return_thunk(
@@ -419,12 +416,11 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk)
     assert_ctx_ok();
     NIX_VALUE_CALL(ctx, state, result, primopValue, toString, four);
 
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR);
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
     ASSERT_THAT(
-        ctx->last_err,
-        testing::Optional(
-            testing::HasSubstr("Implementation error in custom function: return value must not be a thunk")));
-    ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badReturnThunk")));
+        nix_err_msg(nullptr, ctx, nullptr),
+        testing::HasSubstr("Implementation error in custom function: return value must not be a thunk"));
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("badReturnThunk"));
 }
 
 TEST_F(nix_api_expr_test, nix_value_call_multi_no_args)
@@ -441,4 +437,105 @@ TEST_F(nix_api_expr_test, nix_value_call_multi_no_args)
     assert_ctx_ok();
     ASSERT_EQ(3, rInt);
 }
+
+// The following is a test case for retryable thunks.
+// This is a requirement for the current way in which NixOps4 evaluates its deployment expressions.
+// An alternative strategy could be implemented, but unwinding the stack may be a more efficient way to deal with many
+// suspensions/resumptions, compared to e.g. using a thread or coroutine stack for each suspended dependency. This test
+// models the essential bits of a deployment tool that uses such a strategy.
+
+// State for the retryable primop - simulates deployment resource availability
+struct DeploymentResourceState
+{
+    bool vm_created = false;
+};
+
+static void primop_load_resource_input(
+    void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret)
+{
+    assert(context);
+    assert(state);
+    auto * resource_state = static_cast<DeploymentResourceState *>(user_data);
+
+    // Get the resource input name argument
+    std::string input_name;
+    if (nix_get_string(context, args[0], OBSERVE_STRING(input_name)) != NIX_OK)
+        return;
+
+    // Only handle "vm_id" input - throw for anything else
+    if (input_name != "vm_id") {
+        std::string error_msg = "unknown resource input: " + input_name;
+        nix_set_err_msg(context, NIX_ERR_NIX_ERROR, error_msg.c_str());
+        return;
+    }
+
+    if (resource_state->vm_created) {
+        // VM has been created, return the ID
+        nix_init_string(context, ret, "vm-12345");
+    } else {
+        // VM not created yet, fail with dependency error
+        nix_set_err_msg(context, NIX_ERR_RECOVERABLE, "VM not yet created");
+    }
+}
+
+TEST_F(nix_api_expr_test, nix_expr_thunk_re_evaluation_after_deployment)
+{
+    // This test demonstrates NixOps4's requirement: a thunk calling a primop should be
+    // re-evaluable when deployment resources become available that were not available initially.
+
+    DeploymentResourceState resource_state;
+
+    PrimOp * primop = nix_alloc_primop(
+        ctx,
+        primop_load_resource_input,
+        1,
+        "loadResourceInput",
+        nullptr,
+        "load a deployment resource input",
+        &resource_state);
+    assert_ctx_ok();
+
+    nix_value * primopValue = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_init_primop(ctx, primopValue, primop);
+    assert_ctx_ok();
+
+    nix_value * inputName = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_init_string(ctx, inputName, "vm_id");
+    assert_ctx_ok();
+
+    // Create a single thunk by using nix_init_apply instead of nix_value_call
+    // This creates a lazy application that can be forced multiple times
+    nix_value * thunk = nix_alloc_value(ctx, state);
+    assert_ctx_ok();
+    nix_init_apply(ctx, thunk, primopValue, inputName);
+    assert_ctx_ok();
+
+    // First force: VM not created yet, should fail
+    nix_value_force(ctx, state, thunk);
+    ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx));
+    ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("VM not yet created"));
+
+    // Clear the error context for the next attempt
+    nix_c_context_free(ctx);
+    ctx = nix_c_context_create();
+
+    // Simulate deployment process: VM gets created
+    resource_state.vm_created = true;
+
+    // Second force of the SAME thunk: this is where the "failed" value issue appears
+    // With failed value caching, this should fail because the thunk is marked as permanently failed
+    // Without failed value caching (or with retryable failures), this should succeed
+    nix_value_force(ctx, state, thunk);
+
+    // If we get here without error, the thunk was successfully re-evaluated
+    assert_ctx_ok();
+
+    std::string result;
+    nix_get_string(ctx, thunk, OBSERVE_STRING(result));
+    assert_ctx_ok();
+    ASSERT_STREQ("vm-12345", result.c_str());
+}
+
 } // namespace nixC

src/libexpr-tests/nix_api_external.cc

@@ -1,9 +1,6 @@
 #include "nix_api_store.h"
-#include "nix_api_store_internal.h"
 #include "nix_api_util.h"
-#include "nix_api_util_internal.h"
 #include "nix_api_expr.h"
-#include "nix_api_expr_internal.h"
 #include "nix_api_value.h"
 #include "nix_api_external.h"
 
@@ -39,7 +36,7 @@ private:
         std::string type_string = "nix-external<MyExternalValueDesc( ";
         type_string += std::to_string(obj->_x);
         type_string += " )>";
-        res->str = &*type_string.begin();
+        nix_set_string_return(res, &*type_string.begin());
     }
 };
 

src/libexpr-tests/nix_api_value.cc

@@ -1,10 +1,7 @@
 #include "nix_api_store.h"
-#include "nix_api_store_internal.h"
 #include "nix_api_util.h"
-#include "nix_api_util_internal.h"
 #include "nix_api_expr.h"
 #include "nix_api_value.h"
-#include "nix_api_expr_internal.h"
 
 #include "nix/expr/tests/nix_api_expr.hh"
 #include "nix/util/tests/string_callback.hh"
@@ -16,14 +13,6 @@
 
 namespace nixC {
 
-TEST_F(nix_api_expr_test, as_nix_value_ptr)
-{
-    // nix_alloc_value casts nix::Value to nix_value
-    // It should be obvious from the decl that that works, but if it doesn't,
-    // the whole implementation would be utterly broken.
-    ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value));
-}
-
 TEST_F(nix_api_expr_test, nix_value_get_int_invalid)
 {
     ASSERT_EQ(0, nix_get_int(ctx, nullptr));
@@ -320,8 +309,10 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_error)
 
     // Evaluate it
     nix_value_force(ctx, state, v);
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR);
-    ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("attempt to call something which is not a function but"));
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr),
+        testing::HasSubstr("attempt to call something which is not a function but"));
 
     // Clean up
     nix_gc_decref(ctx, some_string);
@@ -380,7 +371,9 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg)
     // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception
     nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo");
     ASSERT_EQ(nullptr, foo);
-    ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg"));
+    ASSERT_THAT(
+        nix_err_msg(nullptr, ctx, nullptr),
+        testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg"));
 
     // Clean up
     nix_gc_decref(ctx, f);

src/libexpr-tests/nix_api_value_internal.cc

@@ -0,0 +1,25 @@
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+#include "nix_api_expr.h"
+#include "nix_api_value.h"
+#include "nix_api_expr_internal.h"
+
+#include "nix/expr/tests/nix_api_expr.hh"
+#include "nix/util/tests/string_callback.hh"
+
+#include <gmock/gmock.h>
+#include <cstddef>
+#include <cstdlib>
+#include <gtest/gtest.h>
+
+namespace nixC {
+
+TEST_F(nix_api_expr_test, as_nix_value_ptr)
+{
+    // nix_alloc_value casts nix::Value to nix_value
+    // It should be obvious from the decl that that works, but if it doesn't,
+    // the whole implementation would be utterly broken.
+    ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value));
+}
+
+} // namespace nixC

src/libexpr-tests/value/print.cc

@@ -35,14 +35,14 @@ TEST_F(ValuePrintingTests, tBool)
 TEST_F(ValuePrintingTests, tString)
 {
     Value vString;
-    vString.mkString("some-string");
+    vString.mkStringNoCopy("some-string");
     test(vString, "\"some-string\"");
 }
 
 TEST_F(ValuePrintingTests, tPath)
 {
     Value vPath;
-    vPath.mkString("/foo");
+    vPath.mkStringNoCopy("/foo");
     test(vPath, "\"/foo\"");
 }
 
@@ -61,7 +61,7 @@ TEST_F(ValuePrintingTests, tAttrs)
     Value vTwo;
     vTwo.mkInt(2);
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("one"), &vOne);
     builder.insert(state.symbols.create("two"), &vTwo);
 
@@ -196,11 +196,11 @@ TEST_F(ValuePrintingTests, depthAttrs)
     Value vTwo;
     vTwo.mkInt(2);
 
-    BindingsBuilder builderEmpty(state, state.allocBindings(0));
+    BindingsBuilder builderEmpty = state.buildBindings(0);
     Value vAttrsEmpty;
     vAttrsEmpty.mkAttrs(builderEmpty.finish());
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("one"), &vOne);
     builder.insert(state.symbols.create("two"), &vTwo);
     builder.insert(state.symbols.create("nested"), &vAttrsEmpty);
@@ -208,7 +208,7 @@ TEST_F(ValuePrintingTests, depthAttrs)
     Value vAttrs;
     vAttrs.mkAttrs(builder.finish());
 
-    BindingsBuilder builder2(state, state.allocBindings(10));
+    BindingsBuilder builder2 = state.buildBindings(10);
     builder2.insert(state.symbols.create("one"), &vOne);
     builder2.insert(state.symbols.create("two"), &vTwo);
     builder2.insert(state.symbols.create("nested"), &vAttrs);
@@ -233,14 +233,14 @@ TEST_F(ValuePrintingTests, depthList)
     Value vTwo;
     vTwo.mkInt(2);
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("one"), &vOne);
     builder.insert(state.symbols.create("two"), &vTwo);
 
     Value vAttrs;
     vAttrs.mkAttrs(builder.finish());
 
-    BindingsBuilder builder2(state, state.allocBindings(10));
+    BindingsBuilder builder2 = state.buildBindings(10);
     builder2.insert(state.symbols.create("one"), &vOne);
     builder2.insert(state.symbols.create("two"), &vTwo);
     builder2.insert(state.symbols.create("nested"), &vAttrs);
@@ -290,12 +290,12 @@ TEST_F(StringPrintingTests, maxLengthTruncation)
 TEST_F(ValuePrintingTests, attrsTypeFirst)
 {
     Value vType;
-    vType.mkString("puppy");
+    vType.mkStringNoCopy("puppy");
 
     Value vApple;
-    vApple.mkString("apple");
+    vApple.mkStringNoCopy("apple");
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("type"), &vType);
     builder.insert(state.symbols.create("apple"), &vApple);
 
@@ -334,7 +334,7 @@ TEST_F(ValuePrintingTests, ansiColorsBool)
 TEST_F(ValuePrintingTests, ansiColorsString)
 {
     Value v;
-    v.mkString("puppy");
+    v.mkStringNoCopy("puppy");
 
     test(v, ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, PrintOptions{.ansiColors = true});
 }
@@ -342,7 +342,7 @@ TEST_F(ValuePrintingTests, ansiColorsString)
 TEST_F(ValuePrintingTests, ansiColorsStringElided)
 {
     Value v;
-    v.mkString("puppy");
+    v.mkStringNoCopy("puppy");
 
     test(
         v,
@@ -374,7 +374,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrs)
     Value vTwo;
     vTwo.mkInt(2);
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("one"), &vOne);
     builder.insert(state.symbols.create("two"), &vTwo);
 
@@ -390,9 +390,9 @@ TEST_F(ValuePrintingTests, ansiColorsAttrs)
 TEST_F(ValuePrintingTests, ansiColorsDerivation)
 {
     Value vDerivation;
-    vDerivation.mkString("derivation");
+    vDerivation.mkStringNoCopy("derivation");
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.s.type, &vDerivation);
 
     Value vAttrs;
@@ -413,7 +413,7 @@ TEST_F(ValuePrintingTests, ansiColorsError)
 {
     Value throw_ = state.getBuiltin("throw");
     Value message;
-    message.mkString("uh oh!");
+    message.mkStringNoCopy("uh oh!");
     Value vError;
     vError.mkApp(&throw_, &message);
 
@@ -430,14 +430,14 @@ TEST_F(ValuePrintingTests, ansiColorsDerivationError)
 {
     Value throw_ = state.getBuiltin("throw");
     Value message;
-    message.mkString("uh oh!");
+    message.mkStringNoCopy("uh oh!");
     Value vError;
     vError.mkApp(&throw_, &message);
 
     Value vDerivation;
-    vDerivation.mkString("derivation");
+    vDerivation.mkStringNoCopy("derivation");
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.s.type, &vDerivation);
     builder.insert(state.s.drvPath, &vError);
 
@@ -553,12 +553,12 @@ TEST_F(ValuePrintingTests, ansiColorsBlackhole)
 
 TEST_F(ValuePrintingTests, ansiColorsAttrsRepeated)
 {
-    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
+    BindingsBuilder emptyBuilder = state.buildBindings(1);
 
     Value vEmpty;
     vEmpty.mkAttrs(emptyBuilder.finish());
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("a"), &vEmpty);
     builder.insert(state.symbols.create("b"), &vEmpty);
 
@@ -570,7 +570,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsRepeated)
 
 TEST_F(ValuePrintingTests, ansiColorsListRepeated)
 {
-    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
+    BindingsBuilder emptyBuilder = state.buildBindings(1);
 
     Value vEmpty;
     vEmpty.mkAttrs(emptyBuilder.finish());
@@ -586,7 +586,7 @@ TEST_F(ValuePrintingTests, ansiColorsListRepeated)
 
 TEST_F(ValuePrintingTests, listRepeated)
 {
-    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
+    BindingsBuilder emptyBuilder = state.buildBindings(1);
 
     Value vEmpty;
     vEmpty.mkAttrs(emptyBuilder.finish());
@@ -609,7 +609,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided)
     Value vTwo;
     vTwo.mkInt(2);
 
-    BindingsBuilder builder(state, state.allocBindings(10));
+    BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("one"), &vOne);
     builder.insert(state.symbols.create("two"), &vTwo);
 
@@ -635,8 +635,6 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided)
 
 TEST_F(ValuePrintingTests, ansiColorsListElided)
 {
-    BindingsBuilder emptyBuilder(state, state.allocBindings(1));
-
     Value vOne;
     vOne.mkInt(1);
 

src/libexpr/attr-set.cc

@@ -16,19 +16,19 @@ Bindings * EvalState::allocBindings(size_t capacity)
         throw Error("attribute set of size %d is too big", capacity);
     nrAttrsets++;
     nrAttrsInAttrsets += capacity;
-    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity);
+    return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings();
 }
 
 Value & BindingsBuilder::alloc(Symbol name, PosIdx pos)
 {
-    auto value = state.allocValue();
+    auto value = state.get().allocValue();
     bindings->push_back(Attr(name, value, pos));
     return *value;
 }
 
 Value & BindingsBuilder::alloc(std::string_view name, PosIdx pos)
 {
-    return alloc(state.symbols.create(name), pos);
+    return alloc(state.get().symbols.create(name), pos);
 }
 
 void Bindings::sort()

src/libexpr/eval-error.cc

@@ -113,5 +113,6 @@ template class EvalErrorBuilder<MissingArgumentError>;
 template class EvalErrorBuilder<InfiniteRecursionError>;
 template class EvalErrorBuilder<InvalidPathError>;
 template class EvalErrorBuilder<IFDError>;
+template class EvalErrorBuilder<RecoverableEvalError>;
 
 } // namespace nix

src/libexpr/eval-gc.cc

@@ -16,6 +16,7 @@
 #  endif
 
 #  include <gc/gc_allocator.h>
+#  include <gc/gc_tiny_fl.h> // For GC_GRANULE_BYTES
 
 #  include <boost/coroutine2/coroutine.hpp>
 #  include <boost/coroutine2/protected_fixedsize_stack.hpp>
@@ -23,6 +24,17 @@
 
 #endif
 
+/*
+ * Ensure that Boehm satisfies our alignment requirements. This is the default configuration [^]
+ * and this assertion should never break for any platform. Let's assert it just in case.
+ *
+ * This alignment is particularly useful to be able to use aligned
+ * load/store instructions for loading/writing Values.
+ *
+ * [^]: https://github.com/bdwgc/bdwgc/blob/54ac18ccbc5a833dd7edaff94a10ab9b65044d61/include/gc/gc_tiny_fl.h#L31-L33
+ */
+static_assert(sizeof(void *) * 2 == GC_GRANULE_BYTES, "Boehm GC must use GC_GRANULE_WORDS = 2");
+
 namespace nix {
 
 #if NIX_USE_BOEHMGC

src/libexpr/eval.cc

@@ -1,4 +1,5 @@
 #include "nix/expr/eval.hh"
+#include "nix/expr/eval-error.hh"
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/primops.hh"
 #include "nix/expr/print-options.hh"
@@ -22,10 +23,12 @@
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/tarball.hh"
 #include "nix/fetchers/input-cache.hh"
+#include "nix/util/current-process.hh"
 
 #include "parser-tab.hh"
 
 #include <algorithm>
+#include <exception>
 #include <iostream>
 #include <sstream>
 #include <cstring>
@@ -38,10 +41,6 @@
 #include <nlohmann/json.hpp>
 #include <boost/container/small_vector.hpp>
 
-#ifndef _WIN32 // TODO use portable implementation
-#  include <sys/resource.h>
-#endif
-
 #include "nix/util/strings-inline.hh"
 
 using json = nlohmann::json;
@@ -127,6 +126,8 @@ std::string_view showType(ValueType type, bool withArticle)
         return WA("a", "float");
     case nThunk:
         return WA("a", "thunk");
+    case nFailed:
+        return WA("a", "failure");
     }
     unreachable();
 }
@@ -205,7 +206,7 @@ EvalState::EvalState(
     , settings{settings}
     , symbols(StaticEvalSymbols::staticSymbolTable())
     , repair(NoRepair)
-    , emptyBindings(0)
+    , emptyBindings(Bindings())
     , storeFS(makeMountedSourceAccessor({
           {CanonPath::root, makeEmptySourceAccessor()},
           /* In the pure eval case, we can simply require
@@ -292,10 +293,10 @@ EvalState::EvalState(
     vNull.mkNull();
     vTrue.mkBool(true);
     vFalse.mkBool(false);
-    vStringRegular.mkString("regular");
-    vStringDirectory.mkString("directory");
-    vStringSymlink.mkString("symlink");
-    vStringUnknown.mkString("unknown");
+    vStringRegular.mkStringNoCopy("regular");
+    vStringDirectory.mkStringNoCopy("directory");
+    vStringSymlink.mkStringNoCopy("symlink");
+    vStringUnknown.mkStringNoCopy("unknown");
 
     /* Construct the Nix expression search path. */
     assert(lookupPath.elements.empty());
@@ -824,7 +825,7 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
 
 void Value::mkString(std::string_view s)
 {
-    mkString(makeImmutableString(s));
+    mkStringNoCopy(makeImmutableString(s));
 }
 
 static const char ** encodeContext(const NixStringContext & context)
@@ -843,12 +844,12 @@ static const char ** encodeContext(const NixStringContext & context)
 
 void Value::mkString(std::string_view s, const NixStringContext & context)
 {
-    mkString(makeImmutableString(s), encodeContext(context));
+    mkStringNoCopy(makeImmutableString(s), encodeContext(context));
 }
 
 void Value::mkStringMove(const char * s, const NixStringContext & context)
 {
-    mkString(s, encodeContext(context));
+    mkStringNoCopy(s, encodeContext(context));
 }
 
 void Value::mkPath(const SourcePath & path)
@@ -1218,7 +1219,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
                 *vOverrides,
                 [&]() { return vOverrides->determinePos(noPos); },
                 "while evaluating the `__overrides` attribute");
-            bindings.grow(state.allocBindings(bindings.capacity() + vOverrides->attrs()->size()));
+            bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size()));
             for (auto & i : *vOverrides->attrs()) {
                 AttrDefs::iterator j = attrs.find(i.name);
                 if (j != attrs.end()) {
@@ -2063,6 +2064,54 @@ void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value &
 // always force this to be separate, otherwise forceValue may inline it and take
 // a massive perf hit
 [[gnu::noinline]]
+void EvalState::handleEvalExceptionForThunk(Env * env, Expr * expr, Value & v, const PosIdx pos)
+{
+    v.mkThunk(env, expr);
+    tryFixupBlackHolePos(v, pos);
+
+    auto e = std::current_exception();
+    Value * recovery = nullptr;
+    try {
+        std::rethrow_exception(e);
+    } catch (const RecoverableEvalError & e) {
+        recovery = allocValue();
+    } catch (...) {
+    }
+    if (recovery) {
+        recovery->mkThunk(env, expr);
+    }
+    v.mkFailed(e, recovery);
+}
+
+[[gnu::noinline]]
+void EvalState::handleEvalExceptionForApp(Value & v)
+{
+    auto e = std::current_exception();
+    Value * recovery = nullptr;
+    try {
+        std::rethrow_exception(e);
+    } catch (const RecoverableEvalError & e) {
+        recovery = allocValue();
+    } catch (...) {
+    }
+    if (recovery) {
+        *recovery = v;
+    }
+    v.mkFailed(e, recovery);
+}
+
+[[gnu::noinline]]
+void EvalState::handleEvalFailed(Value & v, const PosIdx pos)
+{
+    assert(v.isFailed());
+    if (auto recoveryValue = v.failed()->recoveryValue) {
+        v = *recoveryValue;
+        forceValue(v, pos);
+    } else {
+        std::rethrow_exception(v.failed()->ex);
+    }
+}
+
 void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
 {
     if (!v.isBlackhole())
@@ -2071,7 +2120,8 @@ void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
     try {
         std::rethrow_exception(e);
     } catch (InfiniteRecursionError & e) {
-        e.atPos(positions[pos]);
+        if (!e.hasPos())
+            e.atPos(positions[pos]);
     } catch (...) {
     }
 }
@@ -2696,8 +2746,11 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st
         }
         return;
 
-    case nThunk: // Must not be left by forceValue
-        assert(false);
+    // Cannot be returned by forceValue().
+    case nThunk:
+    case nFailed:
+        unreachable();
+
     default: // Note that we pass compiler flags that should make `default:` unreachable.
         // Also note that this probably ran after `eqValues`, which implements
         // the same logic more efficiently (without having to unwind stacks),
@@ -2789,8 +2842,11 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
         // !!!
         return v1.fpoint() == v2.fpoint();
 
-    case nThunk: // Must not be left by forceValue
-        assert(false);
+    // Cannot be returned by forceValue().
+    case nThunk:
+    case nFailed:
+        unreachable();
+
     default: // Note that we pass compiler flags that should make `default:` unreachable.
         error<EvalError>("eqValues: cannot compare %1% with %2%", showType(v1), showType(v2))
             .withTrace(pos, errorCtx)
@@ -2830,11 +2886,8 @@ void EvalState::maybePrintStats()
 
 void EvalState::printStatistics()
 {
-#ifndef _WIN32 // TODO use portable implementation
-    struct rusage buf;
-    getrusage(RUSAGE_SELF, &buf);
-    float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000);
-#endif
+    std::chrono::microseconds cpuTimeDuration = getCpuUserTime();
+    float cpuTime = std::chrono::duration_cast<std::chrono::duration<float>>(cpuTimeDuration).count();
 
     uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *);
     uint64_t bLists = nrListElems * sizeof(Value *);
@@ -2856,18 +2909,12 @@ void EvalState::printStatistics()
     if (outPath != "-")
         fs.open(outPath, std::fstream::out);
     json topObj = json::object();
-#ifndef _WIN32 // TODO implement
     topObj["cpuTime"] = cpuTime;
-#endif
     topObj["time"] = {
-#ifndef _WIN32 // TODO implement
         {"cpu", cpuTime},
-#endif
 #if NIX_USE_BOEHMGC
         {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime},
-#  ifndef _WIN32 // TODO implement
         {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime},
-#  endif
 #endif
     };
     topObj["envs"] = {

src/libexpr/include/nix/expr/attr-set.hh

@@ -5,6 +5,7 @@
 #include "nix/expr/symbol-table.hh"
 
 #include <algorithm>
+#include <functional>
 
 namespace nix {
 
@@ -54,16 +55,14 @@ public:
     PosIdx pos;
 
 private:
-    size_t size_, capacity_;
+    size_t size_ = 0;
     Attr attrs[0];
 
-    Bindings(size_t capacity)
-        : size_(0)
-        , capacity_(capacity)
-    {
-    }
-
-    Bindings(const Bindings & bindings) = delete;
+    Bindings() = default;
+    Bindings(const Bindings &) = delete;
+    Bindings(Bindings &&) = delete;
+    Bindings & operator=(const Bindings &) = delete;
+    Bindings & operator=(Bindings &&) = delete;
 
 public:
     size_t size() const
@@ -82,7 +81,6 @@ public:
 
     void push_back(const Attr & attr)
     {
-        assert(size_ < capacity_);
         attrs[size_++] = attr;
     }
 
@@ -136,11 +134,6 @@ public:
 
     void sort();
 
-    size_t capacity() const
-    {
-        return capacity_;
-    }
-
     /**
      * Returns the attributes in lexicographically sorted order.
      */
@@ -165,22 +158,29 @@ public:
  * order at the end. The only way to consume a BindingsBuilder is to
  * call finish(), which sorts the bindings.
  */
-class BindingsBuilder
+class BindingsBuilder final
 {
-    Bindings * bindings;
-
 public:
     // needed by std::back_inserter
     using value_type = Attr;
+    using size_type = Bindings::size_t;
 
-    EvalState & state;
+private:
+    Bindings * bindings;
+    Bindings::size_t capacity_;
 
-    BindingsBuilder(EvalState & state, Bindings * bindings)
+    friend class EvalState;
+
+    BindingsBuilder(EvalState & state, Bindings * bindings, size_type capacity)
         : bindings(bindings)
+        , capacity_(capacity)
         , state(state)
     {
     }
 
+public:
+    std::reference_wrapper<EvalState> state;
+
     void insert(Symbol name, Value * value, PosIdx pos = noPos)
     {
         insert(Attr(name, value, pos));
@@ -193,6 +193,7 @@ public:
 
     void push_back(const Attr & attr)
     {
+        assert(bindings->size() < capacity_);
         bindings->push_back(attr);
     }
 
@@ -211,16 +212,16 @@ public:
         return bindings;
     }
 
-    size_t capacity()
+    size_t capacity() const noexcept
     {
-        return bindings->capacity();
+        return capacity_;
     }
 
-    void grow(Bindings * newBindings)
+    void grow(BindingsBuilder newBindings)
     {
         for (auto & i : *bindings)
-            newBindings->push_back(i);
-        bindings = newBindings;
+            newBindings.push_back(i);
+        std::swap(*this, newBindings);
     }
 
     friend struct ExprAttrs;

src/libexpr/include/nix/expr/eval-error.hh

@@ -56,6 +56,14 @@ MakeError(MissingArgumentError, EvalError);
 MakeError(InfiniteRecursionError, EvalError);
 MakeError(IFDError, EvalBaseError);
 
+/**
+ * An evaluation error which should be retried instead of rethrown
+ *
+ * A RecoverableEvalError is not an EvalError, because we shouldn't cache it in the eval cache, as it should be retried
+ * anyway.
+ */
+MakeError(RecoverableEvalError, EvalBaseError);
+
 struct InvalidPathError : public EvalError
 {
 public:

src/libexpr/include/nix/expr/eval-inline.hh

@@ -5,6 +5,7 @@
 #include "nix/expr/eval.hh"
 #include "nix/expr/eval-error.hh"
 #include "nix/expr/eval-settings.hh"
+#include <exception>
 
 namespace nix {
 
@@ -97,12 +98,19 @@ void EvalState::forceValue(Value & v, const PosIdx pos)
             else
                 ExprBlackHole::throwInfiniteRecursionError(*this, v);
         } catch (...) {
-            v.mkThunk(env, expr);
-            tryFixupBlackHolePos(v, pos);
+            handleEvalExceptionForThunk(env, expr, v, pos);
             throw;
         }
-    } else if (v.isApp())
-        callFunction(*v.app().left, *v.app().right, v, pos);
+    } else if (v.isApp()) {
+        try {
+            callFunction(*v.app().left, *v.app().right, v, pos);
+        } catch (...) {
+            handleEvalExceptionForApp(v);
+            throw;
+        }
+    } else if (v.isFailed()) {
+        handleEvalFailed(v, pos);
+    }
 }
 
 [[gnu::always_inline]]

src/libexpr/include/nix/expr/eval.hh

@@ -610,8 +610,28 @@ public:
      */
     inline void forceValue(Value & v, const PosIdx pos);
 
+private:
+
+    /**
+     * Internal support function for forceValue
+     *
+     * This code is factored out so that it's not in the heavily inlined hot path.
+     */
+    void handleEvalExceptionForThunk(Env * env, Expr * expr, Value & v, const PosIdx pos);
+
+    /**
+     * Internal support function for forceValue
+     *
+     * This code is factored out so that it's not in the heavily inlined hot path.
+     */
+    void handleEvalExceptionForApp(Value & v);
+
+    void handleEvalFailed(Value & v, PosIdx pos);
+
     void tryFixupBlackHolePos(Value & v, PosIdx pos);
 
+public:
+
     /**
      * Force a value, then recursively force list elements and
      * attributes.
@@ -879,7 +899,7 @@ public:
 
     BindingsBuilder buildBindings(size_t capacity)
     {
-        return BindingsBuilder(*this, allocBindings(capacity));
+        return BindingsBuilder(*this, allocBindings(capacity), capacity);
     }
 
     ListBuilder buildList(size_t size)

src/libexpr/include/nix/expr/nixexpr.hh

@@ -158,7 +158,7 @@ struct ExprString : Expr
     ExprString(std::string && s)
         : s(std::move(s))
     {
-        v.mkString(this->s.data());
+        v.mkStringNoCopy(this->s.data());
     };
 
     Value * maybeThunk(EvalState & state, Env & env) override;

src/libexpr/include/nix/expr/symbol-table.hh

@@ -61,6 +61,15 @@ public:
         return id > 0;
     }
 
+    /**
+     * The ID is a private implementation detail that should generally not be observed. However, we expose here just for
+     * sake of `switch...case`, which needs to dispatch on numbers. */
+    [[gnu::always_inline]]
+    constexpr uint32_t getId() const noexcept
+    {
+        return id;
+    }
+
     constexpr auto operator<=>(const Symbol & other) const noexcept = default;
 
     friend class std::hash<Symbol>;
@@ -113,12 +122,12 @@ public:
         // for multi-threaded implementations: lock store and allocator here
         const auto & [v, idx] = key.store.add(SymbolValue{});
         if (size == 0) {
-            v.mkString("", nullptr);
+            v.mkStringNoCopy("", nullptr);
         } else {
             auto s = key.alloc.allocate(size + 1);
             memcpy(s, key.s.data(), size);
             s[size] = '\0';
-            v.mkString(s, nullptr);
+            v.mkStringNoCopy(s, nullptr);
         }
         v.size_ = size;
         v.idx = idx;

src/libexpr/include/nix/expr/value.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include <cassert>
+#include <exception>
 #include <span>
 #include <type_traits>
 #include <concepts>
@@ -35,6 +36,7 @@ typedef enum {
     tBool,
     tNull,
     tFloat,
+    tFailed,
     tExternal,
     tPrimOp,
     tAttrs,
@@ -57,6 +59,7 @@ typedef enum {
  */
 typedef enum {
     nThunk,
+    nFailed,
     nInt,
     nFloat,
     nBool,
@@ -265,6 +268,30 @@ struct ValueBase
         size_t size;
         Value * const * elems;
     };
+
+    /**
+      Representation of an evaluation that previously failed.
+
+      `Value` references `Failed` by packed pointer, and its `new` is GC managed.
+
+      @see gc_cleanup std::exception_ptr
+     */
+    class Failed : public gc_cleanup
+    {
+    public:
+        std::exception_ptr ex;
+        /**
+         * Optional value for recovering `RecoverableEvalError`
+         * Must be set iff `ex` is an instance of `RecoverableEvalError`.
+         */
+        Value * recoveryValue;
+
+        Failed(std::exception_ptr ex, Value * recoveryValue)
+            : ex(ex)
+            , recoveryValue(recoveryValue)
+        {
+        }
+    };
 };
 
 template<typename T>
@@ -291,6 +318,7 @@ struct PayloadTypeToInternalType
     MACRO(PrimOp *, primOp, tPrimOp)                                \
     MACRO(ValueBase::PrimOpApplicationThunk, primOpApp, tPrimOpApp) \
     MACRO(ExternalValueBase *, external, tExternal)                 \
+    MACRO(ValueBase::Failed *, failed, tFailed)                     \
     MACRO(NixFloat, fpoint, tFloat)
 
 #define NIX_VALUE_PAYLOAD_TYPE(T, FIELD_NAME, DISCRIMINATOR) \
@@ -369,7 +397,7 @@ namespace detail {
 /* Whether to use a specialization of ValueStorage that does bitpacking into
    alignment niches. */
 template<std::size_t ptrSize>
-inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEFAULT_NEW_ALIGNMENT__ >= 8);
+inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEFAULT_NEW_ALIGNMENT__ >= 16);
 
 } // namespace detail
 
@@ -378,7 +406,8 @@ inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEF
  * Packs discriminator bits into the pointer alignment niches.
  */
 template<std::size_t ptrSize>
-class ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>> : public detail::ValueBase
+class alignas(16) ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<ptrSize>>>
+    : public detail::ValueBase
 {
     /* Needs a dependent type name in order for member functions (and
      * potentially ill-formed bit casts) to be SFINAE'd out.
@@ -594,6 +623,11 @@ protected:
         path.path = std::bit_cast<const char *>(payload[1]);
     }
 
+    void getStorage(Failed *& failed) const noexcept
+    {
+        failed = std::bit_cast<Failed *>(payload[1]);
+    }
+
     void setStorage(NixInt integer) noexcept
     {
         setSingleDWordPayload<tInt>(integer.value);
@@ -643,6 +677,11 @@ protected:
     {
         setUntaggablePayload<pdPath>(path.accessor, path.path);
     }
+
+    void setStorage(Failed * failed) noexcept
+    {
+        setSingleDWordPayload<tFailed>(std::bit_cast<PackedPointer>(failed));
+    }
 };
 
 /**
@@ -865,12 +904,12 @@ public:
     inline bool isThunk() const
     {
         return isa<tThunk>();
-    };
+    }
 
     inline bool isApp() const
     {
         return isa<tApp>();
-    };
+    }
 
     inline bool isBlackhole() const;
 
@@ -878,17 +917,22 @@ public:
     inline bool isLambda() const
     {
         return isa<tLambda>();
-    };
+    }
 
     inline bool isPrimOp() const
     {
         return isa<tPrimOp>();
-    };
+    }
 
     inline bool isPrimOpApp() const
     {
         return isa<tPrimOpApp>();
-    };
+    }
+
+    inline bool isFailed() const
+    {
+        return isa<tFailed>();
+    }
 
     /**
      * Returns the normal type of a Value. This only returns nThunk if
@@ -925,6 +969,8 @@ public:
             return nExternal;
         case tFloat:
             return nFloat;
+        case tFailed:
+            return nFailed;
         case tThunk:
         case tApp:
             return nThunk;
@@ -960,7 +1006,7 @@ public:
         setStorage(b);
     }
 
-    inline void mkString(const char * s, const char ** context = 0) noexcept
+    void mkStringNoCopy(const char * s, const char ** context = 0) noexcept
     {
         setStorage(StringWithContext{.c_str = s, .context = context});
     }
@@ -972,7 +1018,6 @@ public:
     void mkStringMove(const char * s, const NixStringContext & context);
 
     void mkPath(const SourcePath & path);
-    void mkPath(std::string_view path);
 
     inline void mkPath(SourceAccessor * accessor, const char * path) noexcept
     {
@@ -993,12 +1038,20 @@ public:
 
     void mkList(const ListBuilder & builder) noexcept
     {
-        if (builder.size == 1)
+        switch (builder.size) {
+        case 0:
+            setStorage(List{.size = 0, .elems = nullptr});
+            break;
+        case 1:
             setStorage(std::array<Value *, 2>{builder.inlineElems[0], nullptr});
-        else if (builder.size == 2)
+            break;
+        case 2:
             setStorage(std::array<Value *, 2>{builder.inlineElems[0], builder.inlineElems[1]});
-        else
+            break;
+        default:
             setStorage(List{.size = builder.size, .elems = builder.elems});
+            break;
+        }
     }
 
     inline void mkThunk(Env * e, Expr * ex) noexcept
@@ -1040,6 +1093,11 @@ public:
         setStorage(n);
     }
 
+    inline void mkFailed(std::exception_ptr e, Value * recovery) noexcept
+    {
+        setStorage(new Value::Failed(e, recovery));
+    }
+
     bool isList() const noexcept
     {
         return isa<tListSmall, tListN>();
@@ -1143,6 +1201,11 @@ public:
     {
         return getStorage<Path>().accessor;
     }
+
+    Failed * failed() const noexcept
+    {
+        return getStorage<Failed *>();
+    }
 };
 
 extern ExprBlackHole eBlackHole;

src/libexpr/meson.build

@@ -40,7 +40,10 @@ endforeach
 
 boost = dependency(
   'boost',
-  modules : [ 'container', 'context' ],
+  modules : [
+    'container',
+    'context',
+  ],
   include_type : 'system',
 )
 # boost is a public dependency, but not a pkg-config dependency unfortunately, so we

src/libexpr/primops.cc

@@ -483,42 +483,41 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Value & v)
 {
     state.forceValue(*args[0], pos);
-    std::string t;
     switch (args[0]->type()) {
     case nInt:
-        t = "int";
+        v.mkStringNoCopy("int");
         break;
     case nBool:
-        t = "bool";
+        v.mkStringNoCopy("bool");
         break;
     case nString:
-        t = "string";
+        v.mkStringNoCopy("string");
         break;
     case nPath:
-        t = "path";
+        v.mkStringNoCopy("path");
         break;
     case nNull:
-        t = "null";
+        v.mkStringNoCopy("null");
         break;
     case nAttrs:
-        t = "set";
+        v.mkStringNoCopy("set");
         break;
     case nList:
-        t = "list";
+        v.mkStringNoCopy("list");
         break;
     case nFunction:
-        t = "lambda";
+        v.mkStringNoCopy("lambda");
         break;
     case nExternal:
-        t = args[0]->external()->typeOf();
+        v.mkString(args[0]->external()->typeOf());
         break;
     case nFloat:
-        t = "float";
+        v.mkStringNoCopy("float");
         break;
     case nThunk:
+    case nFailed:
         unreachable();
     }
-    v.mkString(t);
 }
 
 static RegisterPrimOp primop_typeOf({
@@ -1454,19 +1453,22 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
                     continue;
             }
 
-            if (i->name == state.s.contentAddressed && state.forceBool(*i->value, pos, context_below)) {
-                contentAddressed = true;
-                experimentalFeatureSettings.require(Xp::CaDerivations);
-            }
-
-            else if (i->name == state.s.impure && state.forceBool(*i->value, pos, context_below)) {
-                isImpure = true;
-                experimentalFeatureSettings.require(Xp::ImpureDerivations);
-            }
-
+            switch (i->name.getId()) {
+            case EvalState::s.contentAddressed.getId():
+                if (state.forceBool(*i->value, pos, context_below)) {
+                    contentAddressed = true;
+                    experimentalFeatureSettings.require(Xp::CaDerivations);
+                }
+                break;
+            case EvalState::s.impure.getId():
+                if (state.forceBool(*i->value, pos, context_below)) {
+                    isImpure = true;
+                    experimentalFeatureSettings.require(Xp::ImpureDerivations);
+                }
+                break;
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
-            else if (i->name == state.s.args) {
+            case EvalState::s.args.getId():
                 state.forceList(*i->value, pos, context_below);
                 for (auto elem : i->value->listView()) {
                     auto s = state
@@ -1475,11 +1477,10 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
                                  .toOwned();
                     drv.args.push_back(s);
                 }
-            }
-
+                break;
             /* All other attributes are passed to the builder through
                the environment. */
-            else {
+            default:
 
                 if (jsonObject) {
 
@@ -1488,49 +1489,69 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
 
                     jsonObject->structuredAttrs.emplace(key, printValueAsJSON(state, true, *i->value, pos, context));
 
-                    if (i->name == state.s.builder)
+                    switch (i->name.getId()) {
+                    case EvalState::s.builder.getId():
                         drv.builder = state.forceString(*i->value, context, pos, context_below);
-                    else if (i->name == state.s.system)
+                        break;
+                    case EvalState::s.system.getId():
                         drv.platform = state.forceStringNoCtx(*i->value, pos, context_below);
-                    else if (i->name == state.s.outputHash)
+                        break;
+                    case EvalState::s.outputHash.getId():
                         outputHash = state.forceStringNoCtx(*i->value, pos, context_below);
-                    else if (i->name == state.s.outputHashAlgo)
+                        break;
+                    case EvalState::s.outputHashAlgo.getId():
                         outputHashAlgo = parseHashAlgoOpt(state.forceStringNoCtx(*i->value, pos, context_below));
-                    else if (i->name == state.s.outputHashMode)
+                        break;
+                    case EvalState::s.outputHashMode.getId():
                         handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below));
-                    else if (i->name == state.s.outputs) {
-                        /* Require ‘outputs’ to be a list of strings. */
+                        break;
+                    case EvalState::s.outputs.getId(): {
+                        /* Require 'outputs' to be a list of strings. */
                         state.forceList(*i->value, pos, context_below);
                         Strings ss;
                         for (auto elem : i->value->listView())
                             ss.emplace_back(state.forceStringNoCtx(*elem, pos, context_below));
                         handleOutputs(ss);
+                        break;
+                    }
+                    default:
+                        break;
                     }
 
-                    if (i->name == state.s.allowedReferences)
+                    switch (i->name.getId()) {
+                    case EvalState::s.allowedReferences.getId():
                         warn(
                             "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks.<output>.allowedReferences' instead",
                             drvName);
-                    if (i->name == state.s.allowedRequisites)
+                        break;
+                    case EvalState::s.allowedRequisites.getId():
                         warn(
                             "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks.<output>.allowedRequisites' instead",
                             drvName);
-                    if (i->name == state.s.disallowedReferences)
+                        break;
+                    case EvalState::s.disallowedReferences.getId():
                         warn(
                             "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks.<output>.disallowedReferences' instead",
                             drvName);
-                    if (i->name == state.s.disallowedRequisites)
+                        break;
+                    case EvalState::s.disallowedRequisites.getId():
                         warn(
                             "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks.<output>.disallowedRequisites' instead",
                             drvName);
-                    if (i->name == state.s.maxSize)
+                        break;
+                    case EvalState::s.maxSize.getId():
                         warn(
                             "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks.<output>.maxSize' instead",
                             drvName);
-                    if (i->name == state.s.maxClosureSize)
+                        break;
+                    case EvalState::s.maxClosureSize.getId():
                         warn(
                             "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead",
                             drvName);
+                        break;
+                    default:
+                        break;
+                    }
 
                 } else {
                     auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned();
@@ -1541,20 +1562,31 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
                         drv.structuredAttrs = StructuredAttrs::parse(s);
                     } else {
                         drv.env.emplace(key, s);
-                        if (i->name == state.s.builder)
+                        switch (i->name.getId()) {
+                        case EvalState::s.builder.getId():
                             drv.builder = std::move(s);
-                        else if (i->name == state.s.system)
+                            break;
+                        case EvalState::s.system.getId():
                             drv.platform = std::move(s);
-                        else if (i->name == state.s.outputHash)
+                            break;
+                        case EvalState::s.outputHash.getId():
                             outputHash = std::move(s);
-                        else if (i->name == state.s.outputHashAlgo)
+                            break;
+                        case EvalState::s.outputHashAlgo.getId():
                             outputHashAlgo = parseHashAlgoOpt(s);
-                        else if (i->name == state.s.outputHashMode)
+                            break;
+                        case EvalState::s.outputHashMode.getId():
                             handleHashMode(s);
-                        else if (i->name == state.s.outputs)
+                            break;
+                        case EvalState::s.outputs.getId():
                             handleOutputs(tokenizeString<Strings>(s));
+                            break;
+                        default:
+                            break;
+                        }
                     }
                 }
+                break;
             }
 
         } catch (Error & e) {
@@ -4349,7 +4381,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V
     if (len == 0) {
         state.forceValue(*args[2], pos);
         if (args[2]->type() == nString) {
-            v.mkString("", args[2]->context());
+            v.mkStringNoCopy("", args[2]->context());
             return;
         }
     }

src/libexpr/primops/fromTOML.cc

@@ -136,7 +136,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 normalizeDatetimeFormat(t);
 #endif
                 auto attrs = state.buildBindings(2);
-                attrs.alloc("_type").mkString("timestamp");
+                attrs.alloc("_type").mkStringNoCopy("timestamp");
                 std::ostringstream s;
                 s << t;
                 auto str = toView(s);

src/libexpr/print-ambiguous.cc

@@ -75,6 +75,9 @@ void printAmbiguous(
             str << "«potential infinite recursion»";
         }
         break;
+    case nFailed:
+        str << "«failed»";
+        break;
     case nFunction:
         if (v.isLambda()) {
             str << "<LAMBDA>";

src/libexpr/print.cc

@@ -508,6 +508,11 @@ private:
         }
     }
 
+    void printFailed(Value & v)
+    {
+        output << "«failed»";
+    }
+
     void printExternal(Value & v)
     {
         v.external()->print(output);
@@ -583,6 +588,10 @@ private:
                 printThunk(v);
                 break;
 
+            case nFailed:
+                printFailed(v);
+                break;
+
             case nExternal:
                 printExternal(v);
                 break;

src/libexpr/value-to-json.cc

@@ -96,6 +96,7 @@ json printValueAsJSON(
         break;
 
     case nThunk:
+    case nFailed:
     case nFunction:
         state.error<TypeError>("cannot convert %1% to JSON", showType(v)).atPos(v.determinePos(pos)).debugThrow();
     }

src/libexpr/value-to-xml.cc

@@ -170,6 +170,11 @@ static void printValueAsXML(
 
     case nThunk:
         doc.writeEmptyElement("unevaluated");
+        break;
+
+    case nFailed:
+        doc.writeEmptyElement("failed");
+        break;
     }
 }
 

src/libfetchers-c/nix_api_fetchers.cc

@@ -2,6 +2,8 @@
 #include "nix_api_fetchers_internal.hh"
 #include "nix_api_util_internal.h"
 
+extern "C" {
+
 nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context)
 {
     try {
@@ -17,3 +19,5 @@ void nix_fetchers_settings_free(nix_fetchers_settings * settings)
 {
     delete settings;
 }
+
+} // extern "C"

src/libfetchers/git-utils.cc

@@ -568,23 +568,34 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     void verifyCommit(const Hash & rev, const std::vector<fetchers::PublicKey> & publicKeys) override
     {
+        // Map of SSH key types to their internal OpenSSH representations
+        static const std::unordered_map<std::string_view, std::string_view> keyTypeMap = {
+            {"ssh-dsa", "ssh-dsa"},
+            {"ssh-ecdsa", "ssh-ecdsa"},
+            {"ssh-ecdsa-sk", "sk-ecdsa-sha2-nistp256@openssh.com"},
+            {"ssh-ed25519", "ssh-ed25519"},
+            {"ssh-ed25519-sk", "sk-ssh-ed25519@openssh.com"},
+            {"ssh-rsa", "ssh-rsa"}};
+
         // Create ad-hoc allowedSignersFile and populate it with publicKeys
         auto allowedSignersFile = createTempFile().second;
         std::string allowedSigners;
+
         for (const fetchers::PublicKey & k : publicKeys) {
-            if (k.type != "ssh-dsa" && k.type != "ssh-ecdsa" && k.type != "ssh-ecdsa-sk" && k.type != "ssh-ed25519"
-                && k.type != "ssh-ed25519-sk" && k.type != "ssh-rsa")
+            auto it = keyTypeMap.find(k.type);
+            if (it == keyTypeMap.end()) {
+                std::string supportedTypes;
+                for (const auto & [type, _] : keyTypeMap) {
+                    supportedTypes += fmt("  %s\n", type);
+                }
                 throw Error(
-                    "Unknown key type '%s'.\n"
-                    "Please use one of\n"
-                    "- ssh-dsa\n"
-                    "  ssh-ecdsa\n"
-                    "  ssh-ecdsa-sk\n"
-                    "  ssh-ed25519\n"
-                    "  ssh-ed25519-sk\n"
-                    "  ssh-rsa",
-                    k.type);
-            allowedSigners += "* " + k.type + " " + k.key + "\n";
+                    "Invalid SSH key type '%s' in publicKeys.\n"
+                    "Please use one of:\n%s",
+                    k.type,
+                    supportedTypes);
+            }
+
+            allowedSigners += fmt("* %s %s\n", it->second, k.key);
         }
         writeFile(allowedSignersFile, allowedSigners);
 

src/libfetchers/git.cc

@@ -163,8 +163,8 @@ struct GitInputScheme : InputScheme
 {
     std::optional<Input> inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override
     {
-        if (url.scheme != "git" && url.scheme != "git+http" && url.scheme != "git+https" && url.scheme != "git+ssh"
-            && url.scheme != "git+file")
+        auto parsedScheme = parseUrlScheme(url.scheme);
+        if (parsedScheme.application != "git")
             return {};
 
         auto url2(url);

src/libfetchers/include/nix/fetchers/input-cache.hh

@@ -11,6 +11,7 @@ struct InputCache
         ref<SourceAccessor> accessor;
         Input resolvedInput;
         Input lockedInput;
+        Attrs extraAttrs;
     };
 
     CachedResult getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
@@ -19,6 +20,7 @@ struct InputCache
     {
         Input lockedInput;
         ref<SourceAccessor> accessor;
+        Attrs extraAttrs;
     };
 
     virtual std::optional<CachedInput> lookup(const Input & originalInput) const = 0;

src/libfetchers/input-cache.cc

@@ -22,7 +22,8 @@ InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegist
                 fetched = lookup(resolvedInput);
                 if (!fetched) {
                     auto [accessor, lockedInput] = resolvedInput.getAccessor(store);
-                    fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
+                    fetched.emplace(
+                        CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs});
                 }
                 upsert(resolvedInput, *fetched);
             } else {
@@ -36,7 +37,7 @@ InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegist
 
     debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedInput.to_string());
 
-    return {fetched->accessor, resolvedInput, fetched->lockedInput};
+    return {fetched->accessor, resolvedInput, fetched->lockedInput, fetched->extraAttrs};
 }
 
 struct InputCacheImpl : InputCache

src/libflake-c/nix_api_flake.cc

@@ -10,6 +10,8 @@
 
 #include "nix/flake/flake.hh"
 
+extern "C" {
+
 nix_flake_settings * nix_flake_settings_new(nix_c_context * context)
 {
     nix_clear_err(context);
@@ -203,3 +205,5 @@ nix_value * nix_locked_flake_get_output_attrs(
     }
     NIXC_CATCH_ERRS_NULL
 }
+
+} // extern "C"

src/libflake/flake.cc

@@ -341,8 +341,9 @@ static Flake getFlake(
     // Fetch a lazy tree first.
     auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
 
-    auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir);
-    auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir);
+    auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
+    auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
+    auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), subdir);
 
     // Parse/eval flake.nix to get at the input.self attributes.
     auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {cachedInput.accessor}, lockRootAttrPath);

src/libmain-c/nix_api_main.cc

@@ -5,6 +5,8 @@
 
 #include "nix/main/plugin.hh"
 
+extern "C" {
+
 nix_err nix_init_plugins(nix_c_context * context)
 {
     if (context)
@@ -14,3 +16,5 @@ nix_err nix_init_plugins(nix_c_context * context)
     }
     NIXC_CATCH_ERRS
 }
+
+} // extern "C"

src/libstore-c/nix_api_store.cc

@@ -10,6 +10,8 @@
 
 #include "nix/store/globals.hh"
 
+extern "C" {
+
 nix_err nix_libstore_init(nix_c_context * context)
 {
     if (context)
@@ -91,7 +93,7 @@ nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_cal
     NIXC_CATCH_ERRS
 }
 
-bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path)
+bool nix_store_is_valid_path(nix_c_context * context, Store * store, const StorePath * path)
 {
     if (context)
         context->last_err_code = NIX_OK;
@@ -129,7 +131,7 @@ nix_err nix_store_realise(
     Store * store,
     StorePath * path,
     void * userdata,
-    void (*callback)(void * userdata, const char *, const char *))
+    void (*callback)(void * userdata, const char *, const StorePath *))
 {
     if (context)
         context->last_err_code = NIX_OK;
@@ -144,8 +146,8 @@ nix_err nix_store_realise(
         if (callback) {
             for (const auto & result : results) {
                 for (const auto & [outputName, realisation] : result.builtOutputs) {
-                    auto op = store->ptr->printStorePath(realisation.outPath);
-                    callback(userdata, outputName.c_str(), op.c_str());
+                    StorePath p{realisation.outPath};
+                    callback(userdata, outputName.c_str(), &p);
                 }
             }
         }
@@ -180,3 +182,5 @@ nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store
     }
     NIXC_CATCH_ERRS
 }
+
+} // extern "C"

src/libstore-c/nix_api_store.h

@@ -148,7 +148,7 @@ void nix_store_path_free(StorePath * p);
  * @param[in] path Path to check
  * @return true or false, error info in context
  */
-bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path);
+bool nix_store_is_valid_path(nix_c_context * context, Store * store, const StorePath * path);
 
 /**
  * @brief Get the physical location of a store path
@@ -190,7 +190,7 @@ nix_err nix_store_realise(
     Store * store,
     StorePath * path,
     void * userdata,
-    void (*callback)(void * userdata, const char * outname, const char * out));
+    void (*callback)(void * userdata, const char * outname, const StorePath * out));
 
 /**
  * @brief get the version of a nix store.

src/libstore-c/nix_api_store_internal.h

@@ -2,6 +2,8 @@
 #define NIX_API_STORE_INTERNAL_H
 #include "nix/store/store-api.hh"
 
+extern "C" {
+
 struct Store
 {
     nix::ref<nix::Store> ptr;
@@ -12,4 +14,6 @@ struct StorePath
     nix::StorePath path;
 };
 
+} // extern "C"
+
 #endif

src/libstore-tests/data/store-reference/daemon_shorthand.txt

@@ -0,0 +1 @@
+daemon

src/libstore-tests/data/store-reference/local_shorthand_3.txt

@@ -0,0 +1 @@
+local

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_1.txt

@@ -0,0 +1 @@
+ssh://::1

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_2.txt

@@ -0,0 +1 @@
+ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e

src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_3.txt

@@ -0,0 +1 @@
+ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e?a=b&c=d

src/libstore-tests/derivation-parser-bench.cc

@@ -51,18 +51,10 @@ static void BM_UnparseRealDerivationFile(benchmark::State & state, const std::st
 
 // Register benchmarks for actual test derivation files if they exist
 BENCHMARK_CAPTURE(
-    BM_ParseRealDerivationFile,
-    hello,
-    getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value_or(NIX_UNIT_TEST_DATA) + "/derivation/hello.drv");
+    BM_ParseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
 BENCHMARK_CAPTURE(
-    BM_ParseRealDerivationFile,
-    firefox,
-    getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value_or(NIX_UNIT_TEST_DATA) + "/derivation/firefox.drv");
+    BM_ParseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");
 BENCHMARK_CAPTURE(
-    BM_UnparseRealDerivationFile,
-    hello,
-    getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value_or(NIX_UNIT_TEST_DATA) + "/derivation/hello.drv");
+    BM_UnparseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv");
 BENCHMARK_CAPTURE(
-    BM_UnparseRealDerivationFile,
-    firefox,
-    getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value_or(NIX_UNIT_TEST_DATA) + "/derivation/firefox.drv");
+    BM_UnparseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv");

src/libstore-tests/local-store.cc

@@ -33,4 +33,10 @@ TEST(LocalStore, constructConfig_rootPath)
     EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"});
 }
 
+TEST(LocalStore, constructConfig_to_string)
+{
+    LocalStoreConfig config{"local", "", {}};
+    EXPECT_EQ(config.getReference().to_string(), "local");
+}
+
 } // namespace nix

src/libstore-tests/meson.build

@@ -130,10 +130,13 @@ if get_option('benchmarks')
     link_args : linker_export_flags,
     install : true,
     cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [],
-    cpp_args : [
-      '-DNIX_UNIT_TEST_DATA="' + meson.current_source_dir() + '/data"',
-    ],
   )
 
-  benchmark('nix-store-benchmarks', benchmark_exe)
+  benchmark(
+    'nix-store-benchmarks',
+    benchmark_exe,
+    env : {
+      '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data',
+    },
+  )
 endif

src/libstore-tests/nix_api_store.cc

@@ -1,7 +1,5 @@
 #include "nix_api_util.h"
-#include "nix_api_util_internal.h"
 #include "nix_api_store.h"
-#include "nix_api_store_internal.h"
 
 #include "nix/store/tests/nix_api_store.hh"
 #include "nix/util/tests/string_callback.hh"
@@ -65,7 +63,7 @@ TEST_F(nix_api_store_test, nix_store_get_storedir)
 TEST_F(nix_api_store_test, InvalidPathFails)
 {
     nix_store_parse_path(ctx, store, "invalid-path");
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR);
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
 }
 
 TEST_F(nix_api_store_test, ReturnsValidStorePath)
@@ -80,7 +78,7 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath)
 TEST_F(nix_api_store_test, SetsLastErrCodeToNixOk)
 {
     StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str());
-    ASSERT_EQ(ctx->last_err_code, NIX_OK);
+    ASSERT_EQ(nix_err_code(ctx), NIX_OK);
     nix_store_path_free(path);
 }
 
@@ -103,7 +101,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy)
 {
     nix_libstore_init(ctx);
     Store * store = nix_store_open(ctx, "dummy://", nullptr);
-    ASSERT_EQ(NIX_OK, ctx->last_err_code);
+    ASSERT_EQ(NIX_OK, nix_err_code(ctx));
     ASSERT_STREQ("dummy://", store->ptr->config.getReference().render(/*withParams=*/true).c_str());
 
     std::string str;
@@ -117,7 +115,7 @@ TEST_F(nix_api_util_context, nix_store_open_invalid)
 {
     nix_libstore_init(ctx);
     Store * store = nix_store_open(ctx, "invalid://", nullptr);
-    ASSERT_EQ(NIX_ERR_NIX_ERROR, ctx->last_err_code);
+    ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx));
     ASSERT_EQ(nullptr, store);
     nix_store_free(store);
 }

src/libstore-tests/store-reference.cc

@@ -107,6 +107,13 @@ URI_TEST_READ(local_shorthand_1, localExample_1)
 
 URI_TEST_READ(local_shorthand_2, localExample_2)
 
+URI_TEST(
+    local_shorthand_3,
+    (StoreReference{
+        .variant = StoreReference::Local{},
+        .params = {},
+    }))
+
 static StoreReference unixExample{
     .variant =
         StoreReference::Specified{
@@ -134,4 +141,46 @@ URI_TEST(
         .params = {},
     }))
 
+URI_TEST(
+    daemon_shorthand,
+    (StoreReference{
+        .variant = StoreReference::Daemon{},
+        .params = {},
+    }))
+
+static StoreReference sshLoopbackIPv6{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "[::1]",
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_1, sshLoopbackIPv6)
+
+static StoreReference sshIPv6AuthorityWithUserinfo{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e]",
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_2, sshIPv6AuthorityWithUserinfo)
+
+static StoreReference sshIPv6AuthorityWithUserinfoAndParams{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "ssh",
+            .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e]",
+        },
+    .params =
+        {
+            {"a", "b"},
+            {"c", "d"},
+        },
+};
+
+URI_TEST_READ(ssh_unbracketed_ipv6_3, sshIPv6AuthorityWithUserinfoAndParams)
+
 } // namespace nix

src/libstore-tests/uds-remote-store.cc

@@ -16,4 +16,10 @@ TEST(UDSRemoteStore, constructConfigWrongScheme)
     EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError);
 }
 
+TEST(UDSRemoteStore, constructConfig_to_string)
+{
+    UDSRemoteStoreConfig config{"unix", "", {}};
+    EXPECT_EQ(config.getReference().to_string(), "daemon");
+}
+
 } // namespace nix

src/libstore/binary-cache-store.cc

@@ -125,11 +125,8 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
 
     upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
 
-    {
-        auto state_(state.lock());
-        state_->pathInfoCache.upsert(
-            std::string(narInfo->path.to_string()), PathInfoCacheValue{.value = std::shared_ptr<NarInfo>(narInfo)});
-    }
+    pathInfoCache->lock()->upsert(
+        std::string(narInfo->path.to_string()), PathInfoCacheValue{.value = std::shared_ptr<NarInfo>(narInfo)});
 
     if (diskCache)
         diskCache->upsertNarInfo(

src/libstore/build/derivation-building-goal.cc

@@ -127,31 +127,6 @@ static void runPostBuildHook(
    produced using a substitute.  So we have to build instead. */
 Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
 {
-    /* Recheck at goal start. In particular, whereas before we were
-       given this information by the downstream goal, that cannot happen
-       anymore if the downstream goal only cares about one output, but
-       we care about all outputs. */
-    auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
-    for (auto & [outputName, outputHash] : outputHashes) {
-        InitialOutput v{.outputHash = outputHash};
-
-        /* TODO we might want to also allow randomizing the paths
-           for regular CA derivations, e.g. for sake of checking
-           determinism. */
-        if (drv->type().isImpure()) {
-            v.known = InitialOutputStatus{
-                .path = StorePath::random(outputPathName(drv->name, outputName)),
-                .status = PathStatus::Absent,
-            };
-        }
-
-        initialOutputs.insert({
-            outputName,
-            std::move(v),
-        });
-    }
-    checkPathValidity();
-
     Goals waitees;
 
     std::map<ref<const SingleDerivedPath>, GoalPtr, value_comparison> inputGoals;
@@ -334,14 +309,15 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
             if (resolvedResult.success()) {
                 SingleDrvOutputs builtOutputs;
 
+                auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
                 auto resolvedHashes = staticOutputHashes(worker.store, drvResolved);
 
                 StorePathSet outputPaths;
 
                 for (auto & outputName : drvResolved.outputNames()) {
-                    auto initialOutput = get(initialOutputs, outputName);
+                    auto outputHash = get(outputHashes, outputName);
                     auto resolvedHash = get(resolvedHashes, outputName);
-                    if ((!initialOutput) || (!resolvedHash))
+                    if ((!outputHash) || (!resolvedHash))
                         throw Error(
                             "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)",
                             worker.store.printStorePath(drvPath),
@@ -368,7 +344,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
 
                     if (!drv->type().isImpure()) {
                         auto newRealisation = realisation;
-                        newRealisation.id = DrvOutput{initialOutput->outputHash, outputName};
+                        newRealisation.id = DrvOutput{*outputHash, outputName};
                         newRealisation.signatures.clear();
                         if (!drv->type().isFixed()) {
                             auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
@@ -439,136 +415,263 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution()
     co_return tryToBuild();
 }
 
-void DerivationBuildingGoal::started()
-{
-    auto msg =
-        fmt(buildMode == bmRepair  ? "repairing outputs of '%s'"
-            : buildMode == bmCheck ? "checking outputs of '%s'"
-                                   : "building '%s'",
-            worker.store.printStorePath(drvPath));
-    fmt("building '%s'", worker.store.printStorePath(drvPath));
-#ifndef _WIN32 // TODO enable build hook on Windows
-    if (hook)
-        msg += fmt(" on '%s'", machineName);
-#endif
-    act = std::make_unique<Activity>(
-        *logger,
-        lvlInfo,
-        actBuild,
-        msg,
-        Logger::Fields{
-            worker.store.printStorePath(drvPath),
-#ifndef _WIN32 // TODO enable build hook on Windows
-            hook ? machineName :
-#endif
-                 "",
-            1,
-            1});
-    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-    worker.updateProgress();
-}
-
 Goal::Co DerivationBuildingGoal::tryToBuild()
 {
-    trace("trying to build");
+    std::map<std::string, InitialOutput> initialOutputs;
 
-    /* Obtain locks on all output paths, if the paths are known a priori.
+    /* Recheck at this point. In particular, whereas before we were
+       given this information by the downstream goal, that cannot happen
+       anymore if the downstream goal only cares about one output, but
+       we care about all outputs. */
+    auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
+    for (auto & [outputName, outputHash] : outputHashes) {
+        InitialOutput v{.outputHash = outputHash};
 
-       The locks are automatically released when we exit this function or Nix
-       crashes.  If we can't acquire the lock, then continue; hopefully some
-       other goal can start a build, and if not, the main loop will sleep a few
-       seconds and then retry this goal. */
-    PathSet lockFiles;
-    /* FIXME: Should lock something like the drv itself so we don't build same
-       CA drv concurrently */
-    if (dynamic_cast<LocalStore *>(&worker.store)) {
-        /* If we aren't a local store, we might need to use the local store as
-           a build remote, but that would cause a deadlock. */
-        /* FIXME: Make it so we can use ourselves as a build remote even if we
-           are the local store (separate locking for building vs scheduling? */
-        /* FIXME: find some way to lock for scheduling for the other stores so
-           a forking daemon with --store still won't farm out redundant builds.
-           */
-        for (auto & i : drv->outputsAndOptPaths(worker.store)) {
-            if (i.second.second)
-                lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
-            else
-                lockFiles.insert(worker.store.Store::toRealPath(drvPath) + "." + i.first);
+        /* TODO we might want to also allow randomizing the paths
+           for regular CA derivations, e.g. for sake of checking
+           determinism. */
+        if (drv->type().isImpure()) {
+            v.known = InitialOutputStatus{
+                .path = StorePath::random(outputPathName(drv->name, outputName)),
+                .status = PathStatus::Absent,
+            };
         }
+
+        initialOutputs.insert({
+            outputName,
+            std::move(v),
+        });
     }
+    checkPathValidity(initialOutputs);
 
-    if (!outputLocks.lockPaths(lockFiles, "", false)) {
-        Activity act(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles))));
+    auto started = [&]() {
+        auto msg =
+            fmt(buildMode == bmRepair  ? "repairing outputs of '%s'"
+                : buildMode == bmCheck ? "checking outputs of '%s'"
+                                       : "building '%s'",
+                worker.store.printStorePath(drvPath));
+        fmt("building '%s'", worker.store.printStorePath(drvPath));
+#ifndef _WIN32 // TODO enable build hook on Windows
+        if (hook)
+            msg += fmt(" on '%s'", hook->machineName);
+#endif
+        act = std::make_unique<Activity>(
+            *logger,
+            lvlInfo,
+            actBuild,
+            msg,
+            Logger::Fields{
+                worker.store.printStorePath(drvPath),
+#ifndef _WIN32 // TODO enable build hook on Windows
+                hook ? hook->machineName :
+#endif
+                     "",
+                1,
+                1});
+        mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
+        worker.updateProgress();
+    };
 
-        /* Wait then try locking again, repeat until success (returned
-           boolean is true). */
-        do {
-            co_await waitForAWhile();
-        } while (!outputLocks.lockPaths(lockFiles, "", false));
-    }
+    /**
+     * Activity that denotes waiting for a lock.
+     */
+    std::unique_ptr<Activity> actLock;
 
-    /* Now check again whether the outputs are valid.  This is because
-       another process may have started building in parallel.  After
-       it has finished and released the locks, we can (and should)
-       reuse its results.  (Strictly speaking the first check can be
-       omitted, but that would be less efficient.)  Note that since we
-       now hold the locks on the output paths, no other process can
-       build this derivation, so no further checks are necessary. */
-    auto [allValid, validOutputs] = checkPathValidity();
+    /**
+     * Locks on (fixed) output paths.
+     */
+    PathLocks outputLocks;
 
-    if (buildMode != bmCheck && allValid) {
-        debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
-        outputLocks.setDeletion(true);
-        outputLocks.unlock();
-        co_return doneSuccess(BuildResult::AlreadyValid, std::move(validOutputs));
-    }
+    bool useHook;
 
-    /* If any of the outputs already exist but are not valid, delete
-       them. */
-    for (auto & [_, status] : initialOutputs) {
-        if (!status.known || status.known->isValid())
-            continue;
-        auto storePath = status.known->path;
-        debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
-        deletePath(worker.store.Store::toRealPath(storePath));
-    }
+    while (true) {
+        trace("trying to build");
 
-    /* Don't do a remote build if the derivation has the attribute
-       `preferLocalBuild' set.  Also, check and repair modes are only
-       supported for local builds. */
-    bool buildLocally =
-        (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv)) && settings.maxBuildJobs.get() != 0;
+        /* Obtain locks on all output paths, if the paths are known a priori.
 
-    if (!buildLocally) {
-        switch (tryBuildHook()) {
-        case rpAccept:
-            /* Yes, it has started doing so.  Wait until we get
-               EOF from the hook. */
-            actLock.reset();
-            buildResult.startTime = time(0); // inexact
-            started();
-            co_await Suspend{};
-            co_return hookDone();
-        case rpPostpone:
-            /* Not now; wait until at least one child finishes or
-               the wake-up timeout expires. */
-            if (!actLock)
-                actLock = std::make_unique<Activity>(
-                    *logger,
-                    lvlWarn,
-                    actBuildWaiting,
-                    fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
+           The locks are automatically released when we exit this function or Nix
+           crashes.  If we can't acquire the lock, then continue; hopefully some
+           other goal can start a build, and if not, the main loop will sleep a few
+           seconds and then retry this goal. */
+        PathSet lockFiles;
+        /* FIXME: Should lock something like the drv itself so we don't build same
+           CA drv concurrently */
+        if (dynamic_cast<LocalStore *>(&worker.store)) {
+            /* If we aren't a local store, we might need to use the local store as
+               a build remote, but that would cause a deadlock. */
+            /* FIXME: Make it so we can use ourselves as a build remote even if we
+               are the local store (separate locking for building vs scheduling? */
+            /* FIXME: find some way to lock for scheduling for the other stores so
+               a forking daemon with --store still won't farm out redundant builds.
+               */
+            for (auto & i : drv->outputsAndOptPaths(worker.store)) {
+                if (i.second.second)
+                    lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
+                else
+                    lockFiles.insert(worker.store.Store::toRealPath(drvPath) + "." + i.first);
+            }
+        }
+
+        if (!outputLocks.lockPaths(lockFiles, "", false)) {
+            Activity act(
+                *logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles))));
+
+            /* Wait then try locking again, repeat until success (returned
+               boolean is true). */
+            do {
+                co_await waitForAWhile();
+            } while (!outputLocks.lockPaths(lockFiles, "", false));
+        }
+
+        /* Now check again whether the outputs are valid.  This is because
+           another process may have started building in parallel.  After
+           it has finished and released the locks, we can (and should)
+           reuse its results.  (Strictly speaking the first check can be
+           omitted, but that would be less efficient.)  Note that since we
+           now hold the locks on the output paths, no other process can
+           build this derivation, so no further checks are necessary. */
+        auto [allValid, validOutputs] = checkPathValidity(initialOutputs);
+
+        if (buildMode != bmCheck && allValid) {
+            debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath));
+            outputLocks.setDeletion(true);
             outputLocks.unlock();
-            co_await waitForAWhile();
-            co_return tryToBuild();
-        case rpDecline:
-            /* We should do it ourselves. */
-            break;
+            co_return doneSuccess(BuildResult::AlreadyValid, std::move(validOutputs));
         }
+
+        /* If any of the outputs already exist but are not valid, delete
+           them. */
+        for (auto & [_, status] : initialOutputs) {
+            if (!status.known || status.known->isValid())
+                continue;
+            auto storePath = status.known->path;
+            debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path));
+            deletePath(worker.store.Store::toRealPath(storePath));
+        }
+
+        /* Don't do a remote build if the derivation has the attribute
+           `preferLocalBuild' set.  Also, check and repair modes are only
+           supported for local builds. */
+        bool buildLocally = (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv))
+                            && settings.maxBuildJobs.get() != 0;
+
+        if (buildLocally) {
+            useHook = false;
+        } else {
+            switch (tryBuildHook(initialOutputs)) {
+            case rpAccept:
+                /* Yes, it has started doing so.  Wait until we get
+                   EOF from the hook. */
+                useHook = true;
+                break;
+            case rpPostpone:
+                /* Not now; wait until at least one child finishes or
+                   the wake-up timeout expires. */
+                if (!actLock)
+                    actLock = std::make_unique<Activity>(
+                        *logger,
+                        lvlWarn,
+                        actBuildWaiting,
+                        fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
+                outputLocks.unlock();
+                co_await waitForAWhile();
+                continue;
+            case rpDecline:
+                /* We should do it ourselves. */
+                useHook = false;
+                break;
+            }
+        }
+        break;
     }
 
     actLock.reset();
 
+    if (useHook) {
+        buildResult.startTime = time(0); // inexact
+        started();
+        co_await Suspend{};
+
+#ifndef _WIN32
+        assert(hook);
+#endif
+
+        trace("hook build done");
+
+        /* Since we got an EOF on the logger pipe, the builder is presumed
+           to have terminated.  In fact, the builder could also have
+           simply have closed its end of the pipe, so just to be sure,
+           kill it. */
+        int status =
+#ifndef _WIN32 // TODO enable build hook on Windows
+            hook->pid.kill();
+#else
+            0;
+#endif
+
+        debug("build hook for '%s' finished", worker.store.printStorePath(drvPath));
+
+        buildResult.timesBuilt++;
+        buildResult.stopTime = time(0);
+
+        /* So the child is gone now. */
+        worker.childTerminated(this);
+
+        /* Close the read side of the logger pipe. */
+#ifndef _WIN32 // TODO enable build hook on Windows
+        hook->builderOut.readSide.close();
+        hook->fromHook.readSide.close();
+#endif
+
+        /* Close the log file. */
+        closeLogFile();
+
+        /* Check the exit status. */
+        if (!statusOk(status)) {
+            auto e = fixupBuilderFailureErrorMessage({BuildResult::MiscFailure, status, ""});
+
+            outputLocks.unlock();
+
+            /* TODO (once again) support fine-grained error codes, see issue #12641. */
+
+            co_return doneFailure(std::move(e));
+        }
+
+        /* Compute the FS closure of the outputs and register them as
+           being valid. */
+        auto builtOutputs =
+            /* When using a build hook, the build hook can register the output
+               as valid (by doing `nix-store --import').  If so we don't have
+               to do anything here.
+
+               We can only early return when the outputs are known a priori. For
+               floating content-addressing derivations this isn't the case.
+
+               Aborts if any output is not valid or corrupt, and otherwise
+               returns a 'SingleDrvOutputs' structure containing all outputs.
+             */
+            [&] {
+                auto [allValid, validOutputs] = checkPathValidity(initialOutputs);
+                if (!allValid)
+                    throw Error("some outputs are unexpectedly invalid");
+                return validOutputs;
+            }();
+
+        StorePathSet outputPaths;
+        for (auto & [_, output] : builtOutputs)
+            outputPaths.insert(output.outPath);
+        runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
+
+        /* It is now safe to delete the lock files, since all future
+           lockers will see that the output paths are valid; they will
+           not create new lock files with the same names as the old
+           (unlinked) lock files. */
+        outputLocks.setDeletion(true);
+        outputLocks.unlock();
+
+        co_return doneSuccess(BuildResult::Built, std::move(builtOutputs));
+    }
+
     co_await yield();
 
     if (!dynamic_cast<LocalStore *>(&worker.store)) {
@@ -584,12 +687,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
 #ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows
     throw UnimplementedError("building derivations is not yet implemented on Windows");
 #else
+    assert(!hook);
+
+    Descriptor builderOut;
 
     // Will continue here while waiting for a build user below
     while (true) {
 
-        assert(!hook);
-
         unsigned int curBuilds = worker.getNrLocalBuilds();
         if (curBuilds >= settings.maxBuildJobs) {
             outputLocks.unlock();
@@ -614,11 +718,6 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
 
                 ~DerivationBuildingGoalCallbacks() override = default;
 
-                void childStarted(Descriptor builderOut) override
-                {
-                    goal.worker.childStarted(goal.shared_from_this(), {builderOut}, true, true);
-                }
-
                 void childTerminated() override
                 {
                     goal.worker.childTerminated(&goal);
@@ -684,7 +783,17 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
                 });
         }
 
-        if (!builder->prepareBuild()) {
+        std::optional<Descriptor> builderOutOpt;
+        try {
+            /* Okay, we have to build. */
+            builderOutOpt = builder->startBuild();
+        } catch (BuildError & e) {
+            builder.reset();
+            outputLocks.unlock();
+            worker.permanentFailure = true;
+            co_return doneFailure(std::move(e)); // InputRejected
+        }
+        if (!builderOutOpt) {
             if (!actLock)
                 actLock = std::make_unique<Activity>(
                     *logger,
@@ -693,24 +802,16 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
                     fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath))));
             co_await waitForAWhile();
             continue;
-        }
+        } else {
+            builderOut = *std::move(builderOutOpt);
+        };
 
         break;
     }
 
     actLock.reset();
 
-    try {
-
-        /* Okay, we have to build. */
-        builder->startBuilder();
-
-    } catch (BuildError & e) {
-        builder.reset();
-        outputLocks.unlock();
-        worker.permanentFailure = true;
-        co_return doneFailure(std::move(e)); // InputRejected
-    }
+    worker.childStarted(shared_from_this(), {builderOut}, true, true);
 
     started();
     co_await Suspend{};
@@ -868,81 +969,7 @@ BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailur
     return BuildError{e.status, msg};
 }
 
-Goal::Co DerivationBuildingGoal::hookDone()
-{
-#ifndef _WIN32
-    assert(hook);
-#endif
-
-    trace("hook build done");
-
-    /* Since we got an EOF on the logger pipe, the builder is presumed
-       to have terminated.  In fact, the builder could also have
-       simply have closed its end of the pipe, so just to be sure,
-       kill it. */
-    int status =
-#ifndef _WIN32 // TODO enable build hook on Windows
-        hook->pid.kill();
-#else
-        0;
-#endif
-
-    debug("build hook for '%s' finished", worker.store.printStorePath(drvPath));
-
-    buildResult.timesBuilt++;
-    buildResult.stopTime = time(0);
-
-    /* So the child is gone now. */
-    worker.childTerminated(this);
-
-    /* Close the read side of the logger pipe. */
-#ifndef _WIN32 // TODO enable build hook on Windows
-    hook->builderOut.readSide.close();
-    hook->fromHook.readSide.close();
-#endif
-
-    /* Close the log file. */
-    closeLogFile();
-
-    /* Check the exit status. */
-    if (!statusOk(status)) {
-        auto e = fixupBuilderFailureErrorMessage({BuildResult::MiscFailure, status, ""});
-
-        outputLocks.unlock();
-
-        /* TODO (once again) support fine-grained error codes, see issue #12641. */
-
-        co_return doneFailure(std::move(e));
-    }
-
-    /* Compute the FS closure of the outputs and register them as
-       being valid. */
-    auto builtOutputs =
-        /* When using a build hook, the build hook can register the output
-           as valid (by doing `nix-store --import').  If so we don't have
-           to do anything here.
-
-           We can only early return when the outputs are known a priori. For
-           floating content-addressing derivations this isn't the case.
-         */
-        assertPathValidity();
-
-    StorePathSet outputPaths;
-    for (auto & [_, output] : builtOutputs)
-        outputPaths.insert(output.outPath);
-    runPostBuildHook(worker.store, *logger, drvPath, outputPaths);
-
-    /* It is now safe to delete the lock files, since all future
-       lockers will see that the output paths are valid; they will
-       not create new lock files with the same names as the old
-       (unlinked) lock files. */
-    outputLocks.setDeletion(true);
-    outputLocks.unlock();
-
-    co_return doneSuccess(BuildResult::Built, std::move(builtOutputs));
-}
-
-HookReply DerivationBuildingGoal::tryBuildHook()
+HookReply DerivationBuildingGoal::tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs)
 {
 #ifdef _WIN32 // TODO enable build hook on Windows
     return rpDecline;
@@ -1010,7 +1037,7 @@ HookReply DerivationBuildingGoal::tryBuildHook()
     hook = std::move(worker.hook);
 
     try {
-        machineName = readLine(hook->fromHook.readSide.get());
+        hook->machineName = readLine(hook->fromHook.readSide.get());
     } catch (Error & e) {
         e.addTrace({}, "while reading the machine name from the build hook");
         throw;
@@ -1221,7 +1248,8 @@ std::map<std::string, std::optional<StorePath>> DerivationBuildingGoal::queryPar
     return res;
 }
 
-std::pair<bool, SingleDrvOutputs> DerivationBuildingGoal::checkPathValidity()
+std::pair<bool, SingleDrvOutputs>
+DerivationBuildingGoal::checkPathValidity(std::map<std::string, InitialOutput> & initialOutputs)
 {
     if (drv->type().isImpure())
         return {false, {}};
@@ -1278,17 +1306,8 @@ std::pair<bool, SingleDrvOutputs> DerivationBuildingGoal::checkPathValidity()
     return {allValid, validOutputs};
 }
 
-SingleDrvOutputs DerivationBuildingGoal::assertPathValidity()
-{
-    auto [allValid, validOutputs] = checkPathValidity();
-    if (!allValid)
-        throw Error("some outputs are unexpectedly invalid");
-    return validOutputs;
-}
-
 Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Status status, SingleDrvOutputs builtOutputs)
 {
-    outputLocks.unlock();
     buildResult.status = status;
 
     assert(buildResult.success());
@@ -1306,7 +1325,6 @@ Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Status status, Singl
 
 Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex)
 {
-    outputLocks.unlock();
     buildResult.status = ex.status;
     buildResult.errorMsg = fmt("%s", Uncolored(ex.info().msg));
     if (buildResult.status == BuildResult::TimedOut)

src/libstore/daemon.cc

@@ -546,47 +546,22 @@ static void performOp(
         break;
     }
 
-    case WorkerProto::Op::ExportPath: {
-        auto path = store->parseStorePath(readString(conn.from));
-        readInt(conn.from); // obsolete
-        logger->startWork();
-        TunnelSink sink(conn.to);
-        store->exportPath(path, sink);
-        logger->stopWork();
-        conn.to << 1;
-        break;
-    }
-
-    case WorkerProto::Op::ImportPaths: {
-        logger->startWork();
-        TunnelSource source(conn.from, conn.to);
-        auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs);
-        logger->stopWork();
-        Strings paths2;
-        for (auto & i : paths)
-            paths2.push_back(store->printStorePath(i));
-        conn.to << paths2;
-        break;
-    }
-
     case WorkerProto::Op::BuildPaths: {
         auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
         BuildMode mode = bmNormal;
-        if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 15) {
-            mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
+        mode = WorkerProto::Serialise<BuildMode>::read(*store, rconn);
 
-            /* Repairing is not atomic, so disallowed for "untrusted"
-               clients.
+        /* Repairing is not atomic, so disallowed for "untrusted"
+           clients.
 
-               FIXME: layer violation in this message: the daemon code (i.e.
-               this file) knows whether a client/connection is trusted, but it
-               does not how how the client was authenticated. The mechanism
-               need not be getting the UID of the other end of a Unix Domain
-               Socket.
-              */
-            if (mode == bmRepair && !trusted)
-                throw Error("repairing is not allowed because you are not in 'trusted-users'");
-        }
+           FIXME: layer violation in this message: the daemon code (i.e.
+           this file) knows whether a client/connection is trusted, but it
+           does not how how the client was authenticated. The mechanism
+           need not be getting the UID of the other end of a Unix Domain
+           Socket.
+          */
+        if (mode == bmRepair && !trusted)
+            throw Error("repairing is not allowed because you are not in 'trusted-users'");
         logger->startWork();
         store->buildPaths(drvs, mode);
         logger->stopWork();
@@ -805,13 +780,11 @@ static void performOp(
         clientSettings.buildCores = readInt(conn.from);
         clientSettings.useSubstitutes = readInt(conn.from);
 
-        if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) {
-            unsigned int n = readInt(conn.from);
-            for (unsigned int i = 0; i < n; i++) {
-                auto name = readString(conn.from);
-                auto value = readString(conn.from);
-                clientSettings.overrides.emplace(name, value);
-            }
+        unsigned int n = readInt(conn.from);
+        for (unsigned int i = 0; i < n; i++) {
+            auto name = readString(conn.from);
+            auto value = readString(conn.from);
+            clientSettings.overrides.emplace(name, value);
         }
 
         logger->startWork();
@@ -876,19 +849,12 @@ static void performOp(
         auto path = store->parseStorePath(readString(conn.from));
         std::shared_ptr<const ValidPathInfo> info;
         logger->startWork();
-        try {
-            info = store->queryPathInfo(path);
-        } catch (InvalidPath &) {
-            if (GET_PROTOCOL_MINOR(conn.protoVersion) < 17)
-                throw;
-        }
+        info = store->queryPathInfo(path);
         logger->stopWork();
         if (info) {
-            if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 17)
-                conn.to << 1;
+            conn.to << 1;
             WorkerProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
         } else {
-            assert(GET_PROTOCOL_MINOR(conn.protoVersion) >= 17);
             conn.to << 0;
         }
         break;
@@ -1063,7 +1029,7 @@ void processConnection(ref<Store> store, FdSource && from, FdSink && to, Trusted
     auto [protoVersion, features] =
         WorkerProto::BasicServerConnection::handshake(to, from, PROTOCOL_VERSION, WorkerProto::allFeatures);
 
-    if (protoVersion < 0x10a)
+    if (protoVersion < 256 + 18)
         throw Error("the Nix client version is too old");
 
     WorkerProto::BasicServerConnection conn;

src/libstore/gc.cc

@@ -931,7 +931,7 @@ void LocalStore::autoGC(bool sync)
     std::shared_future<void> future;
 
     {
-        auto state(_state.lock());
+        auto state(_state->lock());
 
         if (state->gcRunning) {
             future = state->gcFuture;
@@ -964,7 +964,7 @@ void LocalStore::autoGC(bool sync)
 
                 /* Wake up any threads waiting for the auto-GC to finish. */
                 Finally wakeup([&]() {
-                    auto state(_state.lock());
+                    auto state(_state->lock());
                     state->gcRunning = false;
                     state->lastGCCheck = std::chrono::steady_clock::now();
                     promise.set_value();
@@ -979,7 +979,7 @@ void LocalStore::autoGC(bool sync)
 
                 collectGarbage(options, results);
 
-                _state.lock()->availAfterGC = getAvail();
+                _state->lock()->availAfterGC = getAvail();
 
             } catch (...) {
                 // FIXME: we could propagate the exception to the

src/libstore/include/nix/store/build/derivation-builder.hh

@@ -76,10 +76,7 @@ struct DerivationBuilderParams
      */
     const StorePathSet & inputPaths;
 
-    /**
-     * @note we do in fact mutate this
-     */
-    std::map<std::string, InitialOutput> & initialOutputs;
+    const std::map<std::string, InitialOutput> & initialOutputs;
 
     const BuildMode & buildMode;
 
@@ -117,13 +114,6 @@ struct DerivationBuilderCallbacks
      */
     virtual void closeLogFile() = 0;
 
-    /**
-     * Hook up `builderOut` to some mechanism to ingest the log
-     *
-     * @todo this should be reworked
-     */
-    virtual void childStarted(Descriptor builderOut) = 0;
-
     /**
      * @todo this should be reworked
      */
@@ -157,15 +147,15 @@ struct DerivationBuilder : RestrictionContext
      * locks as needed). After this is run, the builder should be
      * started.
      *
-     * @returns true if successful, false if we could not acquire a build
-     * user. In that case, the caller must wait and then try again.
+     * @returns logging pipe if successful, `std::nullopt` if we could
+     * not acquire a build user. In that case, the caller must wait and
+     * then try again.
+     *
+     * @note "success" just means that we were able to set up the environment
+     * and start the build. The builder could have immediately exited with
+     * failure, and that would still be considered a successful start.
      */
-    virtual bool prepareBuild() = 0;
-
-    /**
-     * Start building a derivation.
-     */
-    virtual void startBuilder() = 0;
+    virtual std::optional<Descriptor> startBuild() = 0;
 
     /**
      * Tear down build environment after the builder exits (either on

src/libstore/include/nix/store/build/derivation-building-goal.hh

@@ -29,6 +29,12 @@ typedef enum { rpAccept, rpDecline, rpPostpone } HookReply;
  */
 struct DerivationBuildingGoal : public Goal
 {
+    DerivationBuildingGoal(
+        const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode = bmNormal);
+    ~DerivationBuildingGoal();
+
+private:
+
     /** The path of the derivation. */
     StorePath drvPath;
 
@@ -43,19 +49,12 @@ struct DerivationBuildingGoal : public Goal
      * The remainder is state held during the build.
      */
 
-    /**
-     * Locks on (fixed) output paths.
-     */
-    PathLocks outputLocks;
-
     /**
      * All input paths (that is, the union of FS closures of the
      * immediate input paths).
      */
     StorePathSet inputPaths;
 
-    std::map<std::string, InitialOutput> initialOutputs;
-
     /**
      * File descriptor for the log file.
      */
@@ -92,22 +91,8 @@ struct DerivationBuildingGoal : public Goal
 
     std::unique_ptr<Activity> act;
 
-    /**
-     * Activity that denotes waiting for a lock.
-     */
-    std::unique_ptr<Activity> actLock;
-
     std::map<ActivityId, Activity> builderActivities;
 
-    /**
-     * The remote machine on which we're building.
-     */
-    std::string machineName;
-
-    DerivationBuildingGoal(
-        const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode = bmNormal);
-    ~DerivationBuildingGoal();
-
     void timedOut(Error && ex) override;
 
     std::string key() override;
@@ -117,12 +102,11 @@ struct DerivationBuildingGoal : public Goal
      */
     Co gaveUpOnSubstitution();
     Co tryToBuild();
-    Co hookDone();
 
     /**
      * Is the build hook willing to perform the build?
      */
-    HookReply tryBuildHook();
+    HookReply tryBuildHook(const std::map<std::string, InitialOutput> & initialOutputs);
 
     /**
      * Open a log file and a pipe to it.
@@ -156,21 +140,13 @@ struct DerivationBuildingGoal : public Goal
      * whether all outputs are valid and non-corrupt, and a
      * 'SingleDrvOutputs' structure containing the valid outputs.
      */
-    std::pair<bool, SingleDrvOutputs> checkPathValidity();
-
-    /**
-     * Aborts if any output is not valid or corrupt, and otherwise
-     * returns a 'SingleDrvOutputs' structure containing all outputs.
-     */
-    SingleDrvOutputs assertPathValidity();
+    std::pair<bool, SingleDrvOutputs> checkPathValidity(std::map<std::string, InitialOutput> & initialOutputs);
 
     /**
      * Forcibly kill the child process, if any.
      */
     void killChild();
 
-    void started();
-
     Done doneSuccess(BuildResult::Status status, SingleDrvOutputs builtOutputs);
 
     Done doneFailure(BuildError ex);

src/libstore/include/nix/store/legacy-ssh-store.hh

@@ -179,12 +179,6 @@ public:
      */
     StorePathSet queryValidPaths(const StorePathSet & paths, bool lock, SubstituteFlag maybeSubstitute = NoSubstitute);
 
-    /**
-     * Just exists because this is exactly what Hydra was doing, and we
-     * don't yet want an algorithmic change.
-     */
-    void addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths);
-
     void connect() override;
 
     unsigned int getProtocol() override;

src/libstore/include/nix/store/local-store.hh

@@ -174,7 +174,11 @@ private:
         std::unique_ptr<PublicKeys> publicKeys;
     };
 
-    Sync<State> _state;
+    /**
+     * Mutable state. It's behind a `ref` to reduce false sharing
+     * between immutable and mutable fields.
+     */
+    ref<Sync<State>> _state;
 
 public:
 

src/libstore/include/nix/store/ssh.hh

@@ -1,6 +1,7 @@
 #pragma once
 ///@file
 
+#include "nix/util/ref.hh"
 #include "nix/util/sync.hh"
 #include "nix/util/url.hh"
 #include "nix/util/processes.hh"
@@ -26,12 +27,13 @@ private:
     const bool compress;
     const Descriptor logFD;
 
+    const ref<const AutoDelete> tmpDir;
+
     struct State
     {
 #ifndef _WIN32 // TODO re-enable on Windows, once we can start processes.
         Pid sshMaster;
 #endif
-        std::unique_ptr<AutoDelete> tmpDir;
         Path socketPath;
     };
 

src/libstore/include/nix/store/store-api.hh

@@ -310,14 +310,11 @@ protected:
         }
     };
 
-    struct State
-    {
-        LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
-    };
-
     void invalidatePathInfoCacheFor(const StorePath & path);
 
-    SharedSync<State> state;
+    // Note: this is a `ref` to avoid false sharing with immutable
+    // bits of `Store`.
+    ref<SharedSync<LRUCache<std::string, PathInfoCacheValue>>> pathInfoCache;
 
     std::shared_ptr<NarInfoDiskCache> diskCache;
 
@@ -860,7 +857,7 @@ public:
      */
     void clearPathInfoCache()
     {
-        state.lock()->pathInfoCache.clear();
+        pathInfoCache->lock()->clear();
     }
 
     /**

src/libstore/include/nix/store/store-reference.hh

@@ -64,7 +64,29 @@ struct StoreReference
         auto operator<=>(const Specified & rhs) const = default;
     };
 
-    typedef std::variant<Auto, Specified> Variant;
+    /**
+     * Special case for `daemon` to avoid normalization.
+     */
+    struct Daemon : Specified
+    {
+        Daemon()
+            : Specified({.scheme = "unix"})
+        {
+        }
+    };
+
+    /**
+     * Special case for `local` to avoid normalization.
+     */
+    struct Local : Specified
+    {
+        Local()
+            : Specified({.scheme = "local"})
+        {
+        }
+    };
+
+    typedef std::variant<Auto, Specified, Daemon, Local> Variant;
 
     Variant variant;
 

src/libstore/include/nix/store/worker-protocol-connection.hh

@@ -130,8 +130,6 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection
         bool * daemonException,
         const StorePath & path,
         std::function<void(Source &)> fun);
-
-    void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source);
 };
 
 struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection

src/libstore/include/nix/store/worker-protocol.hh

@@ -152,7 +152,6 @@ enum struct WorkerProto::Op : uint64_t {
     AddIndirectRoot = 12,
     SyncWithGC = 13,
     FindRoots = 14,
-    ExportPath = 16,   // obsolete
     QueryDeriver = 18, // obsolete
     SetOptions = 19,
     CollectGarbage = 20,
@@ -162,7 +161,6 @@ enum struct WorkerProto::Op : uint64_t {
     QueryFailedPaths = 24,
     ClearFailedPaths = 25,
     QueryPathInfo = 26,
-    ImportPaths = 27,                // obsolete
     QueryDerivationOutputNames = 28, // obsolete
     QueryPathFromHashPart = 29,
     QuerySubstitutablePathInfos = 30,

src/libstore/legacy-ssh-store.cc

@@ -302,22 +302,6 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, bool lo
     return conn->queryValidPaths(*this, lock, paths, maybeSubstitute);
 }
 
-void LegacySSHStore::addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths)
-{
-    auto conn(connections->get());
-    conn->to << ServeProto::Command::ImportPaths;
-    try {
-        srcStore.exportPaths(paths, conn->to);
-    } catch (...) {
-        conn->good = false;
-        throw;
-    }
-    conn->to.flush();
-
-    if (readInt(conn->from) != 1)
-        throw Error("remote machine failed to import closure");
-}
-
 void LegacySSHStore::connect()
 {
     auto conn(connections->get());

src/libstore/local-store.cc

@@ -118,6 +118,7 @@ LocalStore::LocalStore(ref<const Config> config)
     : Store{*config}
     , LocalFSStore{*config}
     , config{config}
+    , _state(make_ref<Sync<State>>())
     , dbDir(config->stateDir + "/db")
     , linksDir(config->realStoreDir + "/.links")
     , reservedPath(dbDir + "/reserved")
@@ -125,7 +126,7 @@ LocalStore::LocalStore(ref<const Config> config)
     , tempRootsDir(config->stateDir + "/temproots")
     , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
 {
-    auto state(_state.lock());
+    auto state(_state->lock());
     state->stmts = std::make_unique<State::Stmts>();
 
     /* Create missing state directories if they don't already exist. */
@@ -433,7 +434,7 @@ LocalStore::~LocalStore()
     std::shared_future<void> future;
 
     {
-        auto state(_state.lock());
+        auto state(_state->lock());
         if (state->gcRunning)
             future = state->gcFuture;
     }
@@ -456,12 +457,17 @@ LocalStore::~LocalStore()
 
 StoreReference LocalStoreConfig::getReference() const
 {
+    auto params = getQueryParams();
+    /* Back-compatibility kludge. Tools like nix-output-monitor expect 'local'
+       and can't parse 'local://'. */
+    if (params.empty())
+        return {.variant = StoreReference::Local{}};
     return {
         .variant =
             StoreReference::Specified{
                 .scheme = *uriSchemes().begin(),
             },
-        .params = getQueryParams(),
+        .params = std::move(params),
     };
 }
 
@@ -624,7 +630,7 @@ void LocalStore::registerDrvOutput(const Realisation & info)
 {
     experimentalFeatureSettings.require(Xp::CaDerivations);
     retrySQLite<void>([&]() {
-        auto state(_state.lock());
+        auto state(_state->lock());
         if (auto oldR = queryRealisation_(*state, info.id)) {
             if (info.isCompatibleWith(*oldR)) {
                 auto combinedSignatures = oldR->signatures;
@@ -716,12 +722,8 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, boo
         }
     }
 
-    {
-        auto state_(Store::state.lock());
-        state_->pathInfoCache.upsert(
-            std::string(info.path.to_string()),
-            PathInfoCacheValue{.value = std::make_shared<const ValidPathInfo>(info)});
-    }
+    pathInfoCache->lock()->upsert(
+        std::string(info.path.to_string()), PathInfoCacheValue{.value = std::make_shared<const ValidPathInfo>(info)});
 
     return id;
 }
@@ -731,8 +733,7 @@ void LocalStore::queryPathInfoUncached(
 {
     try {
         callback(retrySQLite<std::shared_ptr<const ValidPathInfo>>([&]() {
-            auto state(_state.lock());
-            return queryPathInfoInternal(*state, path);
+            return queryPathInfoInternal(*_state->lock(), path);
         }));
 
     } catch (...) {
@@ -814,10 +815,7 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path)
 
 bool LocalStore::isValidPathUncached(const StorePath & path)
 {
-    return retrySQLite<bool>([&]() {
-        auto state(_state.lock());
-        return isValidPath_(*state, path);
-    });
+    return retrySQLite<bool>([&]() { return isValidPath_(*_state->lock(), path); });
 }
 
 StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
@@ -832,7 +830,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF
 StorePathSet LocalStore::queryAllValidPaths()
 {
     return retrySQLite<StorePathSet>([&]() {
-        auto state(_state.lock());
+        auto state(_state->lock());
         auto use(state->stmts->QueryValidPaths.use());
         StorePathSet res;
         while (use.next())
@@ -851,16 +849,13 @@ void LocalStore::queryReferrers(State & state, const StorePath & path, StorePath
 
 void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers)
 {
-    return retrySQLite<void>([&]() {
-        auto state(_state.lock());
-        queryReferrers(*state, path, referrers);
-    });
+    return retrySQLite<void>([&]() { queryReferrers(*_state->lock(), path, referrers); });
 }
 
 StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
 {
     return retrySQLite<StorePathSet>([&]() {
-        auto state(_state.lock());
+        auto state(_state->lock());
 
         auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path)));
 
@@ -876,7 +871,7 @@ std::map<std::string, std::optional<StorePath>>
 LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path)
 {
     return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
-        auto state(_state.lock());
+        auto state(_state->lock());
         std::map<std::string, std::optional<StorePath>> outputs;
         uint64_t drvId;
         drvId = queryValidPathId(*state, path);
@@ -896,7 +891,7 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h
     Path prefix = storeDir + "/" + hashPart;
 
     return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
-        auto state(_state.lock());
+        auto state(_state->lock());
 
         auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix));
 
@@ -961,7 +956,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
 #endif
 
     return retrySQLite<void>([&]() {
-        auto state(_state.lock());
+        auto state(_state->lock());
 
         SQLiteTxn txn(state->db);
         StorePathSet paths;
@@ -1023,15 +1018,12 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
     /* Note that the foreign key constraints on the Refs table take
        care of deleting the references entries for `path'. */
 
-    {
-        auto state_(Store::state.lock());
-        state_->pathInfoCache.erase(std::string(path.to_string()));
-    }
+    pathInfoCache->lock()->erase(std::string(path.to_string()));
 }
 
 const PublicKeys & LocalStore::getPublicKeys()
 {
-    auto state(_state.lock());
+    auto state(_state->lock());
     if (!state->publicKeys)
         state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys());
     return *state->publicKeys;
@@ -1354,7 +1346,7 @@ std::pair<std::filesystem::path, AutoCloseFD> LocalStore::createTempDirInStore()
 void LocalStore::invalidatePathChecked(const StorePath & path)
 {
     retrySQLite<void>([&]() {
-        auto state(_state.lock());
+        auto state(_state->lock());
 
         SQLiteTxn txn(state->db);
 
@@ -1454,10 +1446,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                         update = true;
                     }
 
-                    if (update) {
-                        auto state(_state.lock());
-                        updatePathInfo(*state, *info);
-                    }
+                    if (update)
+                        updatePathInfo(*_state->lock(), *info);
                 }
 
             } catch (Error & e) {
@@ -1544,8 +1534,7 @@ void LocalStore::verifyPath(
 
         if (canInvalidate) {
             printInfo("path '%s' disappeared, removing from database...", pathS);
-            auto state(_state.lock());
-            invalidatePath(*state, path);
+            invalidatePath(*_state->lock(), path);
         } else {
             printError("path '%s' disappeared, but it still has valid referrers!", pathS);
             if (repair)
@@ -1577,14 +1566,13 @@ std::optional<TrustedFlag> LocalStore::isTrustedClient()
 
 void LocalStore::vacuumDB()
 {
-    auto state(_state.lock());
-    state->db.exec("vacuum");
+    _state->lock()->db.exec("vacuum");
 }
 
 void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs)
 {
     retrySQLite<void>([&]() {
-        auto state(_state.lock());
+        auto state(_state->lock());
 
         SQLiteTxn txn(state->db);
 
@@ -1646,10 +1634,8 @@ void LocalStore::queryRealisationUncached(
     const DrvOutput & id, Callback<std::shared_ptr<const Realisation>> callback) noexcept
 {
     try {
-        auto maybeRealisation = retrySQLite<std::optional<const Realisation>>([&]() {
-            auto state(_state.lock());
-            return queryRealisation_(*state, id);
-        });
+        auto maybeRealisation =
+            retrySQLite<std::optional<const Realisation>>([&]() { return queryRealisation_(*_state->lock(), id); });
         if (maybeRealisation)
             callback(std::make_shared<const Realisation>(maybeRealisation.value()));
         else

src/libstore/meson.build

@@ -101,7 +101,12 @@ subdir('nix-meson-build-support/libatomic')
 
 boost = dependency(
   'boost',
-  modules : [ 'container', 'regex' ],
+  modules : [
+    'container',
+    # Shouldn't list, because can header-only, and Meson currently looks for libs
+    #'regex',
+    'url',
+  ],
   include_type : 'system',
 )
 # boost is a public dependency, but not a pkg-config dependency unfortunately, so we

src/libstore/remote-store.cc

@@ -73,6 +73,8 @@ void RemoteStore::initConnection(Connection & conn)
         try {
             auto [protoVersion, features] =
                 WorkerProto::BasicClientConnection::handshake(conn.to, tee, PROTOCOL_VERSION, WorkerProto::allFeatures);
+            if (protoVersion < 256 + 18)
+                throw Error("the Nix daemon version is too old");
             conn.protoVersion = protoVersion;
             conn.features = features;
         } catch (SerialisationError & e) {
@@ -109,24 +111,22 @@ void RemoteStore::setOptions(Connection & conn)
             << 0                                                  /* obsolete print build trace */
             << settings.buildCores << settings.useSubstitutes;
 
-    if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) {
-        std::map<std::string, nix::Config::SettingInfo> overrides;
-        settings.getSettings(overrides, true); // libstore settings
-        fileTransferSettings.getSettings(overrides, true);
-        overrides.erase(settings.keepFailed.name);
-        overrides.erase(settings.keepGoing.name);
-        overrides.erase(settings.tryFallback.name);
-        overrides.erase(settings.maxBuildJobs.name);
-        overrides.erase(settings.maxSilentTime.name);
-        overrides.erase(settings.buildCores.name);
-        overrides.erase(settings.useSubstitutes.name);
-        overrides.erase(loggerSettings.showTrace.name);
-        overrides.erase(experimentalFeatureSettings.experimentalFeatures.name);
-        overrides.erase("plugin-files");
-        conn.to << overrides.size();
-        for (auto & i : overrides)
-            conn.to << i.first << i.second.value;
-    }
+    std::map<std::string, nix::Config::SettingInfo> overrides;
+    settings.getSettings(overrides, true); // libstore settings
+    fileTransferSettings.getSettings(overrides, true);
+    overrides.erase(settings.keepFailed.name);
+    overrides.erase(settings.keepGoing.name);
+    overrides.erase(settings.tryFallback.name);
+    overrides.erase(settings.maxBuildJobs.name);
+    overrides.erase(settings.maxSilentTime.name);
+    overrides.erase(settings.buildCores.name);
+    overrides.erase(settings.useSubstitutes.name);
+    overrides.erase(loggerSettings.showTrace.name);
+    overrides.erase(experimentalFeatureSettings.experimentalFeatures.name);
+    overrides.erase("plugin-files");
+    conn.to << overrides.size();
+    for (auto & i : overrides)
+        conn.to << i.first << i.second.value;
 
     auto ex = conn.processStderrReturn();
     if (ex)
@@ -167,15 +167,7 @@ bool RemoteStore::isValidPathUncached(const StorePath & path)
 StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
 {
     auto conn(getConnection());
-    if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) {
-        StorePathSet res;
-        for (auto & i : paths)
-            if (isValidPath(i))
-                res.insert(i);
-        return res;
-    } else {
-        return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute);
-    }
+    return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute);
 }
 
 StorePathSet RemoteStore::queryAllValidPaths()
@@ -189,21 +181,10 @@ StorePathSet RemoteStore::queryAllValidPaths()
 StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
 {
     auto conn(getConnection());
-    if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) {
-        StorePathSet res;
-        for (auto & i : paths) {
-            conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i);
-            conn.processStderr();
-            if (readInt(conn->from))
-                res.insert(i);
-        }
-        return res;
-    } else {
-        conn->to << WorkerProto::Op::QuerySubstitutablePaths;
-        WorkerProto::write(*this, *conn, paths);
-        conn.processStderr();
-        return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
-    }
+    conn->to << WorkerProto::Op::QuerySubstitutablePaths;
+    WorkerProto::write(*this, *conn, paths);
+    conn.processStderr();
+    return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
 }
 
 void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
@@ -213,45 +194,24 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
 
     auto conn(getConnection());
 
-    if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) {
-
-        for (auto & i : pathsMap) {
-            SubstitutablePathInfo info;
-            conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first);
-            conn.processStderr();
-            unsigned int reply = readInt(conn->from);
-            if (reply == 0)
-                continue;
-            auto deriver = readString(conn->from);
-            if (deriver != "")
-                info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
-            info.downloadSize = readLongLong(conn->from);
-            info.narSize = readLongLong(conn->from);
-            infos.insert_or_assign(i.first, std::move(info));
-        }
-
-    } else {
-
-        conn->to << WorkerProto::Op::QuerySubstitutablePathInfos;
-        if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) {
-            StorePathSet paths;
-            for (auto & path : pathsMap)
-                paths.insert(path.first);
-            WorkerProto::write(*this, *conn, paths);
-        } else
-            WorkerProto::write(*this, *conn, pathsMap);
-        conn.processStderr();
-        size_t count = readNum<size_t>(conn->from);
-        for (size_t n = 0; n < count; n++) {
-            SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
-            auto deriver = readString(conn->from);
-            if (deriver != "")
-                info.deriver = parseStorePath(deriver);
-            info.references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
-            info.downloadSize = readLongLong(conn->from);
-            info.narSize = readLongLong(conn->from);
-        }
+    conn->to << WorkerProto::Op::QuerySubstitutablePathInfos;
+    if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) {
+        StorePathSet paths;
+        for (auto & path : pathsMap)
+            paths.insert(path.first);
+        WorkerProto::write(*this, *conn, paths);
+    } else
+        WorkerProto::write(*this, *conn, pathsMap);
+    conn.processStderr();
+    size_t count = readNum<size_t>(conn->from);
+    for (size_t n = 0; n < count; n++) {
+        SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
+        auto deriver = readString(conn->from);
+        if (deriver != "")
+            info.deriver = parseStorePath(deriver);
+        info.references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
+        info.downloadSize = readLongLong(conn->from);
+        info.narSize = readLongLong(conn->from);
     }
 }
 
@@ -466,36 +426,20 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair
 {
     auto conn(getConnection());
 
-    if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) {
-        auto source2 = sinkToSource([&](Sink & sink) {
-            sink << 1 // == path follows
-                ;
-            copyNAR(source, sink);
-            sink << exportMagic << printStorePath(info.path);
-            WorkerProto::write(*this, *conn, info.references);
-            sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature
-                 << 0                                                        // == no path follows
-                ;
-        });
-        conn->importPaths(*this, &conn.daemonException, *source2);
-    }
+    conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path)
+             << (info.deriver ? printStorePath(*info.deriver) : "")
+             << info.narHash.to_string(HashFormat::Base16, false);
+    WorkerProto::write(*this, *conn, info.references);
+    conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca)
+             << repair << !checkSigs;
 
-    else {
-        conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path)
-                 << (info.deriver ? printStorePath(*info.deriver) : "")
-                 << info.narHash.to_string(HashFormat::Base16, false);
-        WorkerProto::write(*this, *conn, info.references);
-        conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca)
-                 << repair << !checkSigs;
-
-        if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) {
-            conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); });
-        } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) {
-            conn.processStderr(0, &source);
-        } else {
-            copyNAR(source, conn->to);
-            conn.processStderr(0, nullptr);
-        }
+    if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) {
+        conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); });
+    } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) {
+        conn.processStderr(0, &source);
+    } else {
+        copyNAR(source, conn->to);
+        conn.processStderr(0, nullptr);
     }
 }
 
@@ -618,15 +562,8 @@ void RemoteStore::buildPaths(
 
     auto conn(getConnection());
     conn->to << WorkerProto::Op::BuildPaths;
-    assert(GET_PROTOCOL_MINOR(conn->protoVersion) >= 13);
     WorkerProto::write(*this, *conn, drvPaths);
-    if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 15)
-        conn->to << buildMode;
-    else
-        /* Old daemons did not take a 'buildMode' parameter, so we
-           need to validate it here on the client side.  */
-        if (buildMode != bmNormal)
-            throw Error("repairing or checking is not supported when building through the Nix daemon");
+    conn->to << buildMode;
     conn.processStderr();
     readInt(conn->from);
 }
@@ -764,10 +701,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
     results.bytesFreed = readLongLong(conn->from);
     readLongLong(conn->from); // obsolete
 
-    {
-        auto state_(Store::state.lock());
-        state_->pathInfoCache.clear();
-    }
+    pathInfoCache->lock()->clear();
 }
 
 void RemoteStore::optimiseStore()

src/libstore/ssh.cc

@@ -84,23 +84,20 @@ SSHMaster::SSHMaster(
     , useMaster(useMaster && !fakeSSH)
     , compress(compress)
     , logFD(logFD)
+    , tmpDir(make_ref<AutoDelete>(createTempDir("", "nix", 0700)))
 {
     checkValidAuthority(authority);
-    auto state(state_.lock());
-    state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", 0700));
 }
 
 void SSHMaster::addCommonSSHOpts(Strings & args)
 {
-    auto state(state_.lock());
-
     auto sshArgs = getNixSshOpts();
     args.insert(args.end(), sshArgs.begin(), sshArgs.end());
 
     if (!keyFile.empty())
         args.insert(args.end(), {"-i", keyFile});
     if (!sshPublicHostKey.empty()) {
-        std::filesystem::path fileName = state->tmpDir->path() / "host-key";
+        std::filesystem::path fileName = tmpDir->path() / "host-key";
         writeFile(fileName.string(), authority.host + " " + sshPublicHostKey + "\n");
         args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName.string()});
     }
@@ -241,7 +238,7 @@ Path SSHMaster::startMaster()
     if (state->sshMaster != INVALID_DESCRIPTOR)
         return state->socketPath;
 
-    state->socketPath = (Path) *state->tmpDir + "/ssh.sock";
+    state->socketPath = (Path) *tmpDir + "/ssh.sock";
 
     Pipe out;
     out.create();

src/libstore/store-api.cc

@@ -306,7 +306,7 @@ StringSet Store::Config::getDefaultSystemFeatures()
 Store::Store(const Store::Config & config)
     : StoreDirConfig{config}
     , config{config}
-    , state({(size_t) config.pathInfoCacheSize})
+    , pathInfoCache(make_ref<decltype(pathInfoCache)::element_type>((size_t) config.pathInfoCacheSize))
 {
     assertLibStoreInitialized();
 }
@@ -326,7 +326,7 @@ bool Store::PathInfoCacheValue::isKnownNow()
 
 void Store::invalidatePathInfoCacheFor(const StorePath & path)
 {
-    state.lock()->pathInfoCache.erase(path.to_string());
+    pathInfoCache->lock()->erase(path.to_string());
 }
 
 std::map<std::string, std::optional<StorePath>> Store::queryStaticPartialDerivationOutputMap(const StorePath & path)
@@ -448,13 +448,10 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta
 
 bool Store::isValidPath(const StorePath & storePath)
 {
-    {
-        auto state_(state.lock());
-        auto res = state_->pathInfoCache.get(storePath.to_string());
-        if (res && res->isKnownNow()) {
-            stats.narInfoReadAverted++;
-            return res->didExist();
-        }
+    auto res = pathInfoCache->lock()->get(storePath.to_string());
+    if (res && res->isKnownNow()) {
+        stats.narInfoReadAverted++;
+        return res->didExist();
     }
 
     if (diskCache) {
@@ -462,8 +459,7 @@ bool Store::isValidPath(const StorePath & storePath)
             config.getReference().render(/*FIXME withParams=*/false), std::string(storePath.hashPart()));
         if (res.first != NarInfoDiskCache::oUnknown) {
             stats.narInfoReadAverted++;
-            auto state_(state.lock());
-            state_->pathInfoCache.upsert(
+            pathInfoCache->lock()->upsert(
                 storePath.to_string(),
                 res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{}
                                                         : PathInfoCacheValue{.value = res.second});
@@ -518,30 +514,25 @@ std::optional<std::shared_ptr<const ValidPathInfo>> Store::queryPathInfoFromClie
 {
     auto hashPart = std::string(storePath.hashPart());
 
-    {
-        auto res = state.lock()->pathInfoCache.get(storePath.to_string());
-        if (res && res->isKnownNow()) {
-            stats.narInfoReadAverted++;
-            if (res->didExist())
-                return std::make_optional(res->value);
-            else
-                return std::make_optional(nullptr);
-        }
+    auto res = pathInfoCache->lock()->get(storePath.to_string());
+    if (res && res->isKnownNow()) {
+        stats.narInfoReadAverted++;
+        if (res->didExist())
+            return std::make_optional(res->value);
+        else
+            return std::make_optional(nullptr);
     }
 
     if (diskCache) {
         auto res = diskCache->lookupNarInfo(config.getReference().render(/*FIXME withParams=*/false), hashPart);
         if (res.first != NarInfoDiskCache::oUnknown) {
             stats.narInfoReadAverted++;
-            {
-                auto state_(state.lock());
-                state_->pathInfoCache.upsert(
-                    storePath.to_string(),
-                    res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{}
-                                                            : PathInfoCacheValue{.value = res.second});
-                if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path))
-                    return std::make_optional(nullptr);
-            }
+            pathInfoCache->lock()->upsert(
+                storePath.to_string(),
+                res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{}
+                                                        : PathInfoCacheValue{.value = res.second});
+            if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path))
+                return std::make_optional(nullptr);
             assert(res.second);
             return std::make_optional(res.second);
         }
@@ -577,10 +568,7 @@ void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidP
                 if (diskCache)
                     diskCache->upsertNarInfo(config.getReference().render(/*FIXME withParams=*/false), hashPart, info);
 
-                {
-                    auto state_(state.lock());
-                    state_->pathInfoCache.upsert(storePath.to_string(), PathInfoCacheValue{.value = info});
-                }
+                pathInfoCache->lock()->upsert(storePath.to_string(), PathInfoCacheValue{.value = info});
 
                 if (!info || !goodStorePath(storePath, info->path)) {
                     stats.narInfoMissing++;
@@ -803,10 +791,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor
 
 const Store::Stats & Store::getStats()
 {
-    {
-        auto state_(state.readLock());
-        stats.pathInfoCacheSize = state_->pathInfoCache.size();
-    }
+    stats.pathInfoCacheSize = pathInfoCache->readLock()->size();
     return stats;
 }
 
@@ -818,7 +803,13 @@ makeCopyPathMessage(const StoreConfig & srcCfg, const StoreConfig & dstCfg, std:
 
     auto isShorthand = [](const StoreReference & ref) {
         /* At this point StoreReference **must** be resolved. */
-        const auto & specified = std::get<StoreReference::Specified>(ref.variant);
+        const auto & specified = std::visit(
+            overloaded{
+                [](const StoreReference::Auto &) -> const StoreReference::Specified & { unreachable(); },
+                [](const StoreReference::Specified & specified) -> const StoreReference::Specified & {
+                    return specified;
+                }},
+            ref.variant);
         const auto & scheme = specified.scheme;
         return (scheme == "local" || scheme == "unix") && specified.authority.empty();
     };

src/libstore/store-reference.cc

@@ -1,11 +1,12 @@
-#include <regex>
-
 #include "nix/util/error.hh"
+#include "nix/util/split.hh"
 #include "nix/util/url.hh"
 #include "nix/store/store-reference.hh"
 #include "nix/util/file-system.hh"
 #include "nix/util/util.hh"
 
+#include <boost/url/ipv6_address.hpp>
+
 namespace nix {
 
 static bool isNonUriPath(const std::string & spec)
@@ -25,6 +26,8 @@ std::string StoreReference::render(bool withParams) const
     std::visit(
         overloaded{
             [&](const StoreReference::Auto &) { res = "auto"; },
+            [&](const StoreReference::Daemon &) { res = "daemon"; },
+            [&](const StoreReference::Local &) { res = "local"; },
             [&](const StoreReference::Specified & g) {
                 res = g.scheme;
                 res += "://";
@@ -41,6 +44,29 @@ std::string StoreReference::render(bool withParams) const
     return res;
 }
 
+namespace {
+
+struct SchemeAndAuthorityWithPath
+{
+    std::string_view scheme;
+    std::string_view authority;
+};
+
+} // namespace
+
+/**
+ * Return the 'scheme' and remove the '://' or ':' separator.
+ */
+static std::optional<SchemeAndAuthorityWithPath> splitSchemePrefixTo(std::string_view string)
+{
+    auto scheme = splitPrefixTo(string, ':');
+    if (!scheme)
+        return std::nullopt;
+
+    splitPrefix(string, "//");
+    return SchemeAndAuthorityWithPath{.scheme = *scheme, .authority = string};
+}
+
 StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams)
 {
     auto params = extraParams;
@@ -66,21 +92,17 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen
                 .params = std::move(params),
             };
         } else if (baseURI == "daemon") {
+            if (params.empty())
+                return {.variant = Daemon{}};
             return {
-                .variant =
-                    Specified{
-                        .scheme = "unix",
-                        .authority = "",
-                    },
+                .variant = Specified{.scheme = "unix", .authority = ""},
                 .params = std::move(params),
             };
         } else if (baseURI == "local") {
+            if (params.empty())
+                return {.variant = Local{}};
             return {
-                .variant =
-                    Specified{
-                        .scheme = "local",
-                        .authority = "",
-                    },
+                .variant = Specified{.scheme = "local", .authority = ""},
                 .params = std::move(params),
             };
         } else if (isNonUriPath(baseURI)) {
@@ -92,6 +114,32 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen
                     },
                 .params = std::move(params),
             };
+        } else if (auto schemeAndAuthority = splitSchemePrefixTo(baseURI)) {
+            /* Back-compatibility shim to accept unbracketed IPv6 addresses after the scheme.
+             * Old versions of nix allowed that. Note that this is ambiguous and does not allow
+             * specifying the port number. For that the address must be bracketed, otherwise it's
+             * greedily assumed to be the part of the host address. */
+            auto authorityString = schemeAndAuthority->authority;
+            auto userinfo = splitPrefixTo(authorityString, '@');
+            auto maybeIpv6 = boost::urls::parse_ipv6_address(authorityString);
+            if (maybeIpv6) {
+                std::string fixedAuthority;
+                if (userinfo) {
+                    fixedAuthority += *userinfo;
+                    fixedAuthority += '@';
+                }
+                fixedAuthority += '[';
+                fixedAuthority += authorityString;
+                fixedAuthority += ']';
+                return {
+                    .variant =
+                        Specified{
+                            .scheme = std::string(schemeAndAuthority->scheme),
+                            .authority = fixedAuthority,
+                        },
+                    .params = std::move(params),
+                };
+            }
         }
     }
 

src/libstore/uds-remote-store.cc

@@ -57,15 +57,16 @@ UDSRemoteStore::UDSRemoteStore(ref<const Config> config)
 
 StoreReference UDSRemoteStoreConfig::getReference() const
 {
+    /* We specifically return "daemon" here instead of "unix://" or "unix://${path}"
+     * to be more compatible with older versions of nix. Some tooling out there
+     * tries hard to parse store references and it might not be able to handle "unix://". */
+    if (path == settings.nixDaemonSocketFile)
+        return {.variant = StoreReference::Daemon{}};
     return {
         .variant =
             StoreReference::Specified{
                 .scheme = *uriSchemes().begin(),
-                // We return the empty string when the path looks like the
-                // default path, but we could also just return the path
-                // verbatim always, to be robust to overall config changes
-                // at the cost of some verbosity.
-                .authority = path == settings.nixDaemonSocketFile ? "" : path,
+                .authority = path,
             },
     };
 }

src/libstore/unix/build/derivation-builder.cc

@@ -214,9 +214,7 @@ protected:
 
 public:
 
-    bool prepareBuild() override;
-
-    void startBuilder() override;
+    std::optional<Descriptor> startBuild() override;
 
     SingleDrvOutputs unprepareBuild() override;
 
@@ -470,19 +468,6 @@ bool DerivationBuilderImpl::killChild()
     return ret;
 }
 
-bool DerivationBuilderImpl::prepareBuild()
-{
-    if (useBuildUsers()) {
-        if (!buildUser)
-            buildUser = getBuildUser();
-
-        if (!buildUser)
-            return false;
-    }
-
-    return true;
-}
-
 SingleDrvOutputs DerivationBuilderImpl::unprepareBuild()
 {
     /* Since we got an EOF on the logger pipe, the builder is presumed
@@ -679,8 +664,16 @@ static bool checkNotWorldWritable(std::filesystem::path path)
     return true;
 }
 
-void DerivationBuilderImpl::startBuilder()
+std::optional<Descriptor> DerivationBuilderImpl::startBuild()
 {
+    if (useBuildUsers()) {
+        if (!buildUser)
+            buildUser = getBuildUser();
+
+        if (!buildUser)
+            return std::nullopt;
+    }
+
     /* Make sure that no other processes are executing under the
        sandbox uids. This must be done before any chownToBuilder()
        calls. */
@@ -720,7 +713,7 @@ void DerivationBuilderImpl::startBuilder()
 
     /* Create a temporary directory where the build will take
        place. */
-    topTmpDir = createTempDir(buildDir, "nix-build-" + std::string(drvPath.name()), 0700);
+    topTmpDir = createTempDir(buildDir, "nix", 0700);
     setBuildTmpDir();
     assert(!tmpDir.empty());
 
@@ -841,9 +834,10 @@ void DerivationBuilderImpl::startBuilder()
     startChild();
 
     pid.setSeparatePG(true);
-    miscMethods->childStarted(builderOut.get());
 
     processSandboxSetupMessages();
+
+    return builderOut.get();
 }
 
 PathsInChroot DerivationBuilderImpl::getPathsInSandbox()

src/libstore/unix/build/linux-derivation-builder.cc

@@ -362,9 +362,21 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
 
         userNamespaceSync.readSide = -1;
 
-        /* Close the write side to prevent runChild() from hanging
-           reading from this. */
-        Finally cleanup([&]() { userNamespaceSync.writeSide = -1; });
+        /* Make sure that we write *something* to the child in case of
+           an exception. Note that merely closing
+           `userNamespaceSync.writeSide` doesn't work in
+           multi-threaded Nix, since several child processes may have
+           inherited `writeSide` (and O_CLOEXEC doesn't help because
+           the children may not do an execve). */
+        bool userNamespaceSyncDone = false;
+        Finally cleanup([&]() {
+            try {
+                if (!userNamespaceSyncDone)
+                    writeFull(userNamespaceSync.writeSide.get(), "0\n");
+            } catch (...) {
+            }
+            userNamespaceSync.writeSide = -1;
+        });
 
         auto ss = tokenizeString<std::vector<std::string>>(readLine(sendPid.readSide.get()));
         assert(ss.size() == 1);
@@ -419,14 +431,15 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu
             writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
 
         /* Signal the builder that we've updated its user namespace. */
-        writeFull(userNamespaceSync.writeSide.get(), "1");
+        writeFull(userNamespaceSync.writeSide.get(), "1\n");
+        userNamespaceSyncDone = true;
     }
 
     void enterChroot() override
     {
         userNamespaceSync.writeSide = -1;
 
-        if (drainFD(userNamespaceSync.readSide.get()) != "1")
+        if (readLine(userNamespaceSync.readSide.get()) != "1")
             throw Error("user namespace initialisation failed");
 
         userNamespaceSync.readSide = -1;

src/libstore/unix/include/nix/store/build/hook-instance.hh

@@ -7,6 +7,14 @@
 
 namespace nix {
 
+/**
+ * @note Sometimes this is owned by the `Worker`, and sometimes it is
+ * owned by a `Goal`. This is for efficiency: rather than starting the
+ * hook every time we want to ask whether we can run a remote build
+ * (which can be very often), we reuse a hook process for answering
+ * those queries until it accepts a build.  So if there are N
+ * derivations to be built, at most N hooks will be started.
+ */
 struct HookInstance
 {
     /**
@@ -29,6 +37,15 @@ struct HookInstance
      */
     Pid pid;
 
+    /**
+     * The remote machine on which we're building.
+     *
+     * @Invariant When the hook instance is owned by the `Worker`, this
+     * is the empty string. When it is owned by a `Goal`, this should be
+     * set.
+     */
+    std::string machineName;
+
     FdSink sink;
 
     std::map<ActivityId, Activity> activities;

src/libstore/worker-protocol-connection.cc

@@ -313,12 +313,4 @@ void WorkerProto::BasicClientConnection::narFromPath(
     fun(from);
 }
 
-void WorkerProto::BasicClientConnection::importPaths(
-    const StoreDirConfig & store, bool * daemonException, Source & source)
-{
-    to << WorkerProto::Op::ImportPaths;
-    processStderr(daemonException, 0, &source);
-    auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(store, *this);
-    assert(importedPaths.size() <= importedPaths.size());
-}
 } // namespace nix

src/libutil-c/nix_api_util.cc

@@ -9,6 +9,8 @@
 
 #include "nix_api_util_config.h"
 
+extern "C" {
+
 nix_c_context * nix_c_context_create()
 {
     return new nix_c_context();
@@ -156,3 +158,5 @@ nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callb
     callback(str.c_str(), str.size(), user_data);
     return NIX_OK;
 }
+
+} // extern "C"

src/libutil-c/nix_api_util.h

@@ -98,6 +98,13 @@ enum nix_err {
      */
     NIX_ERR_NIX_ERROR = -4,
 
+    /**
+     * @brief A recoverable error occurred.
+     *
+     * This is used primarily by C API *consumers* to communicate that a failed
+     * primop call should be retried on the next evaluation attempt.
+     */
+    NIX_ERR_RECOVERABLE = -5,
 };
 
 typedef enum nix_err nix_err;

src/libutil-c/nix_api_util_internal.h

@@ -7,6 +7,8 @@
 #include "nix/util/error.hh"
 #include "nix_api_util.h"
 
+extern "C" {
+
 struct nix_c_context
 {
     nix_err last_err_code = NIX_OK;
@@ -47,4 +49,6 @@ nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callb
     }
 #define NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_RES(nullptr)
 
+} // extern "C"
+
 #endif // NIX_API_UTIL_INTERNAL_H

src/libutil-test-support/include/nix/util/tests/nix_api_util.hh

@@ -54,4 +54,12 @@ protected:
 #define assert_ctx_err() assert_ctx_err(__FILE__, __LINE__)
 };
 
+static inline auto createOwnedNixContext()
+{
+    return std::unique_ptr<nix_c_context, decltype([](nix_c_context * ctx) {
+                               if (ctx)
+                                   nix_c_context_free(ctx);
+                           })>(nix_c_context_create(), {});
+}
+
 } // namespace nixC

src/libutil-tests/meson.build

@@ -63,6 +63,7 @@ sources = files(
   'lru-cache.cc',
   'monitorfdhup.cc',
   'nix_api_util.cc',
+  'nix_api_util_internal.cc',
   'pool.cc',
   'position.cc',
   'processes.cc',

src/libutil-tests/nix_api_util.cc

@@ -1,7 +1,6 @@
 #include "nix/util/config-global.hh"
 #include "nix/util/args.hh"
 #include "nix_api_util.h"
-#include "nix_api_util_internal.h"
 #include "nix/util/tests/nix_api_util.hh"
 #include "nix/util/tests/string_callback.hh"
 
@@ -13,41 +12,6 @@
 
 namespace nixC {
 
-TEST_F(nix_api_util_context, nix_context_error)
-{
-    std::string err_msg_ref;
-    try {
-        throw nix::Error("testing error");
-    } catch (nix::Error & e) {
-        err_msg_ref = e.what();
-        nix_context_error(ctx);
-    }
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR);
-    ASSERT_EQ(ctx->name, "nix::Error");
-    ASSERT_EQ(*ctx->last_err, err_msg_ref);
-    ASSERT_EQ(ctx->info->msg.str(), "testing error");
-
-    try {
-        throw std::runtime_error("testing exception");
-    } catch (std::exception & e) {
-        err_msg_ref = e.what();
-        nix_context_error(ctx);
-    }
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN);
-    ASSERT_EQ(*ctx->last_err, err_msg_ref);
-
-    nix_clear_err(ctx);
-    ASSERT_EQ(ctx->last_err_code, NIX_OK);
-}
-
-TEST_F(nix_api_util_context, nix_set_err_msg)
-{
-    ASSERT_EQ(ctx->last_err_code, NIX_OK);
-    nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error");
-    ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN);
-    ASSERT_EQ(*ctx->last_err, "unknown test error");
-}
-
 TEST(nix_api_util, nix_version_get)
 {
     ASSERT_EQ(std::string(nix_version_get()), PACKAGE_VERSION);
@@ -61,17 +25,9 @@ struct MySettings : nix::Config
 MySettings mySettings;
 static nix::GlobalConfig::Register rs(&mySettings);
 
-static auto createOwnedNixContext()
-{
-    return std::unique_ptr<nix_c_context, decltype([](nix_c_context * ctx) {
-                               if (ctx)
-                                   nix_c_context_free(ctx);
-                           })>(nix_c_context_create(), {});
-}
-
 TEST_F(nix_api_util_context, nix_setting_get)
 {
-    ASSERT_EQ(ctx->last_err_code, NIX_OK);
+    ASSERT_EQ(nix_err_code(ctx), NIX_OK);
     std::string setting_value;
     nix_err result = nix_setting_get(ctx, "invalid-key", OBSERVE_STRING(setting_value));
     ASSERT_EQ(result, NIX_ERR_KEY);
@@ -114,40 +70,6 @@ TEST_F(nix_api_util_context, nix_err_msg)
     ASSERT_EQ(sz, err_msg.size());
 }
 
-TEST_F(nix_api_util_context, nix_err_info_msg)
-{
-    std::string err_info;
-
-    // no error
-    EXPECT_THROW(nix_err_info_msg(NULL, ctx, OBSERVE_STRING(err_info)), nix::Error);
-
-    try {
-        throw nix::Error("testing error");
-    } catch (...) {
-        nix_context_error(ctx);
-    }
-    auto new_ctx = createOwnedNixContext();
-    nix_err_info_msg(new_ctx.get(), ctx, OBSERVE_STRING(err_info));
-    ASSERT_STREQ("testing error", err_info.c_str());
-}
-
-TEST_F(nix_api_util_context, nix_err_name)
-{
-    std::string err_name;
-
-    // no error
-    EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error);
-
-    try {
-        throw nix::Error("testing error");
-    } catch (...) {
-        nix_context_error(ctx);
-    }
-    auto new_ctx = createOwnedNixContext();
-    nix_err_name(new_ctx.get(), ctx, OBSERVE_STRING(err_name));
-    ASSERT_EQ(std::string(err_name), "nix::Error");
-}
-
 TEST_F(nix_api_util_context, nix_err_code)
 {
     ASSERT_EQ(nix_err_code(ctx), NIX_OK);

src/libutil-tests/nix_api_util_internal.cc

@@ -0,0 +1,85 @@
+#include "nix/util/config-global.hh"
+#include "nix/util/args.hh"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix/util/tests/nix_api_util.hh"
+#include "nix/util/tests/string_callback.hh"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+
+#include "util-tests-config.hh"
+
+namespace nixC {
+
+TEST_F(nix_api_util_context, nix_context_error)
+{
+    std::string err_msg_ref;
+    try {
+        throw nix::Error("testing error");
+    } catch (nix::Error & e) {
+        err_msg_ref = e.what();
+        nix_context_error(ctx);
+    }
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR);
+    ASSERT_EQ(ctx->name, "nix::Error");
+    ASSERT_EQ(*ctx->last_err, err_msg_ref);
+    ASSERT_EQ(ctx->info->msg.str(), "testing error");
+
+    try {
+        throw std::runtime_error("testing exception");
+    } catch (std::exception & e) {
+        err_msg_ref = e.what();
+        nix_context_error(ctx);
+    }
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_UNKNOWN);
+    ASSERT_EQ(*ctx->last_err, err_msg_ref);
+
+    nix_clear_err(ctx);
+    ASSERT_EQ(nix_err_code(ctx), NIX_OK);
+}
+
+TEST_F(nix_api_util_context, nix_set_err_msg)
+{
+    ASSERT_EQ(nix_err_code(ctx), NIX_OK);
+    nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error");
+    ASSERT_EQ(nix_err_code(ctx), NIX_ERR_UNKNOWN);
+    ASSERT_EQ(*ctx->last_err, "unknown test error");
+}
+
+TEST_F(nix_api_util_context, nix_err_info_msg)
+{
+    std::string err_info;
+
+    // no error
+    EXPECT_THROW(nix_err_info_msg(NULL, ctx, OBSERVE_STRING(err_info)), nix::Error);
+
+    try {
+        throw nix::Error("testing error");
+    } catch (...) {
+        nix_context_error(ctx);
+    }
+    auto new_ctx = createOwnedNixContext();
+    nix_err_info_msg(new_ctx.get(), ctx, OBSERVE_STRING(err_info));
+    ASSERT_STREQ("testing error", err_info.c_str());
+}
+
+TEST_F(nix_api_util_context, nix_err_name)
+{
+    std::string err_name;
+
+    // no error
+    EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error);
+
+    try {
+        throw nix::Error("testing error");
+    } catch (...) {
+        nix_context_error(ctx);
+    }
+    auto new_ctx = createOwnedNixContext();
+    nix_err_name(new_ctx.get(), ctx, OBSERVE_STRING(err_name));
+    ASSERT_EQ(std::string(err_name), "nix::Error");
+}
+
+} // namespace nixC

src/libutil-tests/url.cc

@@ -169,55 +169,124 @@ TEST(FixGitURLTestSuite, relativePathParsesPoorly)
             .path = {"relative", "repo"}}));
 }
 
-TEST(parseURL, parsesSimpleHttpUrl)
+struct ParseURLSuccessCase
 {
-    auto s = "http://www.example.org/file.tar.gz";
-    auto parsed = parseURL(s);
+    std::string_view input;
+    ParsedURL expected;
+};
 
-    ParsedURL expected{
-        .scheme = "http",
-        .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
-        .path = {"", "file.tar.gz"},
-        .query = (StringMap) {},
-        .fragment = "",
-    };
+class ParseURLSuccess : public ::testing::TestWithParam<ParseURLSuccessCase>
+{};
 
-    ASSERT_EQ(parsed, expected);
-    ASSERT_EQ(s, parsed.to_string());
+INSTANTIATE_TEST_SUITE_P(
+    ParseURLSuccessCases,
+    ParseURLSuccess,
+    ::testing::Values(
+        ParseURLSuccessCase{
+            .input = "http://www.example.org/file.tar.gz",
+            .expected =
+                ParsedURL{
+                    .scheme = "http",
+                    .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
+                    .path = {"", "file.tar.gz"},
+                    .query = (StringMap) {},
+                    .fragment = "",
+                },
+        },
+        ParseURLSuccessCase{
+            .input = "https://www.example.org/file.tar.gz",
+            .expected =
+                ParsedURL{
+                    .scheme = "https",
+                    .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
+                    .path = {"", "file.tar.gz"},
+                    .query = (StringMap) {},
+                    .fragment = "",
+                },
+        },
+        ParseURLSuccessCase{
+            .input = "https://www.example.org/file.tar.gz?download=fast&when=now#hello",
+            .expected =
+                ParsedURL{
+                    .scheme = "https",
+                    .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
+                    .path = {"", "file.tar.gz"},
+                    .query = (StringMap) {{"download", "fast"}, {"when", "now"}},
+                    .fragment = "hello",
+                },
+        },
+        ParseURLSuccessCase{
+            .input = "file+https://www.example.org/video.mp4",
+            .expected =
+                ParsedURL{
+                    .scheme = "file+https",
+                    .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
+                    .path = {"", "video.mp4"},
+                    .query = (StringMap) {},
+                    .fragment = "",
+                },
+        },
+        ParseURLSuccessCase{
+            .input = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello",
+            .expected =
+                ParsedURL{
+                    .scheme = "http",
+                    .authority = Authority{.hostType = HostType::IPv4, .host = "127.0.0.1", .port = 8080},
+                    .path = {"", "file.tar.gz"},
+                    .query = (StringMap) {{"download", "fast"}, {"when", "now"}},
+                    .fragment = "hello",
+                },
+        },
+        ParseURLSuccessCase{
+            .input = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080",
+            .expected =
+                ParsedURL{
+                    .scheme = "http",
+                    .authority =
+                        Authority{
+                            .hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080},
+                    .path = {""},
+                    .query = (StringMap) {},
+                    .fragment = "",
+                },
+
+        },
+        ParseURLSuccessCase{
+            .input = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
+            .expected =
+                ParsedURL{
+                    .scheme = "http",
+                    .authority =
+                        Authority{
+                            .hostType = HostType::IPv6,
+                            .host = "2a02:8071:8192:c100:311d:192d:81ac:11ea",
+                            .port = 8080,
+                        },
+                    .path = {""},
+                    .query = (StringMap) {},
+                    .fragment = "",
+                },
+        }));
+
+TEST_P(ParseURLSuccess, parsesAsExpected)
+{
+    auto & p = GetParam();
+    const auto parsed = parseURL(p.input);
+    EXPECT_EQ(parsed, p.expected);
 }
 
-TEST(parseURL, parsesSimpleHttpsUrl)
+TEST_P(ParseURLSuccess, toStringRoundTrips)
 {
-    auto s = "https://www.example.org/file.tar.gz";
-    auto parsed = parseURL(s);
-
-    ParsedURL expected{
-        .scheme = "https",
-        .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
-        .path = {"", "file.tar.gz"},
-        .query = (StringMap) {},
-        .fragment = "",
-    };
-
-    ASSERT_EQ(parsed, expected);
-    ASSERT_EQ(s, parsed.to_string());
+    auto & p = GetParam();
+    const auto parsed = parseURL(p.input);
+    EXPECT_EQ(p.input, parsed.to_string());
 }
 
-TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment)
+TEST_P(ParseURLSuccess, makeSureFixGitURLDoesNotModify)
 {
-    auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello";
-    auto parsed = parseURL(s);
-
-    ParsedURL expected{
-        .scheme = "https",
-        .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
-        .path = {"", "file.tar.gz"},
-        .query = (StringMap) {{"download", "fast"}, {"when", "now"}},
-        .fragment = "hello",
-    };
-
-    ASSERT_EQ(parsed, expected);
-    ASSERT_EQ(s, parsed.to_string());
+    auto & p = GetParam();
+    const auto parsed = fixGitURL(std::string{p.input});
+    EXPECT_EQ(p.input, parsed.to_string());
 }
 
 TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment)
@@ -236,23 +305,6 @@ TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment)
     ASSERT_EQ(parsed, expected);
 }
 
-TEST(parseURL, parsesFilePlusHttpsUrl)
-{
-    auto s = "file+https://www.example.org/video.mp4";
-    auto parsed = parseURL(s);
-
-    ParsedURL expected{
-        .scheme = "file+https",
-        .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"},
-        .path = {"", "video.mp4"},
-        .query = (StringMap) {},
-        .fragment = "",
-    };
-
-    ASSERT_EQ(parsed, expected);
-    ASSERT_EQ(s, parsed.to_string());
-}
-
 TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation)
 {
     EXPECT_THAT(
@@ -261,62 +313,6 @@ TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation)
             testing::HasSubstrIgnoreANSIMatcher("has unexpected authority 'www.example.org'")));
 }
 
-TEST(parseURL, parseIPv4Address)
-{
-    auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello";
-    auto parsed = parseURL(s);
-
-    ParsedURL expected{
-        .scheme = "http",
-        .authority = Authority{.hostType = HostType::IPv4, .host = "127.0.0.1", .port = 8080},
-        .path = {"", "file.tar.gz"},
-        .query = (StringMap) {{"download", "fast"}, {"when", "now"}},
-        .fragment = "hello",
-    };
-
-    ASSERT_EQ(parsed, expected);
-    ASSERT_EQ(s, parsed.to_string());
-}
-
-TEST(parseURL, parseScopedRFC6874IPv6Address)
-{
-    auto s = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080";
-    auto parsed = parseURL(s);
-
-    ParsedURL expected{
-        .scheme = "http",
-        .authority = Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080},
-        .path = {""},
-        .query = (StringMap) {},
-        .fragment = "",
-    };
-
-    ASSERT_EQ(parsed, expected);
-    ASSERT_EQ(s, parsed.to_string());
-}
-
-TEST(parseURL, parseIPv6Address)
-{
-    auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080";
-    auto parsed = parseURL(s);
-
-    ParsedURL expected{
-        .scheme = "http",
-        .authority =
-            Authority{
-                .hostType = HostType::IPv6,
-                .host = "2a02:8071:8192:c100:311d:192d:81ac:11ea",
-                .port = 8080,
-            },
-        .path = {""},
-        .query = (StringMap) {},
-        .fragment = "",
-    };
-
-    ASSERT_EQ(parsed, expected);
-    ASSERT_EQ(s, parsed.to_string());
-}
-
 TEST(parseURL, parseEmptyQueryParams)
 {
     auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&";

src/libutil/include/nix/util/current-process.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include <optional>
+#include <chrono>
 
 #ifndef _WIN32
 #  include <sys/resource.h>
@@ -11,6 +12,11 @@
 
 namespace nix {
 
+/**
+ * Get the current process's user space CPU time.
+ */
+std::chrono::microseconds getCpuUserTime();
+
 /**
  * If cgroups are active, attempt to calculate the number of CPUs available.
  * If cgroups are unavailable or if cpu.max is set to "max", return 0.

src/libutil/include/nix/util/error.hh

@@ -51,6 +51,7 @@ struct LinesOfCode
    FIXME: Untangle this mess. Should there be AbstractPos as there used to be before
    4feb7d9f71? */
 struct Pos;
+bool isEmpty(const Pos & pos);
 
 void printCodeLines(std::ostream & out, const std::string & prefix, const Pos & errPos, const LinesOfCode & loc);
 
@@ -187,6 +188,11 @@ public:
         err.pos = pos;
     }
 
+    bool hasPos()
+    {
+        return err.pos.get() && !isEmpty(*err.pos.get());
+    }
+
     void pushTrace(Trace trace)
     {
         err.traces.push_front(trace);

src/libutil/include/nix/util/ref.hh

@@ -18,6 +18,9 @@ private:
     std::shared_ptr<T> p;
 
 public:
+
+    using element_type = T;
+
     explicit ref(const std::shared_ptr<T> & p)
         : p(p)
     {

src/libutil/include/nix/util/sync.hh

@@ -36,6 +36,8 @@ private:
 
 public:
 
+    using element_type = T;
+
     SyncBase() {}
 
     SyncBase(const T & data)

src/libutil/meson.build

@@ -57,7 +57,12 @@ deps_private += blake3
 
 boost = dependency(
   'boost',
-  modules : [ 'context', 'coroutine', 'iostreams', 'url' ],
+  modules : [
+    'context',
+    'coroutine',
+    'iostreams',
+    'url',
+  ],
   include_type : 'system',
   version : '>=1.82.0',
 )

src/libutil/position.cc

@@ -7,6 +7,11 @@ Pos::operator std::shared_ptr<const Pos>() const
     return std::make_shared<const Pos>(*this);
 }
 
+bool isEmpty(const Pos & pos)
+{
+    return !pos.operator bool();
+}
+
 std::optional<LinesOfCode> Pos::getCodeLines() const
 {
     if (line == 0)

src/libutil/unix/current-process.cc

@@ -0,0 +1,23 @@
+#include "nix/util/current-process.hh"
+#include "nix/util/error.hh"
+#include <cmath>
+
+#include <sys/resource.h>
+
+namespace nix {
+
+std::chrono::microseconds getCpuUserTime()
+{
+    struct rusage buf;
+
+    if (getrusage(RUSAGE_SELF, &buf) != 0) {
+        throw SysError("failed to get CPU time");
+    }
+
+    std::chrono::seconds seconds(buf.ru_utime.tv_sec);
+    std::chrono::microseconds microseconds(buf.ru_utime.tv_usec);
+
+    return seconds + microseconds;
+}
+
+} // namespace nix

src/libutil/unix/include/nix/util/monitor-fd.hh

@@ -2,15 +2,18 @@
 ///@file
 
 #include <thread>
-#include <atomic>
+#include <cassert>
 
-#include <cstdlib>
 #include <poll.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <signal.h>
+#include <errno.h>
+
+#ifdef __APPLE__
+#  include <sys/types.h>
+#  include <sys/event.h>
+#endif
 
 #include "nix/util/signals.hh"
+#include "nix/util/file-descriptor.hh"
 
 namespace nix {
 
@@ -20,111 +23,113 @@ private:
     std::thread thread;
     Pipe notifyPipe;
 
+    void runThread(int watchFd, int notifyFd);
+
 public:
-    MonitorFdHup(int fd)
-    {
-        notifyPipe.create();
-        thread = std::thread([this, fd]() {
-            while (true) {
-                // There is a POSIX violation on macOS: you have to listen for
-                // at least POLLHUP to receive HUP events for a FD. POSIX says
-                // this is not so, and you should just receive them regardless.
-                // However, as of our testing on macOS 14.5, the events do not
-                // get delivered if in the all-bits-unset case, but do get
-                // delivered if `POLLHUP` is set.
-                //
-                // This bug filed as rdar://37537852
-                // (https://openradar.appspot.com/37537852).
-                //
-                // macOS's own man page
-                // (https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/poll.2.html)
-                // additionally says that `POLLHUP` is ignored as an input. It
-                // seems the likely order of events here was
-                //
-                // 1. macOS did not follow the POSIX spec
-                //
-                // 2. Somebody ninja-fixed this other spec violation to make
-                // sure `POLLHUP` was not forgotten about, even though they
-                // "fixed" this issue in a spec-non-compliant way. Whatever,
-                // we'll use the fix.
-                //
-                // Relevant code, current version, which shows the :
-                // https://github.com/apple-oss-distributions/xnu/blob/94d3b452840153a99b38a3a9659680b2a006908e/bsd/kern/sys_generic.c#L1751-L1758
-                //
-                // The `POLLHUP` detection was added in
-                // https://github.com/apple-oss-distributions/xnu/commit/e13b1fa57645afc8a7b2e7d868fe9845c6b08c40#diff-a5aa0b0e7f4d866ca417f60702689fc797e9cdfe33b601b05ccf43086c35d395R1468
-                // That means added in 2007 or earlier. Should be good enough
-                // for us.
-                short hangup_events =
-#ifdef __APPLE__
-                    POLLHUP
-#else
-                    0
-#endif
-                    ;
-
-                /* Wait indefinitely until a POLLHUP occurs. */
-                constexpr size_t num_fds = 2;
-                struct pollfd fds[num_fds] = {
-                    {
-                        .fd = fd,
-                        .events = hangup_events,
-                    },
-                    {
-                        .fd = notifyPipe.readSide.get(),
-                        .events = hangup_events,
-                    },
-                };
-
-                auto count = poll(fds, num_fds, -1);
-                if (count == -1) {
-                    if (errno == EINTR || errno == EAGAIN)
-                        continue;
-                    throw SysError("failed to poll() in MonitorFdHup");
-                }
-                /* This shouldn't happen, but can on macOS due to a bug.
-                   See rdar://37550628.
-
-                   This may eventually need a delay or further
-                   coordination with the main thread if spinning proves
-                   too harmful.
-                */
-                if (count == 0)
-                    continue;
-                if (fds[0].revents & POLLHUP) {
-                    unix::triggerInterrupt();
-                    break;
-                }
-                if (fds[1].revents & POLLHUP) {
-                    break;
-                }
-                // On macOS, (jade thinks that) it is possible (although not
-                // observed on macOS 14.5) that in some limited cases on buggy
-                // kernel versions, all the non-POLLHUP events for the socket
-                // get delivered.
-                //
-                // We could sleep to avoid pointlessly spinning a thread on
-                // those, but this opens up a different problem, which is that
-                // if do sleep, it will be longer before the daemon fork for a
-                // client exits. Imagine a sequential shell script, running Nix
-                // commands, each of which talk to the daemon. If the previous
-                // command registered a temp root, exits, and then the next
-                // command issues a delete request before the temp root is
-                // cleaned up, that delete request might fail.
-                //
-                // Not sleeping doesn't actually fix the race condition --- we
-                // would need to block on the old connections' tempt roots being
-                // cleaned up in in the new connection --- but it does make it
-                // much less likely.
-            }
-        });
-    };
+    MonitorFdHup(int fd);
 
     ~MonitorFdHup()
     {
+        // Close the write side to signal termination via POLLHUP
         notifyPipe.writeSide.close();
         thread.join();
     }
 };
 
+#ifdef __APPLE__
+/* This custom kqueue usage exists because Apple's poll implementation is
+ * broken and loses event subscriptions if EVFILT_READ fires without matching
+ * the requested `events` in the pollfd.
+ *
+ * We use EVFILT_READ, which causes some spurious wakeups (at most one per write
+ * from the client, in addition to the socket lifecycle events), because the
+ * alternate API, EVFILT_SOCK, doesn't work on pipes, which this is also used
+ * to monitor in certain situations.
+ *
+ * See (EVFILT_SOCK):
+ * https://github.com/netty/netty/blob/64bd2f4eb62c2fb906bc443a2aabf894c8b7dce9/transport-classes-kqueue/src/main/java/io/netty/channel/kqueue/AbstractKQueueChannel.java#L434
+ *
+ * See: https://git.lix.systems/lix-project/lix/issues/729
+ * Apple bug in poll(2): FB17447257, available at https://openradar.appspot.com/FB17447257
+ */
+inline void MonitorFdHup::runThread(int watchFd, int notifyFd)
+{
+    int kqResult = kqueue();
+    if (kqResult < 0) {
+        throw SysError("MonitorFdHup kqueue");
+    }
+    AutoCloseFD kq{kqResult};
+
+    std::array<struct kevent, 2> kevs;
+
+    // kj uses EVFILT_WRITE for this, but it seems that it causes more spurious
+    // wakeups in our case of doing blocking IO from another thread compared to
+    // EVFILT_READ.
+    //
+    // EVFILT_WRITE and EVFILT_READ (for sockets at least, where I am familiar
+    // with the internals) both go through a common filter which catches EOFs
+    // and generates spurious wakeups for either readable/writable events.
+    EV_SET(&kevs[0], watchFd, EVFILT_READ, EV_ADD | EV_ENABLE | EV_CLEAR, 0, 0, nullptr);
+    EV_SET(&kevs[1], notifyFd, EVFILT_READ, EV_ADD | EV_ENABLE | EV_CLEAR, 0, 0, nullptr);
+
+    int result = kevent(kq.get(), kevs.data(), kevs.size(), nullptr, 0, nullptr);
+    if (result < 0) {
+        throw SysError("MonitorFdHup kevent add");
+    }
+
+    while (true) {
+        struct kevent event;
+        int numEvents = kevent(kq.get(), nullptr, 0, &event, 1, nullptr);
+        if (numEvents < 0) {
+            throw SysError("MonitorFdHup kevent watch");
+        }
+
+        if (numEvents > 0 && (event.flags & EV_EOF)) {
+            if (event.ident == uintptr_t(watchFd)) {
+                unix::triggerInterrupt();
+            }
+            // Either watched fd or notify fd closed, exit
+            return;
+        }
+    }
+}
+#else
+inline void MonitorFdHup::runThread(int watchFd, int notifyFd)
+{
+    while (true) {
+        struct pollfd fds[2];
+        fds[0].fd = watchFd;
+        fds[0].events = 0; // POSIX: POLLHUP is always reported
+        fds[1].fd = notifyFd;
+        fds[1].events = 0;
+
+        auto count = poll(fds, 2, -1);
+        if (count == -1) {
+            if (errno == EINTR || errno == EAGAIN) {
+                continue;
+            } else {
+                throw SysError("in MonitorFdHup poll()");
+            }
+        }
+
+        if (fds[0].revents & POLLHUP) {
+            unix::triggerInterrupt();
+            break;
+        }
+
+        if (fds[1].revents & POLLHUP) {
+            // Notify pipe closed, exit thread
+            break;
+        }
+    }
+}
+#endif
+
+inline MonitorFdHup::MonitorFdHup(int fd)
+{
+    notifyPipe.create();
+    int notifyFd = notifyPipe.readSide.get();
+    thread = std::thread([this, fd, notifyFd]() { this->runThread(fd, notifyFd); });
+};
+
 } // namespace nix

src/libutil/unix/include/nix/util/signals-impl.hh

@@ -42,13 +42,6 @@ extern thread_local std::function<bool()> interruptCheck;
 
 void _interrupted();
 
-/**
- * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't
- * necessarily match the current thread's mask.
- * See saveSignalMask() to set the saved mask to the current mask.
- */
-void setChildSignalMask(sigset_t * sigs);
-
 /**
  * Start a thread that handles various signals. Also block those signals
  * on the current thread (and thus any threads created by it).
@@ -60,8 +53,6 @@ void startSignalHandlerThread();
 /**
  * Saves the signal mask, which is the signal mask that nix will restore
  * before creating child processes.
- * See setChildSignalMask() to set an arbitrary signal mask instead of the
- * current mask.
  */
 void saveSignalMask();
 

src/libutil/unix/meson.build

@@ -49,6 +49,7 @@ config_unix_priv_h = configure_file(
 sources += config_unix_priv_h
 
 sources += files(
+  'current-process.cc',
   'environment-variables.cc',
   'file-descriptor.cc',
   'file-path.cc',