Merge pull request #11408 from NixOS/mergify/bp/2.24-maintenance/pr-11402

Respect max-substitution-jobs again (backport #11402)

.github/workflows/ci.yml

@@ -49,6 +49,7 @@ jobs:
           done
         ) &
     - run: nix --experimental-features 'nix-command flakes' flake check -L
+    - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json
 
   # Steps to test CI automation in your own fork.
   # Cachix:

.version

@@ -1 +1 @@
-2.24.0
+2.24.5

doc/manual/src/installation/prerequisites-source.md

@@ -39,8 +39,6 @@
     `pkgconfig` and the Boehm garbage collector, and pass the flag
     `--enable-gc` to `configure`.
 
-    For `bdw-gc` <= 8.2.4 Nix needs a [small patch](https://github.com/NixOS/nix/blob/ac4d2e7b857acdfeac35ac8a592bdecee2d29838/boehmgc-traceable_allocator-public.diff) to be applied.
-
   - The `boost` library of version 1.66.0 or higher. It can be obtained
     from the official web site <https://www.boost.org/>.
 

flake.lock

@@ -80,11 +80,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1721548954,
-        "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=",
+        "lastModified": 1723688146,
+        "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a",
+        "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0",
         "type": "github"
       },
       "original": {

flake.nix

@@ -24,7 +24,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =

src/libexpr-c/nix_api_expr.h

@@ -14,6 +14,16 @@
 #include "nix_api_util.h"
 #include <stddef.h>
 
+#ifndef __has_c_attribute
+#  define __has_c_attribute(x) 0
+#endif
+
+#if __has_c_attribute(deprecated)
+#  define NIX_DEPRECATED(msg) [[deprecated(msg)]]
+#else
+#  define NIX_DEPRECATED(msg)
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -45,7 +55,7 @@ typedef struct EvalState EvalState; // nix::EvalState
  * @see nix_value_incref, nix_value_decref
  */
 typedef struct nix_value nix_value;
-[[deprecated("use nix_value instead")]] typedef nix_value Value;
+NIX_DEPRECATED("use nix_value instead") typedef nix_value Value;
 
 // Function prototypes
 /**

src/libexpr/eval-gc.cc

@@ -32,122 +32,6 @@ static void * oomHandler(size_t requested)
     throw std::bad_alloc();
 }
 
-class BoehmGCStackAllocator : public StackAllocator
-{
-    boost::coroutines2::protected_fixedsize_stack stack{
-        // We allocate 8 MB, the default max stack size on NixOS.
-        // A smaller stack might be quicker to allocate but reduces the stack
-        // depth available for source filter expressions etc.
-        std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))};
-
-    // This is specific to boost::coroutines2::protected_fixedsize_stack.
-    // The stack protection page is included in sctx.size, so we have to
-    // subtract one page size from the stack size.
-    std::size_t pfss_usable_stack_size(boost::context::stack_context & sctx)
-    {
-        return sctx.size - boost::context::stack_traits::page_size();
-    }
-
-public:
-    boost::context::stack_context allocate() override
-    {
-        auto sctx = stack.allocate();
-
-        // Stacks generally start at a high address and grow to lower addresses.
-        // Architectures that do the opposite are rare; in fact so rare that
-        // boost_routine does not implement it.
-        // So we subtract the stack size.
-        GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        return sctx;
-    }
-
-    void deallocate(boost::context::stack_context sctx) override
-    {
-        GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp);
-        stack.deallocate(sctx);
-    }
-};
-
-static BoehmGCStackAllocator boehmGCStackAllocator;
-
-/**
- * When a thread goes into a coroutine, we lose its original sp until
- * control flow returns to the thread.
- * While in the coroutine, the sp points outside the thread stack,
- * so we can detect this and push the entire thread stack instead,
- * as an approximation.
- * The coroutine's stack is covered by `BoehmGCStackAllocator`.
- * This is not an optimal solution, because the garbage is scanned when a
- * coroutine is active, for both the coroutine and the original thread stack.
- * However, the implementation is quite lean, and usually we don't have active
- * coroutines during evaluation, so this is acceptable.
- */
-void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id)
-{
-    void *& sp = *sp_ptr;
-    auto pthread_id = reinterpret_cast<pthread_t>(_pthread_id);
-#  ifndef __APPLE__
-    pthread_attr_t pattr;
-#  endif
-    size_t osStackSize;
-    // The low address of the stack, which grows down.
-    void * osStackLimit;
-    void * osStackBase;
-
-#  ifdef __APPLE__
-    osStackSize = pthread_get_stacksize_np(pthread_id);
-    osStackLimit = pthread_get_stackaddr_np(pthread_id);
-#  else
-    if (pthread_attr_init(&pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_init failed");
-    }
-#    ifdef HAVE_PTHREAD_GETATTR_NP
-    if (pthread_getattr_np(pthread_id, &pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_getattr_np failed");
-    }
-#    elif HAVE_PTHREAD_ATTR_GET_NP
-    if (!pthread_attr_init(&pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_init failed");
-    }
-    if (!pthread_attr_get_np(pthread_id, &pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed");
-    }
-#    else
-#      error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`"
-#    endif
-    if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed");
-    }
-    if (pthread_attr_destroy(&pattr)) {
-        throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed");
-    }
-#  endif
-    osStackBase = (char *) osStackLimit + osStackSize;
-    // NOTE: We assume the stack grows down, as it does on all architectures we support.
-    //       Architectures that grow the stack up are rare.
-    if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack
-        sp = osStackLimit;
-    }
-}
-
-/* Disable GC while this object lives. Used by CoroutineContext.
- *
- * Boehm keeps a count of GC_disable() and GC_enable() calls,
- * and only enables GC when the count matches.
- */
-class BoehmDisableGC
-{
-public:
-    BoehmDisableGC()
-    {
-        GC_disable();
-    };
-    ~BoehmDisableGC()
-    {
-        GC_enable();
-    };
-};
-
 static inline void initGCReal()
 {
     /* Initialise the Boehm garbage collector. */
@@ -168,24 +52,6 @@ static inline void initGCReal()
 
     GC_set_oom_fn(oomHandler);
 
-    StackAllocator::defaultAllocator = &boehmGCStackAllocator;
-
-// TODO: Remove __APPLE__ condition.
-//       Comment suggests an implementation that works on darwin and windows
-//       https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196
-#  if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__)
-    GC_set_sp_corrector(&fixupBoehmStackPointer);
-
-    if (!GC_get_sp_corrector()) {
-        printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines");
-        /* Used to disable GC when entering coroutines on macOS */
-        create_coro_gc_hook = []() -> std::shared_ptr<void> { return std::make_shared<BoehmDisableGC>(); };
-    }
-#  else
-#    warning \
-        "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later."
-#  endif
-
     /* Set the initial heap size to something fairly big (25% of
        physical RAM, up to a maximum of 384 MiB) so that in most cases
        we don't need to garbage collect at all.  (Collection has a

src/libstore/build/substitution-goal.cc

@@ -145,8 +145,10 @@ Goal::Co PathSubstitutionGoal::init()
     /* None left.  Terminate this goal and let someone else deal
        with it. */
 
-    worker.failedSubstitutions++;
-    worker.updateProgress();
+    if (substituterFailed) {
+        worker.failedSubstitutions++;
+        worker.updateProgress();
+    }
 
     /* Hack: don't indicate failure if there were no substituters.
        In that case the calling derivation should just do a
@@ -158,7 +160,7 @@ Goal::Co PathSubstitutionGoal::init()
 }
 
 
-Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool& substituterFailed)
+Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed)
 {
     trace("all references realised");
 
@@ -181,7 +183,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref<Store> sub,
     /* Make sure that we are allowed to start a substitution.  Note that even
        if maxSubstitutionJobs == 0, we still allow a substituter to run. This
        prevents infinite waiting. */
-    if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
+    while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
         worker.waitForBuildSlot(shared_from_this());
         co_await Suspend{};
     }

src/libstore/build/substitution-goal.hh

@@ -66,7 +66,7 @@ public:
      */
     Co init() override;
     Co gotInfo();
-    Co tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool& substituterFailed);
+    Co tryToRun(StorePath subPath, nix::ref<Store> sub, std::shared_ptr<const ValidPathInfo> info, bool & substituterFailed);
     Co finished();
 
     /**

src/libstore/build/worker.cc

@@ -184,13 +184,13 @@ void Worker::wakeUp(GoalPtr goal)
 }
 
 
-unsigned Worker::getNrLocalBuilds()
+size_t Worker::getNrLocalBuilds()
 {
     return nrLocalBuilds;
 }
 
 
-unsigned Worker::getNrSubstitutions()
+size_t Worker::getNrSubstitutions()
 {
     return nrSubstitutions;
 }

src/libstore/build/worker.hh

@@ -92,12 +92,12 @@ private:
      * Number of build slots occupied.  This includes local builds but does not
      * include substitutions or remote builds via the build hook.
      */
-    unsigned int nrLocalBuilds;
+    size_t nrLocalBuilds;
 
     /**
      * Number of substitution slots occupied.
      */
-    unsigned int nrSubstitutions;
+    size_t nrSubstitutions;
 
     /**
      * Maps used to prevent multiple instantiations of a goal for the
@@ -235,12 +235,12 @@ public:
      * Return the number of local build processes currently running (but not
      * remote builds via the build hook).
      */
-    unsigned int getNrLocalBuilds();
+    size_t getNrLocalBuilds();
 
     /**
      * Return the number of substitution processes currently running.
      */
-    unsigned int getNrSubstitutions();
+    size_t getNrSubstitutions();
 
     /**
      * Registers a running child process.  `inBuildSlot` means that

src/libstore/s3-binary-cache-store.cc

@@ -220,8 +220,6 @@ std::string S3BinaryCacheStoreConfig::doc()
 
 struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore
 {
-    std::string bucketName;
-
     Stats stats;
 
     S3Helper s3Helper;

src/libstore/store-api.cc

@@ -210,14 +210,16 @@ StorePath Store::addToStore(
         fsm = FileSerialisationMethod::NixArchive;
         break;
     }
-    auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(path, sink, fsm, filter);
+    std::optional<StorePath> storePath;
+    auto sink = sourceToSink([&](Source & source) {
+        LengthSource lengthSource(source);
+        storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair);
+        if (lengthSource.total >= settings.warnLargePathThreshold)
+            warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total));
     });
-    LengthSource lengthSource(*source);
-    auto storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair);
-    if (lengthSource.total >= settings.warnLargePathThreshold)
-        warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total));
-    return storePath;
+    dumpPath(path, *sink, fsm, filter);
+    sink->finish();
+    return storePath.value();
 }
 
 void Store::addMultipleToStore(

src/libstore/unix/build/sandbox-defaults.sb

@@ -49,6 +49,7 @@ R""(
 (if (param "_ALLOW_LOCAL_NETWORKING")
     (begin
       (allow network* (remote ip "localhost:*"))
+      (allow network-inbound (local ip "*:*")) ; required to bind and listen
 
       ; Allow access to /etc/resolv.conf (which is a symlink to
       ; /private/var/run/resolv.conf).

src/libutil/serialise.cc

@@ -171,55 +171,6 @@ size_t StringSource::read(char * data, size_t len)
 #error Coroutines are broken in this version of Boost!
 #endif
 
-/* A concrete datatype allow virtual dispatch of stack allocation methods. */
-struct VirtualStackAllocator {
-    StackAllocator *allocator = StackAllocator::defaultAllocator;
-
-    boost::context::stack_context allocate() {
-        return allocator->allocate();
-    }
-
-    void deallocate(boost::context::stack_context sctx) {
-        allocator->deallocate(sctx);
-    }
-};
-
-
-/* This class reifies the default boost coroutine stack allocation strategy with
-   a virtual interface. */
-class DefaultStackAllocator : public StackAllocator {
-    boost::coroutines2::default_stack stack;
-
-    boost::context::stack_context allocate() override {
-        return stack.allocate();
-    }
-
-    void deallocate(boost::context::stack_context sctx) override {
-        stack.deallocate(sctx);
-    }
-};
-
-static DefaultStackAllocator defaultAllocatorSingleton;
-
-StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
-
-
-std::shared_ptr<void> (*create_coro_gc_hook)() = []() -> std::shared_ptr<void> {
-    return {};
-};
-
-/* This class is used for entry and exit hooks on coroutines */
-class CoroutineContext {
-    /* Disable GC when entering the coroutine without the boehm patch,
-     * since it doesn't find the main thread stack in this case.
-     * std::shared_ptr<void> performs type-erasure, so it will call the right
-     * deleter. */
-    const std::shared_ptr<void> coro_gc_hook = create_coro_gc_hook();
-public:
-    CoroutineContext() {};
-    ~CoroutineContext() {};
-};
-
 std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
 {
     struct SourceToSink : FinishSink
@@ -241,14 +192,12 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
             cur = in;
 
             if (!coro) {
-                CoroutineContext ctx;
-                coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
-                    LambdaSource source([&](char *out, size_t out_len) {
+                coro = coro_t::push_type([&](coro_t::pull_type & yield) {
+                    LambdaSource source([&](char * out, size_t out_len) {
                         if (cur.empty()) {
                             yield();
-                            if (yield.get()) {
-                                return (size_t)0;
-                            }
+                            if (yield.get())
+                                throw EndOfFile("coroutine has finished");
                         }
 
                         size_t n = std::min(cur.size(), out_len);
@@ -263,20 +212,14 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
             if (!*coro) { unreachable(); }
 
             if (!cur.empty()) {
-                CoroutineContext ctx;
                 (*coro)(false);
             }
         }
 
         void finish() override
         {
-            if (!coro) return;
-            if (!*coro) unreachable();
-            {
-                CoroutineContext ctx;
+            if (coro && *coro)
                 (*coro)(true);
-            }
-            if (*coro) unreachable();
         }
     };
 
@@ -307,8 +250,7 @@ std::unique_ptr<Source> sinkToSource(
         size_t read(char * data, size_t len) override
         {
             if (!coro) {
-                CoroutineContext ctx;
-                coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
+                coro = coro_t::pull_type([&](coro_t::push_type & yield) {
                     LambdaSink sink([&](std::string_view data) {
                         if (!data.empty()) yield(std::string(data));
                     });
@@ -320,7 +262,6 @@ std::unique_ptr<Source> sinkToSource(
 
             if (pos == cur.size()) {
                 if (!cur.empty()) {
-                    CoroutineContext ctx;
                     (*coro)();
                 }
                 cur = coro->get();

src/libutil/serialise.hh

@@ -557,27 +557,4 @@ struct FramedSink : nix::BufferedSink
     };
 };
 
-/**
- * Stack allocation strategy for sinkToSource.
- * Mutable to avoid a boehm gc dependency in libutil.
- *
- * boost::context doesn't provide a virtual class, so we define our own.
- */
-struct StackAllocator {
-    virtual boost::context::stack_context allocate() = 0;
-    virtual void deallocate(boost::context::stack_context sctx) = 0;
-
-    /**
-     * The stack allocator to use in sinkToSource and potentially elsewhere.
-     * It is reassigned by the initGC() method in libexpr.
-     */
-    static StackAllocator *defaultAllocator;
-};
-
-/* Disabling GC when entering a coroutine (without the boehm patch).
-   mutable to avoid boehm gc dependency in libutil.
- */
-extern std::shared_ptr<void> (*create_coro_gc_hook)();
-
-
 }

src/nix-build/nix-build.cc

@@ -163,7 +163,7 @@ static void main_nix_build(int argc, char * * argv)
         script = argv[1];
         try {
             auto lines = tokenizeString<Strings>(readFile(script), "\n");
-            if (std::regex_search(lines.front(), std::regex("^#!"))) {
+            if (!lines.empty() && std::regex_search(lines.front(), std::regex("^#!"))) {
                 lines.pop_front();
                 inShebang = true;
                 for (int i = 2; i < argc; ++i)