libstore: support searching for roots from an external daemon

This comes in two parts: a `nix store roots-daemon` command that
can run as root and list runtime roots,
and client logic to find runtime roots for a `LocalStore` by connecting
to that daemon.

This may be useful with an unprivileged nix daemon, as it would
otherwise be unable to find runtime roots from process open files
and maps.

doc/manual/rl-next/roots-daemon.md

@@ -0,0 +1,11 @@
+---
+synopsis: New command `nix store roots-daemon` for serving GC roots
+prs: [15143]
+---
+
+New command [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md) runs a daemon that serves garbage collector roots over a Unix domain socket.
+It enables the garbage collector to discover runtime roots when the main Nix daemon doesn't have `CAP_SYS_PTRACE` capability and therefore cannot scan `/proc`.
+
+The garbage collector can be configured to use this daemon via the [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon) store setting.
+
+This feature requires the [`local-overlay-store` experimental feature](@docroot@/development/experimental-features.md#xp-feature-local-overlay-store).

src/libstore/gc.cc

@@ -3,6 +3,7 @@
 #include "nix/store/local-gc.hh"
 #include "nix/store/local-store.hh"
 #include "nix/store/path.hh"
+#include "nix/util/configuration.hh"
 #include "nix/util/finally.hh"
 #include "nix/util/unix-domain-socket.hh"
 #include "nix/util/signals.hh"
@@ -306,9 +307,33 @@ Roots LocalStore::findRoots(bool censor)
     return roots;
 }
 
+static Roots requestRuntimeRoots(const LocalStoreConfig & config, const std::filesystem::path & socketPath)
+{
+    Roots roots;
+
+    auto socket = connect(socketPath);
+    auto socketSource = FdSource(socket.get());
+
+    while (1) {
+        auto line = socketSource.readLine(true, '\0');
+        if (line == "")
+            break;
+        roots[config.parseStorePath(line)].insert(censored);
+    };
+
+    return roots;
+}
+
 void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 {
-    auto unchecked = findRuntimeRootsUnchecked(*config);
+    Roots unchecked;
+
+    if (config->useRootsDaemon) {
+        experimentalFeatureSettings.require(Xp::LocalOverlayStore);
+        unchecked = requestRuntimeRoots(*config, config->getRootsSocketPath());
+    } else {
+        unchecked = findRuntimeRootsUnchecked(*config);
+    }
 
     for (auto & [path, links] : unchecked) {
         if (!isValidPath(path))

src/libstore/include/nix/store/local-store.hh

@@ -133,6 +133,27 @@ public:
         Xp::LocalOverlayStore,
     };
 
+    Setting<bool> useRootsDaemon{
+        this,
+        false,
+        "use-roots-daemon",
+        R"(
+          Whether to request garbage collector roots from an external daemon.
+
+          When enabled, the garbage collector connects to a Unix domain socket
+          at [`<state-dir>`](@docroot@/store/types/local-store.md#store-option-state)`/gc-roots-socket/socket` to discover additional roots
+          that should not be collected. This is useful when the Nix daemon runs
+          without root privileges and cannot scan `/proc` for runtime roots.
+
+          The daemon can be started with [`nix store roots-daemon`](@docroot@/command-ref/new-cli/nix3-store-roots-daemon.md).
+        )",
+        {},
+        true,
+        Xp::LocalOverlayStore,
+    };
+
+    std::filesystem::path getRootsSocketPath() const;
+
     static const std::string name()
     {
         return "Local Store";

src/libstore/local-store.cc

@@ -457,6 +457,11 @@ LocalStore::~LocalStore()
     }
 }
 
+std::filesystem::path LocalStoreConfig::getRootsSocketPath() const
+{
+    return std::filesystem::path(stateDir.get()) / "gc-roots-socket" / "socket";
+}
+
 StoreReference LocalStoreConfig::getReference() const
 {
     auto params = getQueryParams();

src/nix/meson.build

@@ -113,6 +113,7 @@ nix_sources = [ config_priv_h ] + files(
 if host_machine.system() != 'windows'
   nix_sources += files(
     'unix/daemon.cc',
+    'unix/store-roots-daemon.cc',
   )
 endif
 

src/nix/unix/daemon.md

@@ -52,8 +52,8 @@ sockets:
 
 ```
 [Socket]
-ListenStream=/nix/var/nix/unix/daemon-socket/socket
-ListenStream=/nix/var/nix/unix/daemon-socket/socket-2
+ListenStream=/nix/var/nix/daemon-socket/socket
+ListenStream=/nix/var/nix/daemon-socket/socket-2
 ```
 
 )""

src/nix/unix/store-roots-daemon.cc

@@ -0,0 +1,66 @@
+#include "nix/cmd/command.hh"
+#include "nix/cmd/unix-socket-server.hh"
+#include "nix/store/local-store.hh"
+#include "nix/store/store-api.hh"
+#include "nix/store/local-gc.hh"
+#include "nix/util/file-descriptor.hh"
+
+#include <thread>
+
+using namespace nix;
+
+struct CmdRootsDaemon : StoreConfigCommand
+{
+    CmdRootsDaemon() {}
+
+    std::string description() override
+    {
+        return "run a daemon that returns garbage collector roots on request";
+    }
+
+    std::string doc() override
+    {
+        return
+#include "store-roots-daemon.md"
+            ;
+    }
+
+    std::optional<ExperimentalFeature> experimentalFeature() override
+    {
+        return Xp::LocalOverlayStore;
+    }
+
+    void run(ref<StoreConfig> storeConfig) override
+    {
+        auto localStoreConfig = dynamic_cast<LocalStoreConfig *>(&*storeConfig);
+        if (!localStoreConfig) {
+            throw UsageError(
+                "Roots daemon only functions with a local store, not '%s'", storeConfig->getHumanReadableURI());
+        }
+
+        auto gcSocketPath = localStoreConfig->getRootsSocketPath();
+
+        unix::serveUnixSocket(
+            {
+                .socketPath = gcSocketPath,
+                .socketMode = 0666,
+            },
+            [&](AutoCloseFD remote, std::function<void()> closeListeners) {
+                std::thread([&, remote = std::move(remote)]() mutable {
+                    auto roots = findRuntimeRootsUnchecked(*localStoreConfig);
+
+                    FdSink sink(remote.get());
+
+                    for (auto & [key, _] : roots) {
+                        sink(localStoreConfig->printStorePath(key));
+                        sink(std::string_view("\0", 1));
+                    }
+
+                    sink.flush();
+                    remote.close();
+                }).detach();
+            });
+    }
+};
+
+static auto rCmdStoreRootsDaemon = registerCommand2<CmdRootsDaemon>({"store", "roots-daemon"});

src/nix/unix/store-roots-daemon.md

@@ -0,0 +1,46 @@
+R""(
+
+# Examples
+
+* Run the daemon:
+
+  ```console
+  # nix store roots-daemon
+  ```
+
+# Description
+
+This command runs a daemon that serves garbage collector roots from a Unix domain socket.
+It is not required in all Nix installations, but is useful when the main Nix daemon
+is not running as root and therefore cannot find runtime roots by scanning `/proc`.
+
+When the garbage collector runs with [`use-roots-daemon`](@docroot@/store/types/local-store.md#store-experimental-option-use-roots-daemon)
+enabled, it connects to this daemon to discover additional roots that should not be collected.
+
+The daemon listens on [`<state-dir>`](@docroot@/store/types/local-store.md#store-option-state)`/gc-roots-socket/socket` (typically `/nix/var/nix/gc-roots-socket/socket`).
+
+# Protocol
+
+The protocol is simple.
+For each client-initiated Unix socket connection, the server:
+
+1. Sends zero or more [store paths](@docroot@/store/store-path.md) as NUL-terminated (`\0`) strings.
+2. Closes the connection.
+
+Example (with `\0` shown as newlines for clarity):
+```
+/nix/store/s66mzxpvicwk07gjbjfw9izjfa797vsw-hello-2.12.1
+/nix/store/fvpr7x8l3illdnziggvkhdpf6vikg65w-git-2.44.0
+```
+
+# Security
+
+No information is provided as to which processes are opening which store paths.
+While only the main Nix daemon needs to use this daemon, any user able to talk to the main Nix daemon can already obtain the same information with [`nix-store --gc --print-roots`](@docroot@/command-ref/nix-store/gc.md).
+
+Therefore, restricting this daemon to only accept the Nix daemon as a client is, while recommended for defense-in-depth reasons, strictly speaking not reducing what information can be extracted versus merely restricting this daemon to accept connections from any [allowed user](@docroot@/command-ref/conf-file.md#conf-allowed-users).
+
+# Systemd socket activation
+
+`nix store roots-daemon` supports systemd socket-based activation, [just like `nix-daemon`](@docroot@/command-ref/nix-daemon.md#systemd-socket-activation).
+)""

tests/nixos/functional/unprivileged-daemon.nix

@@ -64,7 +64,7 @@ in
 
         environment = {
           CURL_CA_BUNDLE = config.security.pki.caBundle;
-          NIX_REMOTE = "local?ignore-gc-delete-failure=true";
+          NIX_REMOTE = "local?ignore-gc-delete-failure=true&use-roots-daemon=true";
           NIX_CONFIG = ''
             experimental-features = local-overlay-store auto-allocate-uids
             build-users-group =
@@ -85,6 +85,32 @@ in
         };
       };
 
+      systemd.services.nix-roots-daemon = {
+        environment = {
+          # `use-roots-daemon` is not needed because it is only relevant
+          # for the *client* of this daemon (i.e. the nix daemon opening
+          # the local store in this case). The Nix roots daemon *itself*
+          # doesn't care about this setting --- there's no problem if
+          # someone else opens the local store and directly scans for
+          # roots instead of using this daemon, for example.
+          NIX_REMOTE = "local";
+          NIX_CONFIG = ''
+            extra-experimental-features = local-overlay-store
+          '';
+        };
+        serviceConfig.ExecStart = "${config.nix.package.out}/bin/nix --extra-experimental-features nix-command store roots-daemon";
+      };
+
+      systemd.sockets.nix-roots-daemon = {
+        wantedBy = [
+          "nix-daemon.service"
+        ];
+        listenStreams = [ "/nix/var/nix/gc-roots-socket/socket" ];
+        unitConfig = {
+          ConditionPathIsReadWrite = "/nix/var/nix/gc-roots-socket";
+          RequiresMountsFor = "/nix/store";
+        };
+      };
       systemd.sockets.nix-daemon.wantedBy = [ "sockets.target" ];
 
       systemd.tmpfiles.rules = [
@@ -97,6 +123,7 @@ in
         "d  /nix/var/nix/daemon-socket         0755 nix-daemon nix-daemon - -"
         "d  /nix/var/nix/gcroots               0755 nix-daemon nix-daemon - -"
         "L+ /nix/var/nix/gcroots/booted-system 0755 nix-daemon nix-daemon - /run/booted-system"
+        "d  /nix/var/nix/gc-roots-socket       0755 nix-daemon nix-daemon - -"
         "d  /var/empty/.cache/nix              0755 nix-daemon nix-daemon - -"
       ];
     };