Reword the experimental feature description.

Co-Authored-By: Robert Hensing <robert@roberthensing.nl>

.gitignore

@@ -108,6 +108,9 @@ perl/Makefile.config
 
 /misc/systemd/nix-daemon.service
 /misc/systemd/nix-daemon.socket
+/misc/systemd/nix-gc-trace.service
+/misc/systemd/nix-gc-trace.socket
+
 /misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 

Makefile

@@ -12,6 +12,7 @@ makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
   src/libutil/local.mk \
+  src/nix-find-roots/local.mk \
   src/libstore/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
@@ -41,6 +42,7 @@ endif
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 makefiles += \
   tests/functional/local.mk \
+  tests/functional/gc-external-daemon/local.mk \
   tests/functional/ca/local.mk \
   tests/functional/dyn-drv/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \

doc/manual/src/installation/installing-rootless-daemon.md

@@ -0,0 +1,138 @@
+# Using Nix in multi-user mode with a non-root daemon
+
+> Experimental blurb
+
+It is experimentally possible to run Nix in multi-user mode without running the whole daemon as root.
+This is done by delegating the only part that requires root access to a separate daemon, with a much smaller attack surface.
+Because of the need for a second daemon, this makes the setup a bit more complex and isn't yet supported by the installer. It is however possible to set this up manually:
+
+1. Create a new user and group for the daemon:
+    ```sh
+    sudo groupadd nix-daemon
+    sudo useradd --gid nix-daemon --system -c "Nix daemon user" nix-daemon
+    ```
+2. Create `/nix` owned by that user:
+    ```sh
+    sudo mkdir -m 0755 /nix
+    sudo chown nix-daemon:nix-daemon /nix
+    ```
+3. Download a statically-compiled Nix version for bootstrapping
+    ```sh
+    curl -L https://hydra.nixos.org/job/nix/master/buildStatic.x86_64-linux/latest/download-by-type/file/binary-dist -o /tmp/nix-env
+    chmod +x /tmp/nix-env
+    ```
+4. Install a proper Nix and the tracing daemon in the store
+    ```sh
+    export DAEMON_HOME=$(sudo -u nix-daemon mktemp -d)
+    sudo -u nix-daemon HOME="$DAEMON_HOME" \
+        /tmp/nix-env \
+        -f https://github.com/nixos/nix/archive/rootless-daemon.tar.gz \
+        -iA default packages.x86_64-linux.nix-find-roots \
+        --option extra-substituters https://nixos-nix-install-tests.cachix.org \
+        --option extra-trusted-public-keys nixos-nix-install-tests.cachix.org-1:Le57vOUJjOcdzLlbwmZVBuLGoDC+Xg2rQDtmIzALgFU= \
+        --store / \
+        --profile /nix/var/nix/profiles/default
+    sudo -u nix-daemon mkdir -p /nix/var/nix/gc-socket
+    sudo -u nix-daemon rm -rf "$DAEMON_HOME"
+    ```
+5. Move the tracing daemon executable out of the store (as we don't want Nix
+   to own it)
+   ```sh
+   sudo cp /nix/var/nix/profiles/default/bin/nix-find-roots /usr/bin/
+   ```
+6. Install the systemd services for the daemon:
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-daemon.service
+    [Unit]
+    Description=Nix Daemon
+    Documentation=man:nix-daemon https://nixos.org/manual
+    RequiresMountsFor=/nix/store
+    RequiresMountsFor=/nix/var
+    RequiresMountsFor=/nix/var/nix/db
+    ConditionPathIsReadWrite=/nix/var/nix/daemon-socket
+
+    [Service]
+    ExecStart=@/nix/var/nix/profiles/default/bin/nix-daemon nix-daemon --daemon
+    KillMode=process
+    LimitNOFILE=1048576
+    TasksMax=1048576
+    User=nix-daemon
+    Group=nix-daemon
+
+    [Install]
+    WantedBy=multi-user.target
+    EOF
+    ```
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-daemon.socket
+    [Unit]
+    Description=Nix Daemon Socket
+    Before=multi-user.target
+    RequiresMountsFor=/nix/store
+    ConditionPathIsReadWrite=/nix/var/nix/daemon-socket
+
+    [Socket]
+    ListenStream=/nix/var/nix/daemon-socket/socket
+    SocketUser=nix-daemon
+
+    [Install]
+    WantedBy=sockets.target
+    EOF
+    ```
+7. Install the systemd services for the tracing daemon:
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-find-roots.service
+    [Unit]
+    Description=Nix GC tracer daemon
+    RequiresMountsFor=/nix/store
+    RequiresMountsFor=/nix/var
+    ConditionPathIsReadWrite=/nix/var/nix/gc-socket
+    ProcSubset=pid
+
+    [Service]
+    ExecStart=@/usr/bin/nix-find-roots nix-find-roots
+    Type=simple
+    StandardError=journal
+    ProtectSystem=full
+    ReadWritePaths=/nix/var/nix/gc-socket
+    SystemCallFilter=@system-service
+    SystemCallErrorNumber=EPERM
+    PrivateNetwork=true
+    PrivateDevices=true
+    ProtectKernelTunables=true
+
+    [Install]
+    WantedBy=multi-user.target
+    EOF
+    ```
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-find-roots.socket
+    [Unit]
+    Description=Nix Daemon Socket
+    Before=multi-user.target
+    RequiresMountsFor=/nix/store
+    ConditionPathIsReadWrite=/nix/var/nix/gc-socket
+
+    [Socket]
+    ListenStream=/nix/var/nix/gc-socket/socket
+    Accept=false
+
+    [Install]
+    WantedBy=sockets.target
+    EOF
+    ```
+8. Enable the required experimental Nix feature and basic configuration:
+    ```sh
+    sudo mkdir /etc/nix
+    cat <<EOF | sudo tee /etc/nix/nix.conf
+    experimental-features = external-gc-daemon
+    trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+    substituters = https://cache.nixos.org/
+    EOF
+    ```
+9. Start the systemd sockets:
+    ```sh
+    sudo systemctl start nix-daemon.socket
+    sudo systemctl start nix-find-roots.socket
+    ```
+10. Profit

flake.nix

@@ -152,6 +152,30 @@
             '';
           };
 
+          nix-find-roots = prev.stdenv.mkDerivation {
+            pname = "nix-find-roots";
+            inherit version;
+
+            src = fileset.toSource {
+              root = ./src/nix-find-roots;
+              fileset = /*fileset.intersect baseFiles (*/fileset.unions [
+                ./src/nix-find-roots/main.cc
+                ./src/nix-find-roots/lib
+              ]/*)*/;
+            };
+
+            CXXFLAGS = prev.lib.optionalString prev.stdenv.hostPlatform.isStatic "-static";
+
+            buildPhase = ''
+              $CXX $CXXFLAGS -std=c++17 *.cc **/*.cc -I lib -o nix-find-roots
+            '';
+
+            installPhase = ''
+              mkdir -p $out/bin
+              cp nix-find-roots $out/bin/
+            '';
+          };
+
           libgit2-nix = final.libgit2.overrideAttrs (attrs: {
             src = libgit2;
             version = libgit2.lastModifiedDate;
@@ -366,6 +390,9 @@
         default = nix;
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = nixpkgsFor.${system}.static.nix;
+
+        inherit (nixpkgsFor.${system}.static) nix-find-roots;
+
         dockerImage =
           let
             pkgs = nixpkgsFor.${system}.native;

misc/systemd/local.mk

@@ -1,6 +1,8 @@
 ifdef HOST_LINUX
 
-  $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
+  $(foreach n,\
+    nix-daemon.socket nix-daemon.service nix-gc-trace.socket nix-gc-trace.service,\
+    $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
   $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644)))
 
   clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf

misc/systemd/nix-gc-trace.service.in

@@ -0,0 +1,21 @@
+[Unit]
+Description=Nix GC Tracer Daemon
+RequiresMountsFor=@storedir@
+RequiresMountsFor=@localstatedir@
+ConditionPathIsReadWrite=@localstatedir@/nix/gc-socket
+ProcSubset=pid
+
+[Service]
+ExecStart=@@libexecdir@/nix/nix-find-roots nix-find-roots
+Type=simple
+StandardError=journal
+ProtectSystem=full
+ReadWritePaths=@localstatedir@/nix/gc-socket
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+PrivateNetwork=true
+PrivateDevices=true
+ProtectKernelTunables=true
+
+[Install]
+WantedBy=multi-user.target

misc/systemd/nix-gc-trace.socket.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Nix Daemon Socket
+Before=multi-user.target
+RequiresMountsFor=@storedir@
+ConditionPathIsReadWrite=@localstatedir@/nix/gc-socket
+
+[Socket]
+ListenStream=@localstatedir@/nix/gc-socket/socket
+Accept=false
+
+[Install]
+WantedBy=sockets.target

src/libstore/gc.cc

@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "local-store.hh"
 #include "finally.hh"
+#include "find-roots.hh"
 #include "unix-domain-socket.hh"
 #include "signals.hh"
 
@@ -31,6 +32,7 @@ namespace nix {
 
 
 static std::string gcSocketPath = "/gc-socket/socket";
+static std::string rootsSocketPath = "/gc-roots-socket/socket";
 static std::string gcRootsDir = "gcroots";
 
 
@@ -143,7 +145,7 @@ void LocalStore::addTempRoot(const StorePath & path)
         auto fdRootsSocket(_fdRootsSocket.lock());
 
         if (!*fdRootsSocket) {
-            auto socketPath = stateDir.get() + gcSocketPath;
+            auto socketPath = stateDir.get() + rootsSocketPath;
             debug("connecting to '%s'", socketPath);
             *fdRootsSocket = createUnixDomainSocket();
             try {
@@ -241,79 +243,32 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
     }
 }
 
-
-void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
+void LocalStore::findRootsNoTempNoExternalDaemon(Roots & roots, bool censor)
 {
-    auto foundRoot = [&](const Path & path, const Path & target) {
-        try {
-            auto storePath = toStorePath(target).first;
-            if (isValidPath(storePath))
-                roots[std::move(storePath)].emplace(path);
-            else
-                printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
-        } catch (BadStorePath &) { }
+    debug("Can’t connect to the tracing daemon socket, fallback to the internal trace");
+
+    using namespace nix::roots_tracer;
+
+    const TracerConfig opts {
+      .storeDir = fs::path(storeDir),
+      .stateDir = fs::path(stateDir.get())
     };
-
-    try {
-
-        if (type == DT_UNKNOWN)
-            type = getFileType(path);
-
-        if (type == DT_DIR) {
-            for (auto & i : readDirectory(path))
-                findRoots(path + "/" + i.name, i.type, roots);
-        }
-
-        else if (type == DT_LNK) {
-            Path target = readLink(path);
-            if (isInStore(target))
-                foundRoot(path, target);
-
-            /* Handle indirect roots. */
-            else {
-                target = absPath(target, dirOf(path));
-                if (!pathExists(target)) {
-                    if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
-                        printInfo("removing stale link from '%1%' to '%2%'", path, target);
-                        unlink(path.c_str());
-                    }
-                } else {
-                    struct stat st2 = lstat(target);
-                    if (!S_ISLNK(st2.st_mode)) return;
-                    Path target2 = readLink(target);
-                    if (isInStore(target2)) foundRoot(target, target2);
-                }
-            }
-        }
-
-        else if (type == DT_REG) {
-            auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path)));
-            if (storePath && isValidPath(*storePath))
-                roots[std::move(*storePath)].emplace(path);
-        }
-
+    const std::set<fs::path> standardRoots = {
+        opts.stateDir / fs::path(gcRootsDir),
+        opts.stateDir / fs::path("gcroots"),
+    };
+    auto traceResult = traceStaticRoots(opts, standardRoots);
+    auto runtimeRoots = getRuntimeRoots(opts);
+    traceResult.storeRoots.insert(runtimeRoots.begin(), runtimeRoots.end());
+    for (auto & [rawRootInStore, externalRoots] : traceResult.storeRoots) {
+        if (!isInStore(rawRootInStore.string())) continue;
+        auto rootInStore = toStorePath(rawRootInStore.string()).first;
+        if (!isValidPath(rootInStore)) continue;
+        std::unordered_set<std::string> primRoots;
+        for (const auto & externalRoot : externalRoots)
+            primRoots.insert(externalRoot.string());
+        roots.emplace(rootInStore, primRoots);
     }
-
-    catch (SysError & e) {
-        /* We only ignore permanent failures. */
-        if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
-            printInfo("cannot read potential root '%1%'", path);
-        else
-            throw;
-    }
-}
-
-
-void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
-{
-    /* Process direct roots in {gcroots,profiles}. */
-    findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
-    findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
-
-    /* Add additional roots returned by different platforms-specific
-       heuristics.  This is typically used to add running programs to
-       the set of roots (to prevent them from being garbage collected). */
-    findRuntimeRoots(roots, censor);
 }
 
 
@@ -327,148 +282,53 @@ Roots LocalStore::findRoots(bool censor)
     return roots;
 }
 
+void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
+{
+
+    auto fd = createUnixDomainSocket();
+
+    std::string socketPath = settings.gcSocketPath.get() != "auto"
+        ? settings.gcSocketPath.get()
+        : stateDir.get() + gcSocketPath;
+
+    try {
+        nix::connect(fd.get(), socketPath);
+    } catch (SysError & e) {
+        return findRootsNoTempNoExternalDaemon(roots, censor);
+    }
+
+    experimentalFeatureSettings.require(Xp::ExternalGCDaemon);
+
+    try {
+        StringMap unescapes = {
+            { "\\n", "\n"},
+            { "\\t", "\t"},
+        };
+        while (true) {
+            auto line = readLine(fd.get());
+            if (line.empty()) break; // TODO: Handle the broken symlinks
+            auto parsedLine = tokenizeString<std::vector<std::string>>(line, "\t");
+            if (parsedLine.size() != 2)
+                throw Error("Invalid result from the gc helper");
+            auto rawDestPath = rewriteStrings(parsedLine[0], unescapes);
+            auto retainer = rewriteStrings(parsedLine[1], unescapes);
+            if (!isInStore(rawDestPath)) continue;
+            try {
+                auto destPath = toStorePath(rawDestPath).first;
+                if (!isValidPath(destPath)) continue;
+                roots[destPath].insert(
+                    (!censor || isInDir(retainer, stateDir.get())) ? retainer : censored);
+            } catch (Error &) {
+            }
+        }
+    } catch (EndOfFile &) {
+    }
+}
+
 typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
 
-static void readProcLink(const std::string & file, UncheckedRoots & roots)
-{
-    constexpr auto bufsiz = PATH_MAX;
-    char buf[bufsiz];
-    auto res = readlink(file.c_str(), buf, bufsiz);
-    if (res == -1) {
-        if (errno == ENOENT || errno == EACCES || errno == ESRCH)
-            return;
-        throw SysError("reading symlink");
-    }
-    if (res == bufsiz) {
-        throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
-    }
-    if (res > 0 && buf[0] == '/')
-        roots[std::string(static_cast<char *>(buf), res)]
-            .emplace(file);
-}
-
-static std::string quoteRegexChars(const std::string & raw)
-{
-    static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
-    return std::regex_replace(raw, specialRegex, R"(\$&)");
-}
-
-#if __linux__
-static void readFileRoots(const char * path, UncheckedRoots & roots)
-{
-    try {
-        roots[readFile(path)].emplace(path);
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT && e.errNo != EACCES)
-            throw;
-    }
-}
-#endif
-
-void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
-{
-    UncheckedRoots unchecked;
-
-    auto procDir = AutoCloseDir{opendir("/proc")};
-    if (procDir) {
-        struct dirent * ent;
-        auto digitsRegex = std::regex(R"(^\d+$)");
-        auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
-        auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
-        while (errno = 0, ent = readdir(procDir.get())) {
-            checkInterrupt();
-            if (std::regex_match(ent->d_name, digitsRegex)) {
-                try {
-                    readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
-                    readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
-
-                    auto fdStr = fmt("/proc/%s/fd", ent->d_name);
-                    auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
-                    if (!fdDir) {
-                        if (errno == ENOENT || errno == EACCES)
-                            continue;
-                        throw SysError("opening %1%", fdStr);
-                    }
-                    struct dirent * fd_ent;
-                    while (errno = 0, fd_ent = readdir(fdDir.get())) {
-                        if (fd_ent->d_name[0] != '.')
-                            readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
-                    }
-                    if (errno) {
-                        if (errno == ESRCH)
-                            continue;
-                        throw SysError("iterating /proc/%1%/fd", ent->d_name);
-                    }
-                    fdDir.reset();
-
-                    auto mapFile = fmt("/proc/%s/maps", ent->d_name);
-                    auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
-                    for (const auto & line : mapLines) {
-                        auto match = std::smatch{};
-                        if (std::regex_match(line, match, mapRegex))
-                            unchecked[match[1]].emplace(mapFile);
-                    }
-
-                    auto envFile = fmt("/proc/%s/environ", ent->d_name);
-                    auto envString = readFile(envFile);
-                    auto env_end = std::sregex_iterator{};
-                    for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        unchecked[i->str()].emplace(envFile);
-                } catch (SystemError & e) {
-                    if (errno == ENOENT || errno == EACCES || errno == ESRCH)
-                        continue;
-                    throw;
-                }
-            }
-        }
-        if (errno)
-            throw SysError("iterating /proc");
-    }
-
-#if !defined(__linux__)
-    // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
-    // See: https://github.com/NixOS/nix/issues/3011
-    // Because of this we disable lsof when running the tests.
-    if (getEnv("_NIX_TEST_NO_LSOF") != "1") {
-        try {
-            std::regex lsofRegex(R"(^n(/.*)$)");
-            auto lsofLines =
-                tokenizeString<std::vector<std::string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
-            for (const auto & line : lsofLines) {
-                std::smatch match;
-                if (std::regex_match(line, match, lsofRegex))
-                    unchecked[match[1]].emplace("{lsof}");
-            }
-        } catch (ExecError & e) {
-            /* lsof not installed, lsof failed */
-        }
-    }
-#endif
-
-#if __linux__
-    readFileRoots("/proc/sys/kernel/modprobe", unchecked);
-    readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
-    readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
-#endif
-
-    for (auto & [target, links] : unchecked) {
-        if (!isInStore(target)) continue;
-        try {
-            auto path = toStorePath(target).first;
-            if (!isValidPath(path)) continue;
-            debug("got additional root '%1%'", printStorePath(path));
-            if (censor)
-                roots[path].insert(censored);
-            else
-                roots[path].insert(links.begin(), links.end());
-        } catch (BadStorePath &) { }
-    }
-}
-
-
 struct GCLimitReached { };
 
-
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
     bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
@@ -516,7 +376,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         readFile(*p);
 
     /* Start the server for receiving new roots. */
-    auto socketPath = stateDir.get() + gcSocketPath;
+    auto socketPath = stateDir.get() + rootsSocketPath;
     createDirs(dirOf(socketPath));
     auto fdServer = createUnixDomainSocket(socketPath, 0666);
 
@@ -625,7 +485,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printInfo("finding garbage collector roots...");
     Roots rootMap;
     if (!options.ignoreLiveness)
-        findRootsNoTemp(rootMap, true);
+        rootMap = findRoots(true);
 
     for (auto & i : rootMap) roots.insert(i.first);
 

src/libstore/globals.hh

@@ -124,6 +124,13 @@ public:
           section of the manual for supported store types and settings.
         )"};
 
+    Setting<std::string> gcSocketPath {
+        this,
+        "auto",
+        "gc-socket-path",
+        "Path to the socket used to communicate with an external GC."
+    };
+
     Setting<bool> keepFailed{this, false, "keep-failed",
         "Whether to keep temporary directories of failed builds."};
 

src/libstore/local-store.hh

@@ -328,7 +328,7 @@ private:
 
     void findRootsNoTemp(Roots & roots, bool censor);
 
-    void findRuntimeRoots(Roots & roots, bool censor);
+    void findRootsNoTempNoExternalDaemon(Roots & roots, bool censor);
 
     std::pair<Path, AutoCloseFD> createTempDirInStore();
 

src/libstore/local.mk

@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
-libstore_LIBS = libutil
+libstore_LIBS = libutil libfindroots
 
 libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
@@ -28,7 +28,7 @@ ifeq ($(HAVE_SECCOMP), 1)
 endif
 
 libstore_CXXFLAGS += \
- -I src/libutil -I src/libstore -I src/libstore/build \
+ -I src/libutil -I src/libstore -I src/libstore/build -I src/nix-find-roots/lib \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \

src/libutil/experimental-features.cc

@@ -268,6 +268,20 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
         )",
     },
+    {
+        .tag = Xp::ExternalGCDaemon,
+        .name = "external-gc-daemon",
+        .description = R"(
+            Make the garbage collector use an external daemon for the tracing.
+
+            This makes it possible to run a multi-user Nix daemon as a non-root
+            user. Only the tracing daemon needs to be root. This reduces the attack
+            surface.
+
+            This requires more infrastructure and isn't directly supported by the
+            installer.
+        )"
+    },
     {
         .tag = Xp::VerifiedFetches,
         .name = "verified-fetches",

src/libutil/experimental-features.hh

@@ -35,6 +35,7 @@ enum struct ExperimentalFeature
     ReadOnlyLocalStore,
     ConfigurableImpureEnv,
     MountedSSHStore,
+    ExternalGCDaemon,
     VerifiedFetches,
 };
 

src/nix-find-roots/.gitignore

@@ -0,0 +1 @@
+nix-find-roots

src/nix-find-roots/lib/find-roots.cc

@@ -0,0 +1,232 @@
+/**
+ * @file
+ *
+ * A very simple utility to trace all the gc roots through the file-system
+ * The reason for this program is that tracing these roots is the only part of
+ * Nix that requires to run as root (because it requires reading through the
+ * user home directories to resolve the indirect roots)
+ *
+ * This program intentionnally doesnt depend on any Nix library to reduce the attack surface.
+ */
+
+#include <regex>
+#include <unistd.h>
+#include <vector>
+#include <algorithm>
+#include <fstream>
+#include <optional>
+
+#include "find-roots.hh"
+
+
+namespace nix::roots_tracer {
+namespace fs = std::filesystem;
+
+static std::string quoteRegexChars(const std::string & raw)
+{
+    static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
+    return std::regex_replace(raw, specialRegex, R"(\$&)");
+}
+static std::regex storePathRegex(const fs::path storeDir)
+{
+    return std::regex(quoteRegexChars(storeDir) + R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)");
+}
+
+static bool isInStore(fs::path storeDir, fs::path dir)
+{
+    return (std::search(dir.begin(), dir.end(), storeDir.begin(), storeDir.end()) == dir.begin());
+}
+
+static void traceStaticRoot(
+    const TracerConfig & opts,
+    int recursionsLeft,
+    TraceResult & res,
+    const fs::path & root,
+    const fs::file_status & status
+    )
+{
+    opts.debug("Considering file " + root.string());
+
+    if (recursionsLeft < 0)
+        return;
+
+    switch (status.type()) {
+        case fs::file_type::directory:
+            {
+                auto directory_iterator = fs::recursive_directory_iterator(root);
+                for (auto & child : directory_iterator)
+                    traceStaticRoot(opts, recursionsLeft, res, child.path(), child.symlink_status());
+            }
+            break;
+        case fs::file_type::symlink:
+            {
+                auto target = root.parent_path() / fs::read_symlink(root);
+                auto not_found = [&](std::string msg) {
+                    opts.debug("Error accessing the file " + target.string() + ": " + msg);
+                    opts.debug("(When resolving the symlink " + root.string() + ")");
+                    res.deadLinks.insert(root);
+                };
+                try {
+                    auto target_status = fs::symlink_status(target);
+                    if (target_status.type() == fs::file_type::not_found)
+                        not_found("Not found");
+
+                    if (isInStore(opts.storeDir, target)) {
+                        res.storeRoots[target].insert(root);
+                        return;
+                    } else {
+                        traceStaticRoot(opts, recursionsLeft - 1, res, target, target_status);
+                    }
+
+                } catch (fs::filesystem_error & e) {
+                    not_found(e.what());
+                }
+            }
+            break;
+        case fs::file_type::regular:
+            {
+                auto possibleStorePath = opts.storeDir / root.filename();
+                if (fs::exists(possibleStorePath))
+                res.storeRoots[possibleStorePath].insert(root);
+            }
+            break;
+        case fs::file_type::not_found:
+        case fs::file_type::block:
+        case fs::file_type::character:
+        case fs::file_type::fifo:
+        case fs::file_type::socket:
+        case fs::file_type::unknown:
+        case fs::file_type::none:
+        default:
+            break;
+    }
+}
+
+static void traceStaticRoot(
+    const TracerConfig & opts,
+    int recursionsLeft,
+    TraceResult & res,
+    const fs::path & root)
+{
+    try {
+        auto status = fs::symlink_status(root);
+        traceStaticRoot(opts, recursionsLeft, res, root, status);
+    } catch (fs::filesystem_error & e) {
+        opts.debug("Error accessing the file " + root.string() + ": " + e.what());
+    }
+}
+
+TraceResult traceStaticRoots(TracerConfig opts, std::set<fs::path> roots)
+{
+    int maxRecursionLevel = 2;
+    TraceResult res;
+    for (auto & root : roots)
+        traceStaticRoot(opts, maxRecursionLevel, res, root);
+    return res;
+}
+
+/**
+ * Scan the content of the given file for al the occurences of something that looks
+ * like a store path (i.e. that matches `storePathRegex(opts.storeDir)`) and add them
+ * to `res`
+ */
+static void scanFileContent(const TracerConfig & opts, const fs::path & fileToScan, Roots & res)
+{
+    if (!fs::exists(fileToScan))
+        return;
+
+    std::ostringstream contentStream;
+    {
+        std::ifstream fs;
+        fs.open(fileToScan);
+        fs >> contentStream.rdbuf();
+    }
+    std::string content = contentStream.str();
+    auto regex = storePathRegex(opts.storeDir);
+    auto firstMatch
+        = std::sregex_iterator { content.begin(), content.end(), regex };
+    auto fileEnd = std::sregex_iterator{};
+    for (auto i = firstMatch; i != fileEnd; ++i)
+        res[i->str()].emplace(fileToScan);
+}
+
+/**
+ * Scan the content of a `/proc/[pid]/maps` file for regions that are mmaped to
+ * a store path
+ */
+static void scanMapsFile(const TracerConfig & opts, const fs::path & mapsFile, Roots & res)
+{
+    if (!fs::exists(mapsFile))
+        return;
+
+    static auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
+    std::stringstream mappedFile;
+    {
+        std::ifstream fs;
+        fs.open(mapsFile);
+        fs >> mappedFile.rdbuf();
+    }
+    std::string line;
+    while (std::getline(mappedFile, line)) {
+        auto match = std::smatch{};
+        if (std::regex_match(line, match, mapRegex)) {
+            auto matchedPath = fs::path(match[1]);
+            if (isInStore(opts.storeDir, matchedPath))
+                res[fs::path(match[1])].emplace(mapsFile);
+        }
+    }
+
+}
+
+Roots getRuntimeRoots(TracerConfig opts)
+{
+    auto procDir = fs::path("/proc");
+    if (!fs::exists(procDir))
+        return {};
+    Roots res;
+    auto digitsRegex = std::regex(R"(^\d+$)");
+    for (auto & procEntry : fs::directory_iterator(procDir)) {
+        // Only the directories whose name is a sequence of digits represent
+        // pids
+        if (!std::regex_match(procEntry.path().filename().string(), digitsRegex)
+            || !procEntry.is_directory())
+            continue;
+
+        opts.debug("Considering path " + procEntry.path().string());
+
+        // A set of paths used by the executable and possibly symlinks to a
+        // path in the store
+        std::set<fs::path> pathsToConsider;
+        pathsToConsider.insert(procEntry.path()/"exe");
+        pathsToConsider.insert(procEntry.path()/"cwd");
+        try {
+            auto fdDir = procEntry.path()/"fd";
+            for (auto & fdFile : fs::directory_iterator(fdDir))
+                pathsToConsider.insert(fdFile.path());
+        } catch (fs::filesystem_error & e) {
+            if (e.code().value() != ENOENT && e.code().value() != EACCES)
+                throw;
+        }
+        for (auto & path : pathsToConsider) try {
+            auto realPath = fs::read_symlink(path);
+            if (isInStore(opts.storeDir, realPath))
+                res[realPath].insert(path);
+        } catch (fs::filesystem_error &e) {
+            opts.debug(e.what());
+        }
+
+        // Scan the environment of the executable
+        scanFileContent(opts, procEntry.path()/"environ", res);
+        scanMapsFile(opts, procEntry.path()/"maps", res);
+    }
+
+    // Mostly useful for NixOS, but doesn’t hurt to check on other systems
+    // anyways
+    scanFileContent(opts, "/proc/sys/kernel/modprobe", res);
+    scanFileContent(opts, "/proc/sys/kernel/fbsplash", res);
+    scanFileContent(opts, "/proc/sys/kernel/poweroff_cmd", res);
+
+    return res;
+}
+
+}

src/nix-find-roots/lib/find-roots.hh

@@ -0,0 +1,51 @@
+#include <filesystem>
+#include <set>
+#include <map>
+#include <functional>
+
+namespace nix::roots_tracer {
+namespace fs = std::filesystem;
+
+class Error : public std::runtime_error {
+public:
+    using std::runtime_error::runtime_error;
+};
+
+inline void logNone(std::string_view)
+{ }
+
+struct TracerConfig {
+    const fs::path storeDir = "/nix/store";
+    const fs::path stateDir = "/nix/var/nix";
+    const fs::path socketPath = "/nix/var/nix/gc-socket/socket";
+
+    std::function<void(std::string_view msg)> log = logNone;
+    std::function<void(std::string_view msg)> debug = logNone;
+};
+
+/**
+ * A value of type `Roots` is a mapping from a store path to the set of roots that keep it alive
+ */
+typedef std::map<fs::path, std::set<fs::path>> Roots;
+
+struct TraceResult {
+    Roots storeRoots;
+    std::set<fs::path> deadLinks;
+};
+
+/**
+ * Return the set of all the store paths that are reachable from the given set
+ * of filesystem paths, by:
+ * - descending into the directories
+ * - following the symbolic links (at most twice)
+ * - reading the name of regular files (when encountering a file
+ *   `/foo/bar/abcdef`, the algorithm will try to access `/nix/store/abcdef`)
+ *
+ * Also returns the set of all dead links encountered during the process (so
+ * that they can be removed if it makes sense).
+ */
+TraceResult traceStaticRoots(TracerConfig opts, std::set<fs::path> initialRoots);
+
+Roots getRuntimeRoots(TracerConfig opts);
+
+}

src/nix-find-roots/local.mk

@@ -0,0 +1,20 @@
+libraries += libfindroots
+
+libfindroots_NAME = libnixfindroots
+
+libfindroots_DIR := $(d)/lib
+
+libfindroots_SOURCES := $(wildcard $(d)/lib/*.cc)
+
+programs += nix-find-roots
+
+nix-find-roots_DIR := $(d)
+
+nix-find-roots_SOURCES := $(d)/main.cc
+
+nix-find-roots_LIBS := libfindroots
+
+nix-find-roots_CXXFLAGS += \
+	-I src/nix-find-roots/lib
+
+nix-find-roots_INSTALL_DIR := $(libexecdir)/nix

src/nix-find-roots/main.cc

@@ -0,0 +1,188 @@
+#include "find-roots.hh"
+#include <getopt.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <iostream>
+#include <unistd.h>
+#include <cstring>
+#include <signal.h>
+
+using namespace nix::roots_tracer;
+
+void logStderr(std::string_view msg)
+{
+  std::cerr << msg << std::endl;
+}
+
+TracerConfig parseCmdLine(int argc, char** argv)
+{
+    std::function<void(std::string_view msg)> log = logStderr;
+    std::function<void(std::string_view msg)> debug = logNone;
+    fs::path storeDir = "/nix/store";
+    fs::path stateDir = "/nix/var/nix";
+    fs::path socketPath = "/nix/var/nix/gc-trace-socket/socket";
+
+    auto usage = [&]() {
+        std::string programName = argc > 0 ? argv[0] : "nix-find-roots";
+        std::cerr << "Usage: " << programName << " [--verbose|-v] [-s storeDir] [-d stateDir] [-l socketPath]" << std::endl;
+        exit(1);
+    };
+    static struct option long_options[] = {
+        { "verbose", no_argument, 0, 'v' },
+        { "socket_path", required_argument, 0, 'l' },
+        { "store_dir", required_argument, 0, 's' },
+        { "state_dir", required_argument, 0, 'd' },
+        { "help", no_argument, 0, 'h' },
+        { 0, 0, 0, 0 },
+    };
+
+    int option_index = 0;
+    int opt_char;
+    while((opt_char = getopt_long(argc, argv, "vd:s:l:h",
+                    long_options, &option_index)) != -1) {
+        switch (opt_char) {
+            case 0:
+                break;
+          break;
+            case '?':
+            case 'h':
+                usage();
+                break;
+            case 'v':
+                debug = logStderr;
+                break;
+            case 's':
+                storeDir = fs::path(optarg);
+                break;
+            case 'd':
+                stateDir = fs::path(optarg);
+                break;
+            case 'l':
+                socketPath = fs::path(optarg);
+                break;
+            default:
+                std::cerr << "Got invalid char: " << (char)opt_char << std::endl;
+                abort();
+        }
+    };
+    return TracerConfig {
+        .storeDir = storeDir,
+        .stateDir = stateDir,
+        .socketPath = socketPath,
+        .debug = debug,
+    };
+}
+
+/**
+ * Return `original` with every newline or tab character escaped
+ */
+std::string escape(std::string original)
+{
+    std::map<std::string, std::string> replacements = {
+        {"\n", "\\n"},
+        {"\t", "\\t"},
+    };
+    for (auto [oldStr, newStr] : replacements) {
+        size_t currentPos = 0;
+        while ((currentPos = original.find(oldStr, currentPos)) != std::string::npos) {
+            original.replace(currentPos, oldStr.length(), newStr);
+            currentPos += newStr.length();
+        }
+    }
+
+    return original;
+}
+
+#define SD_LISTEN_FDS_START 3 // Like in systemd
+
+int main(int argc, char * * argv)
+{
+    const TracerConfig opts = parseCmdLine(argc, argv);
+    const std::set<fs::path> standardRoots = {
+        opts.stateDir / fs::path("profiles"),
+        opts.stateDir / fs::path("gcroots"),
+    };
+
+    int mySock;
+
+    //  Handle socket-based activation by systemd.
+    auto rawListenFds = std::getenv("LISTEN_FDS");
+    if (rawListenFds) {
+        auto listenFds = std::string(rawListenFds);
+        auto listenPid = std::getenv("LISTEN_PID");
+        if (listenPid == nullptr || listenPid != std::to_string(getpid()) || listenFds != "1")
+            throw Error("unexpected systemd environment variables");
+        mySock = SD_LISTEN_FDS_START;
+    } else {
+        mySock = socket(PF_UNIX, SOCK_STREAM, 0);
+        if (mySock == -1) {
+            throw Error(std::string("Cannot create Unix domain socket, got") +
+                    std::strerror(errno));
+        }
+        struct sockaddr_un addr;
+        addr.sun_family = AF_UNIX;
+
+        auto socketDir = opts.socketPath.parent_path();
+        auto socketFilename = opts.socketPath.filename();
+        if (socketFilename.string().size() > sizeof(addr.sun_path))
+            throw Error(
+                    "Socket filename " + socketFilename.string() +
+                    " is too long, should be at most " +
+                    std::to_string(sizeof(addr.sun_path))
+                    );
+        chdir(socketDir.c_str());
+
+        fs::remove(socketFilename);
+        strcpy(addr.sun_path, socketFilename.c_str());
+        if (bind(mySock, (struct sockaddr*) &addr, sizeof(addr)) == -1) {
+            throw Error("Cannot bind to socket");
+        }
+
+        if (listen(mySock, 5) == -1)
+            throw Error("cannot listen on socket " + opts.socketPath.string());
+    }
+
+    // Ignore SIGPIPE so that an interrupted connection doesn’t stop the daemon
+    signal(SIGPIPE, SIG_IGN);
+
+    while (1) {
+        struct sockaddr_un remoteAddr;
+        socklen_t remoteAddrLen = sizeof(remoteAddr);
+        int remoteSocket = accept(
+                mySock,
+                (struct sockaddr*) & remoteAddr,
+                &remoteAddrLen
+                );
+
+        if (remoteSocket == -1) {
+            if (errno == EINTR) continue;
+            throw Error("Error accepting the connection");
+        }
+
+        opts.log("accepted connection");
+
+        auto printToSocket = [&](std::string_view s) {
+            send(remoteSocket, s.data(), s.size(), 0);
+        };
+
+        auto traceResult = traceStaticRoots(opts, standardRoots);
+        auto runtimeRoots = getRuntimeRoots(opts);
+        traceResult.storeRoots.insert(runtimeRoots.begin(), runtimeRoots.end());
+        for (auto & [rootInStore, externalRoots] : traceResult.storeRoots) {
+            for (auto & externalRoot : externalRoots) {
+                printToSocket(escape(rootInStore.string()));
+                printToSocket("\t");
+                printToSocket(escape(externalRoot.string()));
+                printToSocket("\n");
+            }
+
+        }
+        printToSocket("\n");
+        for (auto & deadLink : traceResult.deadLinks) {
+            printToSocket(escape(deadLink.string()));
+            printToSocket("\n");
+        }
+
+        close(remoteSocket);
+    }
+}

tests/functional/common/vars-and-functions.sh.in

@@ -19,6 +19,7 @@ export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
 export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
 unset NIX_USER_CONF_FILES
+export NIX_GC_SOCKET_PATH=$TEST_ROOT/gc.socket
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1
@@ -73,7 +74,7 @@ clearProfiles() {
 
 clearStore() {
     echo "clearing store..."
-    chmod -R +w "$NIX_STORE_DIR"
+    chmod -R +w "$NIX_STORE_DIR" || true
     rm -rf "$NIX_STORE_DIR"
     mkdir "$NIX_STORE_DIR"
     rm -rf "$NIX_STATE_DIR"
@@ -89,6 +90,15 @@ clearCacheCache() {
     rm -f $TEST_HOME/.cache/nix/binary-cache*
 }
 
+declare -A trapFunctions
+
+callTraps() {
+    for f in "${trapFunctions[@]}"; do
+        $f
+    done
+}
+trap callTraps EXIT
+
 startDaemon() {
     # Don’t start the daemon twice, as this would just make it loop indefinitely
     if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then
@@ -110,6 +120,7 @@ startDaemon() {
       fail "Didn’t manage to start the daemon"
     fi
     trap "killDaemon" EXIT
+    trapFunctions[killDaemon]=killDaemon
     # Save for if daemon is killed
     NIX_REMOTE_OLD=$NIX_REMOTE
     export NIX_REMOTE=daemon
@@ -132,7 +143,7 @@ killDaemon() {
     unset _NIX_TEST_DAEMON_PID
     # Restore old nix remote
     NIX_REMOTE=$NIX_REMOTE_OLD
-    trap "" EXIT
+    trapFunctions[killDaemon]=:
 }
 
 restartDaemon() {
@@ -142,6 +153,36 @@ restartDaemon() {
     startDaemon
 }
 
+startGcDaemon() {
+    # Start the daemon, wait for the socket to appear.  !!!
+    # ‘nix-daemon’ should have an option to fork into the background.
+    rm -f $NIX_GC_SOCKET_PATH
+    $(dirname $(command -v nix))/../libexec/nix/nix-find-roots \
+        -l "$NIX_GC_SOCKET_PATH" \
+        -d "$NIX_STATE_DIR" \
+        -s "$NIX_STORE_DIR" \
+        > /dev/null 2>&1 \
+            &
+    pidGcDaemon=$!
+    for ((i = 0; i < 30; i++)); do
+        if [[ -S $NIX_GC_SOCKET_PATH ]]; then break; fi
+        sleep 1
+    done
+    trapFunctions[killGcDaemon]=killGcDaemon
+}
+
+killGcDaemon() {
+    kill $pidGcDaemon
+    for i in {0..10}; do
+        kill -0 $pidGcDaemon || break
+        sleep 1
+    done
+    kill -9 $pidGcDaemon || true
+    wait $pidGcDaemon || true
+    trapFunctions[killGcDaemon]=:
+}
+
+
 if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi

tests/functional/gc-external-daemon/common.sh

@@ -0,0 +1,6 @@
+source ../common.sh
+
+enableFeatures "external-gc-daemon"
+echo "gc-socket-path = $NIX_GC_SOCKET_PATH" >> "$NIX_CONF_DIR"/nix.conf
+
+startGcDaemon

tests/functional/gc-external-daemon/gc-auto.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+cd ..
+source ./gc-auto.sh
+

tests/functional/gc-external-daemon/gc-concurrent.sh

@@ -0,0 +1,4 @@
+source common.sh
+
+cd ..
+source ./gc-concurrent.sh

tests/functional/gc-external-daemon/gc-runtime.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+cd ..
+source ./gc-runtime.sh
+

tests/functional/gc-external-daemon/gc.sh

@@ -0,0 +1,4 @@
+source common.sh
+
+cd ..
+source ./gc.sh

tests/functional/gc-external-daemon/local.mk

@@ -0,0 +1,7 @@
+gc-external-daemon-tests := \
+  $(d)/gc-auto.sh \
+  $(d)/gc.sh \
+  $(d)/gc-concurrent.sh \
+  $(d)/gc-runtime.sh
+
+install-tests-groups += gc-external-daemon

tests/nixos/default.nix

@@ -109,7 +109,7 @@ in
       nix.package = lib.mkForce pkgs.nixVersions.nix_2_13;
     };
   };
-  
+
   # TODO: (nixpkgs update) remoteBuildsSshNg_remote_2_18 = ...
 
   # Test our Nix as a builder for clients that are older
@@ -137,6 +137,8 @@ in
   # TODO: (nixpkgs update) remoteBuildsSshNg_local_2_18 = ...
   */
 
+  rootless-daemon = runNixOSTestFor "x86_64-linux" ./rootless-daemon.nix;
+
   nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix;
 
   nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix;

tests/nixos/rootless-daemon.nix

@@ -0,0 +1,177 @@
+{ pkgs, ... }:
+
+{
+  name = "rootless-daemon";
+
+  nodes.machine = { config, ... }: {
+    users.users.alice.isNormalUser = true;
+    #users.users.nix-daemon.isSystemUser = true;
+    users.users.nix-daemon.isNormalUser = true;
+    users.users.nix-daemon.group = "nix-daemon";
+    users.groups.nix-daemon = {};
+    environment.variables.NIX_REMOTE = "daemon"; # Even for root
+    virtualisation.writableStore = true;
+
+    boot.readOnlyNixStore = false;
+
+    # No root daemon
+    nix.enable = false;
+
+    # But put Nix on the path anyways
+    environment.systemPackages = [ pkgs.nix ];
+
+    # And install the unit files:
+    #
+    # - nix-gc-trace system-wide
+    # - nix-daemon per-user
+    systemd.packages = [
+       (pkgs.runCommand "shuffled-nix-systemd-unit-files" {} ''
+         mkdir -p $out/lib/systemd/{system,user}
+         ln -s ${pkgs.nix}/lib/systemd/system/nix-gc-trace.* $out/lib/systemd/system
+         ln -s ${pkgs.nix}/lib/systemd/system/nix-daemon.* $out/lib/systemd/user
+       '')
+    ];
+
+    # And prepare the socket dirs anyways
+    systemd.tmpfiles.rules = [
+      "d /nix/var/nix/daemon-socket 0755 nix-daemon root - -"
+      "d /nix/var/nix/gc-socket 0755 nix-daemon root - -"
+    ];
+
+    # Oops isn't working, because cannot disable Nix daemon and enable
+    # Nix settings yet: https://github.com/NixOS/nixpkgs/issues/263250.
+    nix.settings = {
+      experimental-features = [ "external-gc-daemon" ];
+    };
+    # Plan B given the above
+    #
+    # TODO delete once that issue is fixed.
+    environment.etc."nix/nix.conf".source = pkgs.writeTextFile {
+      name = "nix.conf";
+      text = ''
+        experimental-features = external-gc-daemon
+      '';
+    };
+
+    systemd.user.sockets.nix-daemon = {
+    };
+    systemd.user.services.nix-daemon = {
+      path = [ pkgs.nix ];
+      description = "Nix Daemon (non-root)";
+      unitConfig.ConditionUser = "nix-daemon";
+      # Make sure we do not get a "fork bomb" of the daemon trying to
+      # connect to itself.
+      serviceConfig.Environment= "NIX_REMOTE=local";
+    };
+
+    systemd.sockets.nix-gc-trace = {
+      restartTriggers = [ config.environment.etc."nix/nix.conf".source ];
+    };
+    systemd.services.nix-gc-trace = {
+      restartTriggers = [ config.environment.etc."nix/nix.conf".source ];
+      path = [ pkgs.nix ];
+      description = "Nix Find Roots";
+    };
+    # We must enable lingering so that the Systemd User D-Bus is
+    # enabled. We also cannot do this with loginctl enable-linger
+    # because it needs to happen before systemd is loaded.
+    #
+    # See https://github.com/NixOS/nixpkgs/issues/3702
+    #
+    # TODO after upgrading to 23.11 can use new NixOS option for this.
+    system.activationScripts = {
+      enableLingering = ''
+        # remove all existing lingering users
+        rm -rf /var/lib/systemd/linger
+        mkdir -p /var/lib/systemd/linger
+        # enable for the subset of declared users
+        touch /var/lib/systemd/linger/nix-daemon
+      '';
+    };
+  };
+
+  testScript = ''
+    import re
+
+    machine.wait_for_unit("multi-user.target")
+
+  ''
+
+  # Give ownership of the store dir and var to the nix-daemon user.
+  #
+  # We intentionally don't do `-R` on the store because store objects
+  # used by NixOS should still be owned by root.
+  + ''
+    machine.succeed("""
+      set -ex
+
+      chown nix-daemon /nix/store
+      chown -R nix-daemon /nix/var
+    """)
+  ''
+
+  # Test that alice indeed cannot modify the store; we don't want
+  # arbitrary users to have any more permissions than before!
+  + ''
+    machine.fail("su alice -c 'touch /nix/store/foo'")
+
+  ''
+
+  # Start and wait for our units
+  + ''
+    machine.start_job("nix-gc-trace.socket")
+    machine.wait_for_unit("nix-gc-trace.socket")
+
+    machine.start_job("nix-daemon.socket", user="nix-daemon")
+    machine.wait_for_unit("nix-daemon.socket", user="nix-daemon")
+
+  ''
+
+  # Create a store obect, remember its store path in Python.
+  + ''
+    two = machine.succeed("""
+    su --login alice -c "$(cat <<'EOF'
+      set -ex
+      export PS4='+(''${BASH_SOURCE[0]-$0}:$LINENO) '
+
+      echo ehHtmfuULXYyBV6NBk6QUi8iE0 > two
+      nix-store --add two
+    EOF
+    )"
+    """)
+    two = two.strip()
+    assert re.match(r'^/nix/store/.+-two$', two)
+
+  ''
+
+  # Make sure we cannot delete it when it has a GC root, but we can once
+  # that root is destroyed.
+  + ''
+    machine.succeed(f"""
+    su --login alice -c "$(cat <<'EOF'
+      set -ex
+      export PS4='+(''${{BASH_SOURCE[0]-$0}}:$LINENO) '
+
+      test -f {two}
+      nix-store --realize {two} --add-root foo
+      echo $two
+    EOF
+    )"
+    """)
+
+    machine.fail(f"su --login alice -c 'nix-store --delete {two}'")
+
+    machine.succeed(f"""
+    su --login alice -c "$(cat <<'EOF'
+      set -ex
+      export PS4='+(''${{BASH_SOURCE[0]-$0}}:$LINENO) '
+
+      test -f {two}
+      rm foo
+      nix-store --delete {two}
+      test ! -f {two}
+    EOF
+    )"
+    """)
+  '';
+}