Reword the experimental feature description.

Co-Authored-By: Robert Hensing <robert@roberthensing.nl>

.gitignore

@@ -108,6 +108,9 @@ perl/Makefile.config
 
 /misc/systemd/nix-daemon.service
 /misc/systemd/nix-daemon.socket
+/misc/systemd/nix-gc-trace.service
+/misc/systemd/nix-gc-trace.socket
+
 /misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 

Makefile

@@ -12,6 +12,7 @@ makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
   src/libutil/local.mk \
+  src/nix-find-roots/local.mk \
   src/libstore/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
@@ -41,6 +42,7 @@ endif
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 makefiles += \
   tests/functional/local.mk \
+  tests/functional/gc-external-daemon/local.mk \
   tests/functional/ca/local.mk \
   tests/functional/dyn-drv/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \

doc/manual/rl-next/debugger-on-trace.md

@@ -0,0 +1,9 @@
+---
+synopsis: Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set
+prs: 9914
+---
+
+If the `debugger-on-trace` option is set and `--debugger` is given,
+`builtins.trace` calls will behave similarly to `builtins.break` and will enter
+the debug REPL. This is useful for determining where warnings are being emitted
+from.

doc/manual/rl-next/debugger-positions.md

@@ -0,0 +1,25 @@
+---
+synopsis: Debugger prints source position information
+prs: 9913
+---
+
+The `--debugger` now prints source location information, instead of the
+pointers of source location information. Before:
+
+```
+nix-repl> :bt
+0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+0x600001522598
+```
+
+After:
+
+```
+0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
+
+   131|
+   132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
+      |                           ^
+   133|     in
+```

doc/manual/rl-next/lambda-printing.md

@@ -0,0 +1,50 @@
+---
+synopsis: Functions are printed with more detail
+prs: 9606
+issues: 7145
+---
+
+Functions and `builtins` are printed with more detail in `nix repl`, `nix
+eval`, `builtins.trace`, and most other places values are printed.
+
+Before:
+
+```
+$ nix repl nixpkgs
+nix-repl> builtins.map
+«primop»
+
+nix-repl> builtins.map lib.id
+«primop-app»
+
+nix-repl> builtins.trace lib.id "my-value"
+trace: <LAMBDA>
+"my-value"
+
+$ nix eval --file functions.nix
+{ id = <LAMBDA>; primop = <PRIMOP>; primop-app = <PRIMOP-APP>; }
+```
+
+After:
+
+```
+$ nix repl nixpkgs
+nix-repl> builtins.map
+«primop map»
+
+nix-repl> builtins.map lib.id
+«partially applied primop map»
+
+nix-repl> builtins.trace lib.id "my-value"
+trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5»
+"my-value"
+
+$ nix eval --file functions.nix
+{ id = «lambda id @ /Users/wiggles/nix/functions.nix:2:8»; primop = «primop map»; primop-app = «partially applied primop map»; }
+```
+
+This was actually released in Nix 2.20, but wasn't added to the release notes
+so we're announcing it here. The historical release notes have been updated as well.
+
+[type-error]: https://github.com/NixOS/nix/pull/9753
+[coercion-error]: https://github.com/NixOS/nix/pull/9754

doc/manual/rl-next/more-commands-respect-ctrl-c.md

@@ -0,0 +1,13 @@
+---
+synopsis: Nix commands respect Ctrl-C
+prs: 9687 6995
+issues: 7245
+---
+
+Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed
+while performing various operations (including `nix develop`, `nix flake
+update`, and so on). With several fixes to Nix's signal handlers, Nix commands
+will now exit quickly after Ctrl-C is pressed.
+
+This was actually released in Nix 2.20, but wasn't added to the release notes
+so we're announcing it here. The historical release notes have been updated as well.

doc/manual/rl-next/pretty-print-in-nix-repl.md

@@ -0,0 +1,24 @@
+---
+synopsis: "`nix repl` pretty-prints values"
+prs: 9931
+---
+
+`nix repl` will now pretty-print values:
+
+```
+{
+  attrs = {
+    a = {
+      b = {
+        c = { };
+      };
+    };
+  };
+  list = [ 1 ];
+  list' = [
+    1
+    2
+    3
+  ];
+}
+```

doc/manual/rl-next/repl-ctrl-c-while-printing.md

@@ -0,0 +1,8 @@
+---
+synopsis: "`nix repl` now respects Ctrl-C while printing values"
+prs: 9927
+---
+
+`nix repl` will now halt immediately when Ctrl-C is pressed while it's printing
+a value. This is useful if you got curious about what would happen if you
+printed all of Nixpkgs.

doc/manual/rl-next/repl-cycle-detection.md

@@ -0,0 +1,22 @@
+---
+synopsis: Cycle detection in `nix repl` is simpler and more reliable
+prs: 9926
+issues: 8672
+---
+
+The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere
+else values are printed is now simpler and matches the cycle detection in
+`nix-instantiate --eval` output.
+
+Before:
+
+```
+nix eval --expr 'let self = { inherit self; }; in self'
+{ self = { self = «repeated»; }; }
+```
+
+After:
+
+```
+{ self = «repeated»; }
+```

doc/manual/rl-next/stack-size-macos.md

@@ -0,0 +1,9 @@
+---
+synopsis: Stack size is increased on macOS
+prs: 9860
+---
+
+Previously, Nix would set the stack size to 64MiB on Linux, but would leave the
+stack size set to the default (approximately 8KiB) on macOS. Now, the stack
+size is correctly set to 64MiB on macOS as well, which should reduce stack
+overflow segfaults in deeply-recursive Nix expressions.

doc/manual/src/installation/installing-rootless-daemon.md

@@ -0,0 +1,138 @@
+# Using Nix in multi-user mode with a non-root daemon
+
+> Experimental blurb
+
+It is experimentally possible to run Nix in multi-user mode without running the whole daemon as root.
+This is done by delegating the only part that requires root access to a separate daemon, with a much smaller attack surface.
+Because of the need for a second daemon, this makes the setup a bit more complex and isn't yet supported by the installer. It is however possible to set this up manually:
+
+1. Create a new user and group for the daemon:
+    ```sh
+    sudo groupadd nix-daemon
+    sudo useradd --gid nix-daemon --system -c "Nix daemon user" nix-daemon
+    ```
+2. Create `/nix` owned by that user:
+    ```sh
+    sudo mkdir -m 0755 /nix
+    sudo chown nix-daemon:nix-daemon /nix
+    ```
+3. Download a statically-compiled Nix version for bootstrapping
+    ```sh
+    curl -L https://hydra.nixos.org/job/nix/master/buildStatic.x86_64-linux/latest/download-by-type/file/binary-dist -o /tmp/nix-env
+    chmod +x /tmp/nix-env
+    ```
+4. Install a proper Nix and the tracing daemon in the store
+    ```sh
+    export DAEMON_HOME=$(sudo -u nix-daemon mktemp -d)
+    sudo -u nix-daemon HOME="$DAEMON_HOME" \
+        /tmp/nix-env \
+        -f https://github.com/nixos/nix/archive/rootless-daemon.tar.gz \
+        -iA default packages.x86_64-linux.nix-find-roots \
+        --option extra-substituters https://nixos-nix-install-tests.cachix.org \
+        --option extra-trusted-public-keys nixos-nix-install-tests.cachix.org-1:Le57vOUJjOcdzLlbwmZVBuLGoDC+Xg2rQDtmIzALgFU= \
+        --store / \
+        --profile /nix/var/nix/profiles/default
+    sudo -u nix-daemon mkdir -p /nix/var/nix/gc-socket
+    sudo -u nix-daemon rm -rf "$DAEMON_HOME"
+    ```
+5. Move the tracing daemon executable out of the store (as we don't want Nix
+   to own it)
+   ```sh
+   sudo cp /nix/var/nix/profiles/default/bin/nix-find-roots /usr/bin/
+   ```
+6. Install the systemd services for the daemon:
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-daemon.service
+    [Unit]
+    Description=Nix Daemon
+    Documentation=man:nix-daemon https://nixos.org/manual
+    RequiresMountsFor=/nix/store
+    RequiresMountsFor=/nix/var
+    RequiresMountsFor=/nix/var/nix/db
+    ConditionPathIsReadWrite=/nix/var/nix/daemon-socket
+
+    [Service]
+    ExecStart=@/nix/var/nix/profiles/default/bin/nix-daemon nix-daemon --daemon
+    KillMode=process
+    LimitNOFILE=1048576
+    TasksMax=1048576
+    User=nix-daemon
+    Group=nix-daemon
+
+    [Install]
+    WantedBy=multi-user.target
+    EOF
+    ```
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-daemon.socket
+    [Unit]
+    Description=Nix Daemon Socket
+    Before=multi-user.target
+    RequiresMountsFor=/nix/store
+    ConditionPathIsReadWrite=/nix/var/nix/daemon-socket
+
+    [Socket]
+    ListenStream=/nix/var/nix/daemon-socket/socket
+    SocketUser=nix-daemon
+
+    [Install]
+    WantedBy=sockets.target
+    EOF
+    ```
+7. Install the systemd services for the tracing daemon:
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-find-roots.service
+    [Unit]
+    Description=Nix GC tracer daemon
+    RequiresMountsFor=/nix/store
+    RequiresMountsFor=/nix/var
+    ConditionPathIsReadWrite=/nix/var/nix/gc-socket
+    ProcSubset=pid
+
+    [Service]
+    ExecStart=@/usr/bin/nix-find-roots nix-find-roots
+    Type=simple
+    StandardError=journal
+    ProtectSystem=full
+    ReadWritePaths=/nix/var/nix/gc-socket
+    SystemCallFilter=@system-service
+    SystemCallErrorNumber=EPERM
+    PrivateNetwork=true
+    PrivateDevices=true
+    ProtectKernelTunables=true
+
+    [Install]
+    WantedBy=multi-user.target
+    EOF
+    ```
+    ```sh
+    cat <<EOF | sudo tee /etc/systemd/system/nix-find-roots.socket
+    [Unit]
+    Description=Nix Daemon Socket
+    Before=multi-user.target
+    RequiresMountsFor=/nix/store
+    ConditionPathIsReadWrite=/nix/var/nix/gc-socket
+
+    [Socket]
+    ListenStream=/nix/var/nix/gc-socket/socket
+    Accept=false
+
+    [Install]
+    WantedBy=sockets.target
+    EOF
+    ```
+8. Enable the required experimental Nix feature and basic configuration:
+    ```sh
+    sudo mkdir /etc/nix
+    cat <<EOF | sudo tee /etc/nix/nix.conf
+    experimental-features = external-gc-daemon
+    trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+    substituters = https://cache.nixos.org/
+    EOF
+    ```
+9. Start the systemd sockets:
+    ```sh
+    sudo systemctl start nix-daemon.socket
+    sudo systemctl start nix-find-roots.socket
+    ```
+10. Profit

doc/manual/src/release-notes/rl-2.20.md

@@ -167,3 +167,36 @@
 
          error: expected a set but found an integer
   ```
+
+- Functions are printed with more detail [#7145](https://github.com/NixOS/nix/issues/7145) [#9606](https://github.com/NixOS/nix/pull/9606)
+
+  `nix repl`, `nix eval`, `builtins.trace`, and most other places values are
+  printed will now include function names and source location information:
+
+  ```
+  $ nix repl nixpkgs
+  nix-repl> builtins.map
+  «primop map»
+
+  nix-repl> builtins.map lib.id
+  «partially applied primop map»
+
+  nix-repl> builtins.trace lib.id "my-value"
+  trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5»
+  "my-value"
+  ```
+
+- Flake operations like `nix develop` will no longer fail when run in a Git
+  repository where the `flake.lock` file is `.gitignore`d
+  [#8854](https://github.com/NixOS/nix/issues/8854)
+  [#9324](https://github.com/NixOS/nix/pull/9324)
+
+- Nix commands will now respect Ctrl-C
+  [#7145](https://github.com/NixOS/nix/issues/7145)
+  [#6995](https://github.com/NixOS/nix/pull/6995)
+  [#9687](https://github.com/NixOS/nix/pull/9687)
+
+  Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed
+  while performing various operations (including `nix develop`, `nix flake
+  update`, and so on). With several fixes to Nix's signal handlers, Nix
+  commands will now exit quickly after Ctrl-C is pressed.

flake.nix

@@ -152,6 +152,30 @@
             '';
           };
 
+          nix-find-roots = prev.stdenv.mkDerivation {
+            pname = "nix-find-roots";
+            inherit version;
+
+            src = fileset.toSource {
+              root = ./src/nix-find-roots;
+              fileset = /*fileset.intersect baseFiles (*/fileset.unions [
+                ./src/nix-find-roots/main.cc
+                ./src/nix-find-roots/lib
+              ]/*)*/;
+            };
+
+            CXXFLAGS = prev.lib.optionalString prev.stdenv.hostPlatform.isStatic "-static";
+
+            buildPhase = ''
+              $CXX $CXXFLAGS -std=c++17 *.cc **/*.cc -I lib -o nix-find-roots
+            '';
+
+            installPhase = ''
+              mkdir -p $out/bin
+              cp nix-find-roots $out/bin/
+            '';
+          };
+
           libgit2-nix = final.libgit2.overrideAttrs (attrs: {
             src = libgit2;
             version = libgit2.lastModifiedDate;
@@ -366,6 +390,9 @@
         default = nix;
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = nixpkgsFor.${system}.static.nix;
+
+        inherit (nixpkgsFor.${system}.static) nix-find-roots;
+
         dockerImage =
           let
             pkgs = nixpkgsFor.${system}.native;

misc/systemd/local.mk

@@ -1,6 +1,8 @@
 ifdef HOST_LINUX
 
-  $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
+  $(foreach n,\
+    nix-daemon.socket nix-daemon.service nix-gc-trace.socket nix-gc-trace.service,\
+    $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
   $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644)))
 
   clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf

misc/systemd/nix-gc-trace.service.in

@@ -0,0 +1,21 @@
+[Unit]
+Description=Nix GC Tracer Daemon
+RequiresMountsFor=@storedir@
+RequiresMountsFor=@localstatedir@
+ConditionPathIsReadWrite=@localstatedir@/nix/gc-socket
+ProcSubset=pid
+
+[Service]
+ExecStart=@@libexecdir@/nix/nix-find-roots nix-find-roots
+Type=simple
+StandardError=journal
+ProtectSystem=full
+ReadWritePaths=@localstatedir@/nix/gc-socket
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+PrivateNetwork=true
+PrivateDevices=true
+ProtectKernelTunables=true
+
+[Install]
+WantedBy=multi-user.target

misc/systemd/nix-gc-trace.socket.in

@@ -0,0 +1,12 @@
+[Unit]
+Description=Nix Daemon Socket
+Before=multi-user.target
+RequiresMountsFor=@storedir@
+ConditionPathIsReadWrite=@localstatedir@/nix/gc-socket
+
+[Socket]
+ListenStream=@localstatedir@/nix/gc-socket/socket
+Accept=false
+
+[Install]
+WantedBy=sockets.target

src/libcmd/common-eval-args.cc

@@ -9,6 +9,7 @@
 #include "store-api.hh"
 #include "command.hh"
 #include "tarball.hh"
+#include "fetch-to-store.hh"
 
 namespace nix {
 
@@ -167,8 +168,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
 {
     if (EvalSettings::isPseudoUrl(s)) {
-        auto storePath = fetchers::downloadTarball(
-            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath;
+        auto accessor = fetchers::downloadTarball(
+            EvalSettings::resolvePseudoUrl(s)).accessor;
+        auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
         return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
     }
 

src/libcmd/installable-value.cc

@@ -45,7 +45,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = fetchToStore(*state->store, v.path());
+        auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libcmd/misc-store-flags.cc

@@ -0,0 +1,121 @@
+#include "misc-store-flags.hh"
+
+namespace nix::flag
+{
+
+static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & format : hashFormats) {
+        if (hasPrefix(format, prefix)) {
+            completions.add(format);
+        }
+    }
+}
+
+Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf)
+{
+    assert(*hf == nix::HashFormat::SRI);
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.",
+            .labels = {"hash-format"},
+            .handler = {[hf](std::string s) {
+                *hf = parseHashFormat(s);
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).",
+            .labels = {"hash-format"},
+            .handler = {[ohf](std::string s) {
+                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
+            }},
+            .completer = hashFormatCompleter,
+    };
+}
+
+static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
+{
+    for (auto & algo : hashAlgorithms)
+        if (hasPrefix(algo, prefix))
+            completions.add(algo);
+}
+
+Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
+            .labels = {"hash-algo"},
+            .handler = {[ha](std::string s) {
+                *ha = parseHashAlgo(s);
+            }},
+            .completer = hashAlgoCompleter,
+    };
+}
+
+Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha)
+{
+    return Args::Flag {
+            .longName = std::move(longName),
+            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
+            .labels = {"hash-algo"},
+            .handler = {[oha](std::string s) {
+                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
+            }},
+            .completer = hashAlgoCompleter,
+    };
+}
+
+Args::Flag fileIngestionMethod(FileIngestionMethod * method)
+{
+    return Args::Flag {
+        .longName  = "mode",
+        // FIXME indentation carefully made for context, this is messed up.
+        .description = R"(
+    How to compute the hash of the input.
+    One of:
+
+    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+
+    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+        )",
+        .labels = {"file-ingestion-method"},
+        .handler = {[method](std::string s) {
+            *method = parseFileIngestionMethod(s);
+        }},
+    };
+}
+
+Args::Flag contentAddressMethod(ContentAddressMethod * method)
+{
+    return Args::Flag {
+        .longName  = "mode",
+        // FIXME indentation carefully made for context, this is messed up.
+        .description = R"(
+    How to compute the content-address of the store object.
+    One of:
+
+    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
+
+    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
+
+    - `text`: Like `flat`, but used for
+      [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and 
+      [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile).
+      For advanced use-cases only;
+      for regular usage prefer `nar` and `flat.
+        )",
+        .labels = {"content-address-method"},
+        .handler = {[method](std::string s) {
+            *method = ContentAddressMethod::parse(s);
+        }},
+    };
+}
+
+}

src/libcmd/misc-store-flags.hh

@@ -0,0 +1,21 @@
+#include "args.hh"
+#include "content-address.hh"
+
+namespace nix::flag {
+
+Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha);
+static inline Args::Flag hashAlgo(HashAlgorithm * ha)
+{
+    return hashAlgo("hash-algo", ha);
+}
+Args::Flag hashAlgoOpt(std::string && longName, std::optional<HashAlgorithm> * oha);
+Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf);
+Args::Flag hashFormatOpt(std::string && longName, std::optional<HashFormat> * ohf);
+static inline Args::Flag hashAlgoOpt(std::optional<HashAlgorithm> * oha)
+{
+    return hashAlgoOpt("hash-algo", oha);
+}
+Args::Flag fileIngestionMethod(FileIngestionMethod * method);
+Args::Flag contentAddressMethod(ContentAddressMethod * method);
+
+}

src/libcmd/repl.cc

@@ -52,6 +52,27 @@ extern "C" {
 
 namespace nix {
 
+/**
+ * Returned by `NixRepl::processLine`.
+ */
+enum class ProcessLineResult {
+    /**
+     * The user exited with `:quit`. The REPL should exit. The surrounding
+     * program or evaluation (e.g., if the REPL was acting as the debugger)
+     * should also exit.
+     */
+    Quit,
+    /**
+     * The user exited with `:continue`. The REPL should exit, but the program
+     * should continue running.
+     */
+    Continue,
+    /**
+     * The user did not exit. The REPL should request another line of input.
+     */
+    PromptAgain,
+};
+
 struct NixRepl
     : AbstractNixRepl
     #if HAVE_BOEHMGC
@@ -75,13 +96,13 @@ struct NixRepl
             std::function<AnnotatedValues()> getValues);
     virtual ~NixRepl();
 
-    void mainLoop() override;
+    ReplExitStatus mainLoop() override;
     void initEnv() override;
 
     StringSet completePrefix(const std::string & prefix);
     bool getLine(std::string & input, const std::string & prompt);
     StorePath getDerivationPath(Value & v);
-    bool processLine(std::string line);
+    ProcessLineResult processLine(std::string line);
 
     void loadFile(const Path & path);
     void loadFlake(const std::string & flakeRef);
@@ -246,7 +267,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
 
 static bool isFirstRepl = true;
 
-void NixRepl::mainLoop()
+ReplExitStatus NixRepl::mainLoop()
 {
     if (isFirstRepl) {
         std::string_view debuggerNotice = "";
@@ -287,15 +308,25 @@ void NixRepl::mainLoop()
         // When continuing input from previous lines, don't print a prompt, just align to the same
         // number of chars as the prompt.
         if (!getLine(input, input.empty() ? "nix-repl> " : "          ")) {
-            // ctrl-D should exit the debugger.
+            // Ctrl-D should exit the debugger.
             state->debugStop = false;
-            state->debugQuit = true;
             logger->cout("");
-            break;
+            // TODO: Should Ctrl-D exit just the current debugger session or
+            // the entire program?
+            return ReplExitStatus::QuitAll;
         }
         logger->resume();
         try {
-            if (!removeWhitespace(input).empty() && !processLine(input)) return;
+            switch (processLine(input)) {
+                case ProcessLineResult::Quit:
+                    return ReplExitStatus::QuitAll;
+                case ProcessLineResult::Continue:
+                    return ReplExitStatus::Continue;
+                case ProcessLineResult::PromptAgain:
+                    break;
+                default:
+                    abort();
+            }
         } catch (ParseError & e) {
             if (e.msg().find("unexpected end of file") != std::string::npos) {
                 // For parse errors on incomplete input, we continue waiting for the next line of
@@ -483,10 +514,11 @@ void NixRepl::loadDebugTraceEnv(DebugTrace & dt)
     }
 }
 
-bool NixRepl::processLine(std::string line)
+ProcessLineResult NixRepl::processLine(std::string line)
 {
     line = trim(line);
-    if (line == "") return true;
+    if (line.empty())
+        return ProcessLineResult::PromptAgain;
 
     _isInterrupted = false;
 
@@ -581,13 +613,13 @@ bool NixRepl::processLine(std::string line)
     else if (state->debugRepl && (command == ":s" || command == ":step")) {
         // set flag to stop at next DebugTrace; exit repl.
         state->debugStop = true;
-        return false;
+        return ProcessLineResult::Continue;
     }
 
     else if (state->debugRepl && (command == ":c" || command == ":continue")) {
         // set flag to run to next breakpoint or end of program; exit repl.
         state->debugStop = false;
-        return false;
+        return ProcessLineResult::Continue;
     }
 
     else if (command == ":a" || command == ":add") {
@@ -730,8 +762,7 @@ bool NixRepl::processLine(std::string line)
 
     else if (command == ":q" || command == ":quit") {
         state->debugStop = false;
-        state->debugQuit = true;
-        return false;
+        return ProcessLineResult::Quit;
     }
 
     else if (command == ":doc") {
@@ -792,7 +823,7 @@ bool NixRepl::processLine(std::string line)
         }
     }
 
-    return true;
+    return ProcessLineResult::PromptAgain;
 }
 
 void NixRepl::loadFile(const Path & path)
@@ -923,7 +954,7 @@ std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
 }
 
 
-void AbstractNixRepl::runSimple(
+ReplExitStatus AbstractNixRepl::runSimple(
     ref<EvalState> evalState,
     const ValMap & extraEnv)
 {
@@ -945,7 +976,7 @@ void AbstractNixRepl::runSimple(
     for (auto & [name, value] : extraEnv)
         repl->addVarToScope(repl->state->symbols.create(name), *value);
 
-    repl->mainLoop();
+    return repl->mainLoop();
 }
 
 }

src/libcmd/repl.hh

@@ -28,13 +28,13 @@ struct AbstractNixRepl
         const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
         std::function<AnnotatedValues()> getValues);
 
-    static void runSimple(
+    static ReplExitStatus runSimple(
         ref<EvalState> evalState,
         const ValMap & extraEnv);
 
     virtual void initEnv() = 0;
 
-    virtual void mainLoop() = 0;
+    virtual ReplExitStatus mainLoop() = 0;
 };
 
 }

src/libexpr/eval-settings.hh

@@ -127,6 +127,16 @@ struct EvalSettings : Config
 
     Setting<unsigned int> maxCallDepth{this, 10000, "max-call-depth",
         "The maximum function call depth to allow before erroring."};
+
+    Setting<bool> builtinsTraceDebugger{this, false, "debugger-on-trace",
+        R"(
+          If set to true and the `--debugger` flag is given,
+          [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will
+          enter the debugger like
+          [`builtins.break`](@docroot@/language/builtins.md#builtins-break).
+
+          This is useful for debugging warnings in third-party Nix code.
+        )"};
 };
 
 extern EvalSettings evalSettings;

src/libexpr/eval.cc

@@ -3,6 +3,7 @@
 #include "hash.hh"
 #include "primops.hh"
 #include "print-options.hh"
+#include "shared.hh"
 #include "types.hh"
 #include "util.hh"
 #include "store-api.hh"
@@ -416,7 +417,6 @@ EvalState::EvalState(
     , buildStore(buildStore ? buildStore : store)
     , debugRepl(nullptr)
     , debugStop(false)
-    , debugQuit(false)
     , trylevel(0)
     , regexCache(makeRegexCache())
 #if HAVE_BOEHMGC
@@ -467,13 +467,13 @@ EvalState::~EvalState()
 void EvalState::allowPath(const Path & path)
 {
     if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
-        rootFS2->allowPath(CanonPath(path));
+        rootFS2->allowPrefix(CanonPath(path));
 }
 
 void EvalState::allowPath(const StorePath & storePath)
 {
     if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListInputAccessor>())
-        rootFS2->allowPath(CanonPath(store->toRealPath(storePath)));
+        rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath)));
 }
 
 void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
@@ -792,7 +792,17 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
     auto se = getStaticEnv(expr);
     if (se) {
         auto vm = mapStaticEnvBindings(symbols, *se.get(), env);
-        (debugRepl)(ref<EvalState>(shared_from_this()), *vm);
+        auto exitStatus = (debugRepl)(ref<EvalState>(shared_from_this()), *vm);
+        switch (exitStatus) {
+            case ReplExitStatus::QuitAll:
+                if (error)
+                    throw *error;
+                throw Exit(0);
+            case ReplExitStatus::Continue:
+                break;
+            default:
+                abort();
+        }
     }
 }
 
@@ -2339,7 +2349,14 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = fetchToStore(*store, path.resolveSymlinks(), path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = fetchToStore(
+                *store,
+                path.resolveSymlinks(),
+                settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
+                path.baseName(),
+                FileIngestionMethod::Recursive,
+                nullptr,
+                repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@@ -2787,10 +2804,11 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
 
     if (EvalSettings::isPseudoUrl(value)) {
         try {
-            auto storePath = fetchers::downloadTarball(
-                store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath;
+            auto accessor = fetchers::downloadTarball(
+                EvalSettings::resolvePseudoUrl(value)).accessor;
+            auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
             res = { store->toRealPath(storePath) };
-        } catch (FileTransferError & e) {
+        } catch (Error & e) {
             logWarning({
                 .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
             });

src/libexpr/eval.hh

@@ -11,6 +11,7 @@
 #include "experimental-features.hh"
 #include "input-accessor.hh"
 #include "search-path.hh"
+#include "repl-exit-status.hh"
 
 #include <map>
 #include <optional>
@@ -219,9 +220,8 @@ public:
     /**
      * Debugger
      */
-    void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
+    ReplExitStatus (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv);
     bool debugStop;
-    bool debugQuit;
     int trylevel;
     std::list<DebugTrace> debugTraces;
     std::map<const Expr*, const std::shared_ptr<const StaticEnv>> exprEnvs;
@@ -758,7 +758,6 @@ struct DebugTraceStacker {
     DebugTraceStacker(EvalState & evalState, DebugTrace t);
     ~DebugTraceStacker()
     {
-        // assert(evalState.debugTraces.front() == trace);
         evalState.debugTraces.pop_front();
     }
     EvalState & evalState;

src/libexpr/flake/call-flake.nix

@@ -1,20 +1,52 @@
-lockFileStr: rootSrc: rootSubdir:
+# This is a helper to callFlake() to lazily fetch flake inputs.
+
+# The contents of the lock file, in JSON format.
+lockFileStr:
+
+# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets,
+# with sourceInfo.outPath providing an InputAccessor to a previously
+# fetched tree. This is necessary for possibly unlocked inputs, in
+# particular the root input, but also --override-inputs pointing to
+# unlocked trees.
+overrides:
 
 let
 
   lockFile = builtins.fromJSON lockFileStr;
 
+  # Resolve a input spec into a node name. An input spec is
+  # either a node name, or a 'follows' path from the root
+  # node.
+  resolveInput = inputSpec:
+    if builtins.isList inputSpec
+    then getInputByPath lockFile.root inputSpec
+    else inputSpec;
+
+  # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
+  # root node, returning the final node.
+  getInputByPath = nodeName: path:
+    if path == []
+    then nodeName
+    else
+      getInputByPath
+        # Since this could be a 'follows' input, call resolveInput.
+        (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
+        (builtins.tail path);
+
   allNodes =
     builtins.mapAttrs
       (key: node:
         let
 
           sourceInfo =
-            if key == lockFile.root
-            then rootSrc
-            else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
+            if overrides ? ${key}
+            then
+              overrides.${key}.sourceInfo
+            else
+              # FIXME: remove obsolete node.info.
+              fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
 
-          subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
+          subdir = overrides.${key}.dir or node.locked.dir or "";
 
           outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
 
@@ -24,25 +56,6 @@ let
             (inputName: inputSpec: allNodes.${resolveInput inputSpec})
             (node.inputs or {});
 
-          # Resolve a input spec into a node name. An input spec is
-          # either a node name, or a 'follows' path from the root
-          # node.
-          resolveInput = inputSpec:
-              if builtins.isList inputSpec
-              then getInputByPath lockFile.root inputSpec
-              else inputSpec;
-
-          # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
-          # root node, returning the final node.
-          getInputByPath = nodeName: path:
-            if path == []
-            then nodeName
-            else
-              getInputByPath
-                # Since this could be a 'follows' input, call resolveInput.
-                (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
-                (builtins.tail path);
-
           outputs = flake.outputs (inputs // { self = result; });
 
           result =

src/libexpr/flake/flake.cc

@@ -365,6 +365,7 @@ LockedFlake lockFlake(
         std::map<InputPath, FlakeInput> overrides;
         std::set<InputPath> explicitCliOverrides;
         std::set<InputPath> overridesUsed, updatesUsed;
+        std::map<ref<Node>, StorePath> nodePaths;
 
         for (auto & i : lockFlags.inputOverrides) {
             overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
@@ -535,11 +536,13 @@ LockedFlake lockFlake(
                             }
                         }
 
-                        computeLocks(
-                            mustRefetch
-                            ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs
-                            : fakeInputs,
-                            childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch);
+                        if (mustRefetch) {
+                            auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath);
+                            nodePaths.emplace(childNode, inputFlake.storePath);
+                            computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false);
+                        } else {
+                            computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true);
+                        }
 
                     } else {
                         /* We need to create a new lock file entry. So fetch
@@ -584,6 +587,7 @@ LockedFlake lockFlake(
                                flake. Also, unless we already have this flake
                                in the top-level lock file, use this flake's
                                own lock file. */
+                            nodePaths.emplace(childNode, inputFlake.storePath);
                             computeLocks(
                                 inputFlake.inputs, childNode, inputPath,
                                 oldLock
@@ -596,11 +600,13 @@ LockedFlake lockFlake(
                         }
 
                         else {
-                            auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+                            auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
                                 state, *input.ref, useRegistries, flakeCache);
 
                             auto childNode = make_ref<LockedNode>(lockedRef, ref, false);
 
+                            nodePaths.emplace(childNode, storePath);
+
                             node->inputs.insert_or_assign(id, childNode);
                         }
                     }
@@ -615,6 +621,8 @@ LockedFlake lockFlake(
         // Bring in the current ref for relative path resolution if we have it
         auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true);
 
+        nodePaths.emplace(newLockFile.root, flake.storePath);
+
         computeLocks(
             flake.inputs,
             newLockFile.root,
@@ -707,14 +715,6 @@ LockedFlake lockFlake(
                             flake.lockedRef.input.getRev() &&
                             prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
                             warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
-
-                        /* Make sure that we picked up the change,
-                           i.e. the tree should usually be dirty
-                           now. Corner case: we could have reverted from a
-                           dirty to a clean tree! */
-                        if (flake.lockedRef.input == prevLockedRef.input
-                            && !flake.lockedRef.input.isLocked())
-                            throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
                     }
                 } else
                     throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
@@ -724,7 +724,11 @@ LockedFlake lockFlake(
             }
         }
 
-        return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
+        return LockedFlake {
+            .flake = std::move(flake),
+            .lockFile = std::move(newLockFile),
+            .nodePaths = std::move(nodePaths)
+        };
 
     } catch (Error & e) {
         e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string());
@@ -736,30 +740,48 @@ void callFlake(EvalState & state,
     const LockedFlake & lockedFlake,
     Value & vRes)
 {
-    auto vLocks = state.allocValue();
-    auto vRootSrc = state.allocValue();
-    auto vRootSubdir = state.allocValue();
-    auto vTmp1 = state.allocValue();
-    auto vTmp2 = state.allocValue();
+    experimentalFeatureSettings.require(Xp::Flakes);
 
-    vLocks->mkString(lockedFlake.lockFile.to_string());
+    auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string();
 
-    emitTreeAttrs(
-        state,
-        lockedFlake.flake.storePath,
-        lockedFlake.flake.lockedRef.input,
-        *vRootSrc,
-        false,
-        lockedFlake.flake.forceDirty);
+    auto overrides = state.buildBindings(lockedFlake.nodePaths.size());
 
-    vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
+    for (auto & [node, storePath] : lockedFlake.nodePaths) {
+        auto override = state.buildBindings(2);
+
+        auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo"));
+
+        auto lockedNode = node.dynamic_pointer_cast<const LockedNode>();
+
+        emitTreeAttrs(
+            state,
+            storePath,
+            lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input,
+            vSourceInfo,
+            false,
+            !lockedNode && lockedFlake.flake.forceDirty);
+
+        auto key = keyMap.find(node);
+        assert(key != keyMap.end());
+
+        override
+            .alloc(state.symbols.create("dir"))
+            .mkString(lockedNode ? lockedNode->lockedRef.subdir : lockedFlake.flake.lockedRef.subdir);
+
+        overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
+    }
+
+    auto & vOverrides = state.allocValue()->mkAttrs(overrides);
 
     auto vCallFlake = state.allocValue();
     state.evalFile(state.callFlakeInternal, *vCallFlake);
 
+    auto vTmp1 = state.allocValue();
+    auto vLocks = state.allocValue();
+    vLocks->mkString(lockFileStr);
     state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
-    state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
-    state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
+
+    state.callFunction(*vTmp1, vOverrides, vRes, noPos);
 }
 
 static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v)

src/libexpr/flake/flake.hh

@@ -103,6 +103,13 @@ struct LockedFlake
     Flake flake;
     LockFile lockFile;
 
+    /**
+     * Store paths of nodes that have been fetched in
+     * lockFlake(); in particular, the root node and the overriden
+     * inputs.
+     */
+    std::map<ref<Node>, StorePath> nodePaths;
+
     Fingerprint getFingerprint() const;
 };
 

src/libexpr/flake/lockfile.cc

@@ -38,7 +38,7 @@ LockedNode::LockedNode(const nlohmann::json & json)
     , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
 {
     if (!lockedRef.input.isLocked())
-        throw Error("lock file contains mutable lock '%s'",
+        throw Error("lock file contains unlocked input '%s'",
             fetchers::attrsToJSON(lockedRef.input.toAttrs()));
 }
 
@@ -107,7 +107,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
                 std::string inputKey = i.value();
                 auto k = nodeMap.find(inputKey);
                 if (k == nodeMap.end()) {
-                    auto nodes = json["nodes"];
+                    auto & nodes = json["nodes"];
                     auto jsonNode2 = nodes.find(inputKey);
                     if (jsonNode2 == nodes.end())
                         throw Error("lock file references missing node '%s'", inputKey);
@@ -134,10 +134,10 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
     // a bit since we don't need to worry about cycles.
 }
 
-nlohmann::json LockFile::toJSON() const
+std::pair<nlohmann::json, LockFile::KeyMap> LockFile::toJSON() const
 {
     nlohmann::json nodes;
-    std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
+    KeyMap nodeKeys;
     std::unordered_set<std::string> keys;
 
     std::function<std::string(const std::string & key, ref<const Node> node)> dumpNode;
@@ -194,12 +194,13 @@ nlohmann::json LockFile::toJSON() const
     json["root"] = dumpNode("root", root);
     json["nodes"] = std::move(nodes);
 
-    return json;
+    return {json, std::move(nodeKeys)};
 }
 
-std::string LockFile::to_string() const
+std::pair<std::string, LockFile::KeyMap> LockFile::to_string() const
 {
-    return toJSON().dump(2);
+    auto [json, nodeKeys] = toJSON();
+    return {json.dump(2), std::move(nodeKeys)};
 }
 
 LockFile LockFile::read(const Path & path)
@@ -210,7 +211,7 @@ LockFile LockFile::read(const Path & path)
 
 std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
 {
-    stream << lockFile.toJSON().dump(2);
+    stream << lockFile.toJSON().first.dump(2);
     return stream;
 }
 
@@ -243,7 +244,7 @@ std::optional<FlakeRef> LockFile::isUnlocked() const
 bool LockFile::operator ==(const LockFile & other) const
 {
     // FIXME: slow
-    return toJSON() == other.toJSON();
+    return toJSON().first == other.toJSON().first;
 }
 
 bool LockFile::operator !=(const LockFile & other) const

src/libexpr/flake/lockfile.hh

@@ -59,14 +59,15 @@ struct LockFile
 
     typedef std::map<ref<const Node>, std::string> KeyMap;
 
-    nlohmann::json toJSON() const;
+    std::pair<nlohmann::json, KeyMap> toJSON() const;
 
-    std::string to_string() const;
+    std::pair<std::string, KeyMap> to_string() const;
 
     static LockFile read(const Path & path);
 
     /**
-     * Check whether this lock file has any unlocked inputs.
+     * Check whether this lock file has any unlocked inputs. If so,
+     * return one.
      */
     std::optional<FlakeRef> isUnlocked() const;
 

src/libexpr/primops.cc

@@ -760,15 +760,6 @@ static RegisterPrimOp primop_break({
 
             auto & dt = state.debugTraces.front();
             state.runDebugRepl(&error, dt.env, dt.expr);
-
-            if (state.debugQuit) {
-                // If the user elects to quit the repl, throw an exception.
-                throw Error(ErrorInfo{
-                    .level = lvlInfo,
-                    .msg = HintFmt("quit the debugger"),
-                    .pos = nullptr,
-                });
-            }
         }
 
         // Return the value we were passed.
@@ -879,7 +870,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va
     /* increment state.trylevel, and decrement it when this function returns. */
     MaintainCount trylevel(state.trylevel);
 
-    void (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
+    ReplExitStatus (* savedDebugRepl)(ref<EvalState> es, const ValMap & extraEnv) = nullptr;
     if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry)
     {
         /* to prevent starting the repl from exceptions withing a tryEval, null it. */
@@ -995,6 +986,10 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu
         printError("trace: %1%", args[0]->string_view());
     else
         printError("trace: %1%", ValuePrinter(state, *args[0]));
+    if (evalSettings.builtinsTraceDebugger && state.debugRepl && !state.debugTraces.empty()) {
+        const DebugTrace & last = state.debugTraces.front();
+        state.runDebugRepl(nullptr, last.env, last.expr);
+    }
     state.forceValue(*args[1], pos);
     v = *args[1];
 }
@@ -1006,6 +1001,12 @@ static RegisterPrimOp primop_trace({
       Evaluate *e1* and print its abstract syntax representation on
       standard error. Then return *e2*. This function is useful for
       debugging.
+
+      If the
+      [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace)
+      option is set to `true` and the `--debugger` flag is given, the
+      interactive debugger will be started when `trace` is called (like
+      [`break`](@docroot@/language/builtins.md#builtins-break)).
     )",
     .fun = prim_trace,
 });
@@ -2231,7 +2232,14 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = fetchToStore(*state.store, path.resolveSymlinks(), name, method, filter.get(), state.repair);
+            auto dstPath = fetchToStore(
+                *state.store,
+                path.resolveSymlinks(),
+                settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
+                name,
+                method,
+                filter.get(),
+                state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.error<EvalError>(
                     "store path mismatch in (possibly filtered) path added from '%s'",

src/libexpr/primops/fetchTree.cc

@@ -9,6 +9,7 @@
 #include "tarball.hh"
 #include "url.hh"
 #include "value-to-json.hh"
+#include "fetch-to-store.hh"
 
 #include <ctime>
 #include <iomanip>
@@ -24,8 +25,6 @@ void emitTreeAttrs(
     bool emptyRevFallback,
     bool forceDirty)
 {
-    assert(input.isLocked());
-
     auto attrs = state.buildBindings(100);
 
     state.mkStorePathString(storePath, attrs.alloc(state.sOutPath));
@@ -176,8 +175,8 @@ static void fetchTree(
             fetcher = "fetchGit";
 
         state.error<EvalError>(
-            "in pure evaluation mode, %s requires a locked input",
-            fetcher
+            "in pure evaluation mode, '%s' will not fetch unlocked input '%s'",
+            fetcher, input.to_string()
         ).atPos(pos).debugThrow();
     }
 
@@ -473,7 +472,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     //       https://github.com/NixOS/nix/issues/4313
     auto storePath =
         unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+        ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
     if (expectedHash) {

src/libexpr/repl-exit-status.hh

@@ -0,0 +1,20 @@
+#pragma once
+
+namespace nix {
+
+/**
+ * Exit status returned from the REPL.
+ */
+enum class ReplExitStatus {
+    /**
+     * The user exited with `:quit`. The program (e.g., if the REPL was acting
+     * as the debugger) should exit.
+     */
+    QuitAll,
+    /**
+     * The user exited with `:continue`. The program should continue running.
+     */
+    Continue,
+};
+
+}

src/libfetchers/fetch-to-store.cc

@@ -7,6 +7,7 @@ namespace nix {
 StorePath fetchToStore(
     Store & store,
     const SourcePath & path,
+    FetchMode mode,
     std::string_view name,
     ContentAddressMethod method,
     PathFilter * filter,
@@ -33,18 +34,19 @@ StorePath fetchToStore(
     } else
         debug("source path '%s' is uncacheable", path);
 
-    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path));
+    Activity act(*logger, lvlChatty, actUnknown,
+        fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path));
 
     auto filter2 = filter ? *filter : defaultPathFilter;
 
     auto storePath =
-        settings.readOnlyMode
+        mode == FetchMode::DryRun
         ? store.computeStorePath(
             name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first
         : store.addToStore(
             name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
 
-    if (cacheKey)
+    if (cacheKey && mode == FetchMode::Copy)
         fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
 
     return storePath;

src/libfetchers/fetch-to-store.hh

@@ -8,12 +8,15 @@
 
 namespace nix {
 
+enum struct FetchMode { DryRun, Copy };
+
 /**
  * Copy the `path` to the Nix store.
  */
 StorePath fetchToStore(
     Store & store,
     const SourcePath & path,
+    FetchMode mode,
     std::string_view name = "source",
     ContentAddressMethod method = FileIngestionMethod::Recursive,
     PathFilter * filter = nullptr,

src/libfetchers/fetchers.cc

@@ -45,12 +45,8 @@ static void fixupInput(Input & input)
     // Check common attributes.
     input.getType();
     input.getRef();
-    if (input.getRev())
-        input.locked = true;
     input.getRevCount();
     input.getLastModified();
-    if (input.getNarHash())
-        input.locked = true;
 }
 
 Input Input::fromURL(const ParsedURL & url, bool requireTree)
@@ -140,6 +136,11 @@ bool Input::isDirect() const
     return !scheme || scheme->isDirect(*this);
 }
 
+bool Input::isLocked() const
+{
+    return scheme && scheme->isLocked(*this);
+}
+
 Attrs Input::toAttrs() const
 {
     return attrs;
@@ -222,8 +223,6 @@ std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
                 input.to_string(), *prevRevCount);
     }
 
-    input.locked = true;
-
     return {std::move(storePath), input};
 }
 
@@ -376,7 +375,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = fetchToStore(*store, SourcePath(accessor), input2.getName());
+    auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, input2.getName());
     return {storePath, input2};
 }
 

src/libfetchers/fetchers.hh

@@ -29,7 +29,6 @@ struct Input
 
     std::shared_ptr<InputScheme> scheme; // note: can be null
     Attrs attrs;
-    bool locked = false;
 
     /**
      * path of the parent of this input, used for relative path resolution
@@ -71,7 +70,7 @@ public:
      * Check whether this is a "locked" input, that is,
      * one that contains a commit hash or content hash.
      */
-    bool isLocked() const { return locked; }
+    bool isLocked() const;
 
     bool operator ==(const Input & other) const;
 
@@ -121,7 +120,6 @@ public:
     std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
 
-
 /**
  * The `InputScheme` represents a type of fetcher.  Each fetcher
  * registers with nix at startup time.  When processing an `Input`,
@@ -196,6 +194,14 @@ struct InputScheme
      */
     virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     { return std::nullopt; }
+
+    /**
+     * Return `true` if this input is considered "locked", i.e. it has
+     * attributes like a Git revision or NAR hash that uniquely
+     * identify its contents.
+     */
+    virtual bool isLocked(const Input & input) const
+    { return false; }
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

src/libfetchers/filtering-input-accessor.cc

@@ -51,33 +51,33 @@ void FilteringInputAccessor::checkAccess(const CanonPath & path)
 
 struct AllowListInputAccessorImpl : AllowListInputAccessor
 {
-    std::set<CanonPath> allowedPaths;
+    std::set<CanonPath> allowedPrefixes;
 
     AllowListInputAccessorImpl(
         ref<InputAccessor> next,
-        std::set<CanonPath> && allowedPaths,
+        std::set<CanonPath> && allowedPrefixes,
         MakeNotAllowedError && makeNotAllowedError)
         : AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError))
-        , allowedPaths(std::move(allowedPaths))
+        , allowedPrefixes(std::move(allowedPrefixes))
     { }
 
     bool isAllowed(const CanonPath & path) override
     {
-        return path.isAllowed(allowedPaths);
+        return path.isAllowed(allowedPrefixes);
     }
 
-    void allowPath(CanonPath path) override
+    void allowPrefix(CanonPath prefix) override
     {
-        allowedPaths.insert(std::move(path));
+        allowedPrefixes.insert(std::move(prefix));
     }
 };
 
 ref<AllowListInputAccessor> AllowListInputAccessor::create(
     ref<InputAccessor> next,
-    std::set<CanonPath> && allowedPaths,
+    std::set<CanonPath> && allowedPrefixes,
     MakeNotAllowedError && makeNotAllowedError)
 {
-    return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPaths), std::move(makeNotAllowedError));
+    return make_ref<AllowListInputAccessorImpl>(next, std::move(allowedPrefixes), std::move(makeNotAllowedError));
 }
 
 bool CachingFilteringInputAccessor::isAllowed(const CanonPath & path)

src/libfetchers/filtering-input-accessor.hh

@@ -54,18 +54,19 @@ struct FilteringInputAccessor : InputAccessor
 };
 
 /**
- * A wrapping `InputAccessor` that checks paths against an allow-list.
+ * A wrapping `InputAccessor` that checks paths against a set of
+ * allowed prefixes.
  */
 struct AllowListInputAccessor : public FilteringInputAccessor
 {
     /**
-     * Grant access to the specified path.
+     * Grant access to the specified prefix.
      */
-    virtual void allowPath(CanonPath path) = 0;
+    virtual void allowPrefix(CanonPath prefix) = 0;
 
     static ref<AllowListInputAccessor> create(
         ref<InputAccessor> next,
-        std::set<CanonPath> && allowedPaths,
+        std::set<CanonPath> && allowedPrefixes,
         MakeNotAllowedError && makeNotAllowedError);
 
     using FilteringInputAccessor::FilteringInputAccessor;

src/libfetchers/git-utils.cc

@@ -2,6 +2,7 @@
 #include "fs-input-accessor.hh"
 #include "input-accessor.hh"
 #include "filtering-input-accessor.hh"
+#include "memory-input-accessor.hh"
 #include "cache.hh"
 #include "finally.hh"
 #include "processes.hh"
@@ -466,6 +467,22 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         else
             throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output);
     }
+
+    Hash treeHashToNarHash(const Hash & treeHash) override
+    {
+        auto accessor = getAccessor(treeHash, false);
+
+        fetchers::Attrs cacheKey({{"_what", "treeHashToNarHash"}, {"treeHash", treeHash.gitRev()}});
+
+        if (auto res = fetchers::getCache()->lookup(cacheKey))
+            return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
+
+        auto narHash = accessor->hashPath(CanonPath::root);
+
+        fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
+
+        return narHash;
+    }
 };
 
 ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare)
@@ -942,17 +959,21 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
 ref<InputAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
+    /* In case of an empty workdir, return an empty in-memory tree. We
+       cannot use AllowListInputAccessor because it would return an
+       error for the root (and we can't add the root to the allow-list
+       since that would allow access to all its children). */
     ref<InputAccessor> fileAccessor =
-        AllowListInputAccessor::create(
-                makeFSInputAccessor(path),
-                std::set<CanonPath> { wd.files },
-                std::move(makeNotAllowedError));
-    if (exportIgnore) {
+        wd.files.empty()
+        ? makeEmptyInputAccessor()
+        : AllowListInputAccessor::create(
+            makeFSInputAccessor(path),
+            std::set<CanonPath> { wd.files },
+            std::move(makeNotAllowedError)).cast<InputAccessor>();
+    if (exportIgnore)
         return make_ref<GitExportIgnoreInputAccessor>(self, fileAccessor, std::nullopt);
-    }
-    else {
+    else
         return fileAccessor;
-    }
 }
 
 ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()

src/libfetchers/git-utils.hh

@@ -93,6 +93,12 @@ struct GitRepo
     virtual void verifyCommit(
         const Hash & rev,
         const std::vector<fetchers::PublicKey> & publicKeys) = 0;
+
+    /**
+     * Given a Git tree hash, compute the hash of its NAR
+     * serialisation. This is memoised on-disk.
+     */
+    virtual Hash treeHashToNarHash(const Hash & treeHash) = 0;
 };
 
 ref<GitRepo> getTarballCache();

src/libfetchers/git.cc

@@ -158,6 +158,8 @@ std::vector<PublicKey> getPublicKeys(const Attrs & attrs)
 
 }  // end namespace
 
+static const Hash nullRev{HashAlgorithm::SHA1};
+
 struct GitInputScheme : InputScheme
 {
     std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
@@ -708,10 +710,12 @@ struct GitInputScheme : InputScheme
             if (auto ref = repo->getWorkdirRef())
                 input.attrs.insert_or_assign("ref", *ref);
 
-            auto rev = repoInfo.workdirInfo.headRev.value();
+            /* Return a rev of 000... if there are no commits yet. */
+            auto rev = repoInfo.workdirInfo.headRev.value_or(nullRev);
 
             input.attrs.insert_or_assign("rev", rev.gitRev());
-            input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev));
+            input.attrs.insert_or_assign("revCount",
+                rev == nullRev ? 0 : getRevCount(repoInfo, repoInfo.url, rev));
 
             verifyCommit(input, repo);
         } else {
@@ -733,8 +737,6 @@ struct GitInputScheme : InputScheme
             ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev)
             : 0);
 
-        input.locked = true; // FIXME
-
         return {accessor, std::move(input)};
     }
 
@@ -771,6 +773,11 @@ struct GitInputScheme : InputScheme
         else
             return std::nullopt;
     }
+
+    bool isLocked(const Input & input) const override
+    {
+        return (bool) input.getRev();
+    }
 };
 
 static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });

src/libfetchers/github.cc

@@ -280,6 +280,15 @@ struct GitArchiveInputScheme : InputScheme
         return {accessor, input};
     }
 
+    bool isLocked(const Input & input) const override
+    {
+        /* Since we can't verify the integrity of the tarball from the
+           Git revision alone, we also require a NAR hash for
+           locking. FIXME: in the future, we may want to require a Git
+           tree hash instead of a NAR hash. */
+        return input.getRev().has_value() && input.getNarHash().has_value();
+    }
+
     std::optional<ExperimentalFeature> experimentalFeature() const override
     {
         return Xp::Flakes;

src/libfetchers/memory-input-accessor.cc

@@ -20,4 +20,10 @@ ref<MemoryInputAccessor> makeMemoryInputAccessor()
     return make_ref<MemoryInputAccessorImpl>();
 }
 
+ref<InputAccessor> makeEmptyInputAccessor()
+{
+    static auto empty = makeMemoryInputAccessor().cast<InputAccessor>();
+    return empty;
+}
+
 }

src/libfetchers/memory-input-accessor.hh

@@ -13,4 +13,6 @@ struct MemoryInputAccessor : InputAccessor
 
 ref<MemoryInputAccessor> makeMemoryInputAccessor();
 
+ref<InputAccessor> makeEmptyInputAccessor();
+
 }

src/libfetchers/mercurial.cc

@@ -347,6 +347,11 @@ struct MercurialInputScheme : InputScheme
         return makeResult(infoAttrs, std::move(storePath));
     }
 
+    bool isLocked(const Input & input) const override
+    {
+        return (bool) input.getRev();
+    }
+
     std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto rev = input.getRev())

src/libfetchers/path.cc

@@ -87,6 +87,11 @@ struct PathInputScheme : InputScheme
         writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents);
     }
 
+    bool isLocked(const Input & input) const override
+    {
+        return (bool) input.getNarHash();
+    }
+
     CanonPath getAbsPath(const Input & input) const
     {
         auto path = getStrAttr(input.attrs, "path");

src/libfetchers/tarball.cc

@@ -9,6 +9,9 @@
 #include "types.hh"
 #include "split.hh"
 #include "posix-source-accessor.hh"
+#include "fs-input-accessor.hh"
+#include "store-api.hh"
+#include "git-utils.hh"
 
 namespace nix::fetchers {
 
@@ -57,10 +60,8 @@ DownloadFileResult downloadFile(
             throw;
     }
 
-    // FIXME: write to temporary file.
     Attrs infoAttrs({
         {"etag", res.etag},
-        {"url", res.effectiveUri},
     });
 
     if (res.immutableUrl)
@@ -91,96 +92,102 @@ DownloadFileResult downloadFile(
         storePath = std::move(info.path);
     }
 
-    getCache()->add(
-        *store,
-        inAttrs,
-        infoAttrs,
-        *storePath,
-        locked);
-
-    if (url != res.effectiveUri)
+    /* Cache metadata for all URLs in the redirect chain. */
+    for (auto & url : res.urls) {
+        inAttrs.insert_or_assign("url", url);
+        infoAttrs.insert_or_assign("url", *res.urls.rbegin());
         getCache()->add(
             *store,
-            {
-                {"type", "file"},
-                {"url", res.effectiveUri},
-                {"name", name},
-            },
+            inAttrs,
             infoAttrs,
             *storePath,
             locked);
+    }
 
     return {
         .storePath = std::move(*storePath),
         .etag = res.etag,
-        .effectiveUrl = res.effectiveUri,
+        .effectiveUrl = *res.urls.rbegin(),
         .immutableUrl = res.immutableUrl,
     };
 }
 
 DownloadTarballResult downloadTarball(
-    ref<Store> store,
     const std::string & url,
-    const std::string & name,
-    bool locked,
     const Headers & headers)
 {
     Attrs inAttrs({
-        {"type", "tarball"},
+        {"_what", "tarballCache"},
         {"url", url},
-        {"name", name},
     });
 
-    auto cached = getCache()->lookupExpired(*store, inAttrs);
+    auto cached = getCache()->lookupExpired(inAttrs);
+
+    auto attrsToResult = [&](const Attrs & infoAttrs)
+    {
+        auto treeHash = getRevAttr(infoAttrs, "treeHash");
+        return DownloadTarballResult {
+            .treeHash = treeHash,
+            .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
+            .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
+            .accessor = getTarballCache()->getAccessor(treeHash, false),
+        };
+    };
+
+    if (cached && !getTarballCache()->hasObject(getRevAttr(cached->infoAttrs, "treeHash")))
+        cached.reset();
 
     if (cached && !cached->expired)
-        return {
-            .storePath = std::move(cached->storePath),
-            .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
-            .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
-        };
+        /* We previously downloaded this tarball and it's younger than
+           `tarballTtl`, so no need to check the server. */
+        return attrsToResult(cached->infoAttrs);
 
-    auto res = downloadFile(store, url, name, locked, headers);
+    auto _res = std::make_shared<Sync<FileTransferResult>>();
 
-    std::optional<StorePath> unpackedStorePath;
-    time_t lastModified;
-
-    if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) {
-        unpackedStorePath = std::move(cached->storePath);
-        lastModified = getIntAttr(cached->infoAttrs, "lastModified");
-    } else {
-        Path tmpDir = createTempDir();
-        AutoDelete autoDelete(tmpDir, true);
-        unpackTarfile(store->toRealPath(res.storePath), tmpDir);
-        auto members = readDirectory(tmpDir);
-        if (members.size() != 1)
-            throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
-        auto topDir = tmpDir + "/" + members.begin()->name;
-        lastModified = lstat(topDir).st_mtime;
-        PosixSourceAccessor accessor;
-        unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, defaultPathFilter, NoRepair);
-    }
-
-    Attrs infoAttrs({
-        {"lastModified", uint64_t(lastModified)},
-        {"etag", res.etag},
+    auto source = sinkToSource([&](Sink & sink) {
+        FileTransferRequest req(url);
+        req.expectedETag = cached ? getStrAttr(cached->infoAttrs, "etag") : "";
+        getFileTransfer()->download(std::move(req), sink,
+            [_res](FileTransferResult r)
+            {
+                *_res->lock() = r;
+            });
     });
 
-    if (res.immutableUrl)
-        infoAttrs.emplace("immutableUrl", *res.immutableUrl);
+    // TODO: fall back to cached value if download fails.
 
-    getCache()->add(
-        *store,
-        inAttrs,
-        infoAttrs,
-        *unpackedStorePath,
-        locked);
+    /* Note: if the download is cached, `importTarball()` will receive
+       no data, which causes it to import an empty tarball. */
+    TarArchive archive { *source };
+    auto parseSink = getTarballCache()->getFileSystemObjectSink();
+    auto lastModified = unpackTarfileToSink(archive, *parseSink);
 
-    return {
-        .storePath = std::move(*unpackedStorePath),
-        .lastModified = lastModified,
-        .immutableUrl = res.immutableUrl,
-    };
+    auto res(_res->lock());
+
+    Attrs infoAttrs;
+
+    if (res->cached) {
+        /* The server says that the previously downloaded version is
+           still current. */
+        infoAttrs = cached->infoAttrs;
+    } else {
+        infoAttrs.insert_or_assign("etag", res->etag);
+        infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev());
+        infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified));
+        if (res->immutableUrl)
+            infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl);
+    }
+
+    /* Insert a cache entry for every URL in the redirect chain. */
+    for (auto & url : res->urls) {
+        inAttrs.insert_or_assign("url", url);
+        getCache()->upsert(inAttrs, infoAttrs);
+    }
+
+    // FIXME: add a cache entry for immutableUrl? That could allow
+    // cache poisoning.
+
+    return attrsToResult(infoAttrs);
 }
 
 // An input scheme corresponding to a curl-downloadable resource.
@@ -198,6 +205,8 @@ struct CurlInputScheme : InputScheme
 
     virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0;
 
+    static const std::set<std::string> specialParams;
+
     std::optional<Input> inputFromURL(const ParsedURL & _url, bool requireTree) const override
     {
         if (!isValidURL(_url, requireTree))
@@ -220,8 +229,17 @@ struct CurlInputScheme : InputScheme
             if (auto n = string2Int<uint64_t>(*i))
                 input.attrs.insert_or_assign("revCount", *n);
 
-        url.query.erase("rev");
-        url.query.erase("revCount");
+        if (auto i = get(url.query, "lastModified"))
+            if (auto n = string2Int<uint64_t>(*i))
+                input.attrs.insert_or_assign("lastModified", *n);
+
+        /* The URL query parameters serve two roles: specifying fetch
+           settings for Nix itself, and arbitrary data as part of the
+           HTTP request. Now that we've processed the Nix-specific
+           attributes above, remove them so we don't also send them as
+           part of the HTTP request. */
+        for (auto & param : allowedAttrs())
+            url.query.erase(param);
 
         input.attrs.insert_or_assign("type", std::string { schemeName() });
         input.attrs.insert_or_assign("url", url.to_string());
@@ -260,6 +278,11 @@ struct CurlInputScheme : InputScheme
             url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
         return url;
     }
+
+    bool isLocked(const Input & input) const override
+    {
+        return (bool) input.getNarHash();
+    }
 };
 
 struct FileInputScheme : CurlInputScheme
@@ -275,10 +298,24 @@ struct FileInputScheme : CurlInputScheme
                 : (!requireTree && !hasTarballExtension(url.path)));
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
     {
+        auto input(_input);
+
+        /* Unlike TarballInputScheme, this stores downloaded files in
+           the Nix store directly, since there is little deduplication
+           benefit in using the Git cache for single big files like
+           tarballs. */
         auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false);
-        return {std::move(file.storePath), input};
+
+        auto narHash = store->queryPathInfo(file.storePath)->narHash;
+        input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
+
+        auto accessor = makeStorePathAccessor(store, file.storePath);
+
+        accessor->setPathDisplay("«" + input.to_string() + "»");
+
+        return {accessor, input};
     }
 };
 
@@ -296,11 +333,13 @@ struct TarballInputScheme : CurlInputScheme
                 : (requireTree || hasTarballExtension(url.path)));
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
     {
-        Input input(_input);
-        auto url = getStrAttr(input.attrs, "url");
-        auto result = downloadTarball(store, url, input.getName(), false);
+        auto input(_input);
+
+        auto result = downloadTarball(getStrAttr(input.attrs, "url"), {});
+
+        result.accessor->setPathDisplay("«" + input.to_string() + "»");
 
         if (result.immutableUrl) {
             auto immutableInput = Input::fromURL(*result.immutableUrl);
@@ -314,7 +353,10 @@ struct TarballInputScheme : CurlInputScheme
         if (result.lastModified && !input.attrs.contains("lastModified"))
             input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
-        return {result.storePath, std::move(input)};
+        input.attrs.insert_or_assign("narHash",
+            getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true));
+
+        return {result.accessor, input};
     }
 };
 

src/libfetchers/tarball.hh

@@ -2,11 +2,13 @@
 
 #include "types.hh"
 #include "path.hh"
+#include "hash.hh"
 
 #include <optional>
 
 namespace nix {
 class Store;
+struct InputAccessor;
 }
 
 namespace nix::fetchers {
@@ -28,16 +30,18 @@ DownloadFileResult downloadFile(
 
 struct DownloadTarballResult
 {
-    StorePath storePath;
+    Hash treeHash;
     time_t lastModified;
     std::optional<std::string> immutableUrl;
+    ref<InputAccessor> accessor;
 };
 
+/**
+ * Download and import a tarball into the Git cache. The result is the
+ * Git tree hash of the root directory.
+ */
 DownloadTarballResult downloadTarball(
-    ref<Store> store,
     const std::string & url,
-    const std::string & name,
-    bool locked,
     const Headers & headers = {});
 
 }

src/libmain/shared.cc

@@ -408,6 +408,4 @@ PrintFreed::~PrintFreed()
             showBytes(results.bytesFreed));
 }
 
-Exit::~Exit() { }
-
 }

src/libmain/shared.hh

@@ -7,6 +7,7 @@
 #include "common-args.hh"
 #include "path.hh"
 #include "derived-path.hh"
+#include "exit.hh"
 
 #include <signal.h>
 
@@ -15,15 +16,6 @@
 
 namespace nix {
 
-class Exit : public std::exception
-{
-public:
-    int status;
-    Exit() : status(0) { }
-    Exit(int status) : status(status) { }
-    virtual ~Exit();
-};
-
 int handleExceptions(const std::string & programName, std::function<void()> fun);
 
 /**

src/libstore/filetransfer.cc

@@ -106,6 +106,8 @@ struct curlFileTransfer : public FileTransfer
                     this->result.data.append(data);
               })
         {
+            result.urls.push_back(request.uri);
+
             requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz");
             if (!request.expectedETag.empty())
                 requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
@@ -182,6 +184,14 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
         }
 
+        void appendCurrentUrl()
+        {
+            char * effectiveUriCStr = nullptr;
+            curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
+            if (effectiveUriCStr && *result.urls.rbegin() != effectiveUriCStr)
+                result.urls.push_back(effectiveUriCStr);
+        }
+
         size_t headerCallback(void * contents, size_t size, size_t nmemb)
         {
             size_t realSize = size * nmemb;
@@ -196,6 +206,7 @@ struct curlFileTransfer : public FileTransfer
                 statusMsg = trim(match.str(1));
                 acceptRanges = false;
                 encoding = "";
+                appendCurrentUrl();
             } else {
 
                 auto i = line.find(':');
@@ -360,14 +371,11 @@ struct curlFileTransfer : public FileTransfer
         {
             auto httpStatus = getHTTPStatus();
 
-            char * effectiveUriCStr = nullptr;
-            curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
-            if (effectiveUriCStr)
-                result.effectiveUri = effectiveUriCStr;
-
             debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
                 request.verb(), request.uri, code, httpStatus, result.bodySize);
 
+            appendCurrentUrl();
+
             if (decompressionSink) {
                 try {
                     decompressionSink->finish();
@@ -779,7 +787,10 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
     return enqueueFileTransfer(request).get();
 }
 
-void FileTransfer::download(FileTransferRequest && request, Sink & sink)
+void FileTransfer::download(
+    FileTransferRequest && request,
+    Sink & sink,
+    std::function<void(FileTransferResult)> resultCallback)
 {
     /* Note: we can't call 'sink' via request.dataCallback, because
        that would cause the sink to execute on the fileTransfer
@@ -829,11 +840,13 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
     };
 
     enqueueFileTransfer(request,
-        {[_state](std::future<FileTransferResult> fut) {
+        {[_state, resultCallback{std::move(resultCallback)}](std::future<FileTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
             try {
-                fut.get();
+                auto res = fut.get();
+                if (resultCallback)
+                    resultCallback(std::move(res));
             } catch (...) {
                 state->exc = std::current_exception();
             }

src/libstore/filetransfer.hh

@@ -75,14 +75,34 @@ struct FileTransferRequest
 
 struct FileTransferResult
 {
+    /**
+     * Whether this is a cache hit (i.e. the ETag supplied in the
+     * request is still valid). If so, `data` is empty.
+     */
     bool cached = false;
+
+    /**
+     * The ETag of the object.
+     */
     std::string etag;
-    std::string effectiveUri;
+
+    /**
+     * All URLs visited in the redirect chain.
+     */
+    std::vector<std::string> urls;
+
+    /**
+     * The response body.
+     */
     std::string data;
+
     uint64_t bodySize = 0;
-    /* An "immutable" URL for this resource (i.e. one whose contents
-       will never change), as returned by the `Link: <url>;
-       rel="immutable"` header. */
+
+    /**
+     * An "immutable" URL for this resource (i.e. one whose contents
+     * will never change), as returned by the `Link: <url>;
+     * rel="immutable"` header.
+     */
     std::optional<std::string> immutableUrl;
 };
 
@@ -116,7 +136,10 @@ struct FileTransfer
      * Download a file, writing its data to a sink. The sink will be
      * invoked on the thread of the caller.
      */
-    void download(FileTransferRequest && request, Sink & sink);
+    void download(
+        FileTransferRequest && request,
+        Sink & sink,
+        std::function<void(FileTransferResult)> resultCallback = {});
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };

src/libstore/gc.cc

@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "local-store.hh"
 #include "finally.hh"
+#include "find-roots.hh"
 #include "unix-domain-socket.hh"
 #include "signals.hh"
 
@@ -31,6 +32,7 @@ namespace nix {
 
 
 static std::string gcSocketPath = "/gc-socket/socket";
+static std::string rootsSocketPath = "/gc-roots-socket/socket";
 static std::string gcRootsDir = "gcroots";
 
 
@@ -143,7 +145,7 @@ void LocalStore::addTempRoot(const StorePath & path)
         auto fdRootsSocket(_fdRootsSocket.lock());
 
         if (!*fdRootsSocket) {
-            auto socketPath = stateDir.get() + gcSocketPath;
+            auto socketPath = stateDir.get() + rootsSocketPath;
             debug("connecting to '%s'", socketPath);
             *fdRootsSocket = createUnixDomainSocket();
             try {
@@ -241,79 +243,32 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
     }
 }
 
-
-void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
+void LocalStore::findRootsNoTempNoExternalDaemon(Roots & roots, bool censor)
 {
-    auto foundRoot = [&](const Path & path, const Path & target) {
-        try {
-            auto storePath = toStorePath(target).first;
-            if (isValidPath(storePath))
-                roots[std::move(storePath)].emplace(path);
-            else
-                printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
-        } catch (BadStorePath &) { }
+    debug("Can’t connect to the tracing daemon socket, fallback to the internal trace");
+
+    using namespace nix::roots_tracer;
+
+    const TracerConfig opts {
+      .storeDir = fs::path(storeDir),
+      .stateDir = fs::path(stateDir.get())
     };
-
-    try {
-
-        if (type == DT_UNKNOWN)
-            type = getFileType(path);
-
-        if (type == DT_DIR) {
-            for (auto & i : readDirectory(path))
-                findRoots(path + "/" + i.name, i.type, roots);
-        }
-
-        else if (type == DT_LNK) {
-            Path target = readLink(path);
-            if (isInStore(target))
-                foundRoot(path, target);
-
-            /* Handle indirect roots. */
-            else {
-                target = absPath(target, dirOf(path));
-                if (!pathExists(target)) {
-                    if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
-                        printInfo("removing stale link from '%1%' to '%2%'", path, target);
-                        unlink(path.c_str());
-                    }
-                } else {
-                    struct stat st2 = lstat(target);
-                    if (!S_ISLNK(st2.st_mode)) return;
-                    Path target2 = readLink(target);
-                    if (isInStore(target2)) foundRoot(target, target2);
-                }
-            }
-        }
-
-        else if (type == DT_REG) {
-            auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path)));
-            if (storePath && isValidPath(*storePath))
-                roots[std::move(*storePath)].emplace(path);
-        }
-
+    const std::set<fs::path> standardRoots = {
+        opts.stateDir / fs::path(gcRootsDir),
+        opts.stateDir / fs::path("gcroots"),
+    };
+    auto traceResult = traceStaticRoots(opts, standardRoots);
+    auto runtimeRoots = getRuntimeRoots(opts);
+    traceResult.storeRoots.insert(runtimeRoots.begin(), runtimeRoots.end());
+    for (auto & [rawRootInStore, externalRoots] : traceResult.storeRoots) {
+        if (!isInStore(rawRootInStore.string())) continue;
+        auto rootInStore = toStorePath(rawRootInStore.string()).first;
+        if (!isValidPath(rootInStore)) continue;
+        std::unordered_set<std::string> primRoots;
+        for (const auto & externalRoot : externalRoots)
+            primRoots.insert(externalRoot.string());
+        roots.emplace(rootInStore, primRoots);
     }
-
-    catch (SysError & e) {
-        /* We only ignore permanent failures. */
-        if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
-            printInfo("cannot read potential root '%1%'", path);
-        else
-            throw;
-    }
-}
-
-
-void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
-{
-    /* Process direct roots in {gcroots,profiles}. */
-    findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
-    findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
-
-    /* Add additional roots returned by different platforms-specific
-       heuristics.  This is typically used to add running programs to
-       the set of roots (to prevent them from being garbage collected). */
-    findRuntimeRoots(roots, censor);
 }
 
 
@@ -327,148 +282,53 @@ Roots LocalStore::findRoots(bool censor)
     return roots;
 }
 
+void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
+{
+
+    auto fd = createUnixDomainSocket();
+
+    std::string socketPath = settings.gcSocketPath.get() != "auto"
+        ? settings.gcSocketPath.get()
+        : stateDir.get() + gcSocketPath;
+
+    try {
+        nix::connect(fd.get(), socketPath);
+    } catch (SysError & e) {
+        return findRootsNoTempNoExternalDaemon(roots, censor);
+    }
+
+    experimentalFeatureSettings.require(Xp::ExternalGCDaemon);
+
+    try {
+        StringMap unescapes = {
+            { "\\n", "\n"},
+            { "\\t", "\t"},
+        };
+        while (true) {
+            auto line = readLine(fd.get());
+            if (line.empty()) break; // TODO: Handle the broken symlinks
+            auto parsedLine = tokenizeString<std::vector<std::string>>(line, "\t");
+            if (parsedLine.size() != 2)
+                throw Error("Invalid result from the gc helper");
+            auto rawDestPath = rewriteStrings(parsedLine[0], unescapes);
+            auto retainer = rewriteStrings(parsedLine[1], unescapes);
+            if (!isInStore(rawDestPath)) continue;
+            try {
+                auto destPath = toStorePath(rawDestPath).first;
+                if (!isValidPath(destPath)) continue;
+                roots[destPath].insert(
+                    (!censor || isInDir(retainer, stateDir.get())) ? retainer : censored);
+            } catch (Error &) {
+            }
+        }
+    } catch (EndOfFile &) {
+    }
+}
+
 typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
 
-static void readProcLink(const std::string & file, UncheckedRoots & roots)
-{
-    constexpr auto bufsiz = PATH_MAX;
-    char buf[bufsiz];
-    auto res = readlink(file.c_str(), buf, bufsiz);
-    if (res == -1) {
-        if (errno == ENOENT || errno == EACCES || errno == ESRCH)
-            return;
-        throw SysError("reading symlink");
-    }
-    if (res == bufsiz) {
-        throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
-    }
-    if (res > 0 && buf[0] == '/')
-        roots[std::string(static_cast<char *>(buf), res)]
-            .emplace(file);
-}
-
-static std::string quoteRegexChars(const std::string & raw)
-{
-    static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
-    return std::regex_replace(raw, specialRegex, R"(\$&)");
-}
-
-#if __linux__
-static void readFileRoots(const char * path, UncheckedRoots & roots)
-{
-    try {
-        roots[readFile(path)].emplace(path);
-    } catch (SysError & e) {
-        if (e.errNo != ENOENT && e.errNo != EACCES)
-            throw;
-    }
-}
-#endif
-
-void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
-{
-    UncheckedRoots unchecked;
-
-    auto procDir = AutoCloseDir{opendir("/proc")};
-    if (procDir) {
-        struct dirent * ent;
-        auto digitsRegex = std::regex(R"(^\d+$)");
-        auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
-        auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
-        while (errno = 0, ent = readdir(procDir.get())) {
-            checkInterrupt();
-            if (std::regex_match(ent->d_name, digitsRegex)) {
-                try {
-                    readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
-                    readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
-
-                    auto fdStr = fmt("/proc/%s/fd", ent->d_name);
-                    auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
-                    if (!fdDir) {
-                        if (errno == ENOENT || errno == EACCES)
-                            continue;
-                        throw SysError("opening %1%", fdStr);
-                    }
-                    struct dirent * fd_ent;
-                    while (errno = 0, fd_ent = readdir(fdDir.get())) {
-                        if (fd_ent->d_name[0] != '.')
-                            readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
-                    }
-                    if (errno) {
-                        if (errno == ESRCH)
-                            continue;
-                        throw SysError("iterating /proc/%1%/fd", ent->d_name);
-                    }
-                    fdDir.reset();
-
-                    auto mapFile = fmt("/proc/%s/maps", ent->d_name);
-                    auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
-                    for (const auto & line : mapLines) {
-                        auto match = std::smatch{};
-                        if (std::regex_match(line, match, mapRegex))
-                            unchecked[match[1]].emplace(mapFile);
-                    }
-
-                    auto envFile = fmt("/proc/%s/environ", ent->d_name);
-                    auto envString = readFile(envFile);
-                    auto env_end = std::sregex_iterator{};
-                    for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
-                        unchecked[i->str()].emplace(envFile);
-                } catch (SystemError & e) {
-                    if (errno == ENOENT || errno == EACCES || errno == ESRCH)
-                        continue;
-                    throw;
-                }
-            }
-        }
-        if (errno)
-            throw SysError("iterating /proc");
-    }
-
-#if !defined(__linux__)
-    // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
-    // See: https://github.com/NixOS/nix/issues/3011
-    // Because of this we disable lsof when running the tests.
-    if (getEnv("_NIX_TEST_NO_LSOF") != "1") {
-        try {
-            std::regex lsofRegex(R"(^n(/.*)$)");
-            auto lsofLines =
-                tokenizeString<std::vector<std::string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
-            for (const auto & line : lsofLines) {
-                std::smatch match;
-                if (std::regex_match(line, match, lsofRegex))
-                    unchecked[match[1]].emplace("{lsof}");
-            }
-        } catch (ExecError & e) {
-            /* lsof not installed, lsof failed */
-        }
-    }
-#endif
-
-#if __linux__
-    readFileRoots("/proc/sys/kernel/modprobe", unchecked);
-    readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
-    readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
-#endif
-
-    for (auto & [target, links] : unchecked) {
-        if (!isInStore(target)) continue;
-        try {
-            auto path = toStorePath(target).first;
-            if (!isValidPath(path)) continue;
-            debug("got additional root '%1%'", printStorePath(path));
-            if (censor)
-                roots[path].insert(censored);
-            else
-                roots[path].insert(links.begin(), links.end());
-        } catch (BadStorePath &) { }
-    }
-}
-
-
 struct GCLimitReached { };
 
-
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
     bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
@@ -516,7 +376,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         readFile(*p);
 
     /* Start the server for receiving new roots. */
-    auto socketPath = stateDir.get() + gcSocketPath;
+    auto socketPath = stateDir.get() + rootsSocketPath;
     createDirs(dirOf(socketPath));
     auto fdServer = createUnixDomainSocket(socketPath, 0666);
 
@@ -625,7 +485,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printInfo("finding garbage collector roots...");
     Roots rootMap;
     if (!options.ignoreLiveness)
-        findRootsNoTemp(rootMap, true);
+        rootMap = findRoots(true);
 
     for (auto & i : rootMap) roots.insert(i.first);
 

src/libstore/globals.hh

@@ -124,6 +124,13 @@ public:
           section of the manual for supported store types and settings.
         )"};
 
+    Setting<std::string> gcSocketPath {
+        this,
+        "auto",
+        "gc-socket-path",
+        "Path to the socket used to communicate with an external GC."
+    };
+
     Setting<bool> keepFailed{this, false, "keep-failed",
         "Whether to keep temporary directories of failed builds."};
 

src/libstore/local-store.hh

@@ -328,7 +328,7 @@ private:
 
     void findRootsNoTemp(Roots & roots, bool censor);
 
-    void findRuntimeRoots(Roots & roots, bool censor);
+    void findRootsNoTempNoExternalDaemon(Roots & roots, bool censor);
 
     std::pair<Path, AutoCloseFD> createTempDirInStore();
 

src/libstore/local.mk

@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
-libstore_LIBS = libutil
+libstore_LIBS = libutil libfindroots
 
 libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
@@ -28,7 +28,7 @@ ifeq ($(HAVE_SECCOMP), 1)
 endif
 
 libstore_CXXFLAGS += \
- -I src/libutil -I src/libstore -I src/libstore/build \
+ -I src/libutil -I src/libstore -I src/libstore/build -I src/nix-find-roots/lib \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \

src/libutil/args.cc

@@ -544,73 +544,6 @@ nlohmann::json Args::toJSON()
     return res;
 }
 
-static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
-{
-    for (auto & format : hashFormats) {
-        if (hasPrefix(format, prefix)) {
-            completions.add(format);
-        }
-    }
-}
-
-Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) {
-    assert(*hf == nix::HashFormat::SRI);
-    return Flag{
-            .longName = std::move(longName),
-            .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.",
-            .labels = {"hash-format"},
-            .handler = {[hf](std::string s) {
-                *hf = parseHashFormat(s);
-            }},
-            .completer = hashFormatCompleter,
-    };
-}
-
-Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf) {
-    return Flag{
-            .longName = std::move(longName),
-            .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).",
-            .labels = {"hash-format"},
-            .handler = {[ohf](std::string s) {
-                *ohf = std::optional<HashFormat>{parseHashFormat(s)};
-            }},
-            .completer = hashFormatCompleter,
-    };
-}
-
-static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix)
-{
-    for (auto & algo : hashAlgorithms)
-        if (hasPrefix(algo, prefix))
-            completions.add(algo);
-}
-
-Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha)
-{
-    return Flag{
-            .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
-            .labels = {"hash-algo"},
-            .handler = {[ha](std::string s) {
-                *ha = parseHashAlgo(s);
-            }},
-            .completer = hashAlgoCompleter,
-    };
-}
-
-Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha)
-{
-    return Flag{
-            .longName = std::move(longName),
-            .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
-            .labels = {"hash-algo"},
-            .handler = {[oha](std::string s) {
-                *oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};
-            }},
-            .completer = hashAlgoCompleter,
-    };
-}
-
 static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs)
 {
     completions.setType(Completions::Type::Filenames);

src/libutil/args.hh

@@ -155,6 +155,8 @@ protected:
      */
     using CompleterClosure = std::function<CompleterFun>;
 
+public:
+
     /**
      * Description of flags / options
      *
@@ -175,19 +177,10 @@ protected:
         CompleterClosure completer;
 
         std::optional<ExperimentalFeature> experimentalFeature;
-
-        static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
-        static Flag mkHashAlgoFlag(HashAlgorithm * ha) {
-            return mkHashAlgoFlag("hash-algo", ha);
-        }
-        static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
-        static Flag mkHashAlgoOptFlag(std::optional<HashAlgorithm> * oha) {
-            return mkHashAlgoOptFlag("hash-algo", oha);
-        }
-        static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf);
-        static Flag mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf);
     };
 
+protected:
+
     /**
      * Index of all registered "long" flag descriptions (flags like
      * `--long`).
@@ -206,6 +199,8 @@ protected:
      */
     virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
 
+public:
+
     /**
      * Description of positional arguments
      *
@@ -220,6 +215,8 @@ protected:
         CompleterClosure completer;
     };
 
+protected:
+
     /**
      * Queue of expected positional argument forms.
      *

src/libutil/config-impl.hh

@@ -4,7 +4,7 @@
  *
  * Template implementations (as opposed to mere declarations).
  *
- * This file is an exmample of the "impl.hh" pattern. See the
+ * This file is an example of the "impl.hh" pattern. See the
  * contributing guide.
  *
  * One only needs to include this when one is declaring a

src/libutil/config.cc

@@ -84,7 +84,9 @@ void AbstractConfig::reapplyUnknownSettings()
 void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
     for (const auto & opt : _settings)
-        if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
+        if (!opt.second.isAlias
+            && (!overriddenOnly || opt.second.setting->overridden)
+            && experimentalFeatureSettings.isEnabled(opt.second.setting->experimentalFeature))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 

src/libutil/exit.cc

@@ -0,0 +1,7 @@
+#include "exit.hh"
+
+namespace nix {
+
+Exit::~Exit() {}
+
+}

src/libutil/exit.hh

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <exception>
+
+namespace nix {
+
+/**
+ * Exit the program with a given exit code.
+ */
+class Exit : public std::exception
+{
+public:
+    int status;
+    Exit() : status(0) { }
+    explicit Exit(int status) : status(status) { }
+    virtual ~Exit();
+};
+
+}

src/libutil/experimental-features.cc

@@ -268,6 +268,20 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
         )",
     },
+    {
+        .tag = Xp::ExternalGCDaemon,
+        .name = "external-gc-daemon",
+        .description = R"(
+            Make the garbage collector use an external daemon for the tracing.
+
+            This makes it possible to run a multi-user Nix daemon as a non-root
+            user. Only the tracing daemon needs to be root. This reduces the attack
+            surface.
+
+            This requires more infrastructure and isn't directly supported by the
+            installer.
+        )"
+    },
     {
         .tag = Xp::VerifiedFetches,
         .name = "verified-fetches",

src/libutil/experimental-features.hh

@@ -35,6 +35,7 @@ enum struct ExperimentalFeature
     ReadOnlyLocalStore,
     ConfigurableImpureEnv,
     MountedSSHStore,
+    ExternalGCDaemon,
     VerifiedFetches,
 };
 

src/libutil/posix-source-accessor.cc

@@ -30,6 +30,11 @@ std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path)
 {
     return root.empty()
         ? (std::filesystem::path { path.abs() })
+        : path.isRoot()
+        ? /* Don't append a slash for the root of the accessor, since
+             it can be a non-directory (e.g. in the case of `fetchTree
+             { type = "file" }`). */
+          root
         : root / path.rel();
 }
 

src/nix-channel/nix-channel.cc

@@ -138,7 +138,7 @@ static void update(const StringSet & channelNames)
 
     // Unpack the channel tarballs into the Nix store and install them
     // into the channels profile.
-    std::cerr << "unpacking channels...\n";
+    std::cerr << "unpacking " << exprs.size() << " channels...\n";
     Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--remove-all", "--from-expression" };
     for (auto & expr : exprs)
         envArgs.push_back(std::move(expr));

src/nix-env/nix-env.cc

@@ -143,7 +143,7 @@ static void getAllExprs(EvalState & state,
             }
             /* Load the expression on demand. */
             auto vArg = state.allocValue();
-            vArg->mkString(path2.path.abs());
+            vArg->mkPath(path2);
             if (seen.size() == maxAttrs)
                 throw Error("too many Nix expressions in directory '%1%'", path);
             attrs.alloc(attrName).mkApp(&state.getBuiltin("import"), vArg);

src/nix-find-roots/.gitignore

@@ -0,0 +1 @@
+nix-find-roots

src/nix-find-roots/lib/find-roots.cc

@@ -0,0 +1,232 @@
+/**
+ * @file
+ *
+ * A very simple utility to trace all the gc roots through the file-system
+ * The reason for this program is that tracing these roots is the only part of
+ * Nix that requires to run as root (because it requires reading through the
+ * user home directories to resolve the indirect roots)
+ *
+ * This program intentionnally doesnt depend on any Nix library to reduce the attack surface.
+ */
+
+#include <regex>
+#include <unistd.h>
+#include <vector>
+#include <algorithm>
+#include <fstream>
+#include <optional>
+
+#include "find-roots.hh"
+
+
+namespace nix::roots_tracer {
+namespace fs = std::filesystem;
+
+static std::string quoteRegexChars(const std::string & raw)
+{
+    static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
+    return std::regex_replace(raw, specialRegex, R"(\$&)");
+}
+static std::regex storePathRegex(const fs::path storeDir)
+{
+    return std::regex(quoteRegexChars(storeDir) + R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)");
+}
+
+static bool isInStore(fs::path storeDir, fs::path dir)
+{
+    return (std::search(dir.begin(), dir.end(), storeDir.begin(), storeDir.end()) == dir.begin());
+}
+
+static void traceStaticRoot(
+    const TracerConfig & opts,
+    int recursionsLeft,
+    TraceResult & res,
+    const fs::path & root,
+    const fs::file_status & status
+    )
+{
+    opts.debug("Considering file " + root.string());
+
+    if (recursionsLeft < 0)
+        return;
+
+    switch (status.type()) {
+        case fs::file_type::directory:
+            {
+                auto directory_iterator = fs::recursive_directory_iterator(root);
+                for (auto & child : directory_iterator)
+                    traceStaticRoot(opts, recursionsLeft, res, child.path(), child.symlink_status());
+            }
+            break;
+        case fs::file_type::symlink:
+            {
+                auto target = root.parent_path() / fs::read_symlink(root);
+                auto not_found = [&](std::string msg) {
+                    opts.debug("Error accessing the file " + target.string() + ": " + msg);
+                    opts.debug("(When resolving the symlink " + root.string() + ")");
+                    res.deadLinks.insert(root);
+                };
+                try {
+                    auto target_status = fs::symlink_status(target);
+                    if (target_status.type() == fs::file_type::not_found)
+                        not_found("Not found");
+
+                    if (isInStore(opts.storeDir, target)) {
+                        res.storeRoots[target].insert(root);
+                        return;
+                    } else {
+                        traceStaticRoot(opts, recursionsLeft - 1, res, target, target_status);
+                    }
+
+                } catch (fs::filesystem_error & e) {
+                    not_found(e.what());
+                }
+            }
+            break;
+        case fs::file_type::regular:
+            {
+                auto possibleStorePath = opts.storeDir / root.filename();
+                if (fs::exists(possibleStorePath))
+                res.storeRoots[possibleStorePath].insert(root);
+            }
+            break;
+        case fs::file_type::not_found:
+        case fs::file_type::block:
+        case fs::file_type::character:
+        case fs::file_type::fifo:
+        case fs::file_type::socket:
+        case fs::file_type::unknown:
+        case fs::file_type::none:
+        default:
+            break;
+    }
+}
+
+static void traceStaticRoot(
+    const TracerConfig & opts,
+    int recursionsLeft,
+    TraceResult & res,
+    const fs::path & root)
+{
+    try {
+        auto status = fs::symlink_status(root);
+        traceStaticRoot(opts, recursionsLeft, res, root, status);
+    } catch (fs::filesystem_error & e) {
+        opts.debug("Error accessing the file " + root.string() + ": " + e.what());
+    }
+}
+
+TraceResult traceStaticRoots(TracerConfig opts, std::set<fs::path> roots)
+{
+    int maxRecursionLevel = 2;
+    TraceResult res;
+    for (auto & root : roots)
+        traceStaticRoot(opts, maxRecursionLevel, res, root);
+    return res;
+}
+
+/**
+ * Scan the content of the given file for al the occurences of something that looks
+ * like a store path (i.e. that matches `storePathRegex(opts.storeDir)`) and add them
+ * to `res`
+ */
+static void scanFileContent(const TracerConfig & opts, const fs::path & fileToScan, Roots & res)
+{
+    if (!fs::exists(fileToScan))
+        return;
+
+    std::ostringstream contentStream;
+    {
+        std::ifstream fs;
+        fs.open(fileToScan);
+        fs >> contentStream.rdbuf();
+    }
+    std::string content = contentStream.str();
+    auto regex = storePathRegex(opts.storeDir);
+    auto firstMatch
+        = std::sregex_iterator { content.begin(), content.end(), regex };
+    auto fileEnd = std::sregex_iterator{};
+    for (auto i = firstMatch; i != fileEnd; ++i)
+        res[i->str()].emplace(fileToScan);
+}
+
+/**
+ * Scan the content of a `/proc/[pid]/maps` file for regions that are mmaped to
+ * a store path
+ */
+static void scanMapsFile(const TracerConfig & opts, const fs::path & mapsFile, Roots & res)
+{
+    if (!fs::exists(mapsFile))
+        return;
+
+    static auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
+    std::stringstream mappedFile;
+    {
+        std::ifstream fs;
+        fs.open(mapsFile);
+        fs >> mappedFile.rdbuf();
+    }
+    std::string line;
+    while (std::getline(mappedFile, line)) {
+        auto match = std::smatch{};
+        if (std::regex_match(line, match, mapRegex)) {
+            auto matchedPath = fs::path(match[1]);
+            if (isInStore(opts.storeDir, matchedPath))
+                res[fs::path(match[1])].emplace(mapsFile);
+        }
+    }
+
+}
+
+Roots getRuntimeRoots(TracerConfig opts)
+{
+    auto procDir = fs::path("/proc");
+    if (!fs::exists(procDir))
+        return {};
+    Roots res;
+    auto digitsRegex = std::regex(R"(^\d+$)");
+    for (auto & procEntry : fs::directory_iterator(procDir)) {
+        // Only the directories whose name is a sequence of digits represent
+        // pids
+        if (!std::regex_match(procEntry.path().filename().string(), digitsRegex)
+            || !procEntry.is_directory())
+            continue;
+
+        opts.debug("Considering path " + procEntry.path().string());
+
+        // A set of paths used by the executable and possibly symlinks to a
+        // path in the store
+        std::set<fs::path> pathsToConsider;
+        pathsToConsider.insert(procEntry.path()/"exe");
+        pathsToConsider.insert(procEntry.path()/"cwd");
+        try {
+            auto fdDir = procEntry.path()/"fd";
+            for (auto & fdFile : fs::directory_iterator(fdDir))
+                pathsToConsider.insert(fdFile.path());
+        } catch (fs::filesystem_error & e) {
+            if (e.code().value() != ENOENT && e.code().value() != EACCES)
+                throw;
+        }
+        for (auto & path : pathsToConsider) try {
+            auto realPath = fs::read_symlink(path);
+            if (isInStore(opts.storeDir, realPath))
+                res[realPath].insert(path);
+        } catch (fs::filesystem_error &e) {
+            opts.debug(e.what());
+        }
+
+        // Scan the environment of the executable
+        scanFileContent(opts, procEntry.path()/"environ", res);
+        scanMapsFile(opts, procEntry.path()/"maps", res);
+    }
+
+    // Mostly useful for NixOS, but doesn’t hurt to check on other systems
+    // anyways
+    scanFileContent(opts, "/proc/sys/kernel/modprobe", res);
+    scanFileContent(opts, "/proc/sys/kernel/fbsplash", res);
+    scanFileContent(opts, "/proc/sys/kernel/poweroff_cmd", res);
+
+    return res;
+}
+
+}

src/nix-find-roots/lib/find-roots.hh

@@ -0,0 +1,51 @@
+#include <filesystem>
+#include <set>
+#include <map>
+#include <functional>
+
+namespace nix::roots_tracer {
+namespace fs = std::filesystem;
+
+class Error : public std::runtime_error {
+public:
+    using std::runtime_error::runtime_error;
+};
+
+inline void logNone(std::string_view)
+{ }
+
+struct TracerConfig {
+    const fs::path storeDir = "/nix/store";
+    const fs::path stateDir = "/nix/var/nix";
+    const fs::path socketPath = "/nix/var/nix/gc-socket/socket";
+
+    std::function<void(std::string_view msg)> log = logNone;
+    std::function<void(std::string_view msg)> debug = logNone;
+};
+
+/**
+ * A value of type `Roots` is a mapping from a store path to the set of roots that keep it alive
+ */
+typedef std::map<fs::path, std::set<fs::path>> Roots;
+
+struct TraceResult {
+    Roots storeRoots;
+    std::set<fs::path> deadLinks;
+};
+
+/**
+ * Return the set of all the store paths that are reachable from the given set
+ * of filesystem paths, by:
+ * - descending into the directories
+ * - following the symbolic links (at most twice)
+ * - reading the name of regular files (when encountering a file
+ *   `/foo/bar/abcdef`, the algorithm will try to access `/nix/store/abcdef`)
+ *
+ * Also returns the set of all dead links encountered during the process (so
+ * that they can be removed if it makes sense).
+ */
+TraceResult traceStaticRoots(TracerConfig opts, std::set<fs::path> initialRoots);
+
+Roots getRuntimeRoots(TracerConfig opts);
+
+}

src/nix-find-roots/local.mk

@@ -0,0 +1,20 @@
+libraries += libfindroots
+
+libfindroots_NAME = libnixfindroots
+
+libfindroots_DIR := $(d)/lib
+
+libfindroots_SOURCES := $(wildcard $(d)/lib/*.cc)
+
+programs += nix-find-roots
+
+nix-find-roots_DIR := $(d)
+
+nix-find-roots_SOURCES := $(d)/main.cc
+
+nix-find-roots_LIBS := libfindroots
+
+nix-find-roots_CXXFLAGS += \
+	-I src/nix-find-roots/lib
+
+nix-find-roots_INSTALL_DIR := $(libexecdir)/nix

src/nix-find-roots/main.cc

@@ -0,0 +1,188 @@
+#include "find-roots.hh"
+#include <getopt.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <iostream>
+#include <unistd.h>
+#include <cstring>
+#include <signal.h>
+
+using namespace nix::roots_tracer;
+
+void logStderr(std::string_view msg)
+{
+  std::cerr << msg << std::endl;
+}
+
+TracerConfig parseCmdLine(int argc, char** argv)
+{
+    std::function<void(std::string_view msg)> log = logStderr;
+    std::function<void(std::string_view msg)> debug = logNone;
+    fs::path storeDir = "/nix/store";
+    fs::path stateDir = "/nix/var/nix";
+    fs::path socketPath = "/nix/var/nix/gc-trace-socket/socket";
+
+    auto usage = [&]() {
+        std::string programName = argc > 0 ? argv[0] : "nix-find-roots";
+        std::cerr << "Usage: " << programName << " [--verbose|-v] [-s storeDir] [-d stateDir] [-l socketPath]" << std::endl;
+        exit(1);
+    };
+    static struct option long_options[] = {
+        { "verbose", no_argument, 0, 'v' },
+        { "socket_path", required_argument, 0, 'l' },
+        { "store_dir", required_argument, 0, 's' },
+        { "state_dir", required_argument, 0, 'd' },
+        { "help", no_argument, 0, 'h' },
+        { 0, 0, 0, 0 },
+    };
+
+    int option_index = 0;
+    int opt_char;
+    while((opt_char = getopt_long(argc, argv, "vd:s:l:h",
+                    long_options, &option_index)) != -1) {
+        switch (opt_char) {
+            case 0:
+                break;
+          break;
+            case '?':
+            case 'h':
+                usage();
+                break;
+            case 'v':
+                debug = logStderr;
+                break;
+            case 's':
+                storeDir = fs::path(optarg);
+                break;
+            case 'd':
+                stateDir = fs::path(optarg);
+                break;
+            case 'l':
+                socketPath = fs::path(optarg);
+                break;
+            default:
+                std::cerr << "Got invalid char: " << (char)opt_char << std::endl;
+                abort();
+        }
+    };
+    return TracerConfig {
+        .storeDir = storeDir,
+        .stateDir = stateDir,
+        .socketPath = socketPath,
+        .debug = debug,
+    };
+}
+
+/**
+ * Return `original` with every newline or tab character escaped
+ */
+std::string escape(std::string original)
+{
+    std::map<std::string, std::string> replacements = {
+        {"\n", "\\n"},
+        {"\t", "\\t"},
+    };
+    for (auto [oldStr, newStr] : replacements) {
+        size_t currentPos = 0;
+        while ((currentPos = original.find(oldStr, currentPos)) != std::string::npos) {
+            original.replace(currentPos, oldStr.length(), newStr);
+            currentPos += newStr.length();
+        }
+    }
+
+    return original;
+}
+
+#define SD_LISTEN_FDS_START 3 // Like in systemd
+
+int main(int argc, char * * argv)
+{
+    const TracerConfig opts = parseCmdLine(argc, argv);
+    const std::set<fs::path> standardRoots = {
+        opts.stateDir / fs::path("profiles"),
+        opts.stateDir / fs::path("gcroots"),
+    };
+
+    int mySock;
+
+    //  Handle socket-based activation by systemd.
+    auto rawListenFds = std::getenv("LISTEN_FDS");
+    if (rawListenFds) {
+        auto listenFds = std::string(rawListenFds);
+        auto listenPid = std::getenv("LISTEN_PID");
+        if (listenPid == nullptr || listenPid != std::to_string(getpid()) || listenFds != "1")
+            throw Error("unexpected systemd environment variables");
+        mySock = SD_LISTEN_FDS_START;
+    } else {
+        mySock = socket(PF_UNIX, SOCK_STREAM, 0);
+        if (mySock == -1) {
+            throw Error(std::string("Cannot create Unix domain socket, got") +
+                    std::strerror(errno));
+        }
+        struct sockaddr_un addr;
+        addr.sun_family = AF_UNIX;
+
+        auto socketDir = opts.socketPath.parent_path();
+        auto socketFilename = opts.socketPath.filename();
+        if (socketFilename.string().size() > sizeof(addr.sun_path))
+            throw Error(
+                    "Socket filename " + socketFilename.string() +
+                    " is too long, should be at most " +
+                    std::to_string(sizeof(addr.sun_path))
+                    );
+        chdir(socketDir.c_str());
+
+        fs::remove(socketFilename);
+        strcpy(addr.sun_path, socketFilename.c_str());
+        if (bind(mySock, (struct sockaddr*) &addr, sizeof(addr)) == -1) {
+            throw Error("Cannot bind to socket");
+        }
+
+        if (listen(mySock, 5) == -1)
+            throw Error("cannot listen on socket " + opts.socketPath.string());
+    }
+
+    // Ignore SIGPIPE so that an interrupted connection doesn’t stop the daemon
+    signal(SIGPIPE, SIG_IGN);
+
+    while (1) {
+        struct sockaddr_un remoteAddr;
+        socklen_t remoteAddrLen = sizeof(remoteAddr);
+        int remoteSocket = accept(
+                mySock,
+                (struct sockaddr*) & remoteAddr,
+                &remoteAddrLen
+                );
+
+        if (remoteSocket == -1) {
+            if (errno == EINTR) continue;
+            throw Error("Error accepting the connection");
+        }
+
+        opts.log("accepted connection");
+
+        auto printToSocket = [&](std::string_view s) {
+            send(remoteSocket, s.data(), s.size(), 0);
+        };
+
+        auto traceResult = traceStaticRoots(opts, standardRoots);
+        auto runtimeRoots = getRuntimeRoots(opts);
+        traceResult.storeRoots.insert(runtimeRoots.begin(), runtimeRoots.end());
+        for (auto & [rootInStore, externalRoots] : traceResult.storeRoots) {
+            for (auto & externalRoot : externalRoots) {
+                printToSocket(escape(rootInStore.string()));
+                printToSocket("\t");
+                printToSocket(escape(externalRoot.string()));
+                printToSocket("\n");
+            }
+
+        }
+        printToSocket("\n");
+        for (auto & deadLink : traceResult.deadLinks) {
+            printToSocket(escape(deadLink.string()));
+            printToSocket("\n");
+        }
+
+        close(remoteSocket);
+    }
+}

src/nix/add-to-store.cc

@@ -3,6 +3,7 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "posix-source-accessor.hh"
+#include "misc-store-flags.hh"
 
 using namespace nix;
 
@@ -26,23 +27,9 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             .handler = {&namePart},
         });
 
-        addFlag({
-            .longName  = "mode",
-            .description = R"(
-    How to compute the hash of the input.
-    One of:
+        addFlag(flag::contentAddressMethod(&caMethod));
 
-    - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function.
-
-    - `flat`: Assumes that the input is a single file and directly passes it to the hash function;
-            )",
-            .labels = {"hash-mode"},
-            .handler = {[this](std::string s) {
-                this->caMethod = parseFileIngestionMethod(s);
-            }},
-        });
-
-        addFlag(Flag::mkHashAlgoFlag(&hashAlgo));
+        addFlag(flag::hashAlgo(&hashAlgo));
     }
 
     void run(ref<Store> store) override
@@ -63,7 +50,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
 struct CmdAdd : CmdAddToStore
 {
-
     std::string description() override
     {
         return "Add a file or directory to the Nix store";

src/nix/flake.cc

@@ -224,7 +224,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
             if (auto lastModified = flake.lockedRef.input.getLastModified())
                 j["lastModified"] = *lastModified;
             j["path"] = store->printStorePath(flake.storePath);
-            j["locks"] = lockedFlake.lockFile.toJSON();
+            j["locks"] = lockedFlake.lockFile.toJSON().first;
             logger->cout("%s", j.dump());
         } else {
             logger->cout(

src/nix/hash.cc

@@ -6,11 +6,12 @@
 #include "references.hh"
 #include "archive.hh"
 #include "posix-source-accessor.hh"
+#include "misc-store-flags.hh"
 
 using namespace nix;
 
 /**
- * Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy).
+ * Base for `nix hash path`, `nix hash file` (deprecated), and `nix-hash` (legacy).
  *
  * Deprecation Issue: https://github.com/NixOS/nix/issues/8876
  */
@@ -19,12 +20,21 @@ struct CmdHashBase : Command
     FileIngestionMethod mode;
     HashFormat hashFormat = HashFormat::SRI;
     bool truncate = false;
-    HashAlgorithm ha = HashAlgorithm::SHA256;
+    HashAlgorithm hashAlgo = HashAlgorithm::SHA256;
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
     explicit CmdHashBase(FileIngestionMethod mode) : mode(mode)
     {
+        expectArgs({
+            .label = "paths",
+            .handler = {&paths},
+            .completer = completePath
+        });
+
+        // FIXME The following flags should be deprecated, but we don't
+        // yet have a mechanism for that.
+
         addFlag({
             .longName = "sri",
             .description = "Print the hash in SRI format.",
@@ -49,22 +59,7 @@ struct CmdHashBase : Command
             .handler = {&hashFormat, HashFormat::Base16},
         });
 
-        addFlag(Flag::mkHashAlgoFlag("type", &ha));
-
-        #if 0
-        addFlag({
-            .longName = "modulo",
-            .description = "Compute the hash modulo the specified string.",
-            .labels = {"modulus"},
-            .handler = {&modulus},
-        });
-        #endif\
-
-        expectArgs({
-            .label = "paths",
-            .handler = {&paths},
-            .completer = completePath
-        });
+        addFlag(flag::hashAlgo("type", &hashAlgo));
     }
 
     std::string description() override
@@ -85,9 +80,9 @@ struct CmdHashBase : Command
 
             std::unique_ptr<AbstractHashSink> hashSink;
             if (modulus)
-                hashSink = std::make_unique<HashModuloSink>(ha, *modulus);
+                hashSink = std::make_unique<HashModuloSink>(hashAlgo, *modulus);
             else
-                hashSink = std::make_unique<HashSink>(ha);
+                hashSink = std::make_unique<HashSink>(hashAlgo);
 
             auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path);
             dumpPath(accessor, canonPath, *hashSink, mode);
@@ -99,15 +94,53 @@ struct CmdHashBase : Command
     }
 };
 
+/**
+ * `nix hash path`
+ */
+struct CmdHashPath : CmdHashBase
+{
+    CmdHashPath()
+        : CmdHashBase(FileIngestionMethod::Recursive)
+    {
+        addFlag(flag::hashAlgo("algo", &hashAlgo));
+        addFlag(flag::fileIngestionMethod(&mode));
+        addFlag(flag::hashFormatWithDefault("format", &hashFormat));
+        #if 0
+        addFlag({
+            .longName = "modulo",
+            .description = "Compute the hash modulo the specified string.",
+            .labels = {"modulus"},
+            .handler = {&modulus},
+        });
+        #endif
+    }
+};
+
+/**
+ * For deprecated `nix hash file`
+ *
+ * Deprecation Issue: https://github.com/NixOS/nix/issues/8876
+ */
+struct CmdHashFile : CmdHashBase
+{
+    CmdHashFile()
+        : CmdHashBase(FileIngestionMethod::Flat)
+    {
+    }
+};
+
+/**
+ * For deprecated `nix hash to-*`
+ */
 struct CmdToBase : Command
 {
     HashFormat hashFormat;
-    std::optional<HashAlgorithm> ht;
+    std::optional<HashAlgorithm> hashAlgo;
     std::vector<std::string> args;
 
     CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat)
     {
-        addFlag(Flag::mkHashAlgoOptFlag("type", &ht));
+        addFlag(flag::hashAlgoOpt("type", &hashAlgo));
         expectArgs("strings", &args);
     }
 
@@ -124,7 +157,7 @@ struct CmdToBase : Command
     {
         warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`.");
         for (auto s : args)
-            logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI));
+            logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI));
     }
 };
 
@@ -139,9 +172,9 @@ struct CmdHashConvert : Command
     std::vector<std::string> hashStrings;
 
     CmdHashConvert(): to(HashFormat::SRI) {
-        addFlag(Args::Flag::mkHashFormatOptFlag("from", &from));
-        addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to));
-        addFlag(Args::Flag::mkHashAlgoOptFlag(&algo));
+        addFlag(flag::hashFormatOpt("from", &from));
+        addFlag(flag::hashFormatWithDefault("to", &to));
+        addFlag(flag::hashAlgoOpt(&algo));
         expectArgs({
            .label = "hashes",
            .handler = {&hashStrings},
@@ -181,8 +214,8 @@ struct CmdHash : NixMultiCommand
             "hash",
             {
                 {"convert", []() { return make_ref<CmdHashConvert>();}},
-                {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
-                {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
+                {"path", []() { return make_ref<CmdHashPath>(); }},
+                {"file", []() { return make_ref<CmdHashFile>(); }},
                 {"to-base16", []() { return make_ref<CmdToBase>(HashFormat::Base16); }},
                 {"to-base32", []() { return make_ref<CmdToBase>(HashFormat::Nix32); }},
                 {"to-base64", []() { return make_ref<CmdToBase>(HashFormat::Base64); }},
@@ -206,7 +239,7 @@ static int compatNixHash(int argc, char * * argv)
     // Wait until `nix hash convert` is not hidden behind experimental flags anymore.
     // warn("`nix-hash` has been deprecated in favor of `nix hash convert`.");
 
-    std::optional<HashAlgorithm> ha;
+    std::optional<HashAlgorithm> hashAlgo;
     bool flat = false;
     HashFormat hashFormat = HashFormat::Base16;
     bool truncate = false;
@@ -226,7 +259,7 @@ static int compatNixHash(int argc, char * * argv)
         else if (*arg == "--truncate") truncate = true;
         else if (*arg == "--type") {
             std::string s = getArg(*arg, arg, end);
-            ha = parseHashAlgo(s);
+            hashAlgo = parseHashAlgo(s);
         }
         else if (*arg == "--to-base16") {
             op = opTo;
@@ -253,8 +286,8 @@ static int compatNixHash(int argc, char * * argv)
 
     if (op == opHash) {
         CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
-        if (!ha.has_value()) ha = HashAlgorithm::MD5;
-        cmd.ha = ha.value();
+        if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5;
+        cmd.hashAlgo = hashAlgo.value();
         cmd.hashFormat = hashFormat;
         cmd.truncate = truncate;
         cmd.paths = ss;
@@ -264,7 +297,7 @@ static int compatNixHash(int argc, char * * argv)
     else {
         CmdToBase cmd(hashFormat);
         cmd.args = ss;
-        if (ha.has_value()) cmd.ht = ha;
+        if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo;
         cmd.run();
     }
 

src/nix/prefetch.cc

@@ -10,6 +10,7 @@
 #include "eval-inline.hh"
 #include "legacy.hh"
 #include "posix-source-accessor.hh"
+#include "misc-store-flags.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -284,7 +285,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON
             }}
         });
 
-        addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo));
+        addFlag(flag::hashAlgo("hash-type", &hashAlgo));
 
         addFlag({
             .longName = "executable",

src/nix/profile.cc

@@ -400,13 +400,13 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
             //       See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102
             auto findRefByFilePath = [&]<typename Iterator>(Iterator begin, Iterator end) {
                 for (auto it = begin; it != end; it++) {
-                    auto & profileElement = it->second;
+                    auto & [name, profileElement] = *it;
                     for (auto & storePath : profileElement.storePaths) {
                         if (conflictError.fileA.starts_with(store->printStorePath(storePath))) {
-                            return std::pair(conflictError.fileA, profileElement.toInstallables(*store));
+                            return std::tuple(conflictError.fileA, name, profileElement.toInstallables(*store));
                         }
                         if (conflictError.fileB.starts_with(store->printStorePath(storePath))) {
-                            return std::pair(conflictError.fileB, profileElement.toInstallables(*store));
+                            return std::tuple(conflictError.fileB, name, profileElement.toInstallables(*store));
                         }
                     }
                 }
@@ -415,9 +415,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
             // There are 2 conflicting files. We need to find out which one is from the already installed package and
             // which one is the package that is the new package that is being installed.
             // The first matching package is the one that was already installed (original).
-            auto [originalConflictingFilePath, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end());
+            auto [originalConflictingFilePath, originalEntryName, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end());
             // The last matching package is the one that was going to be installed (new).
-            auto [newConflictingFilePath, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend());
+            auto [newConflictingFilePath, newEntryName, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend());
 
             throw Error(
                 "An existing package already provides the following file:\n"
@@ -443,7 +443,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                 "  nix profile install %4% --priority %7%\n",
                 originalConflictingFilePath,
                 newConflictingFilePath,
-                concatStringsSep(" ", originalConflictingRefs),
+                originalEntryName,
                 concatStringsSep(" ", newConflictingRefs),
                 conflictError.priority,
                 conflictError.priority - 1,

tests/functional/add.sh

@@ -45,3 +45,8 @@ clearStore
     [[ "$path1" == "$path2" ]]
     path4=$(nix store add --mode flat --hash-algo sha1 ./dummy)
 )
+(
+    path1=$(nix store add --mode text ./dummy)
+    path2=$(nix eval --impure --raw --expr 'builtins.toFile "dummy" (builtins.readFile ./dummy)')
+    [[ "$path1" == "$path2" ]]
+)

tests/functional/common/vars-and-functions.sh.in

@@ -19,6 +19,7 @@ export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
 export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
 unset NIX_USER_CONF_FILES
+export NIX_GC_SOCKET_PATH=$TEST_ROOT/gc.socket
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1
@@ -73,7 +74,7 @@ clearProfiles() {
 
 clearStore() {
     echo "clearing store..."
-    chmod -R +w "$NIX_STORE_DIR"
+    chmod -R +w "$NIX_STORE_DIR" || true
     rm -rf "$NIX_STORE_DIR"
     mkdir "$NIX_STORE_DIR"
     rm -rf "$NIX_STATE_DIR"
@@ -89,6 +90,15 @@ clearCacheCache() {
     rm -f $TEST_HOME/.cache/nix/binary-cache*
 }
 
+declare -A trapFunctions
+
+callTraps() {
+    for f in "${trapFunctions[@]}"; do
+        $f
+    done
+}
+trap callTraps EXIT
+
 startDaemon() {
     # Don’t start the daemon twice, as this would just make it loop indefinitely
     if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then
@@ -110,6 +120,7 @@ startDaemon() {
       fail "Didn’t manage to start the daemon"
     fi
     trap "killDaemon" EXIT
+    trapFunctions[killDaemon]=killDaemon
     # Save for if daemon is killed
     NIX_REMOTE_OLD=$NIX_REMOTE
     export NIX_REMOTE=daemon
@@ -132,7 +143,7 @@ killDaemon() {
     unset _NIX_TEST_DAEMON_PID
     # Restore old nix remote
     NIX_REMOTE=$NIX_REMOTE_OLD
-    trap "" EXIT
+    trapFunctions[killDaemon]=:
 }
 
 restartDaemon() {
@@ -142,6 +153,36 @@ restartDaemon() {
     startDaemon
 }
 
+startGcDaemon() {
+    # Start the daemon, wait for the socket to appear.  !!!
+    # ‘nix-daemon’ should have an option to fork into the background.
+    rm -f $NIX_GC_SOCKET_PATH
+    $(dirname $(command -v nix))/../libexec/nix/nix-find-roots \
+        -l "$NIX_GC_SOCKET_PATH" \
+        -d "$NIX_STATE_DIR" \
+        -s "$NIX_STORE_DIR" \
+        > /dev/null 2>&1 \
+            &
+    pidGcDaemon=$!
+    for ((i = 0; i < 30; i++)); do
+        if [[ -S $NIX_GC_SOCKET_PATH ]]; then break; fi
+        sleep 1
+    done
+    trapFunctions[killGcDaemon]=killGcDaemon
+}
+
+killGcDaemon() {
+    kill $pidGcDaemon
+    for i in {0..10}; do
+        kill -0 $pidGcDaemon || break
+        sleep 1
+    done
+    kill -9 $pidGcDaemon || true
+    wait $pidGcDaemon || true
+    trapFunctions[killGcDaemon]=:
+}
+
+
 if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi

tests/functional/experimental-features.sh

@@ -31,17 +31,19 @@ source common.sh
 NIX_CONFIG='
   experimental-features = nix-command
   accept-flake-config = true
-' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
-grepQuiet "false" $TEST_ROOT/stdout
+' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+[[ $(cat $TEST_ROOT/stdout) = '' ]]
 grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
+grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr
 
 # 'flakes' experimental-feature is disabled after, ignore and warn
 NIX_CONFIG='
   accept-flake-config = true
   experimental-features = nix-command
-' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
-grepQuiet "false" $TEST_ROOT/stdout
+' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr
+[[ $(cat $TEST_ROOT/stdout) = '' ]]
 grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr
+grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr
 
 # 'flakes' experimental-feature is enabled before, process
 NIX_CONFIG='

tests/functional/fetchGit.sh

@@ -30,7 +30,10 @@ echo hello >> $TEST_ROOT/worktree/hello
 rev2=$(git -C $repo rev-parse HEAD)
 git -C $repo tag -a tag2 -m tag2
 
-# Fetch a worktree
+# Check whether fetching in read-only mode works.
+nix-instantiate --eval -E "builtins.readFile ((builtins.fetchGit file://$TEST_ROOT/worktree) + \"/hello\") == \"utrecht\\n\""
+
+# Fetch a worktree.
 unset _NIX_FORCE_HTTP
 path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
 path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
@@ -67,7 +70,7 @@ path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"
 [[ $(nix eval --raw --expr "builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]]
 
 # But without a hash, it fails
-expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "fetchGit requires a locked input"
+expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "'fetchGit' will not fetch unlocked input"
 
 # Fetch again. This should be cached.
 mv $repo ${repo}-tmp
@@ -208,7 +211,7 @@ path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; ur
 [[ $path3 = $path6 ]]
 [[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]
 
-expectStderr 1 nix eval --expr 'builtins.fetchTree { type = "git"; url = "file:///foo"; }' | grepQuiet "fetchTree requires a locked input"
+expectStderr 1 nix eval --expr 'builtins.fetchTree { type = "git"; url = "file:///foo"; }' | grepQuiet "'fetchTree' will not fetch unlocked input"
 
 # Explicit ref = "HEAD" should work, and produce the same outPath as without ref
 path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath")
@@ -268,3 +271,28 @@ git -C "$repo" add hello .gitignore
 git -C "$repo" commit -m 'Bla1'
 cd "$repo"
 path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath")
+
+# Test a workdir with no commits.
+empty="$TEST_ROOT/empty"
+git init "$empty"
+
+emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }'
+
+[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]]
+
+echo foo > "$empty/x"
+
+[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]]
+
+git -C "$empty" add x
+
+[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = '{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-wzlAGjxKxpaWdqVhlq55q5Gxo4Bf860+kLeEa/v02As="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' ]]
+
+# Test a repo with an empty commit.
+git -C "$empty" rm -f x
+
+git -C "$empty" config user.email "foobar@example.com"
+git -C "$empty" config user.name "Foobar"
+git -C "$empty" commit --allow-empty --allow-empty-message --message ""
+
+nix eval --impure --expr "let attrs = builtins.fetchGit $empty; in assert attrs.lastModified != 0; assert attrs.rev != \"0000000000000000000000000000000000000000\"; assert attrs.revCount == 1; true"

tests/functional/fetchTree-file.sh

@@ -14,6 +14,7 @@ test_fetch_file () {
         tree = builtins.fetchTree { type = "file"; url = "file://$PWD/test_input"; };
     in
     assert (tree.narHash == "$input_hash");
+    assert builtins.readFile tree == "foo\n";
     tree
 EOF
 }

tests/functional/gc-external-daemon/common.sh

@@ -0,0 +1,6 @@
+source ../common.sh
+
+enableFeatures "external-gc-daemon"
+echo "gc-socket-path = $NIX_GC_SOCKET_PATH" >> "$NIX_CONF_DIR"/nix.conf
+
+startGcDaemon

tests/functional/gc-external-daemon/gc-auto.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+cd ..
+source ./gc-auto.sh
+

tests/functional/gc-external-daemon/gc-concurrent.sh

@@ -0,0 +1,4 @@
+source common.sh
+
+cd ..
+source ./gc-concurrent.sh

tests/functional/gc-external-daemon/gc-runtime.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+cd ..
+source ./gc-runtime.sh
+

tests/functional/gc-external-daemon/gc.sh

@@ -0,0 +1,4 @@
+source common.sh
+
+cd ..
+source ./gc.sh

tests/functional/gc-external-daemon/local.mk

@@ -0,0 +1,7 @@
+gc-external-daemon-tests := \
+  $(d)/gc-auto.sh \
+  $(d)/gc.sh \
+  $(d)/gc-concurrent.sh \
+  $(d)/gc-runtime.sh
+
+install-tests-groups += gc-external-daemon

tests/functional/hash-path.sh

@@ -2,19 +2,24 @@ source common.sh
 
 try () {
     printf "%s" "$2" > $TEST_ROOT/vector
-    hash="$(nix-hash --flat ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")"
+    hash="$(nix-hash --flat ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")"
     if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then
         echo "try nix-hash: hash $1, expected $3, got $hash"
         exit 1
     fi
-    hash="$(nix hash file ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")"
+    hash="$(nix hash file ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")"
+    if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then
+        echo "try nix hash: hash $1, expected $3, got $hash"
+        exit 1
+    fi
+    hash="$(nix hash path --mode flat ${FORMAT+--format $FORMAT} --algo "$1" "$TEST_ROOT/vector")"
     if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then
         echo "try nix hash: hash $1, expected $3, got $hash"
         exit 1
     fi
 }
 
-FORMAT_FLAG=--base16
+FORMAT=base16
 try md5 "" "d41d8cd98f00b204e9800998ecf8427e"
 try md5 "a" "0cc175b9c0f1b6a831c399e269772661"
 try md5 "abc" "900150983cd24fb0d6963f7d28e17f72"
@@ -34,18 +39,18 @@ try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "248d6a61d
 try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"
 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
 try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445"
-unset FORMAT_FLAG
+unset FORMAT
 
-FORMAT_FLAG=--base32
+FORMAT=base32
 try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s"
-unset FORMAT_FLAG
+unset FORMAT
 
-FORMAT_FLAG=--sri
+FORMAT=sri
 try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw=="
 try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ=="
 try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE="
-unset FORMAT_FLAG
+unset FORMAT
 
 # nix-hash [--flat] defaults to the Base16 format
 NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"
@@ -56,7 +61,12 @@ NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5
 try2 () {
     hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path)
     if test "$hash" != "$2"; then
-        echo "hash $1, expected $2, got $hash"
+        echo "try nix-hash; hash $1, expected $2, got $hash"
+        exit 1
+    fi
+    hash="$(nix hash path --mode nar --format base16 --algo "$1" "$TEST_ROOT/hash-path")"
+    if test "$hash" != "$2"; then
+        echo "try nix hash: hash $1, expected $2, got $hash"
         exit 1
     fi
 }

tests/functional/nix-profile.sh

@@ -166,7 +166,7 @@ error: An existing package already provides the following file:
 
        To remove the existing package:
 
-         nix profile remove path:${flake1Dir}#packages.${system}.default
+         nix profile remove flake1
 
        The new package can also be installed next to the existing one by assigning a different priority.
        The conflicting packages have a priority of 5.

tests/functional/tarball.sh

@@ -42,11 +42,11 @@ test_tarball() {
     nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
     nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true'
 
-    nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
-    nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
-    (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)
+    nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar$ext
+    nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file:///no-such-tarball$ext
+    (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file:///no-such-tarball$ext)
 
-    nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball$ext -I fnord=.
+    nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file:///no-such-tarball$ext -I fnord=.
 
     # Ensure that the `name` attribute isn’t accepted as that would mess
     # with the content-addressing
@@ -57,8 +57,3 @@ test_tarball() {
 test_tarball '' cat
 test_tarball .xz xz
 test_tarball .gz gzip
-
-rm -rf $TEST_ROOT/tmp
-mkdir -p $TEST_ROOT/tmp
-(! TMPDIR=$TEST_ROOT/tmp XDG_RUNTIME_DIR=$TEST_ROOT/tmp nix-env -f file://$(pwd)/bad.tar.xz -qa --out-path)
-(! [ -e $TEST_ROOT/tmp/bad ])

tests/nixos/default.nix

@@ -109,7 +109,7 @@ in
       nix.package = lib.mkForce pkgs.nixVersions.nix_2_13;
     };
   };
-  
+
   # TODO: (nixpkgs update) remoteBuildsSshNg_remote_2_18 = ...
 
   # Test our Nix as a builder for clients that are older
@@ -137,6 +137,8 @@ in
   # TODO: (nixpkgs update) remoteBuildsSshNg_local_2_18 = ...
   */
 
+  rootless-daemon = runNixOSTestFor "x86_64-linux" ./rootless-daemon.nix;
+
   nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix;
 
   nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix;

tests/nixos/rootless-daemon.nix

@@ -0,0 +1,177 @@
+{ pkgs, ... }:
+
+{
+  name = "rootless-daemon";
+
+  nodes.machine = { config, ... }: {
+    users.users.alice.isNormalUser = true;
+    #users.users.nix-daemon.isSystemUser = true;
+    users.users.nix-daemon.isNormalUser = true;
+    users.users.nix-daemon.group = "nix-daemon";
+    users.groups.nix-daemon = {};
+    environment.variables.NIX_REMOTE = "daemon"; # Even for root
+    virtualisation.writableStore = true;
+
+    boot.readOnlyNixStore = false;
+
+    # No root daemon
+    nix.enable = false;
+
+    # But put Nix on the path anyways
+    environment.systemPackages = [ pkgs.nix ];
+
+    # And install the unit files:
+    #
+    # - nix-gc-trace system-wide
+    # - nix-daemon per-user
+    systemd.packages = [
+       (pkgs.runCommand "shuffled-nix-systemd-unit-files" {} ''
+         mkdir -p $out/lib/systemd/{system,user}
+         ln -s ${pkgs.nix}/lib/systemd/system/nix-gc-trace.* $out/lib/systemd/system
+         ln -s ${pkgs.nix}/lib/systemd/system/nix-daemon.* $out/lib/systemd/user
+       '')
+    ];
+
+    # And prepare the socket dirs anyways
+    systemd.tmpfiles.rules = [
+      "d /nix/var/nix/daemon-socket 0755 nix-daemon root - -"
+      "d /nix/var/nix/gc-socket 0755 nix-daemon root - -"
+    ];
+
+    # Oops isn't working, because cannot disable Nix daemon and enable
+    # Nix settings yet: https://github.com/NixOS/nixpkgs/issues/263250.
+    nix.settings = {
+      experimental-features = [ "external-gc-daemon" ];
+    };
+    # Plan B given the above
+    #
+    # TODO delete once that issue is fixed.
+    environment.etc."nix/nix.conf".source = pkgs.writeTextFile {
+      name = "nix.conf";
+      text = ''
+        experimental-features = external-gc-daemon
+      '';
+    };
+
+    systemd.user.sockets.nix-daemon = {
+    };
+    systemd.user.services.nix-daemon = {
+      path = [ pkgs.nix ];
+      description = "Nix Daemon (non-root)";
+      unitConfig.ConditionUser = "nix-daemon";
+      # Make sure we do not get a "fork bomb" of the daemon trying to
+      # connect to itself.
+      serviceConfig.Environment= "NIX_REMOTE=local";
+    };
+
+    systemd.sockets.nix-gc-trace = {
+      restartTriggers = [ config.environment.etc."nix/nix.conf".source ];
+    };
+    systemd.services.nix-gc-trace = {
+      restartTriggers = [ config.environment.etc."nix/nix.conf".source ];
+      path = [ pkgs.nix ];
+      description = "Nix Find Roots";
+    };
+    # We must enable lingering so that the Systemd User D-Bus is
+    # enabled. We also cannot do this with loginctl enable-linger
+    # because it needs to happen before systemd is loaded.
+    #
+    # See https://github.com/NixOS/nixpkgs/issues/3702
+    #
+    # TODO after upgrading to 23.11 can use new NixOS option for this.
+    system.activationScripts = {
+      enableLingering = ''
+        # remove all existing lingering users
+        rm -rf /var/lib/systemd/linger
+        mkdir -p /var/lib/systemd/linger
+        # enable for the subset of declared users
+        touch /var/lib/systemd/linger/nix-daemon
+      '';
+    };
+  };
+
+  testScript = ''
+    import re
+
+    machine.wait_for_unit("multi-user.target")
+
+  ''
+
+  # Give ownership of the store dir and var to the nix-daemon user.
+  #
+  # We intentionally don't do `-R` on the store because store objects
+  # used by NixOS should still be owned by root.
+  + ''
+    machine.succeed("""
+      set -ex
+
+      chown nix-daemon /nix/store
+      chown -R nix-daemon /nix/var
+    """)
+  ''
+
+  # Test that alice indeed cannot modify the store; we don't want
+  # arbitrary users to have any more permissions than before!
+  + ''
+    machine.fail("su alice -c 'touch /nix/store/foo'")
+
+  ''
+
+  # Start and wait for our units
+  + ''
+    machine.start_job("nix-gc-trace.socket")
+    machine.wait_for_unit("nix-gc-trace.socket")
+
+    machine.start_job("nix-daemon.socket", user="nix-daemon")
+    machine.wait_for_unit("nix-daemon.socket", user="nix-daemon")
+
+  ''
+
+  # Create a store obect, remember its store path in Python.
+  + ''
+    two = machine.succeed("""
+    su --login alice -c "$(cat <<'EOF'
+      set -ex
+      export PS4='+(''${BASH_SOURCE[0]-$0}:$LINENO) '
+
+      echo ehHtmfuULXYyBV6NBk6QUi8iE0 > two
+      nix-store --add two
+    EOF
+    )"
+    """)
+    two = two.strip()
+    assert re.match(r'^/nix/store/.+-two$', two)
+
+  ''
+
+  # Make sure we cannot delete it when it has a GC root, but we can once
+  # that root is destroyed.
+  + ''
+    machine.succeed(f"""
+    su --login alice -c "$(cat <<'EOF'
+      set -ex
+      export PS4='+(''${{BASH_SOURCE[0]-$0}}:$LINENO) '
+
+      test -f {two}
+      nix-store --realize {two} --add-root foo
+      echo $two
+    EOF
+    )"
+    """)
+
+    machine.fail(f"su --login alice -c 'nix-store --delete {two}'")
+
+    machine.succeed(f"""
+    su --login alice -c "$(cat <<'EOF'
+      set -ex
+      export PS4='+(''${{BASH_SOURCE[0]-$0}}:$LINENO) '
+
+      test -f {two}
+      rm foo
+      nix-store --delete {two}
+      test ! -f {two}
+    EOF
+    )"
+    """)
+  '';
+}