Fix the parsing of path flake refs with a query

`/path?foo=bar` was parsed as `/path`, omitting the query altogether
(`/path?foo=bar#` was parsed correctly though).

Rewrite the path flake ref parser to fix this

.github/workflows/ci.yml

@@ -64,7 +64,7 @@ jobs:
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/install-nix-action@v25
       with:
-        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
+        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - uses: cachix/cachix-action@v14
       with:
         name: '${{ env.CACHIX_NAME }}'
@@ -116,7 +116,7 @@ jobs:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v25
       with:
-        install_url: https://releases.nixos.org/nix/nix-2.13.3/install
+        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v14
@@ -153,6 +153,8 @@ jobs:
         IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
 
         docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
-        docker tag nix:$NIX_VERSION $IMAGE_ID:master
+        docker tag nix:$NIX_VERSION $IMAGE_ID:latest
         docker push $IMAGE_ID:$NIX_VERSION
+        docker push $IMAGE_ID:latest
+        # deprecated 2024-02-24
         docker push $IMAGE_ID:master

.gitignore

@@ -45,13 +45,16 @@ perl/Makefile.config
 /src/libexpr/parser-tab.hh
 /src/libexpr/parser-tab.output
 /src/libexpr/nix.tbl
+/src/libexpr/tests
 /tests/unit/libexpr/libnixexpr-tests
 
 # /src/libstore/
 *.gen.*
+/src/libstore/tests
 /tests/unit/libstore/libnixstore-tests
 
 # /src/libutil/
+/src/libutil/tests
 /tests/unit/libutil/libnixutil-tests
 
 /src/nix/nix
@@ -108,9 +111,6 @@ perl/Makefile.config
 
 /misc/systemd/nix-daemon.service
 /misc/systemd/nix-daemon.socket
-/misc/systemd/nix-gc-trace.service
-/misc/systemd/nix-gc-trace.socket
-
 /misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 

Makefile

@@ -12,7 +12,6 @@ makefiles = \
   mk/precompiled-headers.mk \
   local.mk \
   src/libutil/local.mk \
-  src/nix-find-roots/local.mk \
   src/libstore/local.mk \
   src/libfetchers/local.mk \
   src/libmain/local.mk \
@@ -42,8 +41,8 @@ endif
 ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes)
 makefiles += \
   tests/functional/local.mk \
-  tests/functional/gc-external-daemon/local.mk \
   tests/functional/ca/local.mk \
+  tests/functional/git-hashing/local.mk \
   tests/functional/dyn-drv/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
@@ -69,6 +68,7 @@ ifeq ($(OPTIMIZE), 1)
   GLOBAL_LDFLAGS += $(CXXLTO)
 else
   GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE
+  unexport NIX_HARDENING_ENABLE
 endif
 
 include mk/platform.mk
@@ -83,7 +83,7 @@ ifdef HOST_WINDOWS
   GLOBAL_LDFLAGS += -Wl,--export-all-symbols
 endif
 
-GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src
+GLOBAL_CXXFLAGS += -g -Wall -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src
 
 # Include the main lib, causing rules to be defined
 

doc/manual/rl-next/inherit-from-by-need.md

@@ -0,0 +1,7 @@
+---
+synopsis: "`inherit (x) ...` evaluates `x` only once"
+prs: 9847
+---
+
+`inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute.
+This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown.

doc/manual/rl-next/source-location-in-while-evaluating-attribute.md

@@ -0,0 +1,23 @@
+---
+synopsis: "In the debugger, `while evaluating the attribute` errors now include position information"
+prs: 9915
+---
+
+Before:
+
+```
+0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+0x600001522598
+```
+
+After:
+
+```
+0: while evaluating the attribute 'python311.pythonForBuild.pkgs'
+/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27
+
+   131|
+   132|       bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs;
+      |                           ^
+   133|     in
+```

doc/manual/src/installation/installing-rootless-daemon.md

@@ -1,138 +0,0 @@
-# Using Nix in multi-user mode with a non-root daemon
-
-> Experimental blurb
-
-It is experimentally possible to run Nix in multi-user mode without running the whole daemon as root.
-This is done by delegating the only part that requires root access to a separate daemon, with a much smaller attack surface.
-Because of the need for a second daemon, this makes the setup a bit more complex and isn't yet supported by the installer. It is however possible to set this up manually:
-
-1. Create a new user and group for the daemon:
-    ```sh
-    sudo groupadd nix-daemon
-    sudo useradd --gid nix-daemon --system -c "Nix daemon user" nix-daemon
-    ```
-2. Create `/nix` owned by that user:
-    ```sh
-    sudo mkdir -m 0755 /nix
-    sudo chown nix-daemon:nix-daemon /nix
-    ```
-3. Download a statically-compiled Nix version for bootstrapping
-    ```sh
-    curl -L https://hydra.nixos.org/job/nix/master/buildStatic.x86_64-linux/latest/download-by-type/file/binary-dist -o /tmp/nix-env
-    chmod +x /tmp/nix-env
-    ```
-4. Install a proper Nix and the tracing daemon in the store
-    ```sh
-    export DAEMON_HOME=$(sudo -u nix-daemon mktemp -d)
-    sudo -u nix-daemon HOME="$DAEMON_HOME" \
-        /tmp/nix-env \
-        -f https://github.com/nixos/nix/archive/rootless-daemon.tar.gz \
-        -iA default packages.x86_64-linux.nix-find-roots \
-        --option extra-substituters https://nixos-nix-install-tests.cachix.org \
-        --option extra-trusted-public-keys nixos-nix-install-tests.cachix.org-1:Le57vOUJjOcdzLlbwmZVBuLGoDC+Xg2rQDtmIzALgFU= \
-        --store / \
-        --profile /nix/var/nix/profiles/default
-    sudo -u nix-daemon mkdir -p /nix/var/nix/gc-socket
-    sudo -u nix-daemon rm -rf "$DAEMON_HOME"
-    ```
-5. Move the tracing daemon executable out of the store (as we don't want Nix
-   to own it)
-   ```sh
-   sudo cp /nix/var/nix/profiles/default/bin/nix-find-roots /usr/bin/
-   ```
-6. Install the systemd services for the daemon:
-    ```sh
-    cat <<EOF | sudo tee /etc/systemd/system/nix-daemon.service
-    [Unit]
-    Description=Nix Daemon
-    Documentation=man:nix-daemon https://nixos.org/manual
-    RequiresMountsFor=/nix/store
-    RequiresMountsFor=/nix/var
-    RequiresMountsFor=/nix/var/nix/db
-    ConditionPathIsReadWrite=/nix/var/nix/daemon-socket
-
-    [Service]
-    ExecStart=@/nix/var/nix/profiles/default/bin/nix-daemon nix-daemon --daemon
-    KillMode=process
-    LimitNOFILE=1048576
-    TasksMax=1048576
-    User=nix-daemon
-    Group=nix-daemon
-
-    [Install]
-    WantedBy=multi-user.target
-    EOF
-    ```
-    ```sh
-    cat <<EOF | sudo tee /etc/systemd/system/nix-daemon.socket
-    [Unit]
-    Description=Nix Daemon Socket
-    Before=multi-user.target
-    RequiresMountsFor=/nix/store
-    ConditionPathIsReadWrite=/nix/var/nix/daemon-socket
-
-    [Socket]
-    ListenStream=/nix/var/nix/daemon-socket/socket
-    SocketUser=nix-daemon
-
-    [Install]
-    WantedBy=sockets.target
-    EOF
-    ```
-7. Install the systemd services for the tracing daemon:
-    ```sh
-    cat <<EOF | sudo tee /etc/systemd/system/nix-find-roots.service
-    [Unit]
-    Description=Nix GC tracer daemon
-    RequiresMountsFor=/nix/store
-    RequiresMountsFor=/nix/var
-    ConditionPathIsReadWrite=/nix/var/nix/gc-socket
-    ProcSubset=pid
-
-    [Service]
-    ExecStart=@/usr/bin/nix-find-roots nix-find-roots
-    Type=simple
-    StandardError=journal
-    ProtectSystem=full
-    ReadWritePaths=/nix/var/nix/gc-socket
-    SystemCallFilter=@system-service
-    SystemCallErrorNumber=EPERM
-    PrivateNetwork=true
-    PrivateDevices=true
-    ProtectKernelTunables=true
-
-    [Install]
-    WantedBy=multi-user.target
-    EOF
-    ```
-    ```sh
-    cat <<EOF | sudo tee /etc/systemd/system/nix-find-roots.socket
-    [Unit]
-    Description=Nix Daemon Socket
-    Before=multi-user.target
-    RequiresMountsFor=/nix/store
-    ConditionPathIsReadWrite=/nix/var/nix/gc-socket
-
-    [Socket]
-    ListenStream=/nix/var/nix/gc-socket/socket
-    Accept=false
-
-    [Install]
-    WantedBy=sockets.target
-    EOF
-    ```
-8. Enable the required experimental Nix feature and basic configuration:
-    ```sh
-    sudo mkdir /etc/nix
-    cat <<EOF | sudo tee /etc/nix/nix.conf
-    experimental-features = external-gc-daemon
-    trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-    substituters = https://cache.nixos.org/
-    EOF
-    ```
-9. Start the systemd sockets:
-    ```sh
-    sudo systemctl start nix-daemon.socket
-    sudo systemctl start nix-find-roots.socket
-    ```
-10. Profit

doc/manual/src/protocols/store-path.md

@@ -89,15 +89,20 @@ where
 
       - `rec` = one of:
 
+        - ```ebnf
+          | ""
+          ```
+          (empty string) for hashes of the flat (single file) serialization
+
         - ```ebnf
           | "r:"
           ```
           hashes of the for [Nix Archive (NAR)] (arbitrary file system object) serialization
 
         - ```ebnf
-          | ""
+          | "git:"
           ```
-          (empty string) for hashes of the flat (single file) serialization
+          hashes of the [Git blob/tree](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) [Merkel tree](https://en.wikipedia.org/wiki/Merkle_tree) format
 
       - ```ebnf
         algo = "md5" | "sha1" | "sha256"

flake.lock

@@ -34,16 +34,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1705033721,
-        "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=",
+        "lastModified": 1709083642,
+        "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea",
+        "rev": "b550fe4b4776908ac2a861124307045f8e717c8e",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05-small",
+        "ref": "release-23.11",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,7 +1,9 @@
 {
   description = "The purely functional package manager";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
+  # TODO switch to nixos-23.11-small
+  #      https://nixpk.gs/pr-tracker.html?pr=291954
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
   inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
@@ -10,20 +12,10 @@
 
     let
       inherit (nixpkgs) lib;
-
-      # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981
-      # Not an "idiomatic" flake input because:
-      #  - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730
-      #  - Subflake would download redundant and huge parent flake
-      #  - No git tree hash support: https://github.com/NixOS/nix/issues/6044
-      inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; }))
-        fileset;
+      inherit (lib) fileset;
 
       officialRelease = false;
 
-      # Set to true to build the release notes for the next release.
-      buildUnreleasedNotes = false;
-
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
         if officialRelease
@@ -152,30 +144,6 @@
             '';
           };
 
-          nix-find-roots = prev.stdenv.mkDerivation {
-            pname = "nix-find-roots";
-            inherit version;
-
-            src = fileset.toSource {
-              root = ./src/nix-find-roots;
-              fileset = /*fileset.intersect baseFiles (*/fileset.unions [
-                ./src/nix-find-roots/main.cc
-                ./src/nix-find-roots/lib
-              ]/*)*/;
-            };
-
-            CXXFLAGS = prev.lib.optionalString prev.stdenv.hostPlatform.isStatic "-static";
-
-            buildPhase = ''
-              $CXX $CXXFLAGS -std=c++17 *.cc **/*.cc -I lib -o nix-find-roots
-            '';
-
-            installPhase = ''
-              mkdir -p $out/bin
-              cp nix-find-roots $out/bin/
-            '';
-          };
-
           libgit2-nix = final.libgit2.overrideAttrs (attrs: {
             src = libgit2;
             version = libgit2.lastModifiedDate;
@@ -390,9 +358,6 @@
         default = nix;
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = nixpkgsFor.${system}.static.nix;
-
-        inherit (nixpkgsFor.${system}.static) nix-find-roots;
-
         dockerImage =
           let
             pkgs = nixpkgsFor.${system}.native;
@@ -431,8 +396,11 @@
             # Make bash completion work.
             XDG_DATA_DIRS+=:$out/share
           '';
+
           nativeBuildInputs = attrs.nativeBuildInputs or []
-            ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear
+            # TODO: Remove the darwin check once
+            # https://github.com/NixOS/nixpkgs/pull/291814 is available
+            ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
             ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools;
         });
         in
@@ -444,8 +412,9 @@
               (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName}));
           in
             (makeShells "native" nixpkgsFor.${system}.native) //
-            (makeShells "static" nixpkgsFor.${system}.static) //
-            (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
+            (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin)
+              (makeShells "static" nixpkgsFor.${system}.static)) //
+              (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) //
             {
               default = self.devShells.${system}.native-stdenvPackages;
             }

misc/systemd/local.mk

@@ -1,8 +1,6 @@
 ifdef HOST_LINUX
 
-  $(foreach n,\
-    nix-daemon.socket nix-daemon.service nix-gc-trace.socket nix-gc-trace.service,\
-    $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
+  $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644)))
   $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644)))
 
   clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf

misc/systemd/nix-gc-trace.service.in

@@ -1,21 +0,0 @@
-[Unit]
-Description=Nix GC Tracer Daemon
-RequiresMountsFor=@storedir@
-RequiresMountsFor=@localstatedir@
-ConditionPathIsReadWrite=@localstatedir@/nix/gc-socket
-ProcSubset=pid
-
-[Service]
-ExecStart=@@libexecdir@/nix/nix-find-roots nix-find-roots
-Type=simple
-StandardError=journal
-ProtectSystem=full
-ReadWritePaths=@localstatedir@/nix/gc-socket
-SystemCallFilter=@system-service
-SystemCallErrorNumber=EPERM
-PrivateNetwork=true
-PrivateDevices=true
-ProtectKernelTunables=true
-
-[Install]
-WantedBy=multi-user.target

misc/systemd/nix-gc-trace.socket.in

@@ -1,12 +0,0 @@
-[Unit]
-Description=Nix Daemon Socket
-Before=multi-user.target
-RequiresMountsFor=@storedir@
-ConditionPathIsReadWrite=@localstatedir@/nix/gc-socket
-
-[Socket]
-ListenStream=@localstatedir@/nix/gc-socket/socket
-Accept=false
-
-[Install]
-WantedBy=sockets.target

package.nix

@@ -154,7 +154,7 @@ in {
     in
       fileset.toSource {
         root = ./.;
-        fileset = fileset.intersect baseFiles (fileset.unions ([
+        fileset = fileset.intersection baseFiles (fileset.unions ([
           # For configure
           ./.version
           ./configure.ac
@@ -209,6 +209,10 @@ in {
     (lib.getBin lowdown)
     mdbook
     mdbook-linkcheck
+  ] ++ lib.optionals doInstallCheck [
+    git
+    mercurial
+    openssh
   ] ++ lib.optionals (doInstallCheck || enableManual) [
     jq # Also for custom mdBook preprocessor.
   ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
@@ -249,12 +253,6 @@ in {
   dontBuild = !attrs.doBuild;
   doCheck = attrs.doCheck;
 
-  nativeCheckInputs = [
-    git
-    mercurial
-    openssh
-  ];
-
   disallowedReferences = [ boost ];
 
   preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) (

perl/lib/Nix/Store.xs

@@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path)
             auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path);
             Hash h = hashPath(
                 accessor, canonPath,
-                FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
+                FileIngestionMethod::Recursive, parseHashAlgo(algo));
             auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {

scripts/install-multi-user.sh

@@ -58,6 +58,31 @@ readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
 
 readonly ROOT_HOME=~root
 
+readonly PROXY_ENVIRONMENT_VARIABLES=(
+    http_proxy
+    https_proxy
+    ftp_proxy
+    no_proxy
+    HTTP_PROXY
+    HTTPS_PROXY
+    FTP_PROXY
+    NO_PROXY
+)
+
+SUDO_EXTRA_ENVIRONMENT_VARIABLES=()
+
+setup_sudo_extra_environment_variables() {
+    local i=${#SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]}
+    for variable in "${PROXY_ENVIRONMENT_VARIABLES[@]}"; do
+        if [ "x${!variable:-}" != "x" ]; then
+            SUDO_EXTRA_ENVIRONMENT_VARIABLES[i]="$variable=${!variable}"
+            i=$((i + 1))
+        fi
+    done
+}
+
+setup_sudo_extra_environment_variables
+
 if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then
     readonly IS_HEADLESS='no'
 else
@@ -361,7 +386,7 @@ _sudo() {
     if is_root; then
         env "$@"
     else
-        sudo "$@"
+        sudo "${SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]}" "$@"
     fi
 }
 

src/libcmd/repl.cc

@@ -383,7 +383,6 @@ bool NixRepl::getLine(std::string & input, const std::string & prompt)
     };
 
     setupSignals();
-    Finally resetTerminal([&]() { rl_deprep_terminal(); });
     char * s = readline(prompt.c_str());
     Finally doFree([&]() { free(s); });
     restoreSignals();

src/libexpr/eval-error.cc

@@ -28,15 +28,7 @@ template<class T>
 EvalErrorBuilder<T> & EvalErrorBuilder<T>::withTrace(PosIdx pos, const std::string_view text)
 {
     error.err.traces.push_front(
-        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = false});
-    return *this;
-}
-
-template<class T>
-EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrameTrace(PosIdx pos, const std::string_view text)
-{
-    error.err.traces.push_front(
-        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text)), .frame = true});
+        Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))});
     return *this;
 }
 
@@ -63,9 +55,9 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrame(const Env & env, const Expr
 }
 
 template<class T>
-EvalErrorBuilder<T> & EvalErrorBuilder<T>::addTrace(PosIdx pos, HintFmt hint, bool frame)
+EvalErrorBuilder<T> & EvalErrorBuilder<T>::addTrace(PosIdx pos, HintFmt hint)
 {
-    error.addTrace(error.state.positions[pos], hint, frame);
+    error.addTrace(error.state.positions[pos], hint);
     return *this;
 }
 

src/libexpr/eval-error.hh

@@ -89,7 +89,7 @@ public:
 
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & withFrame(const Env & e, const Expr & ex);
 
-    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint, bool frame = false);
+    [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> & addTrace(PosIdx pos, HintFmt hint);
 
     template<typename... Args>
     [[nodiscard, gnu::noinline]] EvalErrorBuilder<T> &

src/libexpr/eval.cc

@@ -806,14 +806,16 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
     }
 }
 
-void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const
+template<typename... Args>
+void EvalState::addErrorTrace(Error & e, const Args & ... formatArgs) const
 {
-    e.addTrace(nullptr, s, s2);
+    e.addTrace(nullptr, HintFmt(formatArgs...));
 }
 
-void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame) const
+template<typename... Args>
+void EvalState::addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const
 {
-    e.addTrace(positions[pos], HintFmt(s, s2), frame);
+    e.addTrace(positions[pos], HintFmt(formatArgs...));
 }
 
 template<typename... Args>
@@ -1196,6 +1198,18 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v)
 }
 
 
+Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
+{
+    Env & inheritEnv = state.allocEnv(inheritFromExprs->size());
+    inheritEnv.up = &up;
+
+    Displacement displ = 0;
+    for (auto from : *inheritFromExprs)
+        inheritEnv.values[displ++] = from->maybeThunk(state, up);
+
+    return &inheritEnv;
+}
+
 void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 {
     v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish());
@@ -1207,6 +1221,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Env & env2(state.allocEnv(attrs.size()));
         env2.up = &env;
         dynamicEnv = &env2;
+        Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
 
         AttrDefs::iterator overrides = attrs.find(state.sOverrides);
         bool hasOverrides = overrides != attrs.end();
@@ -1217,11 +1232,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         Displacement displ = 0;
         for (auto & i : attrs) {
             Value * vAttr;
-            if (hasOverrides && !i.second.inherited) {
+            if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
                 vAttr = state.allocValue();
-                mkThunk(*vAttr, env2, i.second.e);
+                mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e);
             } else
-                vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
+                vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
             env2.values[displ++] = vAttr;
             v.attrs->push_back(Attr(i.first, vAttr, i.second.pos));
         }
@@ -1253,9 +1268,15 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
         }
     }
 
-    else
-        for (auto & i : attrs)
-            v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), i.second.pos));
+    else {
+        Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
+        for (auto & i : attrs) {
+            v.attrs->push_back(Attr(
+                    i.first,
+                    i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)),
+                    i.second.pos));
+        }
+    }
 
     /* Dynamic attrs apply *after* rec and __overrides. */
     for (auto & i : dynamicAttrs) {
@@ -1287,12 +1308,17 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
     Env & env2(state.allocEnv(attrs->attrs.size()));
     env2.up = &env;
 
+    Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
+
     /* The recursive attributes are evaluated in the new environment,
        while the inherited attributes are evaluated in the original
        environment. */
     Displacement displ = 0;
-    for (auto & i : attrs->attrs)
-        env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2);
+    for (auto & i : attrs->attrs) {
+        env2.values[displ++] = i.second.e->maybeThunk(
+            state,
+            *i.second.chooseByKind(&env2, &env, inheritEnv));
+    }
 
     auto dts = state.debugRepl
         ? makeDebugTraceStacker(
@@ -1369,7 +1395,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 state,
                 *this,
                 env,
-                state.positions[pos2],
+                state.positions[getPos()],
                 "while evaluating the attribute '%1%'",
                 showAttrPath(state, env, attrPath))
             : nullptr;
@@ -1587,9 +1613,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                         "while calling %s",
                         lambda.name
                         ? concatStrings("'", symbols[lambda.name], "'")
-                        : "anonymous lambda",
-                        true);
-                    if (pos) addErrorTrace(e, pos, "from call site%s", "", true);
+                        : "anonymous lambda");
+                    if (pos) addErrorTrace(e, pos, "from call site");
                 }
                 throw;
             }

src/libexpr/eval.hh

@@ -432,10 +432,12 @@ public:
     std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx);
     std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);
 
+    template<typename... Args>
     [[gnu::noinline]]
-    void addErrorTrace(Error & e, const char * s, const std::string & s2) const;
+    void addErrorTrace(Error & e, const Args & ... formatArgs) const;
+    template<typename... Args>
     [[gnu::noinline]]
-    void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const;
+    void addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const;
 
 public:
     /**

src/libexpr/flake/flakeref.cc

@@ -55,7 +55,7 @@ FlakeRef parseFlakeRef(
 {
     auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake);
     if (fragment != "")
-        throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
+        throw FlakeRefError("unexpected fragment '%s' in flake reference '%s'", fragment, url);
     return flakeRef;
 }
 
@@ -78,19 +78,25 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
     std::string path = url;
     std::string fragment = "";
     std::map<std::string, std::string> query;
-    auto pathEnd = url.find_first_of("#?");
-    auto fragmentStart = pathEnd;
-    if (pathEnd != std::string::npos && url[pathEnd] == '?') {
-        fragmentStart = url.find("#");
-    }
+    auto pathEnd = url.find_first_of("?#");
     if (pathEnd != std::string::npos) {
+        // There's something (either a query string or a fragment) in addition
+        // to the path
         path = url.substr(0, pathEnd);
-    }
-    if (fragmentStart != std::string::npos) {
-        fragment = percentDecode(url.substr(fragmentStart+1));
-    }
-    if (pathEnd != std::string::npos && fragmentStart != std::string::npos) {
-        query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1));
+        std::string non_path_part = url.substr(pathEnd + 1);
+        if (url[pathEnd] == '#') {
+            // Not query, just a fragment
+            fragment = percentDecode(non_path_part);
+        } else {
+            // We have a query, and maybe a fragment too
+            auto fragmentStart = non_path_part.find("#");
+            if (fragmentStart != std::string::npos) {
+                query = decodeQuery(non_path_part.substr(0, fragmentStart));
+                fragment = percentDecode(non_path_part.substr(fragmentStart+1));
+            } else {
+                query = decodeQuery(non_path_part);
+            }
+        }
     }
 
     if (baseDir) {
@@ -113,10 +119,10 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
                         found = true;
                         break;
                     } else if (pathExists(path + "/.git"))
-                        throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
+                        throw FlakeRefError("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path);
                     else {
                         if (lstat(path).st_dev != device)
-                            throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path);
+                            throw FlakeRefError("unable to find a flake before encountering filesystem boundary at '%s'", path);
                     }
                     path = dirOf(path);
                 }
@@ -148,7 +154,7 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
 
                     if (subdir != "") {
                         if (parsedURL.query.count("dir"))
-                            throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+                            throw FlakeRefError("flake URL '%s' has an inconsistent 'dir' parameter", url);
                         parsedURL.query.insert_or_assign("dir", subdir);
                     }
 
@@ -171,11 +177,16 @@ std::pair<FlakeRef, std::string> parsePathFlakeRefWithFragment(
         path = canonPath(path + "/" + getOr(query, "dir", ""));
     }
 
-    fetchers::Attrs attrs;
-    attrs.insert_or_assign("type", "path");
-    attrs.insert_or_assign("path", path);
+    auto parsedURL = ParsedURL{
+        .url = url,
+        .base = url,
+        .scheme = "path",
+        .authority = "",
+        .path = path,
+        .query = query,
+    };
 
-    return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment);
+    return std::make_pair(FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), fragment);
 };
 
 

src/libexpr/flake/flakeref.hh

@@ -15,6 +15,8 @@ class Store;
 
 typedef std::string FlakeId;
 
+MakeError(FlakeRefError, Error);
+
 /**
  * A flake reference specifies how to fetch a flake or raw source
  * (e.g. from a Git repository).  It is created from a URL-like syntax

src/libexpr/flake/url-name.cc

@@ -5,13 +5,12 @@
 namespace nix {
 
 static const std::string attributeNamePattern("[a-zA-Z0-9_-]+");
-static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?");
+static const std::regex lastAttributeRegex("^((?:" + attributeNamePattern + "\\.)*)(" + attributeNamePattern +")(\\^.*)?$");
 static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+");
 static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")");
 static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?");
 static const std::regex gitProviderRegex("github|gitlab|sourcehut");
 static const std::regex gitSchemeRegex("git($|\\+.*)");
-static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)");
 
 std::optional<std::string> getNameFromURL(const ParsedURL & url)
 {
@@ -22,8 +21,11 @@ std::optional<std::string> getNameFromURL(const ParsedURL & url)
         return url.query.at("dir");
 
     /* If the fragment isn't a "default" and contains two attribute elements, use the last one */
-    if (std::regex_match(url.fragment, match, lastAttributeRegex))
-        return match.str(1);
+    if (std::regex_match(url.fragment, match, lastAttributeRegex)
+        && match.str(1) != "defaultPackage."
+        && match.str(2) != "default") {
+        return match.str(2);
+    }
 
     /* If this is a github/gitlab/sourcehut flake, use the repo name */
     if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex))
@@ -33,10 +35,6 @@ std::optional<std::string> getNameFromURL(const ParsedURL & url)
     if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex))
         return match.str(1);
 
-    /* If everything failed but there is a non-default fragment, use it in full */
-    if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex))
-        return url.fragment;
-
     /* If there is no fragment, take the last element of the path */
     if (std::regex_match(url.path, match, lastPathSegmentRegex))
         return match.str(1);

src/libexpr/lexer.l

@@ -94,6 +94,9 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
 
 }
 
+// yacc generates code that uses unannotated fallthrough.
+#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
+
 #define YY_USER_INIT initLoc(yylloc)
 #define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng);
 

src/libexpr/nixexpr.cc

@@ -70,10 +70,8 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
     str << ") ? " << showAttrPath(symbols, attrPath) << ")";
 }
 
-void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
+void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
 {
-    if (recursive) str << "rec ";
-    str << "{ ";
     typedef const decltype(attrs)::value_type * Attr;
     std::vector<Attr> sorted;
     for (auto & i : attrs) sorted.push_back(&i);
@@ -81,10 +79,37 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
         std::string_view sa = symbols[a->first], sb = symbols[b->first];
         return sa < sb;
     });
+    std::vector<Symbol> inherits;
+    std::map<ExprInheritFrom *, std::vector<Symbol>> inheritsFrom;
     for (auto & i : sorted) {
-        if (i->second.inherited)
-            str << "inherit " << symbols[i->first] << " " << "; ";
-        else {
+        switch (i->second.kind) {
+        case AttrDef::Kind::Plain:
+            break;
+        case AttrDef::Kind::Inherited:
+            inherits.push_back(i->first);
+            break;
+        case AttrDef::Kind::InheritedFrom: {
+            auto & select = dynamic_cast<ExprSelect &>(*i->second.e);
+            auto & from = dynamic_cast<ExprInheritFrom &>(*select.e);
+            inheritsFrom[&from].push_back(i->first);
+            break;
+        }
+        }
+    }
+    if (!inherits.empty()) {
+        str << "inherit";
+        for (auto sym : inherits) str << " " << symbols[sym];
+        str << "; ";
+    }
+    for (const auto & [from, syms] : inheritsFrom) {
+        str << "inherit (";
+        (*inheritFromExprs)[from->displ]->show(symbols, str);
+        str << ")";
+        for (auto sym : syms) str << " " << symbols[sym];
+        str << "; ";
+    }
+    for (auto & i : sorted) {
+        if (i->second.kind == AttrDef::Kind::Plain) {
             str << symbols[i->first] << " = ";
             i->second.e->show(symbols, str);
             str << "; ";
@@ -97,6 +122,13 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
         i.valueExpr->show(symbols, str);
         str << "; ";
     }
+}
+
+void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
+{
+    if (recursive) str << "rec ";
+    str << "{ ";
+    showBindings(symbols, str);
     str << "}";
 }
 
@@ -152,15 +184,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
 void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const
 {
     str << "(let ";
-    for (auto & i : attrs->attrs)
-        if (i.second.inherited) {
-            str << "inherit " << symbols[i.first] << "; ";
-        }
-        else {
-            str << symbols[i.first] << " = ";
-            i.second.e->show(symbols, str);
-            str << "; ";
-        }
+    attrs->showBindings(symbols, str);
     str << "in ";
     body->show(symbols, str);
     str << ")";
@@ -305,6 +329,12 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     this->level = withLevel;
 }
 
+void ExprInheritFrom::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
+{
+    if (es.debugRepl)
+        es.exprEnvs.insert(std::make_pair(this, env));
+}
+
 void ExprSelect::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     if (es.debugRepl)
@@ -328,22 +358,47 @@ void ExprOpHasAttr::bindVars(EvalState & es, const std::shared_ptr<const StaticE
             i.expr->bindVars(es, env);
 }
 
+std::shared_ptr<const StaticEnv> ExprAttrs::bindInheritSources(
+    EvalState & es, const std::shared_ptr<const StaticEnv> & env)
+{
+    if (!inheritFromExprs)
+        return nullptr;
+
+    // the inherit (from) source values are inserted into an env of its own, which
+    // does not introduce any variable names.
+    // analysis must see an empty env, or an env that contains only entries with
+    // otherwise unused names to not interfere with regular names. the parser
+    // has already filled all exprs that access this env with appropriate level
+    // and displacement, and nothing else is allowed to access it. ideally we'd
+    // not even *have* an expr that grabs anything from this env since it's fully
+    // invisible, but the evaluator does not allow for this yet.
+    auto inner = std::make_shared<StaticEnv>(nullptr, env.get(), 0);
+    for (auto from : *inheritFromExprs)
+        from->bindVars(es, env);
+
+    return inner;
+}
+
 void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
-        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), recursive ? attrs.size() : 0);
+        auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
+            auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs.size());
 
-        Displacement displ = 0;
-        for (auto & i : attrs)
-            newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+            Displacement displ = 0;
+            for (auto & i : attrs)
+                newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+            return newEnv;
+        }();
 
         // No need to sort newEnv since attrs is in sorted order.
 
+        auto inheritFromEnv = bindInheritSources(es, newEnv);
         for (auto & i : attrs)
-            i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
+            i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
         for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, newEnv);
@@ -351,8 +406,10 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         }
     }
     else {
+        auto inheritFromEnv = bindInheritSources(es, env);
+
         for (auto & i : attrs)
-            i.second.e->bindVars(es, env);
+            i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
 
         for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, env);
@@ -409,16 +466,20 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
 
 void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
+    auto newEnv = [&] () -> std::shared_ptr<const StaticEnv> {
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env.get(), attrs->attrs.size());
 
-    Displacement displ = 0;
-    for (auto & i : attrs->attrs)
-        newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+        Displacement displ = 0;
+        for (auto & i : attrs->attrs)
+            newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
+        return newEnv;
+    }();
 
     // No need to sort newEnv since attrs->attrs is in sorted order.
 
+    auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
     for (auto & i : attrs->attrs)
-        i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
+        i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, newEnv));

src/libexpr/nixexpr.hh

@@ -135,6 +135,23 @@ struct ExprVar : Expr
     COMMON_METHODS
 };
 
+/**
+ * A pseudo-expression for the purpose of evaluating the `from` expression in `inherit (from)` syntax.
+ * Unlike normal variable references, the displacement is set during parsing, and always refers to
+ * `ExprAttrs::inheritFromExprs` (by itself or in `ExprLet`), whose values are put into their own `Env`.
+ */
+struct ExprInheritFrom : ExprVar
+{
+    ExprInheritFrom(PosIdx pos, Displacement displ): ExprVar(pos, {})
+    {
+        this->level = 0;
+        this->displ = displ;
+        this->fromWith = nullptr;
+    }
+
+    void bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+};
+
 struct ExprSelect : Expr
 {
     PosIdx pos;
@@ -160,16 +177,40 @@ struct ExprAttrs : Expr
     bool recursive;
     PosIdx pos;
     struct AttrDef {
-        bool inherited;
+        enum class Kind {
+            /** `attr = expr;` */
+            Plain,
+            /** `inherit attr1 attrn;` */
+            Inherited,
+            /** `inherit (expr) attr1 attrn;` */
+            InheritedFrom,
+        };
+
+        Kind kind;
         Expr * e;
         PosIdx pos;
         Displacement displ; // displacement
-        AttrDef(Expr * e, const PosIdx & pos, bool inherited=false)
-            : inherited(inherited), e(e), pos(pos) { };
+        AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain)
+            : kind(kind), e(e), pos(pos) { };
         AttrDef() { };
+
+        template<typename T>
+        const T & chooseByKind(const T & plain, const T & inherited, const T & inheritedFrom) const
+        {
+            switch (kind) {
+            case Kind::Plain:
+                return plain;
+            case Kind::Inherited:
+                return inherited;
+            default:
+            case Kind::InheritedFrom:
+                return inheritedFrom;
+            }
+        }
     };
     typedef std::map<Symbol, AttrDef> AttrDefs;
     AttrDefs attrs;
+    std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
     struct DynamicAttrDef {
         Expr * nameExpr, * valueExpr;
         PosIdx pos;
@@ -182,6 +223,11 @@ struct ExprAttrs : Expr
     ExprAttrs() : recursive(false) { };
     PosIdx getPos() const override { return pos; }
     COMMON_METHODS
+
+    std::shared_ptr<const StaticEnv> bindInheritSources(
+        EvalState & es, const std::shared_ptr<const StaticEnv> & env);
+    Env * buildInheritFromEnv(EvalState & state, Env & up);
+    void showBindings(const SymbolTable & symbols, std::ostream & str) const;
 };
 
 struct ExprList : Expr

src/libexpr/parser-state.hh

@@ -89,7 +89,7 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
         if (i->symbol) {
             ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
             if (j != attrs->attrs.end()) {
-                if (!j->second.inherited) {
+                if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) {
                     ExprAttrs * attrs2 = dynamic_cast<ExprAttrs *>(j->second.e);
                     if (!attrs2) dupAttr(attrPath, pos, j->second.pos);
                     attrs = attrs2;
@@ -118,13 +118,24 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr *
             auto ae = dynamic_cast<ExprAttrs *>(e);
             auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e);
             if (jAttrs && ae) {
+                if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
+                    jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
                 for (auto & ad : ae->attrs) {
                     auto j2 = jAttrs->attrs.find(ad.first);
                     if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error.
                         dupAttr(ad.first, j2->second.pos, ad.second.pos);
                     jAttrs->attrs.emplace(ad.first, ad.second);
+                    if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
+                        auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
+                        auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
+                        from.displ += jAttrs->inheritFromExprs->size();
+                    }
                 }
                 jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
+                if (ae->inheritFromExprs) {
+                    jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(),
+                        ae->inheritFromExprs->begin(), ae->inheritFromExprs->end());
+                }
             } else {
                 dupAttr(attrPath, pos, j->second.pos);
             }

src/libexpr/parser.y

@@ -313,17 +313,27 @@ binds
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos);
           auto pos = state->at(@3);
-          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true));
+          $$->attrs.emplace(
+              i.symbol,
+              ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, ExprAttrs::AttrDef::Kind::Inherited));
       }
       delete $3;
     }
   | binds INHERIT '(' expr ')' attrs ';'
     { $$ = $1;
-      /* !!! Should ensure sharing of the expression in $4. */
+      if (!$$->inheritFromExprs)
+          $$->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+      $$->inheritFromExprs->push_back($4);
+      auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1);
       for (auto & i : *$6) {
           if ($$->attrs.find(i.symbol) != $$->attrs.end())
               state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos);
-          $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->at(@6)));
+          $$->attrs.emplace(
+              i.symbol,
+              ExprAttrs::AttrDef(
+                  new ExprSelect(CUR_POS, from, i.symbol),
+                  state->at(@6),
+                  ExprAttrs::AttrDef::Kind::InheritedFrom));
       }
       delete $6;
     }

src/libexpr/primops.cc

@@ -705,38 +705,53 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
     .args = {"attrset"},
     .arity = 1,
     .doc = R"(
-      Take an *attrset* with values named `startSet` and `operator` in order to
-      return a *list of attrsets* by starting with the `startSet` and recursively
-      applying the `operator` function to each `item`. The *attrsets* in the
-      `startSet` and the *attrsets* produced by `operator` must contain a value
-      named `key` which is comparable. The result is produced by calling `operator`
-      for each `item` with a value for `key` that has not been called yet including
-      newly produced `item`s. The function terminates when no new `item`s are
-      produced. The resulting *list of attrsets* contains only *attrsets* with a
-      unique key. For example,
+      `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function.
 
-      ```
-      builtins.genericClosure {
-        startSet = [ {key = 5;} ];
-        operator = item: [{
-          key = if (item.key / 2 ) * 2 == item.key
-               then item.key / 2
-               else 3 * item.key + 1;
-        }];
-      }
-      ```
-      evaluates to
-      ```
-      [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
-      ```
+      It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets:
+
+      - `startSet`:
+        The initial list of attribute sets.
+
+      - `operator`:
+        A function that takes an attribute set and returns a list of attribute sets.
+        It defines how each item in the current set is processed and expanded into more items.
+
+      Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison.
+      The value of `key` can be one of the following types:
 
-      `key` can be one of the following types:
       - [Number](@docroot@/language/values.md#type-number)
       - [Boolean](@docroot@/language/values.md#type-boolean)
       - [String](@docroot@/language/values.md#type-string)
       - [Path](@docroot@/language/values.md#type-path)
       - [List](@docroot@/language/values.md#list)
 
+      The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added.
+      Items are compared by their `key` attribute.
+
+      Common usages are:
+
+      - Generating unique collections of items, such as dependency graphs.
+      - Traversing through structures that may contain cycles or loops.
+      - Processing data structures with complex internal relationships.
+
+      > **Example**
+      >
+      > ```nix
+      > builtins.genericClosure {
+      >   startSet = [ {key = 5;} ];
+      >   operator = item: [{
+      >     key = if (item.key / 2 ) * 2 == item.key
+      >          then item.key / 2
+      >          else 3 * item.key + 1;
+      >   }];
+      > }
+      > ```
+      >
+      > evaluates to
+      >
+      > ```nix
+      > [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
+      > ```
       )",
     .fun = prim_genericClosure,
 });
@@ -811,7 +826,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
         auto message = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
                 false, false).toOwned();
-        e.addTrace(nullptr, HintFmt(message), true);
+        e.addTrace(nullptr, HintFmt(message));
         throw;
     }
 }
@@ -1075,7 +1090,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
         e.addTrace(nullptr, HintFmt(
                 "while evaluating derivation '%s'\n"
                 "  whose name attribute is located at %s",
-                drvName, pos), true);
+                drvName, pos));
         throw;
     }
 }
@@ -1123,7 +1138,10 @@ drvName, Bindings * attrs, Value & v)
         auto handleHashMode = [&](const std::string_view s) {
             if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
             else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
-            else if (s == "text") {
+            else if (s == "git") {
+                experimentalFeatureSettings.require(Xp::GitHashing);
+                ingestionMethod = FileIngestionMethod::Git;
+            } else if (s == "text") {
                 experimentalFeatureSettings.require(Xp::DynamicDerivations);
                 ingestionMethod = TextIngestionMethod {};
             } else
@@ -1233,8 +1251,7 @@ drvName, Bindings * attrs, Value & v)
 
         } catch (Error & e) {
             e.addTrace(state.positions[i->pos],
-                HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName),
-                true);
+                HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName));
             throw;
         }
     }
@@ -2075,7 +2092,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
         })
         : ({
             StringSource s { contents };
-            state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
+            state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
         });
 
     /* Note: we don't need to add `context' to the context of the

src/libstore/binary-cache-store.cc

@@ -305,7 +305,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
 StorePath BinaryCacheStore::addToStoreFromDump(
     Source & dump,
     std::string_view name,
-    ContentAddressMethod method,
+    FileSerialisationMethod dumpMethod,
+    ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     RepairFlag repair)
@@ -313,17 +314,27 @@ StorePath BinaryCacheStore::addToStoreFromDump(
     std::optional<Hash> caHash;
     std::string nar;
 
+    // Calculating Git hash from NAR stream not yet implemented. May not
+    // be possible to implement in single-pass if the NAR is in an
+    // inconvenient order. Could fetch after uploading, however.
+    if (hashMethod.getFileIngestionMethod() == FileIngestionMethod::Git)
+        unsupported("addToStoreFromDump");
+
     if (auto * dump2p = dynamic_cast<StringSource *>(&dump)) {
         auto & dump2 = *dump2p;
         // Hack, this gives us a "replayable" source so we can compute
         // multiple hashes more easily.
-        caHash = hashString(HashAlgorithm::SHA256, dump2.s);
-        switch (method.getFileIngestionMethod()) {
-        case FileIngestionMethod::Recursive:
+        //
+        // Only calculate if the dump is in the right format, however.
+        if (static_cast<FileIngestionMethod>(dumpMethod) == hashMethod.getFileIngestionMethod())
+            caHash = hashString(HashAlgorithm::SHA256, dump2.s);
+        switch (dumpMethod) {
+        case FileSerialisationMethod::Recursive:
             // The dump is already NAR in this case, just use it.
             nar = dump2.s;
             break;
-        case FileIngestionMethod::Flat:
+        case FileSerialisationMethod::Flat:
+        {
             // The dump is Flat, so we need to convert it to NAR with a
             // single file.
             StringSink s;
@@ -331,10 +342,11 @@ StorePath BinaryCacheStore::addToStoreFromDump(
             nar = std::move(s.s);
             break;
         }
+        }
     } else {
         // Otherwise, we have to do th same hashing as NAR so our single
         // hash will suffice for both purposes.
-        if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
+        if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256)
             unsupported("addToStoreFromDump");
     }
     StringSource narDump { nar };
@@ -349,7 +361,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(
             *this,
             name,
             ContentAddressWithReferences::fromParts(
-                method,
+                hashMethod,
                 caHash ? *caHash : nar.first,
                 {
                     .others = references,
@@ -450,7 +462,7 @@ StorePath BinaryCacheStore::addToStore(
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
 
-    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter);
 
     auto source = sinkToSource([&](Sink & sink) {
         accessor.dumpPath(path, sink, filter);

src/libstore/binary-cache-store.hh

@@ -125,7 +125,8 @@ public:
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method,
+        FileSerialisationMethod dumpMethod,
+        ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         RepairFlag repair) override;
@@ -147,7 +148,7 @@ public:
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 
-    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
 
     void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
 

src/libstore/build-result.hh

@@ -123,6 +123,11 @@ struct KeyedBuildResult : BuildResult
      * The derivation we built or the store path we substituted.
      */
     DerivedPath path;
+
+    // Hack to work around a gcc "may be used uninitialized" warning.
+    KeyedBuildResult(BuildResult res, DerivedPath path)
+        : BuildResult(std::move(res)), path(std::move(path))
+    { }
 };
 
 }

src/libstore/build/local-derivation-goal.cc

@@ -8,6 +8,7 @@
 #include "finally.hh"
 #include "util.hh"
 #include "archive.hh"
+#include "git.hh"
 #include "compression.hh"
 #include "daemon.hh"
 #include "topo-sort.hh"
@@ -1311,12 +1312,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method,
+        FileSerialisationMethod dumpMethod,
+        ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         RepairFlag repair) override
     {
-        auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair);
+        auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair);
         goal.addDependency(path);
         return path;
     }
@@ -2457,15 +2459,29 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             rewriteOutput(outputRewrites);
             /* FIXME optimize and deduplicate with addToStore */
             std::string oldHashPart { scratchPath->hashPart() };
-            auto got = ({
-                HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
+            auto got = [&]{
                 PosixSourceAccessor accessor;
-                dumpPath(
-                    accessor, CanonPath { actualPath },
-                    caSink,
-                    outputHash.method.getFileIngestionMethod());
-                caSink.finish().first;
-            });
+                auto fim = outputHash.method.getFileIngestionMethod();
+                switch (fim) {
+                case FileIngestionMethod::Flat:
+                case FileIngestionMethod::Recursive:
+                {
+                    HashModuloSink caSink { outputHash.hashAlgo, oldHashPart };
+                    auto fim = outputHash.method.getFileIngestionMethod();
+                    dumpPath(
+                        accessor, CanonPath { actualPath },
+                        caSink,
+                        (FileSerialisationMethod) fim);
+                    return caSink.finish().first;
+                }
+                case FileIngestionMethod::Git: {
+                    return git::dumpHash(
+                        outputHash.hashAlgo, accessor,
+                        CanonPath { tmpDir + "/tmp" }).hash;
+                }
+                }
+                assert(false);
+            }();
 
             ValidPathInfo newInfo0 {
                 worker.store,
@@ -2491,7 +2507,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 PosixSourceAccessor accessor;
                 HashResult narHashAndSize = hashPath(
                     accessor, CanonPath { actualPath },
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
+                    FileSerialisationMethod::Recursive, HashAlgorithm::SHA256);
                 newInfo0.narHash = narHashAndSize.first;
                 newInfo0.narSize = narHashAndSize.second;
             }
@@ -2515,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                 PosixSourceAccessor accessor;
                 HashResult narHashAndSize = hashPath(
                     accessor, CanonPath { actualPath },
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
+                    FileSerialisationMethod::Recursive, HashAlgorithm::SHA256);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
                 auto refs = rewriteRefs();

src/libstore/build/worker.cc

@@ -529,11 +529,11 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(
+        Hash current = hashPath(
             *store.getFSAccessor(), CanonPath { store.printStorePath(path) },
             FileIngestionMethod::Recursive, info->narHash.algo);
         Hash nullHash(HashAlgorithm::SHA256);
-        res = info->narHash == nullHash || info->narHash == current.first;
+        res = info->narHash == nullHash || info->narHash == current;
     }
     pathContentsGoodCache.insert_or_assign(path, res);
     if (!res)

src/libstore/content-address.cc

@@ -11,6 +11,9 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m)
         return "";
     case FileIngestionMethod::Recursive:
         return "r:";
+    case FileIngestionMethod::Git:
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return "git:";
     default:
         throw Error("impossible, caught both cases");
     }
@@ -51,6 +54,10 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
     if (splitPrefix(m, "r:")) {
         return FileIngestionMethod::Recursive;
     }
+    else if (splitPrefix(m, "git:")) {
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return FileIngestionMethod::Git;
+    }
     else if (splitPrefix(m, "text:")) {
         return TextIngestionMethod {};
     }
@@ -111,10 +118,10 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
     }
 
     auto parseHashAlgorithm_ = [&](){
-        auto hashTypeRaw = splitPrefixTo(rest, ':');
-        if (!hashTypeRaw)
+        auto hashAlgoRaw = splitPrefixTo(rest, ':');
+        if (!hashAlgoRaw)
             throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
-        HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw);
+        HashAlgorithm hashAlgo = parseHashAlgo(*hashAlgoRaw);
         return hashAlgo;
     };
 
@@ -131,6 +138,10 @@ static std::pair<ContentAddressMethod, HashAlgorithm> parseContentAddressMethodP
         auto method = FileIngestionMethod::Flat;
         if (splitPrefix(rest, "r:"))
             method = FileIngestionMethod::Recursive;
+        else if (splitPrefix(rest, "git:")) {
+            experimentalFeatureSettings.require(Xp::GitHashing);
+            method = FileIngestionMethod::Git;
+        }
         HashAlgorithm hashAlgo = parseHashAlgorithm_();
         return {
             std::move(method),

src/libstore/content-address.hh

@@ -91,17 +91,17 @@ struct ContentAddressMethod
     std::string_view renderPrefix() const;
 
     /**
-     * Parse a content addressing method and hash type.
+     * Parse a content addressing method and hash algorithm.
      */
     static std::pair<ContentAddressMethod, HashAlgorithm> parseWithAlgo(std::string_view rawCaMethod);
 
     /**
-     * Render a content addressing method and hash type in a
+     * Render a content addressing method and hash algorithm in a
      * nicer way, prefixing both cases.
      *
      * The rough inverse of `parse()`.
      */
-    std::string renderWithAlgo(HashAlgorithm ht) const;
+    std::string renderWithAlgo(HashAlgorithm ha) const;
 
     /**
      * Get the underlying way to content-address file system objects.
@@ -127,7 +127,7 @@ struct ContentAddressMethod
  *   ‘text:sha256:<sha256 hash of file contents>’
  *
  * - `FixedIngestionMethod`:
- *   ‘fixed:<r?>:<hash type>:<hash of file contents>’
+ *   ‘fixed:<r?>:<hash algorithm>:<hash of file contents>’
  */
 struct ContentAddress
 {

src/libstore/daemon.cc

@@ -13,6 +13,7 @@
 #include "archive.hh"
 #include "derivations.hh"
 #include "args.hh"
+#include "git.hh"
 
 namespace nix::daemon {
 
@@ -400,11 +401,25 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             logger->startWork();
             auto pathInfo = [&]() {
                 // NB: FramedSource must be out of scope before logger->stopWork();
-                auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parseWithAlgo(camStr);
-                auto hashAlgo = hashAlgo_; // work around clang bug
+                auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr);
                 FramedSource source(from);
+                FileSerialisationMethod dumpMethod;
+                switch (contentAddressMethod.getFileIngestionMethod()) {
+                case FileIngestionMethod::Flat:
+                    dumpMethod = FileSerialisationMethod::Flat;
+                    break;
+                case FileIngestionMethod::Recursive:
+                    dumpMethod = FileSerialisationMethod::Recursive;
+                    break;
+                case FileIngestionMethod::Git:
+                    // Use NAR; Git is not a serialization method
+                    dumpMethod = FileSerialisationMethod::Recursive;
+                    break;
+                default:
+                    assert(false);
+                }
                 // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store.
-                auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair);
+                auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair);
                 return store->queryPathInfo(path);
             }();
             logger->stopWork();
@@ -430,30 +445,23 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 hashAlgo = parseHashAlgo(hashAlgoRaw);
             }
 
+            // Old protocol always sends NAR, regardless of hashing method
             auto dumpSource = sinkToSource([&](Sink & saved) {
-                if (method == FileIngestionMethod::Recursive) {
-                    /* We parse the NAR dump through into `saved` unmodified,
-                       so why all this extra work? We still parse the NAR so
-                       that we aren't sending arbitrary data to `saved`
-                       unwittingly`, and we know when the NAR ends so we don't
-                       consume the rest of `from` and can't parse another
-                       command. (We don't trust `addToStoreFromDump` to not
-                       eagerly consume the entire stream it's given, past the
-                       length of the Nar. */
-                    TeeSource savedNARSource(from, saved);
-                    NullFileSystemObjectSink sink; /* just parse the NAR */
-                    parseDump(sink, savedNARSource);
-                } else {
-                    /* Incrementally parse the NAR file, stripping the
-                       metadata, and streaming the sole file we expect into
-                       `saved`. */
-                    RegularFileSink savedRegular { saved };
-                    parseDump(savedRegular, from);
-                    if (!savedRegular.regular) throw Error("regular file expected");
-                }
+                /* We parse the NAR dump through into `saved` unmodified,
+                   so why all this extra work? We still parse the NAR so
+                   that we aren't sending arbitrary data to `saved`
+                   unwittingly`, and we know when the NAR ends so we don't
+                   consume the rest of `from` and can't parse another
+                   command. (We don't trust `addToStoreFromDump` to not
+                   eagerly consume the entire stream it's given, past the
+                   length of the Nar. */
+                TeeSource savedNARSource(from, saved);
+                NullFileSystemObjectSink sink; /* just parse the NAR */
+                parseDump(sink, savedNARSource);
             });
             logger->startWork();
-            auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
+            auto path = store->addToStoreFromDump(
+                *dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo);
             logger->stopWork();
 
             to << store->printStorePath(path);
@@ -485,7 +493,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         auto path = ({
             StringSource source { s };
-            store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
+            store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair);
         });
         logger->stopWork();
         to << store->printStorePath(path);

src/libstore/derivations.cc

@@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store,
         })
         : ({
             StringSource s { contents };
-            store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
+            store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair);
         });
 }
 
@@ -701,7 +701,7 @@ DerivationType BasicDerivation::type() const
                     floatingHashAlgo = dof.hashAlgo;
                 } else {
                     if (*floatingHashAlgo != dof.hashAlgo)
-                        throw Error("all floating outputs must use the same hash type");
+                        throw Error("all floating outputs must use the same hash algorithm");
                 }
             },
             [&](const DerivationOutput::Deferred &) {

src/libstore/dummy-store.cc

@@ -61,7 +61,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
     virtual StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) override

src/libstore/gc.cc

@@ -2,7 +2,6 @@
 #include "globals.hh"
 #include "local-store.hh"
 #include "finally.hh"
-#include "find-roots.hh"
 #include "unix-domain-socket.hh"
 #include "signals.hh"
 
@@ -32,7 +31,6 @@ namespace nix {
 
 
 static std::string gcSocketPath = "/gc-socket/socket";
-static std::string rootsSocketPath = "/gc-roots-socket/socket";
 static std::string gcRootsDir = "gcroots";
 
 
@@ -145,7 +143,7 @@ void LocalStore::addTempRoot(const StorePath & path)
         auto fdRootsSocket(_fdRootsSocket.lock());
 
         if (!*fdRootsSocket) {
-            auto socketPath = stateDir.get() + rootsSocketPath;
+            auto socketPath = stateDir.get() + gcSocketPath;
             debug("connecting to '%s'", socketPath);
             *fdRootsSocket = createUnixDomainSocket();
             try {
@@ -243,32 +241,79 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
     }
 }
 
-void LocalStore::findRootsNoTempNoExternalDaemon(Roots & roots, bool censor)
+
+void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
 {
-    debug("Can’t connect to the tracing daemon socket, fallback to the internal trace");
-
-    using namespace nix::roots_tracer;
-
-    const TracerConfig opts {
-      .storeDir = fs::path(storeDir),
-      .stateDir = fs::path(stateDir.get())
+    auto foundRoot = [&](const Path & path, const Path & target) {
+        try {
+            auto storePath = toStorePath(target).first;
+            if (isValidPath(storePath))
+                roots[std::move(storePath)].emplace(path);
+            else
+                printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+        } catch (BadStorePath &) { }
     };
-    const std::set<fs::path> standardRoots = {
-        opts.stateDir / fs::path(gcRootsDir),
-        opts.stateDir / fs::path("gcroots"),
-    };
-    auto traceResult = traceStaticRoots(opts, standardRoots);
-    auto runtimeRoots = getRuntimeRoots(opts);
-    traceResult.storeRoots.insert(runtimeRoots.begin(), runtimeRoots.end());
-    for (auto & [rawRootInStore, externalRoots] : traceResult.storeRoots) {
-        if (!isInStore(rawRootInStore.string())) continue;
-        auto rootInStore = toStorePath(rawRootInStore.string()).first;
-        if (!isValidPath(rootInStore)) continue;
-        std::unordered_set<std::string> primRoots;
-        for (const auto & externalRoot : externalRoots)
-            primRoots.insert(externalRoot.string());
-        roots.emplace(rootInStore, primRoots);
+
+    try {
+
+        if (type == DT_UNKNOWN)
+            type = getFileType(path);
+
+        if (type == DT_DIR) {
+            for (auto & i : readDirectory(path))
+                findRoots(path + "/" + i.name, i.type, roots);
+        }
+
+        else if (type == DT_LNK) {
+            Path target = readLink(path);
+            if (isInStore(target))
+                foundRoot(path, target);
+
+            /* Handle indirect roots. */
+            else {
+                target = absPath(target, dirOf(path));
+                if (!pathExists(target)) {
+                    if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
+                        printInfo("removing stale link from '%1%' to '%2%'", path, target);
+                        unlink(path.c_str());
+                    }
+                } else {
+                    struct stat st2 = lstat(target);
+                    if (!S_ISLNK(st2.st_mode)) return;
+                    Path target2 = readLink(target);
+                    if (isInStore(target2)) foundRoot(target, target2);
+                }
+            }
+        }
+
+        else if (type == DT_REG) {
+            auto storePath = maybeParseStorePath(storeDir + "/" + std::string(baseNameOf(path)));
+            if (storePath && isValidPath(*storePath))
+                roots[std::move(*storePath)].emplace(path);
+        }
+
     }
+
+    catch (SysError & e) {
+        /* We only ignore permanent failures. */
+        if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR)
+            printInfo("cannot read potential root '%1%'", path);
+        else
+            throw;
+    }
+}
+
+
+void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
+{
+    /* Process direct roots in {gcroots,profiles}. */
+    findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
+    findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
+
+    /* Add additional roots returned by different platforms-specific
+       heuristics.  This is typically used to add running programs to
+       the set of roots (to prevent them from being garbage collected). */
+    findRuntimeRoots(roots, censor);
 }
 
 
@@ -282,53 +327,148 @@ Roots LocalStore::findRoots(bool censor)
     return roots;
 }
 
-void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
+typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
+
+static void readProcLink(const std::string & file, UncheckedRoots & roots)
 {
-
-    auto fd = createUnixDomainSocket();
-
-    std::string socketPath = settings.gcSocketPath.get() != "auto"
-        ? settings.gcSocketPath.get()
-        : stateDir.get() + gcSocketPath;
-
-    try {
-        nix::connect(fd.get(), socketPath);
-    } catch (SysError & e) {
-        return findRootsNoTempNoExternalDaemon(roots, censor);
+    constexpr auto bufsiz = PATH_MAX;
+    char buf[bufsiz];
+    auto res = readlink(file.c_str(), buf, bufsiz);
+    if (res == -1) {
+        if (errno == ENOENT || errno == EACCES || errno == ESRCH)
+            return;
+        throw SysError("reading symlink");
     }
+    if (res == bufsiz) {
+        throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
+    }
+    if (res > 0 && buf[0] == '/')
+        roots[std::string(static_cast<char *>(buf), res)]
+            .emplace(file);
+}
 
-    experimentalFeatureSettings.require(Xp::ExternalGCDaemon);
+static std::string quoteRegexChars(const std::string & raw)
+{
+    static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
+    return std::regex_replace(raw, specialRegex, R"(\$&)");
+}
 
+#if __linux__
+static void readFileRoots(const char * path, UncheckedRoots & roots)
+{
     try {
-        StringMap unescapes = {
-            { "\\n", "\n"},
-            { "\\t", "\t"},
-        };
-        while (true) {
-            auto line = readLine(fd.get());
-            if (line.empty()) break; // TODO: Handle the broken symlinks
-            auto parsedLine = tokenizeString<std::vector<std::string>>(line, "\t");
-            if (parsedLine.size() != 2)
-                throw Error("Invalid result from the gc helper");
-            auto rawDestPath = rewriteStrings(parsedLine[0], unescapes);
-            auto retainer = rewriteStrings(parsedLine[1], unescapes);
-            if (!isInStore(rawDestPath)) continue;
-            try {
-                auto destPath = toStorePath(rawDestPath).first;
-                if (!isValidPath(destPath)) continue;
-                roots[destPath].insert(
-                    (!censor || isInDir(retainer, stateDir.get())) ? retainer : censored);
-            } catch (Error &) {
+        roots[readFile(path)].emplace(path);
+    } catch (SysError & e) {
+        if (e.errNo != ENOENT && e.errNo != EACCES)
+            throw;
+    }
+}
+#endif
+
+void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
+{
+    UncheckedRoots unchecked;
+
+    auto procDir = AutoCloseDir{opendir("/proc")};
+    if (procDir) {
+        struct dirent * ent;
+        auto digitsRegex = std::regex(R"(^\d+$)");
+        auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
+        auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
+        while (errno = 0, ent = readdir(procDir.get())) {
+            checkInterrupt();
+            if (std::regex_match(ent->d_name, digitsRegex)) {
+                try {
+                    readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
+                    readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
+
+                    auto fdStr = fmt("/proc/%s/fd", ent->d_name);
+                    auto fdDir = AutoCloseDir(opendir(fdStr.c_str()));
+                    if (!fdDir) {
+                        if (errno == ENOENT || errno == EACCES)
+                            continue;
+                        throw SysError("opening %1%", fdStr);
+                    }
+                    struct dirent * fd_ent;
+                    while (errno = 0, fd_ent = readdir(fdDir.get())) {
+                        if (fd_ent->d_name[0] != '.')
+                            readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked);
+                    }
+                    if (errno) {
+                        if (errno == ESRCH)
+                            continue;
+                        throw SysError("iterating /proc/%1%/fd", ent->d_name);
+                    }
+                    fdDir.reset();
+
+                    auto mapFile = fmt("/proc/%s/maps", ent->d_name);
+                    auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
+                    for (const auto & line : mapLines) {
+                        auto match = std::smatch{};
+                        if (std::regex_match(line, match, mapRegex))
+                            unchecked[match[1]].emplace(mapFile);
+                    }
+
+                    auto envFile = fmt("/proc/%s/environ", ent->d_name);
+                    auto envString = readFile(envFile);
+                    auto env_end = std::sregex_iterator{};
+                    for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
+                        unchecked[i->str()].emplace(envFile);
+                } catch (SystemError & e) {
+                    if (errno == ENOENT || errno == EACCES || errno == ESRCH)
+                        continue;
+                    throw;
+                }
             }
         }
-    } catch (EndOfFile &) {
+        if (errno)
+            throw SysError("iterating /proc");
+    }
+
+#if !defined(__linux__)
+    // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail.
+    // See: https://github.com/NixOS/nix/issues/3011
+    // Because of this we disable lsof when running the tests.
+    if (getEnv("_NIX_TEST_NO_LSOF") != "1") {
+        try {
+            std::regex lsofRegex(R"(^n(/.*)$)");
+            auto lsofLines =
+                tokenizeString<std::vector<std::string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n");
+            for (const auto & line : lsofLines) {
+                std::smatch match;
+                if (std::regex_match(line, match, lsofRegex))
+                    unchecked[match[1]].emplace("{lsof}");
+            }
+        } catch (ExecError & e) {
+            /* lsof not installed, lsof failed */
+        }
+    }
+#endif
+
+#if __linux__
+    readFileRoots("/proc/sys/kernel/modprobe", unchecked);
+    readFileRoots("/proc/sys/kernel/fbsplash", unchecked);
+    readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked);
+#endif
+
+    for (auto & [target, links] : unchecked) {
+        if (!isInStore(target)) continue;
+        try {
+            auto path = toStorePath(target).first;
+            if (!isValidPath(path)) continue;
+            debug("got additional root '%1%'", printStorePath(path));
+            if (censor)
+                roots[path].insert(censored);
+            else
+                roots[path].insert(links.begin(), links.end());
+        } catch (BadStorePath &) { }
     }
 }
 
-typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
 
 struct GCLimitReached { };
 
+
 void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 {
     bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
@@ -376,7 +516,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         readFile(*p);
 
     /* Start the server for receiving new roots. */
-    auto socketPath = stateDir.get() + rootsSocketPath;
+    auto socketPath = stateDir.get() + gcSocketPath;
     createDirs(dirOf(socketPath));
     auto fdServer = createUnixDomainSocket(socketPath, 0666);
 
@@ -485,7 +625,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
     printInfo("finding garbage collector roots...");
     Roots rootMap;
     if (!options.ignoreLiveness)
-        rootMap = findRoots(true);
+        findRootsNoTemp(rootMap, true);
 
     for (auto & i : rootMap) roots.insert(i.first);
 

src/libstore/globals.hh

@@ -124,13 +124,6 @@ public:
           section of the manual for supported store types and settings.
         )"};
 
-    Setting<std::string> gcSocketPath {
-        this,
-        "auto",
-        "gc-socket-path",
-        "Path to the socket used to communicate with an external GC."
-    };
-
     Setting<bool> keepFailed{this, false, "keep-failed",
         "Whether to keep temporary directories of failed builds."};
 
@@ -1101,8 +1094,8 @@ public:
         this, {}, "hashed-mirrors",
         R"(
           A list of web servers used by `builtins.fetchurl` to obtain files by
-          hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to
-          download the file from *hashed-mirror*/*ht*/*h*. This allows files to
+          hash. Given a hash algorithm *ha* and a base-16 hash *h*, Nix will try to
+          download the file from *hashed-mirror*/*ha*/*h*. This allows files to
           be downloaded even if they have disappeared from their original URI.
           For example, given an example mirror `http://tarballs.nixos.org/`,
           when building the derivation

src/libstore/legacy-ssh-store.hh

@@ -72,7 +72,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     virtual StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) override

src/libstore/local-fs-store.hh

@@ -43,7 +43,7 @@ public:
     LocalFSStore(const Params & params);
 
     void narFromPath(const StorePath & path, Sink & sink) override;
-    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
 
     /**
      * Creates symlink from the `gcRoot` to the `storePath` and

src/libstore/local-store.cc

@@ -1,5 +1,6 @@
 #include "local-store.hh"
 #include "globals.hh"
+#include "git.hh"
 #include "archive.hh"
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
@@ -1097,19 +1098,29 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
             if (info.ca) {
                 auto & specified = *info.ca;
                 auto actualHash = ({
-                    HashModuloSink caSink {
-                        specified.hash.algo,
-                        std::string { info.path.hashPart() },
-                    };
-                    PosixSourceAccessor accessor;
-                    dumpPath(
-                        *getFSAccessor(false),
-                        CanonPath { printStorePath(info.path) },
-                        caSink,
-                        specified.method.getFileIngestionMethod());
+                    auto accessor = getFSAccessor(false);
+                    CanonPath path { printStorePath(info.path) };
+                    Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++
+                    auto fim = specified.method.getFileIngestionMethod();
+                    switch (fim) {
+                    case FileIngestionMethod::Flat:
+                    case FileIngestionMethod::Recursive:
+                    {
+                        HashModuloSink caSink {
+                            specified.hash.algo,
+                            std::string { info.path.hashPart() },
+                        };
+                        dumpPath(*accessor, path, caSink, (FileSerialisationMethod) fim);
+                        h = caSink.finish().first;
+                        break;
+                    }
+                    case FileIngestionMethod::Git:
+                        h = git::dumpHash(specified.hash.algo, *accessor, path).hash;
+                        break;
+                    }
                     ContentAddress {
                         .method = specified.method,
-                        .hash = caSink.finish().first,
+                        .hash = std::move(h),
                     };
                 });
                 if (specified.hash != actualHash.hash) {
@@ -1137,7 +1148,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 StorePath LocalStore::addToStoreFromDump(
     Source & source0,
     std::string_view name,
-    ContentAddressMethod method,
+    FileSerialisationMethod dumpMethod,
+    ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     RepairFlag repair)
@@ -1190,7 +1202,13 @@ StorePath LocalStore::addToStoreFromDump(
     Path tempDir;
     AutoCloseFD tempDirFd;
 
-    if (!inMemory) {
+    bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod;
+
+    /* If the methods don't match, our streaming hash of the dump is the
+       wrong sort, and we need to rehash. */
+    bool inMemoryAndDontNeedRestore = inMemory && methodsMatch;
+
+    if (!inMemoryAndDontNeedRestore) {
         /* Drain what we pulled so far, and then keep on pulling */
         StringSource dumpSource { dump };
         ChainSource bothSource { dumpSource, source };
@@ -1199,17 +1217,23 @@ StorePath LocalStore::addToStoreFromDump(
         delTempDir = std::make_unique<AutoDelete>(tempDir);
         tempPath = tempDir + "/x";
 
-        restorePath(tempPath, bothSource, method.getFileIngestionMethod());
+        restorePath(tempPath, bothSource, dumpMethod);
 
         dumpBuffer.reset();
         dump = {};
     }
 
-    auto [hash, size] = hashSink->finish();
+    auto [dumpHash, size] = hashSink->finish();
+
+    PosixSourceAccessor accessor;
 
     auto desc = ContentAddressWithReferences::fromParts(
-        method,
-        hash,
+        hashMethod,
+        methodsMatch
+            ? dumpHash
+            : hashPath(
+                accessor, CanonPath { tempPath },
+                hashMethod.getFileIngestionMethod(), hashAlgo),
         {
             .others = references,
             // caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -1235,10 +1259,20 @@ StorePath LocalStore::addToStoreFromDump(
 
             autoGC();
 
-            if (inMemory) {
+            if (inMemoryAndDontNeedRestore) {
                 StringSource dumpSource { dump };
                 /* Restore from the buffer in memory. */
-                restorePath(realPath, dumpSource, method.getFileIngestionMethod());
+                auto fim = hashMethod.getFileIngestionMethod();
+                switch (fim) {
+                case FileIngestionMethod::Flat:
+                case FileIngestionMethod::Recursive:
+                    restorePath(realPath, dumpSource, (FileSerialisationMethod) fim);
+                    break;
+                case FileIngestionMethod::Git:
+                    // doesn't correspond to serialization method, so
+                    // this should be unreachable
+                    assert(false);
+                }
             } else {
                 /* Move the temporary path we restored above. */
                 moveFile(tempPath, realPath);
@@ -1246,8 +1280,8 @@ StorePath LocalStore::addToStoreFromDump(
 
             /* For computing the nar hash. In recursive SHA-256 mode, this
                is the same as the store hash, so no need to do it again. */
-            auto narHash = std::pair { hash, size };
-            if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
+            auto narHash = std::pair { dumpHash, size };
+            if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) {
                 HashSink narSink { HashAlgorithm::SHA256 };
                 dumpPath(realPath, narSink);
                 narHash = narSink.finish();
@@ -1367,7 +1401,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             PosixSourceAccessor accessor;
             std::string hash = hashPath(
                 accessor, CanonPath { linkPath },
-                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false);
+                FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false);
             if (hash != link.name) {
                 printError("link '%s' was modified! expected hash '%s', got '%s'",
                     linkPath, link.name, hash);

src/libstore/local-store.hh

@@ -180,7 +180,8 @@ public:
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method,
+        FileSerialisationMethod dumpMethod,
+        ContentAddressMethod hashMethod,
         HashAlgorithm hashAlgo,
         const StorePathSet & references,
         RepairFlag repair) override;
@@ -328,7 +329,7 @@ private:
 
     void findRootsNoTemp(Roots & roots, bool censor);
 
-    void findRootsNoTempNoExternalDaemon(Roots & roots, bool censor);
+    void findRuntimeRoots(Roots & roots, bool censor);
 
     std::pair<Path, AutoCloseFD> createTempDirInStore();
 

src/libstore/local.mk

@@ -6,7 +6,7 @@ libstore_DIR := $(d)
 
 libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc)
 
-libstore_LIBS = libutil libfindroots
+libstore_LIBS = libutil
 
 libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS)
 ifdef HOST_LINUX
@@ -28,7 +28,7 @@ ifeq ($(HAVE_SECCOMP), 1)
 endif
 
 libstore_CXXFLAGS += \
- -I src/libutil -I src/libstore -I src/libstore/build -I src/nix-find-roots/lib \
+ -I src/libutil -I src/libstore -I src/libstore/build \
  -DNIX_PREFIX=\"$(prefix)\" \
  -DNIX_STORE_DIR=\"$(storedir)\" \
  -DNIX_DATA_DIR=\"$(datadir)\" \

src/libstore/optimise-store.cc

@@ -151,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
         PosixSourceAccessor accessor;
         hashPath(
             accessor, CanonPath { path },
-            FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+            FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
     });
     debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true));
 
@@ -166,7 +166,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
                 PosixSourceAccessor accessor;
                 hashPath(
                     accessor, CanonPath { linkPath },
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first;
+                    FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first;
            })))
         {
             // XXX: Consider overwriting linkPath with our valid version.

src/libstore/remote-store.cc

@@ -13,6 +13,7 @@
 #include "derivations.hh"
 #include "pool.hh"
 #include "finally.hh"
+#include "git.hh"
 #include "logging.hh"
 #include "callback.hh"
 #include "filetransfer.hh"
@@ -508,12 +509,30 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
 StorePath RemoteStore::addToStoreFromDump(
     Source & dump,
     std::string_view name,
-    ContentAddressMethod method,
+    FileSerialisationMethod dumpMethod,
+    ContentAddressMethod hashMethod,
     HashAlgorithm hashAlgo,
     const StorePathSet & references,
     RepairFlag repair)
 {
-    return addCAToStore(dump, name, method, hashAlgo, references, repair)->path;
+    FileSerialisationMethod fsm;
+    switch (hashMethod.getFileIngestionMethod()) {
+    case FileIngestionMethod::Flat:
+        fsm = FileSerialisationMethod::Flat;
+        break;
+    case FileIngestionMethod::Recursive:
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    case FileIngestionMethod::Git:
+        // Use NAR; Git is not a serialization method
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    default:
+        assert(false);
+    }
+    if (fsm != dumpMethod)
+        unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination");
+    return addCAToStore(dump, name, hashMethod, hashAlgo, references, repair)->path;
 }
 
 

src/libstore/remote-store.hh

@@ -87,7 +87,8 @@ public:
     StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) override;
@@ -184,7 +185,7 @@ protected:
 
     friend struct ConnectionHandle;
 
-    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override;
+    virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
 
     virtual void narFromPath(const StorePath & path, Sink & sink) override;
 

src/libstore/store-api.cc

@@ -12,7 +12,9 @@
 #include "references.hh"
 #include "archive.hh"
 #include "callback.hh"
+#include "git.hh"
 #include "remote-store.hh"
+#include "posix-source-accessor.hh"
 // FIXME this should not be here, see TODO below on
 // `addMultipleToStore`.
 #include "worker-protocol.hh"
@@ -119,6 +121,9 @@ static std::string makeType(
 
 StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
 {
+    if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1)
+        throw Error("Git file ingestion must use SHA-1 hash");
+
     if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
     } else {
@@ -166,7 +171,7 @@ std::pair<StorePath, Hash> StoreDirConfig::computeStorePath(
     const StorePathSet & references,
     PathFilter & filter) const
 {
-    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter);
     return {
         makeFixedOutputPathFromCA(
             name,
@@ -192,10 +197,23 @@ StorePath Store::addToStore(
     PathFilter & filter,
     RepairFlag repair)
 {
+    FileSerialisationMethod fsm;
+    switch (method.getFileIngestionMethod()) {
+    case FileIngestionMethod::Flat:
+        fsm = FileSerialisationMethod::Flat;
+        break;
+    case FileIngestionMethod::Recursive:
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    case FileIngestionMethod::Git:
+        // Use NAR; Git is not a serialization method
+        fsm = FileSerialisationMethod::Recursive;
+        break;
+    }
     auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter);
+        dumpPath(accessor, path, sink, fsm, filter);
     });
-    return addToStoreFromDump(*source, name, method, hashAlgo, references, repair);
+    return addToStoreFromDump(*source, name, fsm, method, hashAlgo, references, repair);
 }
 
 void Store::addMultipleToStore(
@@ -355,9 +373,7 @@ ValidPathInfo Store::addToStoreSlow(
     NullFileSystemObjectSink blank;
     auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat
         ? (FileSystemObjectSink &) fileSink
-        : method.getFileIngestionMethod() == FileIngestionMethod::Recursive
-        ? (FileSystemObjectSink &) blank
-        : (abort(), (FileSystemObjectSink &)*(FileSystemObjectSink *)nullptr); // handled both cases
+        : (FileSystemObjectSink &) blank; // for recursive or git we do recursive
 
     /* The information that flows from tapped (besides being replicated in
        narSink), is now put in parseSink. */
@@ -369,6 +385,8 @@ ValidPathInfo Store::addToStoreSlow(
 
     auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256
         ? narHash
+        : method == FileIngestionMethod::Git
+        ? git::dumpHash(hashAlgo, accessor, srcPath).hash
         : caHashSink.finish().first;
 
     if (expectedCAHash && expectedCAHash != hash)

src/libstore/store-api.hh

@@ -466,14 +466,23 @@ public:
      * in `dump`, which is either a NAR serialisation (if recursive ==
      * true) or simply the contents of a regular file (if recursive ==
      * false).
-     * `dump` may be drained
      *
-     * \todo remove?
+     * `dump` may be drained.
+     *
+     * @param dumpMethod What serialisation format is `dump`, i.e. how
+     * to deserialize it. Must either match hashMethod or be
+     * `FileSerialisationMethod::Recursive`.
+     *
+     * @param hashMethod How content addressing? Need not match be the
+     * same as `dumpMethod`.
+     *
+     * @todo remove?
      */
     virtual StorePath addToStoreFromDump(
         Source & dump,
         std::string_view name,
-        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive,
+        ContentAddressMethod hashMethod = FileIngestionMethod::Recursive,
         HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
         const StorePathSet & references = StorePathSet(),
         RepairFlag repair = NoRepair) = 0;
@@ -772,7 +781,7 @@ protected:
      * Helper for methods that are not unsupported: this is used for
      * default definitions for virtual methods that are meant to be overriden.
      *
-     * \todo Using this should be a last resort. It is better to make
+     * @todo Using this should be a last resort. It is better to make
      * the method "virtual pure" and/or move it to a subclass.
      */
     [[noreturn]] void unsupported(const std::string & op)

src/libstore/uds-remote-store.hh

@@ -35,7 +35,7 @@ public:
     static std::set<std::string> uriSchemes()
     { return {"unix"}; }
 
-    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override
     { return LocalFSStore::getFSAccessor(requireValidPath); }
 
     void narFromPath(const StorePath & path, Sink & sink) override

src/libutil/canon-path.cc

@@ -8,7 +8,7 @@ CanonPath CanonPath::root = CanonPath("/");
 
 static std::string absPathPure(std::string_view path)
 {
-    return canonPathInner(path, [](auto &, auto &){});
+    return canonPathInner<UnixPathTrait>(path, [](auto &, auto &){});
 }
 
 CanonPath::CanonPath(std::string_view raw)

src/libutil/error.cc

@@ -11,9 +11,9 @@
 
 namespace nix {
 
-void BaseError::addTrace(std::shared_ptr<Pos> && e, HintFmt hint, bool frame)
+void BaseError::addTrace(std::shared_ptr<Pos> && e, HintFmt hint)
 {
-    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
+    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint });
 }
 
 void throwExceptionSelfCheck(){
@@ -61,8 +61,7 @@ inline bool operator<(const Trace& lhs, const Trace& rhs)
     // This formats a freshly formatted hint string and then throws it away, which
     // shouldn't be much of a problem because it only runs when pos is equal, and this function is
     // used for trace printing, which is infrequent.
-    return std::forward_as_tuple(lhs.hint.str(), lhs.frame)
-        < std::forward_as_tuple(rhs.hint.str(), rhs.frame);
+    return lhs.hint.str() < rhs.hint.str();
 }
 inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; }
 inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); }
@@ -373,7 +372,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
     // prepended to each element of the trace
     auto ellipsisIndent = "  ";
 
-    bool frameOnly = false;
     if (!einfo.traces.empty()) {
         // Stack traces seen since we last printed a chunk of `duplicate frames
         // omitted`.
@@ -384,7 +382,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
         for (const auto & trace : einfo.traces) {
             if (trace.hint.str().empty()) continue;
-            if (frameOnly && !trace.frame) continue;
 
             if (!showTrace && count > 3) {
                 oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n";
@@ -400,7 +397,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
 
             count++;
-            frameOnly = trace.frame;
 
             printTrace(oss, ellipsisIndent, count, trace);
         }

src/libutil/error.hh

@@ -64,7 +64,6 @@ void printCodeLines(std::ostream & out,
 struct Trace {
     std::shared_ptr<Pos> pos;
     HintFmt hint;
-    bool frame;
 };
 
 inline bool operator<(const Trace& lhs, const Trace& rhs);
@@ -162,7 +161,7 @@ public:
         addTrace(std::move(e), HintFmt(std::string(fs), args...));
     }
 
-    void addTrace(std::shared_ptr<Pos> && e, HintFmt hint, bool frame = false);
+    void addTrace(std::shared_ptr<Pos> && e, HintFmt hint);
 
     bool hasTrace() const { return !err.traces.empty(); }
 

src/libutil/experimental-features.cc

@@ -268,20 +268,6 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
         )",
     },
-    {
-        .tag = Xp::ExternalGCDaemon,
-        .name = "external-gc-daemon",
-        .description = R"(
-            Make the garbage collector use an external daemon for the tracing.
-
-            This makes it possible to run a multi-user Nix daemon as a non-root
-            user. Only the tracing daemon needs to be root. This reduces the attack
-            surface.
-
-            This requires more infrastructure and isn't directly supported by the
-            installer.
-        )"
-    },
     {
         .tag = Xp::VerifiedFetches,
         .name = "verified-fetches",

src/libutil/experimental-features.hh

@@ -35,7 +35,6 @@ enum struct ExperimentalFeature
     ReadOnlyLocalStore,
     ConfigurableImpureEnv,
     MountedSSHStore,
-    ExternalGCDaemon,
     VerifiedFetches,
 };
 

src/libutil/file-content-address.cc

@@ -1,16 +1,53 @@
 #include "file-content-address.hh"
 #include "archive.hh"
+#include "git.hh"
 
 namespace nix {
 
-FileIngestionMethod parseFileIngestionMethod(std::string_view input)
+static std::optional<FileSerialisationMethod> parseFileSerialisationMethodOpt(std::string_view input)
 {
     if (input == "flat") {
-        return FileIngestionMethod::Flat;
+        return FileSerialisationMethod::Flat;
     } else if (input == "nar") {
-        return FileIngestionMethod::Recursive;
+        return FileSerialisationMethod::Recursive;
     } else {
-        throw UsageError("Unknown file ingestion method '%s', expect `flat` or `nar`");
+        return std::nullopt;
+    }
+}
+
+FileSerialisationMethod parseFileSerialisationMethod(std::string_view input)
+{
+    auto ret = parseFileSerialisationMethodOpt(input);
+    if (ret)
+        return *ret;
+    else
+        throw UsageError("Unknown file serialiation method '%s', expect `flat` or `nar`");
+}
+
+
+FileIngestionMethod parseFileIngestionMethod(std::string_view input)
+{
+    if (input == "git") {
+        return FileIngestionMethod::Git;
+    } else {
+        auto ret = parseFileSerialisationMethodOpt(input);
+        if (ret)
+            return static_cast<FileIngestionMethod>(*ret);
+        else
+            throw UsageError("Unknown file ingestion method '%s', expect `flat`, `nar`, or `git`");
+    }
+}
+
+
+std::string_view renderFileSerialisationMethod(FileSerialisationMethod method)
+{
+    switch (method) {
+    case FileSerialisationMethod::Flat:
+        return "flat";
+    case FileSerialisationMethod::Recursive:
+        return "nar";
+    default:
+        assert(false);
     }
 }
 
@@ -19,9 +56,11 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method)
 {
     switch (method) {
     case FileIngestionMethod::Flat:
-        return "flat";
     case FileIngestionMethod::Recursive:
-        return "nar";
+        return renderFileSerialisationMethod(
+            static_cast<FileSerialisationMethod>(method));
+    case FileIngestionMethod::Git:
+        return "git";
     default:
         abort();
     }
@@ -31,14 +70,14 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method)
 void dumpPath(
     SourceAccessor & accessor, const CanonPath & path,
     Sink & sink,
-    FileIngestionMethod method,
+    FileSerialisationMethod method,
     PathFilter & filter)
 {
     switch (method) {
-    case FileIngestionMethod::Flat:
+    case FileSerialisationMethod::Flat:
         accessor.readFile(path, sink);
         break;
-    case FileIngestionMethod::Recursive:
+    case FileSerialisationMethod::Recursive:
         accessor.dumpPath(path, sink, filter);
         break;
     }
@@ -48,13 +87,13 @@ void dumpPath(
 void restorePath(
     const Path & path,
     Source & source,
-    FileIngestionMethod method)
+    FileSerialisationMethod method)
 {
     switch (method) {
-    case FileIngestionMethod::Flat:
+    case FileSerialisationMethod::Flat:
         writeFile(path, source);
         break;
-    case FileIngestionMethod::Recursive:
+    case FileSerialisationMethod::Recursive:
         restorePath(path, source);
         break;
     }
@@ -63,12 +102,28 @@ void restorePath(
 
 HashResult hashPath(
     SourceAccessor & accessor, const CanonPath & path,
-    FileIngestionMethod method, HashAlgorithm ht,
+    FileSerialisationMethod method, HashAlgorithm ha,
     PathFilter & filter)
 {
-    HashSink sink { ht };
+    HashSink sink { ha };
     dumpPath(accessor, path, sink, method, filter);
     return sink.finish();
 }
 
+
+Hash hashPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    FileIngestionMethod method, HashAlgorithm ht,
+    PathFilter & filter)
+{
+    switch (method) {
+    case FileIngestionMethod::Flat:
+    case FileIngestionMethod::Recursive:
+        return hashPath(accessor, path, (FileSerialisationMethod) method, ht, filter).first;
+    case FileIngestionMethod::Git:
+        return git::dumpHash(ht, accessor, path, filter).hash;
+    }
+    assert(false);
+}
+
 }

src/libutil/file-content-address.hh

@@ -8,37 +8,38 @@
 namespace nix {
 
 /**
- * An enumeration of the main ways we can serialize file system
+ * An enumeration of the ways we can serialize file system
  * objects.
  */
-enum struct FileIngestionMethod : uint8_t {
+enum struct FileSerialisationMethod : uint8_t {
     /**
-     * Flat-file hashing. Directly ingest the contents of a single file
+     * Flat-file. The contents of a single file exactly.
      */
-    Flat = 0,
+    Flat,
+
     /**
-     * Recursive (or NAR) hashing. Serializes the file-system object in
-     * Nix Archive format and ingest that.
+     * Nix Archive. Serializes the file-system object in
+     * Nix Archive format.
      */
-    Recursive = 1,
+    Recursive,
 };
 
 /**
- * Parse a `FileIngestionMethod` by name. Choice of:
+ * Parse a `FileSerialisationMethod` by name. Choice of:
  *
- *  - `flat`: `FileIngestionMethod::Flat`
- *  - `nar`: `FileIngestionMethod::Recursive`
+ *  - `flat`: `FileSerialisationMethod::Flat`
+ *  - `nar`: `FileSerialisationMethod::Recursive`
  *
- * Oppostite of `renderFileIngestionMethod`.
+ * Opposite of `renderFileSerialisationMethod`.
  */
-FileIngestionMethod parseFileIngestionMethod(std::string_view input);
+FileSerialisationMethod parseFileSerialisationMethod(std::string_view input);
 
 /**
- * Render a `FileIngestionMethod` by name.
+ * Render a `FileSerialisationMethod` by name.
  *
- * Oppostite of `parseFileIngestionMethod`.
+ * Opposite of `parseFileSerialisationMethod`.
  */
-std::string_view renderFileIngestionMethod(FileIngestionMethod method);
+std::string_view renderFileSerialisationMethod(FileSerialisationMethod method);
 
 /**
  * Dump a serialization of the given file system object.
@@ -46,28 +47,99 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method);
 void dumpPath(
     SourceAccessor & accessor, const CanonPath & path,
     Sink & sink,
-    FileIngestionMethod method,
+    FileSerialisationMethod method,
     PathFilter & filter = defaultPathFilter);
 
 /**
- * Restore a serialization of the given file system object.
+ * Restore a serialisation of the given file system object.
  *
  * @TODO use an arbitrary `FileSystemObjectSink`.
  */
 void restorePath(
     const Path & path,
     Source & source,
-    FileIngestionMethod method);
+    FileSerialisationMethod method);
+
 
 /**
  * Compute the hash of the given file system object according to the
  * given method.
  *
- * The hash is defined as (essentially) hashString(ht, dumpPath(path)).
+ * the hash is defined as (in pseudocode):
+ *
+ * ```
+ * hashString(ha, dumpPath(...))
+ * ```
  */
 HashResult hashPath(
     SourceAccessor & accessor, const CanonPath & path,
-    FileIngestionMethod method, HashAlgorithm ht,
+    FileSerialisationMethod method, HashAlgorithm ha,
+    PathFilter & filter = defaultPathFilter);
+
+/**
+ * An enumeration of the ways we can ingest file system
+ * objects, producing a hash or digest.
+ */
+enum struct FileIngestionMethod : uint8_t {
+    /**
+     * Hash `FileSerialisationMethod::Flat` serialisation.
+     */
+    Flat,
+
+    /**
+     * Hash `FileSerialisationMethod::Git` serialisation.
+     */
+    Recursive,
+
+    /**
+     * Git hashing. In particular files are hashed as git "blobs", and
+     * directories are hashed as git "trees".
+     *
+     * Unlike `Flat` and `Recursive`, this is not a hash of a single
+     * serialisation but a [Merkle
+     * DAG](https://en.wikipedia.org/wiki/Merkle_tree) of multiple
+     * rounds of serialisation and hashing.
+     *
+     * @note Git's data model is slightly different, in that a plain
+     * file doesn't have an executable bit, directory entries do
+     * instead. We decide treat a bare file as non-executable by fiat,
+     * as we do with `FileIngestionMethod::Flat` which also lacks this
+     * information. Thus, Git can encode some but all of Nix's "File
+     * System Objects", and this sort of hashing is likewise partial.
+     */
+    Git,
+};
+
+/**
+ * Parse a `FileIngestionMethod` by name. Choice of:
+ *
+ *  - `flat`: `FileIngestionMethod::Flat`
+ *  - `nar`: `FileIngestionMethod::Recursive`
+ *  - `git`: `FileIngestionMethod::Git`
+ *
+ * Opposite of `renderFileIngestionMethod`.
+ */
+FileIngestionMethod parseFileIngestionMethod(std::string_view input);
+
+/**
+ * Render a `FileIngestionMethod` by name.
+ *
+ * Opposite of `parseFileIngestionMethod`.
+ */
+std::string_view renderFileIngestionMethod(FileIngestionMethod method);
+
+/**
+ * Compute the hash of the given file system object according to the
+ * given method.
+ *
+ * Unlike the other `hashPath`, this works on an arbitrary
+ * `FileIngestionMethod` instead of `FileSerialisationMethod`, but
+ * doesn't return the size as this is this is not a both simple and
+ * useful defined for a merkle format.
+ */
+Hash hashPath(
+    SourceAccessor & accessor, const CanonPath & path,
+    FileIngestionMethod method, HashAlgorithm ha,
     PathFilter & filter = defaultPathFilter);
 
 }

src/libutil/file-path-impl.hh

@@ -10,6 +10,100 @@
 
 namespace nix {
 
+/**
+ * Unix-style path primives.
+ *
+ * Nix'result own "logical" paths are always Unix-style. So this is always
+ * used for that, and additionally used for native paths on Unix.
+ */
+struct UnixPathTrait
+{
+    using CharT = char;
+
+    using String = std::string;
+
+    using StringView = std::string_view;
+
+    constexpr static char preferredSep = '/';
+
+    static inline bool isPathSep(char c)
+    {
+        return c == '/';
+    }
+
+    static inline size_t findPathSep(StringView path, size_t from = 0)
+    {
+        return path.find('/', from);
+    }
+
+    static inline size_t rfindPathSep(StringView path, size_t from = StringView::npos)
+    {
+        return path.rfind('/', from);
+    }
+};
+
+
+/**
+ * Windows-style path primitives.
+ *
+ * The character type is a parameter because while windows paths rightly
+ * work over UTF-16 (*) using `wchar_t`, at the current time we are
+ * often manipulating them converted to UTF-8 (*) using `char`.
+ *
+ * (Actually neither are guaranteed to be valid unicode; both are
+ * arbitrary non-0 8- or 16-bit bytes. But for charcters with specifical
+ * meaning like '/', '\\', ':', etc., we refer to an encoding scheme,
+ * and also for sake of UIs that display paths a text.)
+ */
+template<class CharT0>
+struct WindowsPathTrait
+{
+    using CharT = CharT0;
+
+    using String = std::basic_string<CharT>;
+
+    using StringView = std::basic_string_view<CharT>;
+
+    constexpr static CharT preferredSep = '\\';
+
+    static inline bool isPathSep(CharT c)
+    {
+        return c == '/' || c == preferredSep;
+    }
+
+    static size_t findPathSep(StringView path, size_t from = 0)
+    {
+        size_t p1 = path.find('/', from);
+        size_t p2 = path.find(preferredSep, from);
+        return p1 == String::npos ? p2 :
+               p2 == String::npos ? p1 :
+               std::min(p1, p2);
+    }
+
+    static size_t rfindPathSep(StringView path, size_t from = String::npos)
+    {
+        size_t p1 = path.rfind('/', from);
+        size_t p2 = path.rfind(preferredSep, from);
+        return p1 == String::npos ? p2 :
+               p2 == String::npos ? p1 :
+               std::max(p1, p2);
+    }
+};
+
+
+/**
+ * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait`
+ * argument.
+ */
+using NativePathTrait =
+#ifdef _WIN32
+    WindowsPathTrait<char>
+#else
+    UnixPathTrait
+#endif
+    ;
+
+
 /**
  * Core pure path canonicalization algorithm.
  *
@@ -24,25 +118,26 @@ namespace nix {
  *   This is a chance to modify those two paths in arbitrary way, e.g. if
  *   "result" points to a symlink.
  */
-typename std::string canonPathInner(
-    std::string_view remaining,
+template<class PathDict>
+typename PathDict::String canonPathInner(
+    typename PathDict::StringView remaining,
     auto && hookComponent)
 {
     assert(remaining != "");
 
-    std::string result;
+    typename PathDict::String result;
     result.reserve(256);
 
     while (true) {
 
         /* Skip slashes. */
-        while (!remaining.empty() && remaining[0] == '/')
+        while (!remaining.empty() && PathDict::isPathSep(remaining[0]))
             remaining.remove_prefix(1);
 
         if (remaining.empty()) break;
 
         auto nextComp = ({
-            auto nextPathSep = remaining.find('/');
+            auto nextPathSep = PathDict::findPathSep(remaining);
             nextPathSep == remaining.npos ? remaining : remaining.substr(0, nextPathSep);
         });
 
@@ -53,14 +148,14 @@ typename std::string canonPathInner(
         /* If `..', delete the last component. */
         else if (nextComp == "..")
         {
-            if (!result.empty()) result.erase(result.rfind('/'));
+            if (!result.empty()) result.erase(PathDict::rfindPathSep(result));
             remaining.remove_prefix(2);
         }
 
         /* Normal component; copy it. */
         else {
-            result += '/';
-            if (const auto slash = remaining.find('/'); slash == result.npos) {
+            result += PathDict::preferredSep;
+            if (const auto slash = PathDict::findPathSep(remaining); slash == result.npos) {
                 result += remaining;
                 remaining = {};
             } else {
@@ -73,7 +168,7 @@ typename std::string canonPathInner(
     }
 
     if (result.empty())
-        result = "/";
+        result = typename PathDict::String { PathDict::preferredSep };
 
     return result;
 }

src/libutil/file-system.cc

@@ -22,10 +22,14 @@ namespace fs = std::filesystem;
 
 namespace nix {
 
-/** Treat the string as possibly an absolute path, by inspecting the start of it. Return whether it was probably intended to be absolute. */
+/**
+ * Treat the string as possibly an absolute path, by inspecting the
+ * start of it. Return whether it was probably intended to be
+ * absolute.
+ */
 static bool isAbsolute(PathView path)
 {
-    return !path.empty() && path[0] == '/';
+    return fs::path { path }.is_absolute();
 }
 
 
@@ -69,6 +73,9 @@ Path canonPath(PathView path, bool resolveSymlinks)
     if (!isAbsolute(path))
         throw Error("not an absolute path: '%1%'", path);
 
+    // For Windows
+    auto rootName = fs::path { path }.root_name();
+
     /* This just exists because we cannot set the target of `remaining`
        (the callback parameter) directly to a newly-constructed string,
        since it is `std::string_view`. */
@@ -78,7 +85,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
        arbitrary (but high) limit to prevent infinite loops. */
     unsigned int followCount = 0, maxFollow = 1024;
 
-    return canonPathInner(
+    auto ret = canonPathInner<NativePathTrait>(
         path,
         [&followCount, &temp, maxFollow, resolveSymlinks]
         (std::string & result, std::string_view & remaining) {
@@ -99,6 +106,10 @@ Path canonPath(PathView path, bool resolveSymlinks)
                 }
             }
         });
+
+    if (!rootName.empty())
+        ret = rootName.string() + std::move(ret);
+    return ret;
 }
 
 

src/libutil/fs-sink.cc

@@ -15,6 +15,7 @@ void copyRecursive(
     case SourceAccessor::tSymlink:
     {
         sink.createSymlink(to, accessor.readLink(from));
+        break;
     }
 
     case SourceAccessor::tRegular:
@@ -38,6 +39,7 @@ void copyRecursive(
                 sink, to + "/" + name);
             break;
         }
+        break;
     }
 
     case SourceAccessor::tMisc:

src/libutil/hash.cc

@@ -274,7 +274,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha
 {
     if (hashStr.empty()) {
         if (!ha)
-            throw BadHash("empty hash requires explicit hash type");
+            throw BadHash("empty hash requires explicit hash algorithm");
         Hash h(*ha);
         warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true));
         return h;

src/libutil/hash.hh

@@ -58,7 +58,7 @@ struct Hash
      * Parse the hash from a string representation in the format
      * "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
      * Subresource Integrity hash expression). If the 'type' argument
-     * is not present, then the hash type must be specified in the
+     * is not present, then the hash algorithm must be specified in the
      * string.
      */
     static Hash parseAny(std::string_view s, std::optional<HashAlgorithm> optAlgo);
@@ -200,7 +200,7 @@ std::optional<HashFormat> parseHashFormatOpt(std::string_view hashFormatName);
 std::string_view printHashFormat(HashFormat hashFormat);
 
 /**
- * Parse a string representing a hash type.
+ * Parse a string representing a hash algorithm.
  */
 HashAlgorithm parseHashAlgo(std::string_view s);
 

src/nix-env/user-env.cc

@@ -113,7 +113,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems,
         std::string str2 = str.str();
         StringSource source { str2 };
         state.store->addToStoreFromDump(
-            source, "env-manifest.nix", TextIngestionMethod {}, HashAlgorithm::SHA256, references);
+            source, "env-manifest.nix", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references);
     });
 
     /* Get the environment builder expression. */

src/nix-find-roots/.gitignore

@@ -1 +0,0 @@
-nix-find-roots

src/nix-find-roots/lib/find-roots.cc

@@ -1,232 +0,0 @@
-/**
- * @file
- *
- * A very simple utility to trace all the gc roots through the file-system
- * The reason for this program is that tracing these roots is the only part of
- * Nix that requires to run as root (because it requires reading through the
- * user home directories to resolve the indirect roots)
- *
- * This program intentionnally doesnt depend on any Nix library to reduce the attack surface.
- */
-
-#include <regex>
-#include <unistd.h>
-#include <vector>
-#include <algorithm>
-#include <fstream>
-#include <optional>
-
-#include "find-roots.hh"
-
-
-namespace nix::roots_tracer {
-namespace fs = std::filesystem;
-
-static std::string quoteRegexChars(const std::string & raw)
-{
-    static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])");
-    return std::regex_replace(raw, specialRegex, R"(\$&)");
-}
-static std::regex storePathRegex(const fs::path storeDir)
-{
-    return std::regex(quoteRegexChars(storeDir) + R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)");
-}
-
-static bool isInStore(fs::path storeDir, fs::path dir)
-{
-    return (std::search(dir.begin(), dir.end(), storeDir.begin(), storeDir.end()) == dir.begin());
-}
-
-static void traceStaticRoot(
-    const TracerConfig & opts,
-    int recursionsLeft,
-    TraceResult & res,
-    const fs::path & root,
-    const fs::file_status & status
-    )
-{
-    opts.debug("Considering file " + root.string());
-
-    if (recursionsLeft < 0)
-        return;
-
-    switch (status.type()) {
-        case fs::file_type::directory:
-            {
-                auto directory_iterator = fs::recursive_directory_iterator(root);
-                for (auto & child : directory_iterator)
-                    traceStaticRoot(opts, recursionsLeft, res, child.path(), child.symlink_status());
-            }
-            break;
-        case fs::file_type::symlink:
-            {
-                auto target = root.parent_path() / fs::read_symlink(root);
-                auto not_found = [&](std::string msg) {
-                    opts.debug("Error accessing the file " + target.string() + ": " + msg);
-                    opts.debug("(When resolving the symlink " + root.string() + ")");
-                    res.deadLinks.insert(root);
-                };
-                try {
-                    auto target_status = fs::symlink_status(target);
-                    if (target_status.type() == fs::file_type::not_found)
-                        not_found("Not found");
-
-                    if (isInStore(opts.storeDir, target)) {
-                        res.storeRoots[target].insert(root);
-                        return;
-                    } else {
-                        traceStaticRoot(opts, recursionsLeft - 1, res, target, target_status);
-                    }
-
-                } catch (fs::filesystem_error & e) {
-                    not_found(e.what());
-                }
-            }
-            break;
-        case fs::file_type::regular:
-            {
-                auto possibleStorePath = opts.storeDir / root.filename();
-                if (fs::exists(possibleStorePath))
-                res.storeRoots[possibleStorePath].insert(root);
-            }
-            break;
-        case fs::file_type::not_found:
-        case fs::file_type::block:
-        case fs::file_type::character:
-        case fs::file_type::fifo:
-        case fs::file_type::socket:
-        case fs::file_type::unknown:
-        case fs::file_type::none:
-        default:
-            break;
-    }
-}
-
-static void traceStaticRoot(
-    const TracerConfig & opts,
-    int recursionsLeft,
-    TraceResult & res,
-    const fs::path & root)
-{
-    try {
-        auto status = fs::symlink_status(root);
-        traceStaticRoot(opts, recursionsLeft, res, root, status);
-    } catch (fs::filesystem_error & e) {
-        opts.debug("Error accessing the file " + root.string() + ": " + e.what());
-    }
-}
-
-TraceResult traceStaticRoots(TracerConfig opts, std::set<fs::path> roots)
-{
-    int maxRecursionLevel = 2;
-    TraceResult res;
-    for (auto & root : roots)
-        traceStaticRoot(opts, maxRecursionLevel, res, root);
-    return res;
-}
-
-/**
- * Scan the content of the given file for al the occurences of something that looks
- * like a store path (i.e. that matches `storePathRegex(opts.storeDir)`) and add them
- * to `res`
- */
-static void scanFileContent(const TracerConfig & opts, const fs::path & fileToScan, Roots & res)
-{
-    if (!fs::exists(fileToScan))
-        return;
-
-    std::ostringstream contentStream;
-    {
-        std::ifstream fs;
-        fs.open(fileToScan);
-        fs >> contentStream.rdbuf();
-    }
-    std::string content = contentStream.str();
-    auto regex = storePathRegex(opts.storeDir);
-    auto firstMatch
-        = std::sregex_iterator { content.begin(), content.end(), regex };
-    auto fileEnd = std::sregex_iterator{};
-    for (auto i = firstMatch; i != fileEnd; ++i)
-        res[i->str()].emplace(fileToScan);
-}
-
-/**
- * Scan the content of a `/proc/[pid]/maps` file for regions that are mmaped to
- * a store path
- */
-static void scanMapsFile(const TracerConfig & opts, const fs::path & mapsFile, Roots & res)
-{
-    if (!fs::exists(mapsFile))
-        return;
-
-    static auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)");
-    std::stringstream mappedFile;
-    {
-        std::ifstream fs;
-        fs.open(mapsFile);
-        fs >> mappedFile.rdbuf();
-    }
-    std::string line;
-    while (std::getline(mappedFile, line)) {
-        auto match = std::smatch{};
-        if (std::regex_match(line, match, mapRegex)) {
-            auto matchedPath = fs::path(match[1]);
-            if (isInStore(opts.storeDir, matchedPath))
-                res[fs::path(match[1])].emplace(mapsFile);
-        }
-    }
-
-}
-
-Roots getRuntimeRoots(TracerConfig opts)
-{
-    auto procDir = fs::path("/proc");
-    if (!fs::exists(procDir))
-        return {};
-    Roots res;
-    auto digitsRegex = std::regex(R"(^\d+$)");
-    for (auto & procEntry : fs::directory_iterator(procDir)) {
-        // Only the directories whose name is a sequence of digits represent
-        // pids
-        if (!std::regex_match(procEntry.path().filename().string(), digitsRegex)
-            || !procEntry.is_directory())
-            continue;
-
-        opts.debug("Considering path " + procEntry.path().string());
-
-        // A set of paths used by the executable and possibly symlinks to a
-        // path in the store
-        std::set<fs::path> pathsToConsider;
-        pathsToConsider.insert(procEntry.path()/"exe");
-        pathsToConsider.insert(procEntry.path()/"cwd");
-        try {
-            auto fdDir = procEntry.path()/"fd";
-            for (auto & fdFile : fs::directory_iterator(fdDir))
-                pathsToConsider.insert(fdFile.path());
-        } catch (fs::filesystem_error & e) {
-            if (e.code().value() != ENOENT && e.code().value() != EACCES)
-                throw;
-        }
-        for (auto & path : pathsToConsider) try {
-            auto realPath = fs::read_symlink(path);
-            if (isInStore(opts.storeDir, realPath))
-                res[realPath].insert(path);
-        } catch (fs::filesystem_error &e) {
-            opts.debug(e.what());
-        }
-
-        // Scan the environment of the executable
-        scanFileContent(opts, procEntry.path()/"environ", res);
-        scanMapsFile(opts, procEntry.path()/"maps", res);
-    }
-
-    // Mostly useful for NixOS, but doesn’t hurt to check on other systems
-    // anyways
-    scanFileContent(opts, "/proc/sys/kernel/modprobe", res);
-    scanFileContent(opts, "/proc/sys/kernel/fbsplash", res);
-    scanFileContent(opts, "/proc/sys/kernel/poweroff_cmd", res);
-
-    return res;
-}
-
-}

src/nix-find-roots/lib/find-roots.hh

@@ -1,51 +0,0 @@
-#include <filesystem>
-#include <set>
-#include <map>
-#include <functional>
-
-namespace nix::roots_tracer {
-namespace fs = std::filesystem;
-
-class Error : public std::runtime_error {
-public:
-    using std::runtime_error::runtime_error;
-};
-
-inline void logNone(std::string_view)
-{ }
-
-struct TracerConfig {
-    const fs::path storeDir = "/nix/store";
-    const fs::path stateDir = "/nix/var/nix";
-    const fs::path socketPath = "/nix/var/nix/gc-socket/socket";
-
-    std::function<void(std::string_view msg)> log = logNone;
-    std::function<void(std::string_view msg)> debug = logNone;
-};
-
-/**
- * A value of type `Roots` is a mapping from a store path to the set of roots that keep it alive
- */
-typedef std::map<fs::path, std::set<fs::path>> Roots;
-
-struct TraceResult {
-    Roots storeRoots;
-    std::set<fs::path> deadLinks;
-};
-
-/**
- * Return the set of all the store paths that are reachable from the given set
- * of filesystem paths, by:
- * - descending into the directories
- * - following the symbolic links (at most twice)
- * - reading the name of regular files (when encountering a file
- *   `/foo/bar/abcdef`, the algorithm will try to access `/nix/store/abcdef`)
- *
- * Also returns the set of all dead links encountered during the process (so
- * that they can be removed if it makes sense).
- */
-TraceResult traceStaticRoots(TracerConfig opts, std::set<fs::path> initialRoots);
-
-Roots getRuntimeRoots(TracerConfig opts);
-
-}

src/nix-find-roots/local.mk

@@ -1,20 +0,0 @@
-libraries += libfindroots
-
-libfindroots_NAME = libnixfindroots
-
-libfindroots_DIR := $(d)/lib
-
-libfindroots_SOURCES := $(wildcard $(d)/lib/*.cc)
-
-programs += nix-find-roots
-
-nix-find-roots_DIR := $(d)
-
-nix-find-roots_SOURCES := $(d)/main.cc
-
-nix-find-roots_LIBS := libfindroots
-
-nix-find-roots_CXXFLAGS += \
-	-I src/nix-find-roots/lib
-
-nix-find-roots_INSTALL_DIR := $(libexecdir)/nix

src/nix-find-roots/main.cc

@@ -1,188 +0,0 @@
-#include "find-roots.hh"
-#include <getopt.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <iostream>
-#include <unistd.h>
-#include <cstring>
-#include <signal.h>
-
-using namespace nix::roots_tracer;
-
-void logStderr(std::string_view msg)
-{
-  std::cerr << msg << std::endl;
-}
-
-TracerConfig parseCmdLine(int argc, char** argv)
-{
-    std::function<void(std::string_view msg)> log = logStderr;
-    std::function<void(std::string_view msg)> debug = logNone;
-    fs::path storeDir = "/nix/store";
-    fs::path stateDir = "/nix/var/nix";
-    fs::path socketPath = "/nix/var/nix/gc-trace-socket/socket";
-
-    auto usage = [&]() {
-        std::string programName = argc > 0 ? argv[0] : "nix-find-roots";
-        std::cerr << "Usage: " << programName << " [--verbose|-v] [-s storeDir] [-d stateDir] [-l socketPath]" << std::endl;
-        exit(1);
-    };
-    static struct option long_options[] = {
-        { "verbose", no_argument, 0, 'v' },
-        { "socket_path", required_argument, 0, 'l' },
-        { "store_dir", required_argument, 0, 's' },
-        { "state_dir", required_argument, 0, 'd' },
-        { "help", no_argument, 0, 'h' },
-        { 0, 0, 0, 0 },
-    };
-
-    int option_index = 0;
-    int opt_char;
-    while((opt_char = getopt_long(argc, argv, "vd:s:l:h",
-                    long_options, &option_index)) != -1) {
-        switch (opt_char) {
-            case 0:
-                break;
-          break;
-            case '?':
-            case 'h':
-                usage();
-                break;
-            case 'v':
-                debug = logStderr;
-                break;
-            case 's':
-                storeDir = fs::path(optarg);
-                break;
-            case 'd':
-                stateDir = fs::path(optarg);
-                break;
-            case 'l':
-                socketPath = fs::path(optarg);
-                break;
-            default:
-                std::cerr << "Got invalid char: " << (char)opt_char << std::endl;
-                abort();
-        }
-    };
-    return TracerConfig {
-        .storeDir = storeDir,
-        .stateDir = stateDir,
-        .socketPath = socketPath,
-        .debug = debug,
-    };
-}
-
-/**
- * Return `original` with every newline or tab character escaped
- */
-std::string escape(std::string original)
-{
-    std::map<std::string, std::string> replacements = {
-        {"\n", "\\n"},
-        {"\t", "\\t"},
-    };
-    for (auto [oldStr, newStr] : replacements) {
-        size_t currentPos = 0;
-        while ((currentPos = original.find(oldStr, currentPos)) != std::string::npos) {
-            original.replace(currentPos, oldStr.length(), newStr);
-            currentPos += newStr.length();
-        }
-    }
-
-    return original;
-}
-
-#define SD_LISTEN_FDS_START 3 // Like in systemd
-
-int main(int argc, char * * argv)
-{
-    const TracerConfig opts = parseCmdLine(argc, argv);
-    const std::set<fs::path> standardRoots = {
-        opts.stateDir / fs::path("profiles"),
-        opts.stateDir / fs::path("gcroots"),
-    };
-
-    int mySock;
-
-    //  Handle socket-based activation by systemd.
-    auto rawListenFds = std::getenv("LISTEN_FDS");
-    if (rawListenFds) {
-        auto listenFds = std::string(rawListenFds);
-        auto listenPid = std::getenv("LISTEN_PID");
-        if (listenPid == nullptr || listenPid != std::to_string(getpid()) || listenFds != "1")
-            throw Error("unexpected systemd environment variables");
-        mySock = SD_LISTEN_FDS_START;
-    } else {
-        mySock = socket(PF_UNIX, SOCK_STREAM, 0);
-        if (mySock == -1) {
-            throw Error(std::string("Cannot create Unix domain socket, got") +
-                    std::strerror(errno));
-        }
-        struct sockaddr_un addr;
-        addr.sun_family = AF_UNIX;
-
-        auto socketDir = opts.socketPath.parent_path();
-        auto socketFilename = opts.socketPath.filename();
-        if (socketFilename.string().size() > sizeof(addr.sun_path))
-            throw Error(
-                    "Socket filename " + socketFilename.string() +
-                    " is too long, should be at most " +
-                    std::to_string(sizeof(addr.sun_path))
-                    );
-        chdir(socketDir.c_str());
-
-        fs::remove(socketFilename);
-        strcpy(addr.sun_path, socketFilename.c_str());
-        if (bind(mySock, (struct sockaddr*) &addr, sizeof(addr)) == -1) {
-            throw Error("Cannot bind to socket");
-        }
-
-        if (listen(mySock, 5) == -1)
-            throw Error("cannot listen on socket " + opts.socketPath.string());
-    }
-
-    // Ignore SIGPIPE so that an interrupted connection doesn’t stop the daemon
-    signal(SIGPIPE, SIG_IGN);
-
-    while (1) {
-        struct sockaddr_un remoteAddr;
-        socklen_t remoteAddrLen = sizeof(remoteAddr);
-        int remoteSocket = accept(
-                mySock,
-                (struct sockaddr*) & remoteAddr,
-                &remoteAddrLen
-                );
-
-        if (remoteSocket == -1) {
-            if (errno == EINTR) continue;
-            throw Error("Error accepting the connection");
-        }
-
-        opts.log("accepted connection");
-
-        auto printToSocket = [&](std::string_view s) {
-            send(remoteSocket, s.data(), s.size(), 0);
-        };
-
-        auto traceResult = traceStaticRoots(opts, standardRoots);
-        auto runtimeRoots = getRuntimeRoots(opts);
-        traceResult.storeRoots.insert(runtimeRoots.begin(), runtimeRoots.end());
-        for (auto & [rootInStore, externalRoots] : traceResult.storeRoots) {
-            for (auto & externalRoot : externalRoots) {
-                printToSocket(escape(rootInStore.string()));
-                printToSocket("\t");
-                printToSocket(escape(externalRoot.string()));
-                printToSocket("\n");
-            }
-
-        }
-        printToSocket("\n");
-        for (auto & deadLink : traceResult.deadLinks) {
-            printToSocket(escape(deadLink.string()));
-            printToSocket("\n");
-        }
-
-        close(remoteSocket);
-    }
-}

src/nix-store/nix-store.cc

@@ -555,7 +555,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
                 HashResult hash = hashPath(
                     *store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) },
 
-                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256);
+                    FileSerialisationMethod::Recursive, HashAlgorithm::SHA256);
                 info->narHash = hash.first;
                 info->narSize = hash.second;
             }

src/nix/add-to-store.cc

@@ -2,6 +2,7 @@
 #include "common-args.hh"
 #include "store-api.hh"
 #include "archive.hh"
+#include "git.hh"
 #include "posix-source-accessor.hh"
 #include "misc-store-flags.hh"
 

src/nix/develop.cc

@@ -226,7 +226,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     auto getEnvShPath = ({
         StringSource source { getEnvSh };
         evalStore->addToStoreFromDump(
-            source, "get-env.sh", TextIngestionMethod {}, HashAlgorithm::SHA256, {});
+            source, "get-env.sh", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, {});
     });
 
     drv.args = {store->printStorePath(getEnvShPath)};

src/nix/hash.cc

@@ -5,6 +5,7 @@
 #include "shared.hh"
 #include "references.hh"
 #include "archive.hh"
+#include "git.hh"
 #include "posix-source-accessor.hh"
 #include "misc-store-flags.hh"
 
@@ -66,9 +67,11 @@ struct CmdHashBase : Command
     {
         switch (mode) {
         case FileIngestionMethod::Flat:
-            return  "print cryptographic hash of a regular file";
+            return "print cryptographic hash of a regular file";
         case FileIngestionMethod::Recursive:
             return "print cryptographic hash of the NAR serialisation of a path";
+        case FileIngestionMethod::Git:
+            return "print cryptographic hash of the Git serialisation of a path";
         default:
             assert(false);
         };
@@ -77,17 +80,41 @@ struct CmdHashBase : Command
     void run() override
     {
         for (auto path : paths) {
+            auto makeSink = [&]() -> std::unique_ptr<AbstractHashSink> {
+                if (modulus)
+                    return std::make_unique<HashModuloSink>(hashAlgo, *modulus);
+                else
+                    return std::make_unique<HashSink>(hashAlgo);
+            };
 
-            std::unique_ptr<AbstractHashSink> hashSink;
-            if (modulus)
-                hashSink = std::make_unique<HashModuloSink>(hashAlgo, *modulus);
-            else
-                hashSink = std::make_unique<HashSink>(hashAlgo);
+            auto [accessor_, canonPath] = PosixSourceAccessor::createAtRoot(path);
+            auto & accessor = accessor_;
+            Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++
+            switch (mode) {
+            case FileIngestionMethod::Flat:
+            case FileIngestionMethod::Recursive:
+            {
+                auto hashSink = makeSink();
+                dumpPath(accessor, canonPath, *hashSink, (FileSerialisationMethod) mode);
+                h = hashSink->finish().first;
+                break;
+            }
+            case FileIngestionMethod::Git: {
+                std::function<git::DumpHook> hook;
+                hook = [&](const CanonPath & path) -> git::TreeEntry {
+                    auto hashSink = makeSink();
+                    auto mode = dump(accessor, path, *hashSink, hook);
+                    auto hash = hashSink->finish().first;
+                    return {
+                        .mode = mode,
+                        .hash = hash,
+                    };
+                };
+                h = hook(canonPath).hash;
+                break;
+            }
+            }
 
-            auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path);
-            dumpPath(accessor, canonPath, *hashSink, mode);
-
-            Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
             logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI));
         }

src/nix/main.cc

@@ -32,6 +32,24 @@ void chrootHelper(int argc, char * * argv);
 
 namespace nix {
 
+static bool haveProxyEnvironmentVariables()
+{
+    static const std::vector<std::string> proxyVariables = {
+        "http_proxy",
+        "https_proxy",
+        "ftp_proxy",
+        "HTTP_PROXY",
+        "HTTPS_PROXY",
+        "FTP_PROXY"
+    };
+    for (auto & proxyVariable: proxyVariables) {
+        if (getEnv(proxyVariable).has_value()) {
+            return true;
+        }
+    }
+    return false;
+}
+
 /* Check if we have a non-loopback/link-local network interface. */
 static bool haveInternet()
 {
@@ -55,6 +73,8 @@ static bool haveInternet()
         }
     }
 
+    if (haveProxyEnvironmentVariables()) return true;
+
     return false;
 }
 

src/nix/profile.cc

@@ -101,6 +101,15 @@ struct ProfileElement
     }
 };
 
+std::string getNameFromElement(const ProfileElement & element)
+{
+    std::optional<std::string> result = std::nullopt;
+    if (element.source) {
+        result = getNameFromURL(parseURL(element.source->to_string()));
+    }
+    return result.value_or(element.identifier());
+}
+
 struct ProfileManifest
 {
     using ProfileElementName = std::string;
@@ -189,12 +198,8 @@ struct ProfileManifest
 
     void addElement(ProfileElement element)
     {
-        auto name =
-            element.source
-            ? getNameFromURL(parseURL(element.source->to_string()))
-            : std::nullopt;
-        auto name2 = name ? *name : element.identifier();
-        addElement(name2, std::move(element));
+        auto name = getNameFromElement(element);
+        addElement(name, std::move(element));
     }
 
     nlohmann::json toJSON(Store & store) const
@@ -390,7 +395,26 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 
             element.updateStorePaths(getEvalStore(), store, res);
 
-            manifest.addElement(std::move(element));
+            auto elementName = getNameFromElement(element);
+
+            // Check if the element already exists.
+            auto existingPair = manifest.elements.find(elementName);
+            if (existingPair != manifest.elements.end()) {
+                auto existingElement = existingPair->second;
+                auto existingSource = existingElement.source;
+                auto elementSource = element.source;
+                if (existingSource
+                    && elementSource
+                    && existingElement.priority == element.priority
+                    && existingSource->originalRef == elementSource->originalRef
+                    && existingSource->attrPath == elementSource->attrPath
+                    ) {
+                    warn("'%s' is already installed", elementName);
+                    continue;
+                }
+            }
+
+            manifest.addElement(elementName, std::move(element));
         }
 
         try {

tests/functional/common/vars-and-functions.sh.in

@@ -19,7 +19,6 @@ export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
 export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
 unset NIX_USER_CONF_FILES
-export NIX_GC_SOCKET_PATH=$TEST_ROOT/gc.socket
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1
@@ -74,7 +73,7 @@ clearProfiles() {
 
 clearStore() {
     echo "clearing store..."
-    chmod -R +w "$NIX_STORE_DIR" || true
+    chmod -R +w "$NIX_STORE_DIR"
     rm -rf "$NIX_STORE_DIR"
     mkdir "$NIX_STORE_DIR"
     rm -rf "$NIX_STATE_DIR"
@@ -90,15 +89,6 @@ clearCacheCache() {
     rm -f $TEST_HOME/.cache/nix/binary-cache*
 }
 
-declare -A trapFunctions
-
-callTraps() {
-    for f in "${trapFunctions[@]}"; do
-        $f
-    done
-}
-trap callTraps EXIT
-
 startDaemon() {
     # Don’t start the daemon twice, as this would just make it loop indefinitely
     if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then
@@ -120,7 +110,6 @@ startDaemon() {
       fail "Didn’t manage to start the daemon"
     fi
     trap "killDaemon" EXIT
-    trapFunctions[killDaemon]=killDaemon
     # Save for if daemon is killed
     NIX_REMOTE_OLD=$NIX_REMOTE
     export NIX_REMOTE=daemon
@@ -143,7 +132,7 @@ killDaemon() {
     unset _NIX_TEST_DAEMON_PID
     # Restore old nix remote
     NIX_REMOTE=$NIX_REMOTE_OLD
-    trapFunctions[killDaemon]=:
+    trap "" EXIT
 }
 
 restartDaemon() {
@@ -153,36 +142,6 @@ restartDaemon() {
     startDaemon
 }
 
-startGcDaemon() {
-    # Start the daemon, wait for the socket to appear.  !!!
-    # ‘nix-daemon’ should have an option to fork into the background.
-    rm -f $NIX_GC_SOCKET_PATH
-    $(dirname $(command -v nix))/../libexec/nix/nix-find-roots \
-        -l "$NIX_GC_SOCKET_PATH" \
-        -d "$NIX_STATE_DIR" \
-        -s "$NIX_STORE_DIR" \
-        > /dev/null 2>&1 \
-            &
-    pidGcDaemon=$!
-    for ((i = 0; i < 30; i++)); do
-        if [[ -S $NIX_GC_SOCKET_PATH ]]; then break; fi
-        sleep 1
-    done
-    trapFunctions[killGcDaemon]=killGcDaemon
-}
-
-killGcDaemon() {
-    kill $pidGcDaemon
-    for i in {0..10}; do
-        kill -0 $pidGcDaemon || break
-        sleep 1
-    done
-    kill -9 $pidGcDaemon || true
-    wait $pidGcDaemon || true
-    trapFunctions[killGcDaemon]=:
-}
-
-
 if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi

tests/functional/flakes/inputs.sh

@@ -24,7 +24,7 @@ test_subdir_self_path() {
 }
 EOF
     (
-        nix build $baseDir?dir=b-low --no-link
+        nix build $baseDir/b-low --no-link
     )
 }
 test_subdir_self_path

tests/functional/gc-external-daemon/common.sh

@@ -1,6 +0,0 @@
-source ../common.sh
-
-enableFeatures "external-gc-daemon"
-echo "gc-socket-path = $NIX_GC_SOCKET_PATH" >> "$NIX_CONF_DIR"/nix.conf
-
-startGcDaemon

tests/functional/gc-external-daemon/gc-auto.sh

@@ -1,5 +0,0 @@
-source common.sh
-
-cd ..
-source ./gc-auto.sh
-

tests/functional/gc-external-daemon/gc-concurrent.sh

@@ -1,4 +0,0 @@
-source common.sh
-
-cd ..
-source ./gc-concurrent.sh

tests/functional/gc-external-daemon/gc-runtime.sh

@@ -1,5 +0,0 @@
-source common.sh
-
-cd ..
-source ./gc-runtime.sh
-

tests/functional/gc-external-daemon/gc.sh

@@ -1,4 +0,0 @@
-source common.sh
-
-cd ..
-source ./gc.sh

tests/functional/gc-external-daemon/local.mk

@@ -1,7 +0,0 @@
-gc-external-daemon-tests := \
-  $(d)/gc-auto.sh \
-  $(d)/gc.sh \
-  $(d)/gc-concurrent.sh \
-  $(d)/gc-runtime.sh
-
-install-tests-groups += gc-external-daemon

tests/functional/git-hashing/common.sh

@@ -0,0 +1,11 @@
+source ../common.sh
+
+clearStore
+clearCache
+
+# Need backend to support git-hashing too
+requireDaemonNewerThan "2.19"
+
+enableFeatures "git-hashing"
+
+restartDaemon

tests/functional/git-hashing/local.mk

@@ -0,0 +1,7 @@
+git-hashing-tests := \
+  $(d)/simple.sh
+
+install-tests-groups += git-hashing
+
+clean-files += \
+  $(d)/config.nix

tests/functional/git-hashing/simple.sh

@@ -0,0 +1,58 @@
+source common.sh
+
+repo="$TEST_ROOT/scratch"
+git init "$repo"
+
+git -C "$repo" config user.email "you@example.com"
+git -C "$repo" config user.name "Your Name"
+
+try () {
+    hash=$(nix hash path --mode git --format base16 --algo sha1 $TEST_ROOT/hash-path)
+    [[ "$hash" == "$1" ]]
+
+    git -C "$repo" rm -rf hash-path || true
+    cp -r "$TEST_ROOT/hash-path" "$TEST_ROOT/scratch/hash-path"
+    git -C "$repo" add hash-path
+    git -C "$repo" commit -m "x"
+    git -C "$repo" status
+    hash2=$(git -C "$TEST_ROOT/scratch" rev-parse HEAD:hash-path)
+    [[ "$hash2" = "$1" ]]
+}
+
+# blob
+rm -rf $TEST_ROOT/hash-path
+echo "Hello World" > $TEST_ROOT/hash-path
+try "557db03de997c86a4a028e1ebd3a1ceb225be238"
+
+# tree with children
+rm -rf $TEST_ROOT/hash-path
+mkdir $TEST_ROOT/hash-path
+echo "Hello World" > $TEST_ROOT/hash-path/hello
+echo "Run Hello World" > $TEST_ROOT/hash-path/executable
+chmod +x $TEST_ROOT/hash-path/executable
+try "e5c0a11a556801a5c9dcf330ca9d7e2c572697f4"
+
+rm -rf $TEST_ROOT/dummy1
+echo Hello World! > $TEST_ROOT/dummy1
+path1=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy1)
+hash1=$(nix-store -q --hash $path1)
+test "$hash1" = "sha256:1brffhvj2c0z6x8qismd43m0iy8dsgfmy10bgg9w11szway2wp9v"
+
+rm -rf $TEST_ROOT/dummy2
+mkdir -p $TEST_ROOT/dummy2
+echo Hello World! > $TEST_ROOT/dummy2/hello
+path2=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy2)
+hash2=$(nix-store -q --hash $path2)
+test "$hash2" = "sha256:1vhv7zxam7x277q0y0jcypm7hwhccbzss81vkdgf0ww5sm2am4y0"
+
+rm -rf $TEST_ROOT/dummy3
+mkdir -p $TEST_ROOT/dummy3
+mkdir -p $TEST_ROOT/dummy3/dir
+touch $TEST_ROOT/dummy3/dir/file
+echo Hello World! > $TEST_ROOT/dummy3/dir/file
+touch $TEST_ROOT/dummy3/dir/executable
+chmod +x $TEST_ROOT/dummy3/dir/executable
+echo Run Hello World! > $TEST_ROOT/dummy3/dir/executable
+path3=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy3)
+hash3=$(nix-store -q --hash $path3)
+test "$hash3" = "sha256:08y3nm3mvn9qvskqnf13lfgax5lh73krxz4fcjd5cp202ggpw9nv"

tests/functional/impure-env.sh

@@ -1,7 +1,7 @@
 source common.sh
 
 # Needs the config option 'impure-env' to work
-requireDaemonNewerThan "2.18.0pre20230816"
+requireDaemonNewerThan "2.19.0"
 
 enableFeatures "configurable-impure-env"
 restartDaemon

tests/functional/lang/eval-fail-duplicate-traces.err.exp

@@ -41,4 +41,11 @@ error:
              |                ^
             5|     if n > 0
 
+       … while calling the 'throw' builtin
+         at /pwd/lang/eval-fail-duplicate-traces.nix:7:10:
+            6|     then throwAfter (n - 1)
+            7|     else throw "Uh oh!";
+             |          ^
+            8| in
+
        error: Uh oh!

tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp

@@ -27,4 +27,11 @@ error:
              |      ^
             6|
 
+       … while calling the 'throw' builtin
+         at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:9:
+            4|   null
+            5|   [ (_: throw "Not the final value, but is still forced!") (_: 23) ]
+             |         ^
+            6|
+
        error: Not the final value, but is still forced!

tests/functional/lang/eval-fail-mutual-recursion.err.exp

@@ -54,4 +54,11 @@ error:
 
        (21 duplicate frames omitted)
 
+       … while calling the 'throw' builtin
+         at /pwd/lang/eval-fail-mutual-recursion.nix:34:10:
+           33|     then throwAfterB true 10
+           34|     else throw "Uh oh!";
+             |          ^
+           35| in
+
        error: Uh oh!

tests/functional/lang/eval-okay-inherit-from.err.exp

@@ -0,0 +1 @@
+trace: used

tests/functional/lang/eval-okay-inherit-from.exp

@@ -0,0 +1 @@
+[ 1 2 { __overrides = { y = { d = [ ]; }; }; c = [ ]; d = 4; x = { c = [ ]; }; y = «repeated»; } { inner = { c = 3; d = 4; }; } ]

tests/functional/lang/eval-okay-inherit-from.nix

@@ -0,0 +1,16 @@
+let
+  inherit (builtins.trace "used" { a = 1; b = 2; }) a b;
+  x.c = 3;
+  y.d = 4;
+
+  merged = {
+    inner = {
+      inherit (y) d;
+    };
+
+    inner = {
+      inherit (x) c;
+    };
+  };
+in
+  [ a b rec { x.c = []; inherit (x) c; inherit (y) d; __overrides.y.d = []; } merged ]

tests/functional/lang/parse-okay-inherits.exp

@@ -0,0 +1 @@
+(let b = 2; c = { }; in { inherit b; inherit (c) d e; a = 1; f = 3; })

tests/functional/lang/parse-okay-inherits.nix

@@ -0,0 +1,9 @@
+let
+  c = {};
+  b = 2;
+in {
+  a = 1;
+  inherit b;
+  inherit (c) d e;
+  f = 3;
+}

tests/functional/lang/parse-okay-subversion.exp

@@ -1 +1 @@
-({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); }))
+({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { inherit expat httpServer javaSwigBindings javahlBindings localServer pythonBindings sslSupport; builder = /foo/bar; db4 = (if localServer then db4 else null); httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); swig = (if (pythonBindings || javaSwigBindings) then swig else null); }))

tests/functional/nix-profile.sh

@@ -64,6 +64,9 @@ nix profile install $flake1Dir
 [[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]]
 unset NIX_CONFIG
 
+# Test conflicting package install.
+nix profile install $flake1Dir 2>&1 | grep "warning: 'flake1' is already installed"
+
 # Test upgrading a package.
 printf NixOS > $flake1Dir/who
 printf 2.0 > $flake1Dir/version

tests/nixos/default.nix

@@ -109,7 +109,7 @@ in
       nix.package = lib.mkForce pkgs.nixVersions.nix_2_13;
     };
   };
-
+  
   # TODO: (nixpkgs update) remoteBuildsSshNg_remote_2_18 = ...
 
   # Test our Nix as a builder for clients that are older
@@ -137,8 +137,6 @@ in
   # TODO: (nixpkgs update) remoteBuildsSshNg_local_2_18 = ...
   */
 
-  rootless-daemon = runNixOSTestFor "x86_64-linux" ./rootless-daemon.nix;
-
   nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix;
 
   nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix;

tests/nixos/rootless-daemon.nix

@@ -1,177 +0,0 @@
-{ pkgs, ... }:
-
-{
-  name = "rootless-daemon";
-
-  nodes.machine = { config, ... }: {
-    users.users.alice.isNormalUser = true;
-    #users.users.nix-daemon.isSystemUser = true;
-    users.users.nix-daemon.isNormalUser = true;
-    users.users.nix-daemon.group = "nix-daemon";
-    users.groups.nix-daemon = {};
-    environment.variables.NIX_REMOTE = "daemon"; # Even for root
-    virtualisation.writableStore = true;
-
-    boot.readOnlyNixStore = false;
-
-    # No root daemon
-    nix.enable = false;
-
-    # But put Nix on the path anyways
-    environment.systemPackages = [ pkgs.nix ];
-
-    # And install the unit files:
-    #
-    # - nix-gc-trace system-wide
-    # - nix-daemon per-user
-    systemd.packages = [
-       (pkgs.runCommand "shuffled-nix-systemd-unit-files" {} ''
-         mkdir -p $out/lib/systemd/{system,user}
-         ln -s ${pkgs.nix}/lib/systemd/system/nix-gc-trace.* $out/lib/systemd/system
-         ln -s ${pkgs.nix}/lib/systemd/system/nix-daemon.* $out/lib/systemd/user
-       '')
-    ];
-
-    # And prepare the socket dirs anyways
-    systemd.tmpfiles.rules = [
-      "d /nix/var/nix/daemon-socket 0755 nix-daemon root - -"
-      "d /nix/var/nix/gc-socket 0755 nix-daemon root - -"
-    ];
-
-    # Oops isn't working, because cannot disable Nix daemon and enable
-    # Nix settings yet: https://github.com/NixOS/nixpkgs/issues/263250.
-    nix.settings = {
-      experimental-features = [ "external-gc-daemon" ];
-    };
-    # Plan B given the above
-    #
-    # TODO delete once that issue is fixed.
-    environment.etc."nix/nix.conf".source = pkgs.writeTextFile {
-      name = "nix.conf";
-      text = ''
-        experimental-features = external-gc-daemon
-      '';
-    };
-
-    systemd.user.sockets.nix-daemon = {
-    };
-    systemd.user.services.nix-daemon = {
-      path = [ pkgs.nix ];
-      description = "Nix Daemon (non-root)";
-      unitConfig.ConditionUser = "nix-daemon";
-      # Make sure we do not get a "fork bomb" of the daemon trying to
-      # connect to itself.
-      serviceConfig.Environment= "NIX_REMOTE=local";
-    };
-
-    systemd.sockets.nix-gc-trace = {
-      restartTriggers = [ config.environment.etc."nix/nix.conf".source ];
-    };
-    systemd.services.nix-gc-trace = {
-      restartTriggers = [ config.environment.etc."nix/nix.conf".source ];
-      path = [ pkgs.nix ];
-      description = "Nix Find Roots";
-    };
-    # We must enable lingering so that the Systemd User D-Bus is
-    # enabled. We also cannot do this with loginctl enable-linger
-    # because it needs to happen before systemd is loaded.
-    #
-    # See https://github.com/NixOS/nixpkgs/issues/3702
-    #
-    # TODO after upgrading to 23.11 can use new NixOS option for this.
-    system.activationScripts = {
-      enableLingering = ''
-        # remove all existing lingering users
-        rm -rf /var/lib/systemd/linger
-        mkdir -p /var/lib/systemd/linger
-        # enable for the subset of declared users
-        touch /var/lib/systemd/linger/nix-daemon
-      '';
-    };
-  };
-
-  testScript = ''
-    import re
-
-    machine.wait_for_unit("multi-user.target")
-
-  ''
-
-  # Give ownership of the store dir and var to the nix-daemon user.
-  #
-  # We intentionally don't do `-R` on the store because store objects
-  # used by NixOS should still be owned by root.
-  + ''
-    machine.succeed("""
-      set -ex
-
-      chown nix-daemon /nix/store
-      chown -R nix-daemon /nix/var
-    """)
-  ''
-
-  # Test that alice indeed cannot modify the store; we don't want
-  # arbitrary users to have any more permissions than before!
-  + ''
-    machine.fail("su alice -c 'touch /nix/store/foo'")
-
-  ''
-
-  # Start and wait for our units
-  + ''
-    machine.start_job("nix-gc-trace.socket")
-    machine.wait_for_unit("nix-gc-trace.socket")
-
-    machine.start_job("nix-daemon.socket", user="nix-daemon")
-    machine.wait_for_unit("nix-daemon.socket", user="nix-daemon")
-
-  ''
-
-  # Create a store obect, remember its store path in Python.
-  + ''
-    two = machine.succeed("""
-    su --login alice -c "$(cat <<'EOF'
-      set -ex
-      export PS4='+(''${BASH_SOURCE[0]-$0}:$LINENO) '
-
-      echo ehHtmfuULXYyBV6NBk6QUi8iE0 > two
-      nix-store --add two
-    EOF
-    )"
-    """)
-    two = two.strip()
-    assert re.match(r'^/nix/store/.+-two$', two)
-
-  ''
-
-  # Make sure we cannot delete it when it has a GC root, but we can once
-  # that root is destroyed.
-  + ''
-    machine.succeed(f"""
-    su --login alice -c "$(cat <<'EOF'
-      set -ex
-      export PS4='+(''${{BASH_SOURCE[0]-$0}}:$LINENO) '
-
-      test -f {two}
-      nix-store --realize {two} --add-root foo
-      echo $two
-    EOF
-    )"
-    """)
-
-    machine.fail(f"su --login alice -c 'nix-store --delete {two}'")
-
-    machine.succeed(f"""
-    su --login alice -c "$(cat <<'EOF'
-      set -ex
-      export PS4='+(''${{BASH_SOURCE[0]-$0}}:$LINENO) '
-
-      test -f {two}
-      rm foo
-      nix-store --delete {two}
-      test ! -f {two}
-    EOF
-    )"
-    """)
-  '';
-}